From 387127404e6c1d84b3468c387d864877ed1c67fe Mon Sep 17 00:00:00 2001 From: Adrian Lizarraga Date: Mon, 24 Jun 2024 11:24:55 -0700 Subject: [PATCH] [ORT 1.18.1 Release] Update ORT numpy dependency to >=1.21.6,<2.0 (#21141) ### Description Updates the version of numpy required by onnxruntime to >=1.21.6,<2.0 ### Motivation and Context Numpy released version 2.0. The onnxruntime 1.18.1 release is using numpy < 2.0, so we need to update requirement files to only install versions between 1.21.6 and 2.0 (non-inclusive). --- cmake/CMakeLists.txt | 8 -------- cmake/onnxruntime_python.cmake | 3 +++ .../tools/transformers/models/llama/requirements.txt | 3 ++- .../tools/transformers/models/phi2/requirements.txt | 1 + onnxruntime/test/python/requirements.txt | 1 + requirements.txt.in => requirements.txt | 2 +- .../templates/jobs/win-ci-prebuild-steps.yml | 2 +- .../linux/docker/migraphx-ci-pipeline-env.Dockerfile | 2 +- .../github/linux/test_custom_ops_pytorch_export.sh | 2 +- tools/ci_build/github/windows/helpers.ps1 | 7 ++++++- 10 files changed, 17 insertions(+), 14 deletions(-) rename requirements.txt.in => requirements.txt (60%) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 550238b192a8d..8a1333206ca8d 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1390,14 +1390,6 @@ string(APPEND ORT_BUILD_INFO "build type=${CMAKE_BUILD_TYPE}") string(APPEND ORT_BUILD_INFO ", cmake cxx flags: ${CMAKE_CXX_FLAGS}") configure_file(onnxruntime_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_config.h) get_property(onnxruntime_GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) -if (onnxruntime_GENERATOR_IS_MULTI_CONFIG) - configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/Debug/requirements.txt) - configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/Release/requirements.txt) - configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/RelWithDebInfo/requirements.txt) - configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/MinSizeRel/requirements.txt) -else() - configure_file(../requirements.txt.in ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt) -endif() if (onnxruntime_USE_CUDA) set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index b3669931d33dd..5f24eb0deeedb 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -565,6 +565,9 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy ${ONNXRUNTIME_ROOT}/__init__.py $/onnxruntime/ + COMMAND ${CMAKE_COMMAND} -E copy + ${REPO_ROOT}/requirements.txt + $ COMMAND ${CMAKE_COMMAND} -E copy ${REPO_ROOT}/ThirdPartyNotices.txt $/onnxruntime/ diff --git a/onnxruntime/python/tools/transformers/models/llama/requirements.txt b/onnxruntime/python/tools/transformers/models/llama/requirements.txt index ce4b3f6a09ba5..678e77f5ee292 100644 --- a/onnxruntime/python/tools/transformers/models/llama/requirements.txt +++ b/onnxruntime/python/tools/transformers/models/llama/requirements.txt @@ -1,7 +1,8 @@ optimum>=1.14.1 transformers>=4.33.2,<= 4.37.2 torch>=2.2.0 +numpy>=1.21.6,<2.0 onnx==1.16.0 datasets>=2.8.0 protobuf==3.20.2 -psutil \ No newline at end of file +psutil diff --git a/onnxruntime/python/tools/transformers/models/phi2/requirements.txt b/onnxruntime/python/tools/transformers/models/phi2/requirements.txt index 0b2ea0df93a96..fac2fd07e7f4a 100644 --- a/onnxruntime/python/tools/transformers/models/phi2/requirements.txt +++ b/onnxruntime/python/tools/transformers/models/phi2/requirements.txt @@ -1,3 +1,4 @@ +numpy>=1.21.6,<2.0 onnx==1.16.0 transformers>=4.36.2 onnxscript>=0.1.0.dev20240126 diff --git a/onnxruntime/test/python/requirements.txt b/onnxruntime/test/python/requirements.txt index 5d8e356d0fc07..74dc704ba5ff9 100644 --- a/onnxruntime/test/python/requirements.txt +++ b/onnxruntime/test/python/requirements.txt @@ -1,2 +1,3 @@ +numpy>=1.21.6,<2.0 onnx==1.16.0 pytest diff --git a/requirements.txt.in b/requirements.txt similarity index 60% rename from requirements.txt.in rename to requirements.txt index 89242061fb119..ccf0a100df474 100644 --- a/requirements.txt.in +++ b/requirements.txt @@ -1,6 +1,6 @@ coloredlogs flatbuffers -numpy >= @Python_NumPy_VERSION@ +numpy>=1.21.6,<2.0 packaging protobuf sympy diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml index ef48244bbb299..8c2aaaa0f5d82 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml @@ -74,7 +74,7 @@ steps: workingFolder: '$(Build.BinariesDirectory)' - script: | - python -m pip install --upgrade "setuptools>=68.2.2" wheel numpy flatbuffers + python -m pip install --upgrade "setuptools>=68.2.2" wheel "numpy>=1.21.6,<2.0" flatbuffers workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' diff --git a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile index 6c71631368822..88369c0e22105 100644 --- a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile +++ b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile @@ -80,4 +80,4 @@ RUN ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ${CONDA_ENVIRONMENT_PATH}/bi # Install migraphx RUN apt update && apt install -y migraphx -RUN pip install numpy packaging ml_dtypes==0.3.0 +RUN pip install "numpy>=1.21.6,<2.0" packaging ml_dtypes==0.3.0 diff --git a/tools/ci_build/github/linux/test_custom_ops_pytorch_export.sh b/tools/ci_build/github/linux/test_custom_ops_pytorch_export.sh index 56f5ff9f9eac0..5efeecd0c9912 100755 --- a/tools/ci_build/github/linux/test_custom_ops_pytorch_export.sh +++ b/tools/ci_build/github/linux/test_custom_ops_pytorch_export.sh @@ -2,7 +2,7 @@ pip3 install --user --upgrade pip -pip3 install --user numpy==1.19.0 torch pytest +pip3 install --user "numpy>=1.21.6,<2.0" torch pytest pip3 install --user /build/Release/dist/*.whl export PYTHONPATH=/onnxruntime_src/tools:/usr/local/lib/python3.8/site-packages:$PYTHONPATH diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1 index 0e7d279c9fa49..74c0800165d1f 100644 --- a/tools/ci_build/github/windows/helpers.ps1 +++ b/tools/ci_build/github/windows/helpers.ps1 @@ -637,7 +637,12 @@ function Install-ONNX { } Write-Host "Installing python packages..." - [string[]]$pip_args = "-m", "pip", "install", "-qq", "--disable-pip-version-check", "setuptools>=68.2.2", "wheel", "numpy", "protobuf==$protobuf_version" + + # Get the required numpy version from onnxruntime's requirements.txt (e.g., numpy>=1.21.6,<2.0) + $numpy_req_match_info = Get-Content "$src_root\requirements.txt" | Select-String -pattern 'numpy' + $numpy_to_install = $numpy_req_match_info.ToString() + + [string[]]$pip_args = "-m", "pip", "install", "-qq", "--disable-pip-version-check", "setuptools>=68.2.2", "wheel", "$numpy_to_install", "protobuf==$protobuf_version" &"python.exe" $pip_args if ($lastExitCode -ne 0) { exit $lastExitCode