CI FWs/PyThon version update (#1272)

Signed-off-by: chensuyue <suyue.chen@intel.com>
Co-authored-by: yiliu30 <yi4.liu@intel.com>
Signed-off-by: bmyrcha <bartosz.myrcha@intel.com>

Author: 2 people

.azure-pipelines/docker/Dockerfile.devel

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG UBUNTU_VER=20.04
+ARG UBUNTU_VER=22.04
 FROM ubuntu:${UBUNTU_VER} as devel
 
 # See http://bugs.python.org/issue19846

.azure-pipelines/docker/DockerfileCodeScan.devel

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG UBUNTU_VER=20.04
+ARG UBUNTU_VER=22.04
 FROM ubuntu:${UBUNTU_VER} as devel
 
 # See http://bugs.python.org/issue19846

.azure-pipelines/docker/DockerfileWithNC.devel

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG UBUNTU_VER=20.04
+ARG UBUNTU_VER=22.04
 FROM ubuntu:${UBUNTU_VER} as devel
 
 # See http://bugs.python.org/issue19846

.azure-pipelines/scripts/codeScan/pylint/pylint.sh

@@ -20,6 +20,7 @@ apt-get install -y --no-install-recommends --fix-missing \
         build-essential
 
 pip install -r /neural-compressor/requirements.txt
+pip install cmake
 
 pip install torch==1.12.0 \
             horovod \
@@ -52,13 +53,14 @@ elif [ "${scan_module}" = "neural_insights" ]; then
 fi
 
 python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \
---ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo /neural-compressor/${scan_module} \
->$log_dir/pylint.json
+--ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo,horovod,transformers \
+/neural-compressor/${scan_module} > $log_dir/pylint.json
 
 exit_code=$?
 
 $BOLD_YELLOW && echo " -----------------  Current pylint cmd start --------------------------" && $RESET
-echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/${scan_module} > $log_dir/pylint.json"
+echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo,horovod,transformers
+/neural-compressor/${scan_module}>$log_dir/pylint.json"
 $BOLD_YELLOW && echo " -----------------  Current pylint cmd end --------------------------" && $RESET
 
 $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------" && $RESET

.azure-pipelines/scripts/models/env_setup.sh

@@ -87,20 +87,14 @@ elif [[ "${framework}" == "pytorch" ]]; then
     pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html
     pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html
 elif [[ "${framework}" == "onnxrt" ]]; then
-    pip install onnx==1.14.0
+    pip install onnx==1.14.1
     pip install onnxruntime==${fwk_ver}
 elif [[ "${framework}" == "mxnet" ]]; then
     pip install numpy==1.23.5
     echo "re-install pycocotools resolve the issue with numpy..."
     pip uninstall pycocotools -y
     pip install --no-cache-dir pycocotools
-    if [[ "${fwk_ver}" == "1.7.0" ]]; then
-        pip install mxnet==${fwk_ver}.post2
-    elif [[ "${fwk_ver}" == "1.6.0" ]]; then
-        pip install mxnet-mkl==${mxnet_version}
-    else
-        pip install mxnet==${fwk_ver}
-    fi
+    pip install mxnet==${fwk_ver}
 fi
 
 if [ -f "requirements.txt" ]; then

.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh

@@ -22,7 +22,7 @@ do
 done
 
 FRAMEWORK="onnxrt"
-FRAMEWORK_VERSION="1.15.0"
+FRAMEWORK_VERSION="1.15.1"
 
 inc_new_api=false
 # ======== set up config for onnxrt models ========

.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh

@@ -22,8 +22,8 @@ do
 done
 
 FRAMEWORK="pytorch"
-FRAMEWORK_VERSION="2.0.0+cpu"
-TORCH_VISION_VERSION="0.15.1+cpu"
+FRAMEWORK_VERSION="2.0.1+cpu"
+TORCH_VISION_VERSION="0.15.2+cpu"
 
 inc_new_api=false
 # ======== set up config for pytorch models ========

.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh

@@ -22,7 +22,7 @@ do
 done
 
 FRAMEWORK="tensorflow"
-FRAMEWORK_VERSION="2.12.0"
+FRAMEWORK_VERSION="2.13.0"
 
 inc_new_api=false
 # ======== set up config for tensorflow models ========

.azure-pipelines/scripts/ut/env_setup.sh

@@ -24,7 +24,8 @@ echo "========= test case is ${test_case}"
 if [[ "${tensorflow_version}" == *"-official" ]]; then
     pip install tensorflow==${tensorflow_version%-official}
 elif [[ "${tensorflow_version}" == "spr-base" ]]; then
-    pip install /tf_dataset/tf_binary/221212/tensorflow*.whl
+    pip install /tf_dataset/tf_binary/230928/tensorflow*.whl
+    pip install cmake
     pip install protobuf==3.20.3
     pip install horovod==0.27.0
     if [[ $? -ne 0 ]]; then
@@ -48,10 +49,13 @@ if [[ "${torchvision_version}" != "" ]]; then
 fi
 
 if [[ "${ipex_version}" == "1.13.0+cpu" ]]; then
-    ipex_whl="https://github.com/intel/intel-extension-for-pytorch/releases/download/v1.13.0%2Bcpu/intel_extension_for_pytorch-1.13.0-cp38-cp38-manylinux2014_x86_64.whl"
+    ipex_whl="https://github.com/intel/intel-extension-for-pytorch/releases/download/v1.13.0%2Bcpu/intel_extension_for_pytorch-1.13.0-cp310-cp310-manylinux2014_x86_64.whl"
     pip install $ipex_whl
 elif [[ "${ipex_version}" == "2.0.0+cpu" ]]; then
-    ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.0%2Bcpu-cp38-cp38-linux_x86_64.whl"
+    ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.0%2Bcpu-cp310-cp310-linux_x86_64.whl"
+    pip install $ipex_whl
+elif [[ "${ipex_version}" == "2.0.1+cpu" ]]; then
+    ipex_whl="https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_stable/cpu/intel_extension_for_pytorch-2.0.100%2Bcpu-cp310-cp310-linux_x86_64.whl"
     pip install $ipex_whl
 elif [[ "${ipex_version}" == "2.1.0" ]]; then
     pip install /tf_dataset/pt_binary/ww32/torch-*.whl
@@ -65,7 +69,11 @@ fi
 
 if [[ "${onnxruntime_version}" != "" ]]; then
     pip install onnxruntime==${onnxruntime_version}
-    pip install onnxruntime-extensions==0.8.0
+    if [[ "${onnxruntime_version}" == "1.14"* ]]; then
+        pip install onnxruntime-extensions==0.8.0
+    else
+        pip install onnxruntime-extensions
+    fi
     pip install optimum
 fi
 
@@ -79,8 +87,10 @@ fi
 
 # install special test env requirements
 # common deps
-pip install transformers
+pip install cmake
 pip install horovod
+pip install transformers
+
 if [[ $(echo "${test_case}" | grep -c "others") != 0 ]];then
     pip install tf_slim xgboost accelerate==0.21.0
 elif [[ $(echo "${test_case}" | grep -c "nas") != 0 ]]; then

.azure-pipelines/scripts/ut/ut_fwk_version.sh

@@ -4,23 +4,28 @@ echo "export UT fwk version..."
 test_mode=$1
 
 if [ "$test_mode" == "coverage" ]; then
-    export tensorflow_version='2.12.0'
-    export pytorch_version='2.0.0+cpu'
-    export torchvision_version='0.15.1+cpu'
-    export ipex_version='2.0.0+cpu'
-    export onnx_version='1.13.1'
-    export onnxruntime_version='1.14.1'
+    export tensorflow_version='2.13.0'
+    export pytorch_version='2.0.1+cpu'
+    export torchvision_version='0.15.2+cpu'
+    export ipex_version='2.0.1+cpu'
+    export onnx_version='1.14.1'
+    export onnxruntime_version='1.15.1'
     export mxnet_version='1.9.1'
 else
-    export tensorflow_version='2.11.0'
+    export tensorflow_version='2.12.0'
     export pytorch_version='1.13.0+cpu'
     export torchvision_version='0.14.0+cpu'
     export ipex_version='1.13.0+cpu'
-    export onnx_version='1.13.0'
-    export onnxruntime_version='1.13.1'
+    export onnx_version='1.13.1'
+    export onnxruntime_version='1.14.1'
     export mxnet_version='1.9.1'
 fi
 
+# import torch before import tensorflow
+cd /neural-compressor/test || exit 1
+find . -name "test*.py" | xargs sed -i 's/import tensorflow as tf/import torch; import tensorflow as tf/g'
+find . -name "test*.py" | xargs sed -i 's/import tensorflow.compat.v1 as tf/import torch; import tensorflow.compat.v1 as tf/g'
+find . -name "test*.py" | xargs sed -i 's/from tensorflow import keras/import torch; from tensorflow import keras/g'