Update Linux Azure CI (openvinotoolkit#10739)

syaifulnizamiphone7 · Mar 2, 2022 · bea352f · bea352f
1 parent 180f15e
commit bea352f
Showing 1 changed file with 143 additions and 51 deletions.
diff --git a/.ci/azure/linux_arm64.yml b/.ci/azure/linux_arm64.yml
@@ -26,16 +26,28 @@ jobs:
     system.debug: true
     VSTS_HTTP_RETRY: 5
     VSTS_HTTP_TIMEOUT: 200
+    PYTHON_ARM_VERSION: "3.8.12"
+    PYTHON_EXEC: "python3.8"
+    OPENVINO_ARCH: 'aarch64'
+    NUM_PROC: 1
     BUILD_TYPE: Release
     OPENVINO_REPO_DIR: $(Build.Repository.LocalPath)
     OPENVINO_CONTRIB_REPO_DIR: $(OPENVINO_REPO_DIR)/../openvino_contrib
     OPENCV_REPO_DIR: $(OPENVINO_REPO_DIR)/../opencv
+    BUILD_PYTHON: $(WORK_DIR)/build_python 
+    BUILD_OPENCV: $(WORK_DIR)/build_opencv
+    BUILD_OPENVINO: $(WORK_DIR)/build
+    BUILD_OPENVINO_PYTHON: $(WORK_DIR)/build_python
+    BUILD_OPEN_MODEL_ZOO: $(WORK_DIR)/build_open_model_zoo
+    INSTALL_OPENVINO: $(WORK_DIR)/install_openvino
+    INSTALL_PYTHON: $(INSTALL_OPENVINO)/extras/python 
+    INSTALL_OPENCV: $(INSTALL_OPENVINO)/extras/opencv
+    INSTALL_OPEN_MODEL_ZOO: $(INSTALL_OPENVINO)/extras/open_model_zoo
     WORK_DIR: $(Pipeline.Workspace)/_w
-    BUILD_DIR: $(WORK_DIR)/build
-    BUILD_DIR_OPENCV: $(WORK_DIR)/build_opencv
-    TMP_DIR: /mnt/tmp
     SHARE_DIR: /mount/cinfsshare/onnxtestdata
-    CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64
+    TMP_DIR: /mnt/tmp
+    OPENVINO_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64
+    OPENCV_CCACHE_DIR: $(SHARE_DIR)/ccache/master/linux_arm64_opencv
 
   steps:
   - script: |
@@ -56,17 +68,21 @@ jobs:
       df
       lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
       free -h
+      echo "##vso[task.setvariable variable=NUM_PROC]$(nproc --all)"
+      echo "NUM_PROC=$(NUM_PROC)"
     displayName: 'System information'
 
   - script: |
       rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
-      rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
+      mkdir -p $(BUILD_OPENCV) $(BUILD_OPENVINO) $(BUILD_OPENVINO_PYTHON) $(BUILD_PYTHON) $(BUILD_OPEN_MODEL_ZOO)
+      mkdir -p $(INSTALL_OPENVINO) $(INSTALL_PYTHON) $(INSTALL_OPENCV) $(INSTALL_OPEN_MODEL_ZOO)
       sudo rm -rf $(TMP_DIR) ; sudo mkdir $(TMP_DIR) ; sudo chmod 777 -R $(TMP_DIR)
       sudo mkdir -p $(SHARE_DIR)
       sudo apt --assume-yes update && sudo apt --assume-yes install nfs-common
       sudo mount -vvv -t nfs cinfsshare.file.core.windows.net:/cinfsshare/onnxtestdata $(SHARE_DIR) -o vers=4,minorversion=1,sec=sys
-      mkdir -p $(CCACHE_DIR)
-    displayName: 'Make directory'
+      mkdir -p $(OPENVINO_CCACHE_DIR)
+      mkdir -p $(OPENCV_CCACHE_DIR)
+    displayName: 'Make directories'
 
   - checkout: self
     clean: true
@@ -83,74 +99,150 @@ jobs:
   - script: |
       set -e
       $(OPENVINO_REPO_DIR)/install_build_dependencies.sh
-      # Move into contrib install_build_dependencies.sh
-      sudo apt --assume-yes install scons crossbuild-essential-arm64 libprotoc-dev protobuf-compiler
-      # OpenCV should provide install_build_dependencies.sh as well
-      # Move into resources
-      git clone https://github.com/opencv/opencv.git --depth 1 $(OPENCV_REPO_DIR)
-      # Speed up build
-      wget https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip
-      unzip ninja-linux.zip
-      sudo cp -v ninja /usr/local/bin/
-    workingDirectory: $(WORK_DIR)
+      export CCACHE_DIR=$(OPENCV_CCACHE_DIR)
+      export CCACHE_TEMPDIR=$(TMP_DIR)/ccache
+      export CCACHE_BASEDIR=$(Pipeline.Workspace)
+      export CCACHE_MAXSIZE=50G
+      export USE_CCACHE=1
+      export PYTHON_ARM_VERSION=$(PYTHON_ARM_VERSION)
+      export NUM_PROC=$(NUM_PROC)
+      export BUILD_PYTHON=$(BUILD_PYTHON)
+      export WORK_DIR=$(WORK_DIR)
+      export INSTALL_PYTHON=$(INSTALL_PYTHON)
+      export BUILD_TYPE=$(BUILD_TYPE)
+      export OPENVINO_REPO_DIR=$(OPENVINO_REPO_DIR)
+      export INSTALL_OPENCV=$(INSTALL_OPENCV)
+      export PYTHON_EXEC=$(PYTHON_EXEC)
+      export OPENCV_REPO_DIR=$(OPENCV_REPO_DIR)
+      export BUILD_OPENCV=$(BUILD_OPENCV)
+      export INSTALL_OPENVINO=$(INSTALL_OPENVINO)
+      $(OPENVINO_CONTRIB_REPO_DIR)/modules/arm_plugin/scripts/install_build_dependencies.sh
+    workingDirectory: $(BUILD_OPENVINO)
     displayName: 'Install dependencies'
 
   - task: CMake@1
     inputs:
       cmakeArgs: >
         -GNinja
         -DVERBOSE_BUILD=ON
+        -DOpenCV_DIR=$(INSTALL_OPENCV)/cmake 
+        -DENABLE_OPENCV=OFF 
+        -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8 
+        -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib/libpython3.8.so 
+        -DENABLE_PYTHON=ON 
+        -DPYTHON_MODULE_EXTENSION=".so" 
+        -DENABLE_TESTS=ON 
+        -DENABLE_FUNCTIONAL_TESTS=ON 
+        -DENABLE_GAPI_TESTS=OFF 
+        -DENABLE_GAPI_PREPROCESSING=OFF 
+        -DENABLE_DATA=OFF 
+        -DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib 
+        -DTHREADING=SEQ -DENABLE_LTO=ON 
+        -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake 
         -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
-        -DBUILD_LIST=imgcodecs,videoio,highgui
-        -DCMAKE_TOOLCHAIN_FILE=$(OPENCV_REPO_DIR)/platforms/linux/aarch64-gnu.toolchain.cmake
-        $(OPENCV_REPO_DIR)
-      workingDirectory: $(BUILD_DIR_OPENCV)
+        -DENABLE_SAMPLES=ON
+        -DBUILD_java_api=OFF
+        -DENABLE_INTEL_MYRIAD=OFF
+        -DTHREADING=SEQ
+        -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DARM_COMPUTE_SCONS_JOBS=$(NUM_PROC)
+        -DOUTPUT_ROOT=$(INSTALL_OPENVINO)
+        -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO)
+        $(OPENVINO_REPO_DIR)
+      workingDirectory: $(BUILD_OPENVINO)
+    displayName: 'CMake OpenVINO ARM plugin'
 
-  - script: ninja
-    workingDirectory: $(BUILD_DIR_OPENCV)
-    displayName: 'Build OpenCV Linux ARM64'
+  - script: |
+      export CCACHE_DIR=$(OPENVINO_CCACHE_DIR)
+      export CCACHE_TEMPDIR=$(TMP_DIR)/ccache
+      export CCACHE_BASEDIR=$(Pipeline.Workspace)
+      export CCACHE_MAXSIZE=50G
+      export USE_CCACHE=1
+      ninja
+    workingDirectory: $(BUILD_OPENVINO)
+    displayName: 'Build OpenVINO ARM plugin'
 
   - script: ninja install
-    workingDirectory: $(BUILD_DIR_OPENCV)
-    displayName: 'Install OpenCV Linux ARM64'
+    workingDirectory: $(BUILD_OPENVINO)
+    displayName: 'Install OpenVINO ARM plugin'
 
   - task: CMake@1
     inputs:
       cmakeArgs: >
         -GNinja
-        -DVERBOSE_BUILD=ON
+        -DInferenceEngineDeveloperPackage_DIR=$(BUILD_OPENVINO)
+        -DENABLE_PYTHON=ON 
+        -DPYTHON_EXECUTABLE=$(INSTALL_PYTHON)/bin/python3.8 
+        -DPYTHON_INCLUDE_DIRS=$(INSTALL_PYTHON)/include/python3.8 
+        -DPYTHON_LIBRARIES=$(INSTALL_PYTHON)/lib
+        -DPYTHON3_NUMPY_INCLUDE_DIRS=/usr/local/lib/python3.8/site-packages/numpy/core/include 
+        -DPYTHON_MODULE_EXTENSION=".so"
+        -DPYBIND11_FINDPYTHON=OFF
+        -DPYBIND11_NOPYTHON=OFF
+        -DPYTHONLIBS_FOUND=TRUE
         -DCMAKE_BUILD_TYPE=$(BUILD_TYPE)
-        -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
-        -DOpenCV_DIR=$(BUILD_DIR_OPENCV)/install/lib/cmake/opencv4
-        -DENABLE_OPENCV=OFF
-        -DENABLE_TESTS=ON
-        -DENABLE_SAMPLES=ON
-        -DBUILD_java_api=OFF
-        -DENABLE_INTEL_MYRIAD=OFF
-        -DTHREADING=SEQ
-        -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
+        -DENABLE_DATA=OFF
+        -DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath-link,$(INSTALL_OPENCV)/lib 
+        -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake 
         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         -DCMAKE_C_COMPILER_LAUNCHER=ccache
-        $(OPENVINO_REPO_DIR)
-      workingDirectory: $(BUILD_DIR)
-
-  - script: ls -alR $(OPENVINO_REPO_DIR)/temp/
-    displayName: 'List temp SDKs'
-
-  - script: ccache --zero-stats --max-size=50G --show-config
-    displayName: 'Clean ccache stats'
+        -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPENVINO)
+        $(OPENVINO_REPO_DIR)/src/bindings/python
+      workingDirectory: $(BUILD_OPENVINO_PYTHON)
+    displayName: 'CMake OpenVINO python binding'
 
   - script: |
-      export CCACHE_DIR=$(CCACHE_DIR)
+      export CCACHE_DIR=$(OPENVINO_CCACHE_DIR)
       export CCACHE_TEMPDIR=$(TMP_DIR)/ccache
       export CCACHE_BASEDIR=$(Pipeline.Workspace)
       export CCACHE_MAXSIZE=50G
+      export USE_CCACHE=1
       ninja
-    workingDirectory: $(BUILD_DIR)
-    displayName: 'Build Linux ARM64'
+    workingDirectory: $(BUILD_OPENVINO_PYTHON)
+    displayName: 'Build OpenVINO python binding'
+
+  - script: ninja install
+    workingDirectory: $(BUILD_OPENVINO_PYTHON)
+    displayName: 'Install OpenVINO python binding'
+
+  - task: CMake@1
+    inputs:
+      cmakeArgs: >
+        -GNinja
+        -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) 
+        -DENABLE_PYTHON=ON 
+        -DPYTHON_EXECUTABLE=/usr/local/bin/python3.8
+        -DPYTHON_INCLUDE_DIR=$(INSTALL_PYTHON)/include/python3.8 
+        -DPYTHON_LIBRARY=$(INSTALL_PYTHON)/lib
+        -DCMAKE_TOOLCHAIN_FILE=$(OPENVINO_REPO_DIR)/cmake/arm64.toolchain.cmake
+        -DOpenVINO_DIR=$(BUILD_OPENVINO)
+        -DInferenceEngine_DIR=$(BUILD_OPENVINO)
+        -DOpenCV_DIR=$(INSTALL_OPENCV)/cmake 
+        -Dngraph_DIR=$(BUILD_OPENVINO)
+        -DIE_EXTRA_MODULES=$(OPENVINO_CONTRIB_REPO_DIR)/modules
+        -DCMAKE_INSTALL_PREFIX=$(INSTALL_OPEN_MODEL_ZOO)
+        $(OPENVINO_REPO_DIR)/thirdparty/open_model_zoo/demos
+      workingDirectory: $(BUILD_OPEN_MODEL_ZOO)
+    displayName: 'CMake Open Model Zoo demos'
 
-  - script: ccache --show-stats
-    displayName: 'Show ccache stats'
+  - script: ninja
+    workingDirectory: $(BUILD_OPEN_MODEL_ZOO)
+    displayName: 'Build Open Model Zoo demos'
 
-  - script: ls -alR $(OPENVINO_REPO_DIR)/bin/
-    displayName: 'List binary files'
+  - script: ninja install
+    workingDirectory: $(BUILD_OPEN_MODEL_ZOO)
+    displayName: 'Install Open Model Zoo demos'
+
+  - script: |
+      cp -r $(BUILD_OPEN_MODEL_ZOO)/$(OPENVINO_ARCH)/$(BUILD_TYPE)/* $(INSTALL_OPEN_MODEL_ZOO)/
+      zip -9 -r $(Build.ArtifactStagingDirectory)/openvino_$(OPENVINO_ARCH)_linux.zip ./*
+    workingDirectory: $(INSTALL_OPENVINO)
+    displayName: 'Create OpenVINO ARM64 linux package'
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathToPublish: $(Build.ArtifactStagingDirectory)
+      artifactName: 'openvino_aarch64_linux'
+    displayName: 'Publish OpenVINO AArch64 linux package'