Enable linking MxNET with MKL static libraries on MacOS

Add proper mkl_threading flags for Mac Os. Enable all tests that are for MacOS + MKL tests. Rebuild numpy with MKL BLAS (instead of OpenBLAS).
apache · leezu · Oct 13, 2021 · Jul 30, 2021 · Jul 30, 2021 · Aug 4, 2021
commit 51957929de2958e10b6ae0d97ab87f104e113443
@@ -44,7 +44,7 @@ jobs:
 
       - name: Test project
         run: |
-          # python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          # MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
-          # python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
+          python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'not test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          MXNET_ENGINE_TYPE=NaiveEngine python3 -m pytest -n 4 --durations=50 --verbose tests/python/unittest/ -k 'test_operator and not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'not serial'
+          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_external_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)' -m 'serial'
           python3 -m pytest -n 4 --durations=50 --verbose tests/python/mkl
@@ -126,20 +126,13 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   # ---[ MKL Options
   file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX "INTEL_MKL_VERSION")
   string(REGEX MATCH "([0-9]+)" MKL_VERSION ${MKL_VERSION_DEF})
-  if(UNIX AND NOT APPLE)
+  if(UNIX)
     # Single dynamic library interface leads to conflicts between intel omp and llvm omp
     # https://github.com/apache/incubator-mxnet/issues/17641
     # Fixed in oneMKL 2021.3: [MKLD-11109] MKL is opening libgomp.so instead of
     # libgomp.so.1 while SDL=1 & MKL_THREADING_LAYER=GNU
     cmake_dependent_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON
       "NOT BLA_STATIC;MKL_VERSION GREATER_EQUAL 20210003" OFF)
-  elseif(APPLE)
-    # TODO: Do more research on MKL on MacOs as there seems to be some library issues.
-    # As a consequence link static MKL libraries.
-    # TODO: Single dynamic library (SDL) interface for MacOS seems to have:
-    # 'INTEL MKL ERROR: dlopen(/opt/intel/oneapi/mkl/2021.3.0/lib/libmkl_cdft_core.1.dylib,
-    # 9):Symbol not found: _mkl_dft_dfticomputebackward' issue.
-    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF)
   else()
     option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
   endif()

diff --git a/config/distribution/darwin_cpu_mkl.cmake b/config/distribution/darwin_cpu_mkl.cmake
@@ -20,6 +20,7 @@ set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
 set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
 
 set(USE_BLAS "mkl" CACHE STRING "BLAS Vendor")
+set(BLA_STATIC ON CACHE BOOL "Use static libraries")
 
 set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
 set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")

diff --git a/config/distribution/linux_cpu_mkl.cmake b/config/distribution/linux_cpu_mkl.cmake
@@ -20,6 +20,8 @@ set(CFLAGS "-mno-avx" CACHE STRING "CFLAGS")
 set(CXXFLAGS "-mno-avx" CACHE STRING "CXXFLAGS")
 
 set(USE_BLAS "mkl" CACHE STRING "BLAS Vendor")
+set(BLA_STATIC ON CACHE BOOL "Use static libraries")
+
 set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
 set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
 set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")

diff --git a/src/initialize.cc b/src/initialize.cc
@@ -238,7 +238,7 @@ void LibraryInitializer::init_mkl_dynamic_library() {
 #else
   int interface = MKL_INTERFACE_LP64;
 #endif
-#if defined(__INTEL_LLVM_COMPILER)
+#if defined(__INTEL_LLVM_COMPILER) || defined(__APPLE__)
   mkl_set_threading_layer(MKL_THREADING_INTEL);
 #else
   mkl_set_threading_layer(MKL_THREADING_GNU);

@@ -66,6 +66,11 @@ source $DIR/zmq.sh
 source $DIR/lz4.sh
 if [[ $BLAS == 'mkl' ]]; then
     source ${DIR}/mkl.sh
+    source ${DIR}/numpy_mkl.sh
+    if [[ $PLATFORM == 'darwin' ]]; then
+        # export this path to find iomp5 needed by MKL according to Intel Link Line Advisor
+        export DYLD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/oneapi/compiler/${INTEL_MKL}/mac/compiler
+    fi
 fi
 
 

@@ -20,8 +20,8 @@
 # This script downloads OneMKL
 # TODO @akarbown: Get more general link to download the OneMKL (instead of the generated one)
 set -ex
-INTEL_MKL="2021.3.0"
-if [[  (! -e /opt/intel/oneapi/mkl/)]]; then
+export INTEL_MKL="2021.3.0"
+if [[  (! -e /opt/intel/oneapi/mkl/) ]]; then
     >&2 echo "Downloading mkl..."
 
     if [[ $PLATFORM == 'darwin' ]]; then
@@ -43,7 +43,7 @@ if [[  (! -e /opt/intel/oneapi/mkl/)]]; then
         rm ${DEPS_PATH}/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
         add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
         apt-get update && \
-        apt install -y intel-oneapi-mkl-${INTEL_MKL} intel-oneapi-mkl-common-${INTEL_MKL}  intel-oneapi-mkl-devel
+        apt install -y intel-oneapi-mkl-${INTEL_MKL} intel-oneapi-mkl-common-${INTEL_MKL} intel-oneapi-mkl-devel-${INTEL_MKL}
     else
         >&2 echo "Not available"
     fi

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script rebuilds numpy so that it will use MKL BLAS instead of OpenBLAS.
+set -x
+
+# check if numpy uses openblas
+set +e
+python3 -c "from numpy import show_config; show_config()" | grep 'openblas_info'
+if [[ $? -eq 0 ]] && [[ -e /opt/intel/oneapi/mkl/ ]] && [[ ! -e ~/.numpy-site.cfg ]]; then
+  # create file and add to it MKL configuration
+  if [[ $PLATFORM == 'darwin' ]]; then
+    echo "[mkl]
+  library_dirs = /opt/intel/oneapi/compiler/$INTEL_MKL/mac/compiler/lib:/opt/intel/oneapi/mkl/$INTEL_MKL/lib
+  include_dirs = /opt/intel/oneapi/mkl/$INTEL_MKL/include
+  libraries = mkl_rt,iomp5
+  extra_link_args = -Wl,-rpath,/opt/intel/oneapi/mkl/$INTEL_MKL/lib,-rpath,/opt/intel/oneapi/compiler/$INTEL_MKL/mac/compiler/lib" >> ~/.numpy-site.cfg
+  else
+    echo "[mkl]
+  library_dirs = /opt/intel/oneapi/compiler/$INTEL_MKL/linux/compiler/lib/intel64_lin:/opt/intel/oneapi/mkl/$INTEL_MKL/lib/intel64
+  include_dirs = /opt/intel/oneapi/mkl/$INTEL_MKL/include
+  libraries = mkl_rt,iomp5
+  extra_link_args = -Wl,-rpath,/opt/intel/oneapi/mkl/$INTEL_MKL/lib/intel64,-rpath,/opt/intel/oneapi/compiler/$INTEL_MKL/linux/compiler/lib/intel64_lin" >> ~/.numpy-site.cfg
+  fi
+
+  # reinstall numpy to use MKL BLAS
+  pip3 install numpy==1.19.5 --no-binary numpy --force-reinstall --no-cache-dir
+fi
+set -e
@@ -22,7 +22,7 @@
 set -ex
 OPENCV_VERSION=3.4.2
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if [[ $PLATFORM == 'linux' ]] && [[ ! $BLAS == 'mkl' ]]; then
+if [[ $PLATFORM == 'linux' ]] && [[ $BLAS == 'open' ]]; then
     OPENCV_LAPACK_OPTIONS=" \
           -D OpenBLAS_HOME=$DEPS_PATH \
           -D OpenBLAS_INCLUDE_DIR=$DEPS_PATH/include \

@@ -42,8 +42,6 @@ cmake -GNinja -C $cmake_config \
 ninja
 if [[ ! $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
     patchelf --set-rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/intel64/:\$ORIGIN" --force-rpath libmxnet.so
-elif [[ $PLATFORM == 'darwin' ]] && [[ $BLAS == 'mkl' ]]; then
-    install_name_tool -add_rpath "/opt/intel/oneapi/mkl/${INTEL_MKL}/lib/:/opt/intel/oneapi/compiler/${INTEL_MKL}/mac/compiler/lib/:\$ORIGIN" libmxnet.dylib
 fi
 cd -