[FEATURE] Enables dynamic linking with MKL and compiler based OpenMP

OneMKL 2021.3 fixed linking OpenMP while using SDL and
MKL_THREADING_LAYER set to GNU.

ci/docker/Dockerfile.build.ubuntu

@@ -38,11 +38,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     export OS_RELEASE="$(cat /etc/os-release)" && \
     apt-get update && \
     apt-get install -y wget software-properties-common && \
-    if [[ ${OS_RELEASE} == *"Bionic"* ]]; then \
-        wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add -; \
-        apt-add-repository "deb https://apt.repos.intel.com/mkl all main"; \
-        INTEL_MKL="-2020.0-088"; \
-    fi && \
+    wget -qO - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -O - | apt-key add -; \
+    add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"; \
+    INTEL_MKL="-2021.3.0"; \
     apt-get update && \
     apt-get install -y \
         ## Utilities
@@ -64,7 +62,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         g++ \
         g++-7 \
         g++-8 \
-        intel-mkl${INTEL_MKL} \
+        intel-oneapi-mkl${INTEL_MKL} \
+        intel-oneapi-mkl-devel${INTEL_MKL} \
         libomp-dev \
         ## Dependencies
         libgomp1 \

cmake/ChooseBlas.cmake

@@ -122,10 +122,19 @@ set(FORTRAN_DIR \\\"\$\{CMAKE_Fortran_IMPLICIT_LINK_DIRECTORIES\}\\\")
   endif()
 elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
   # ---[ MKL Options
+  find_path(MKL_INCLUDE_DIR mkl.h
+    HINTS ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl ${INTEL_OPT_ROOT}/oneapi/mkl/latest
+    PATHS $ENV{MKLROOT} $ENV{MKLROOT}/latest
+    PATH_SUFFIXES include mkl REQUIRED)
+  file(STRINGS ${MKL_INCLUDE_DIR}/mkl_version.h MKL_VERSION_DEF REGEX "INTEL_MKL_VERSION")
+  string(REGEX MATCH "([0-9]+)" MKL_VERSION ${MKL_VERSION_DEF})
   if(UNIX)
     # Single dynamic library interface leads to conflicts between intel omp and llvm omp
     # https://github.com/apache/incubator-mxnet/issues/17641
-    option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" OFF)
+    # Fixed in oneMKL 2021.3: [MKLD-11109] MKL is opening libgomp.so instead of
+    # libgomp.so.1 while SDL=1 & MKL_THREADING_LAYER=GNU
+    cmake_dependent_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON
+      "NOT BLA_STATIC;MKL_VERSION GREATER_EQUAL 20210003" OFF)
   else()
     option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON)
   endif()
@@ -155,14 +164,16 @@ elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
     endif()
   endif()
   # Setting up BLAS_mkl_MKLROOT for non-Ubuntu 20.04 OSes
-  find_path(BLAS_mkl_MKLROOT mkl PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT})
+  find_path(BLAS_mkl_MKLROOT include/mkl.h
+    PATHS $ENV{MKLROOT} ${INTEL_HOME_ROOT} ${INTEL_OPT_ROOT} ${INTEL_OPT_ROOT}/oneapi/mkl
+    PATH_SUFFIXES mkl latest)
   find_package(BLAS)
-  find_path(MKL_INCLUDE_DIR mkl.h HINTS ${INTEL_HOME_ROOT}/mkl ${INTEL_OPT_ROOT}/mkl PATHS ENV MKLROOT PATH_SUFFIXES include mkl REQUIRED)
   include_directories(SYSTEM ${MKL_INCLUDE_DIR})
   list(APPEND mshadow_LINKER_LIBS ${BLAS_LIBRARIES})
   add_definitions(-DMSHADOW_USE_CBLAS=0)
   add_definitions(-DMSHADOW_USE_MKL=1)
   add_definitions(-DMXNET_USE_BLAS_MKL=1)
+  message("-- Found MKL (version: ${MKL_VERSION})")
 elseif(BLAS STREQUAL "apple")
   find_package(Accelerate REQUIRED)
   include_directories(SYSTEM ${Accelerate_INCLUDE_DIR})

cmake/upstream/FindBLAS.cmake

@@ -522,12 +522,16 @@ if(BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
             get_filename_component(BLAS_mkl_MKLROOT "${BLAS_mkl_MKLROOT}" DIRECTORY)
         endif()
       endif()
+      # MXNET NOTE: This change comes form the newest file version
+      # https://gitlab.kitware.com/cmake/cmake/-/issues/22295
       set(BLAS_mkl_LIB_PATH_SUFFIXES
           "compiler/lib" "compiler/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
           "compiler/lib/${BLAS_mkl_ARCH_NAME}"
           "mkl/lib" "mkl/lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
           "mkl/lib/${BLAS_mkl_ARCH_NAME}"
-          "lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}")
+          "lib" "lib/${BLAS_mkl_ARCH_NAME}_${BLAS_mkl_OS_NAME}"
+          "lib/${BLAS_mkl_ARCH_NAME}"
+          )
 
       foreach(IT ${BLAS_SEARCH_LIBS})
         string(REPLACE " " ";" SEARCH_LIBS ${IT})

src/initialize.cc

@@ -64,7 +64,9 @@ void win_err(char **err) {
 #include "common/utils.h"
 #include "engine/openmp.h"
 
-
+#if defined(MKL_USE_SINGLE_DYNAMIC_LIBRARY)
+#include <mkl.h>
+#endif
 
 namespace mxnet {
 
@@ -93,6 +95,13 @@ LibraryInitializer::LibraryInitializer()
     cpu_worker_nthreads_(dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1)),
     mp_cv_num_threads_(dmlc::GetEnv("MXNET_MP_OPENCV_NUM_THREADS", 0)) {
   dmlc::InitLogging("mxnet");
+#if MKL_USE_SINGLE_DYNAMIC_LIBRARY
+  #if defined( __INTEL_LLVM_COMPILER)
+    mkl_set_threading_layer(MKL_THREADING_INTEL);
+  #else
+    mkl_set_threading_layer(MKL_THREADING_GNU);
+  #endif
+#endif
   engine::OpenMP::Get();   // force OpenMP initialization
   install_pthread_atfork_handlers();
 }