arrayfire · 9prady9 · Apr 29, 2021 · Apr 8, 2021 · Apr 9, 2021 · Apr 10, 2021
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -364,6 +364,7 @@ if((USE_CPU_MKL OR USE_OPENCL_MKL) AND AF_INSTALL_STANDALONE)
     endif()
 
     install(FILES
+      $<TARGET_FILE:MKL::RT>
       $<TARGET_FILE:MKL::Shared>
       $<TARGET_FILE:MKL::ThreadLayer>
       ${MKL_RUNTIME_KERNEL_LIBRARIES}

diff --git a/CMakeModules/FindMKL.cmake b/CMakeModules/FindMKL.cmake
@@ -61,6 +61,12 @@
 #
 # ``MKL::{mkl_def;mkl_mc;mkl_mc3;mkl_avx;mkl_avx2;mkl_avx512}{_STATIC}``
 #   Targets for MKL kernel libraries.
+#
+# This module has the following result variables:
+#
+# ``MKL_INTERFACE_INTEGER_SIZE``
+#   This variable is set integer size in bytes on the platform where this module
+#   runs. This is usually 4/8, and set of values this is dependent on MKL library.
 
 include(CheckTypeSize)
 include(FindPackageHandleStandardArgs)
@@ -212,6 +218,7 @@ function(find_mkl_library)
   add_library(MKL::${mkl_args_NAME}_STATIC STATIC IMPORTED)
 
   if(NOT (WIN32 AND mkl_args_DLL_ONLY))
+    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.1")
     find_library(MKL_${mkl_args_NAME}_LINK_LIBRARY
       NAMES
         ${mkl_args_LIBRARY_NAME}${shared_suffix}
@@ -232,6 +239,7 @@ function(find_mkl_library)
         ""
         intel64
         intel64/gcc4.7)
+    list(REMOVE_ITEM CMAKE_FIND_LIBRARY_SUFFIXES ".so.1")
     if(MKL_${mkl_args_NAME}_LINK_LIBRARY)
       if (CMAKE_VERSION VERSION_GREATER 3.14)
         message(VERBOSE "MKL_${mkl_args_NAME}_LINK_LIBRARY: ${MKL_${mkl_args_NAME}_LINK_LIBRARY}")
@@ -261,47 +269,47 @@ function(find_mkl_library)
         IntelSWTools/compilers_and_libraries/windows/compiler/lib/intel64
         IntelSWTools/compilers_and_libraries/windows/tbb/lib/intel64/${msvc_dir}
         )
-      if(MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY)
-        if (CMAKE_VERSION VERSION_GREATER 3.14)
-          message(VERBOSE "MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY: ${MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY}")
-        endif()
+    if(MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY)
+      if(CMAKE_VERSION VERSION_GREATER 3.14)
+        message(VERBOSE "MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY: ${MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY}")
       endif()
-      mark_as_advanced(MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY)
     endif()
+    mark_as_advanced(MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY)
+  endif()
 
-    set_target_properties(MKL::${mkl_args_NAME}
+  set_target_properties(MKL::${mkl_args_NAME}
+    PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}"
+      IMPORTED_LOCATION "${MKL_${mkl_args_NAME}_LINK_LIBRARY}"
+      IMPORTED_NO_SONAME TRUE)
+
+  set_target_properties(MKL::${mkl_args_NAME}_STATIC
       PROPERTIES
-        INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}"
-        IMPORTED_LOCATION "${MKL_${mkl_args_NAME}_LINK_LIBRARY}"
-        IMPORTED_NO_SONAME TRUE)
+      INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}"
+      IMPORTED_LOCATION "${MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY}"
+      IMPORTED_NO_SONAME TRUE)
 
-    set_target_properties(MKL::${mkl_args_NAME}_STATIC
-        PROPERTIES
-        INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}"
-        IMPORTED_LOCATION "${MKL_${mkl_args_NAME}_STATIC_LINK_LIBRARY}"
-        IMPORTED_NO_SONAME TRUE)
+  if(WIN32)
+    find_file(MKL_${mkl_args_NAME}_DLL_LIBRARY
+      NAMES
+        ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        lib${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        $ENV{LIB}
+        $ENV{LIBRARY_PATH}
+      PATH_SUFFIXES
+        IntelSWTools/compilers_and_libraries/windows/redist/intel64/mkl
+        IntelSWTools/compilers_and_libraries/windows/redist/intel64/compiler
+        IntelSWTools/compilers_and_libraries/windows/redist/intel64/tbb/${msvc_dir}
+      NO_SYSTEM_ENVIRONMENT_PATH)
 
-    if(WIN32)
-      find_file(MKL_${mkl_args_NAME}_DLL_LIBRARY
-        NAMES
-          ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
-          ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX}
-          lib${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX}
-          $ENV{LIB}
-          $ENV{LIBRARY_PATH}
-        PATH_SUFFIXES
-          IntelSWTools/compilers_and_libraries/windows/redist/intel64/mkl
-          IntelSWTools/compilers_and_libraries/windows/redist/intel64/compiler
-          IntelSWTools/compilers_and_libraries/windows/redist/intel64/tbb/${msvc_dir}
-        NO_SYSTEM_ENVIRONMENT_PATH)
-
-      set_target_properties(MKL::${mkl_args_NAME}
-        PROPERTIES
-          IMPORTED_LOCATION "${MKL_${mkl_args_NAME}_DLL_LIBRARY}"
-          IMPORTED_IMPLIB "${MKL_${mkl_args_NAME}_LINK_LIBRARY}")
+    set_target_properties(MKL::${mkl_args_NAME}
+      PROPERTIES
+        IMPORTED_LOCATION "${MKL_${mkl_args_NAME}_DLL_LIBRARY}"
+        IMPORTED_IMPLIB "${MKL_${mkl_args_NAME}_LINK_LIBRARY}")
 
-      mark_as_advanced(MKL_${mkl_args_NAME}_DLL_LIBRARY)
-    endif()
+    mark_as_advanced(MKL_${mkl_args_NAME}_DLL_LIBRARY)
+  endif()
 endfunction()
 
 
@@ -334,8 +342,10 @@ elseif(MKL_THREAD_LAYER STREQUAL "Sequential")
 endif()
 
 if("${INT_SIZE}" EQUAL 4)
+  set(MKL_INTERFACE_INTEGER_SIZE 4)
   find_mkl_library(NAME Interface LIBRARY_NAME mkl_intel_lp64 SEARCH_STATIC)
 else()
+  set(MKL_INTERFACE_INTEGER_SIZE 8)
   find_mkl_library(NAME Interface LIBRARY_NAME mkl_intel_ilp64 SEARCH_STATIC)
 endif()
 

diff --git a/src/api/c/CMakeLists.txt b/src/api/c/CMakeLists.txt
@@ -184,6 +184,23 @@ if(FreeImage_FOUND AND AF_WITH_IMAGEIO)
   endif ()
 endif()
 
+if(USE_CPU_MKL OR USE_OPENCL_MKL)
+  target_compile_definitions(c_api_interface
+    INTERFACE
+      AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE}
+    )
+  # Create mkl thread layer compile option based on cmake cache variable
+  if(MKL_THREAD_LAYER STREQUAL "Sequential")
+    target_compile_definitions(c_api_interface INTERFACE AF_MKL_THREAD_LAYER=0)
+  elseif(MKL_THREAD_LAYER STREQUAL "GNU OpenMP")
+    target_compile_definitions(c_api_interface INTERFACE AF_MKL_THREAD_LAYER=1)
+  elseif(MKL_THREAD_LAYER STREQUAL "Intel OpenMP")
+    target_compile_definitions(c_api_interface INTERFACE AF_MKL_THREAD_LAYER=2)
+  else() #default Intel Thread Layer for ArrayFire
+    target_compile_definitions(c_api_interface INTERFACE AF_MKL_THREAD_LAYER=3)
+  endif()
+endif()
+
 target_include_directories(c_api_interface
   INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR}

diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
@@ -20,6 +20,10 @@
 #include <af/dim4.hpp>
 #include <af/version.h>
 
+#if defined(USE_MKL)
+#include <mkl_service.h>
+#endif
+
 #include <cstring>
 #include <string>
 
@@ -102,7 +106,46 @@ af_err af_get_active_backend(af_backend* result) {
 af_err af_init() {
     try {
         thread_local std::once_flag flag;
-        std::call_once(flag, []() { getDeviceInfo(); });
+        std::call_once(flag, []() {
+            getDeviceInfo();
+#if defined(USE_MKL)
+            int errCode = -1;
+            // Have used the AF_MKL_INTERFACE_SIZE as regular if's so that
+            // we will know if these are not defined when using MKL when a
+            // compilation error is generated.
+            if (AF_MKL_INTERFACE_SIZE == 4) {
+                errCode = mkl_set_interface_layer(MKL_INTERFACE_LP64);
+            } else if (AF_MKL_INTERFACE_SIZE == 8) {
+                errCode = mkl_set_interface_layer(MKL_INTERFACE_ILP64);
+            }
+            if (errCode == -1) {
+                AF_ERROR(
+                    "Intel MKL Interface layer was not specified prior to the "
+                    "call and the input parameter is incorrect.",
+                    AF_ERR_RUNTIME);
+            }
+            switch (AF_MKL_THREAD_LAYER) {
+                case 0:
+                    errCode = mkl_set_threading_layer(MKL_THREADING_SEQUENTIAL);
+                    break;
+                case 1:
+                    errCode = mkl_set_threading_layer(MKL_THREADING_GNU);
+                    break;
+                case 2:
+                    errCode = mkl_set_threading_layer(MKL_THREADING_INTEL);
+                    break;
+                case 3:
+                    errCode = mkl_set_threading_layer(MKL_THREADING_TBB);
+                    break;
+            }
+            if (errCode == -1) {
+                AF_ERROR(
+                    "Intel MKL Thread layer was not specified prior to the "
+                    "call and the input parameter is incorrect.",
+                    AF_ERR_RUNTIME);
+            }
+#endif
+        });
     }
     CATCHALL;
     return AF_SUCCESS;

diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
@@ -107,20 +107,6 @@ target_link_libraries(af
     ${CMAKE_DL_LIBS}
   )
 
-
-# NOTE: When loading libraries we only use the RTLD_LAZY flag for the unified
-# backend. This will only load the symbols but will not make those symbols
-# available to libraries loaded in the future. Because we link against MKL
-# and since MKL also dynamically loads libraries at runtime, the linker
-# is not able to load those symbols that are needed by those files. You could
-# pass the RTLD_GLOBAL flag to dlload, but that causes issues with the ArrayFire
-# libraries. To get around this we are also linking the unified backend with
-# the MKL library
-if((USE_CPU_MKL OR USE_OPENCL_MKL) AND TARGET MKL::Shared AND NOT AF_WITH_STATIC_MKL)
-  target_link_libraries(af PRIVATE MKL::Shared)
-endif()
-
-
 install(TARGETS af
   EXPORT ArrayFireUnifiedTargets
   COMPONENT unified

diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
@@ -318,7 +318,7 @@ if(USE_CPU_MKL)
   if(AF_WITH_STATIC_MKL)
       target_link_libraries(afcpu PRIVATE MKL::Static)
   else()
-      target_link_libraries(afcpu PRIVATE MKL::Shared)
+      target_link_libraries(afcpu PRIVATE MKL::RT)
   endif()
 else()
   dependency_check(FFTW_FOUND "FFTW not found")

diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
@@ -469,7 +469,7 @@ if(LAPACK_FOUND OR (USE_OPENCL_MKL AND MKL_Shared_FOUND))
     if(AF_WITH_STATIC_MKL)
         target_link_libraries(afopencl PRIVATE MKL::Static)
     else()
-        target_link_libraries(afopencl PRIVATE MKL::Shared)
+        target_link_libraries(afopencl PRIVATE MKL::RT)
     endif()
   else()
     dependency_check(OpenCL_FOUND "OpenCL not found.")