[DFT] Initial DFT work & MKLGPU backend

CMakeLists.txt

@@ -88,6 +88,10 @@ if(ENABLE_MKLCPU_BACKEND
         OR ENABLE_ROCRAND_BACKEND)
   list(APPEND DOMAINS_LIST "rng")
 endif()
+if(ENABLE_MKLGPU_BACKEND
+        OR ENABLE_MKLCPU_BACKEND)
+  list(APPEND DOMAINS_LIST "dft")
+endif()
 
 # Define required CXX compilers before project
 if(CMAKE_CXX_COMPILER OR NOT ONEMKL_SYCL_IMPLEMENTATION STREQUAL "dpc++")

README.md

@@ -223,6 +223,13 @@ Supported domains: BLAS, LAPACK, RNG
             <td align="center">Dynamic, Static</td>
             <td align="center">LLVM*, hipSYCL</td>
         </tr>
+        <tr>
+            <td rowspan=1 align="center">DFT</td>
+            <td align="center">Intel GPU</td>
+            <td rowspan=1 align="center">Intel(R) oneAPI Math Kernel Library</td>
+            <td align="center">Dynamic, Static</td>
+            <td align="center">DPC++</td>
+        </tr>
     </tbody>
 </table>
 

docs/building_the_project.rst

@@ -565,7 +565,7 @@ CMake.
      - False     
    * - target_domains (list)
      - TARGET_DOMAINS (list)
-     - blas, lapack, rng
+     - blas, lapack, rng, dft
      - All domains 
 
 .. note::

examples/README.md

@@ -3,18 +3,20 @@ oneAPI Math Kernel Library (oneMKL) Interfaces offers examples with the followin
 - blas: level3/gemm_usm  
 - rng: uniform_usm  
 - lapack: getrs_usm
+- dft: complex_fwd_buffer, real_fwd_usm
 
 Each routine has one run-time dispatching example and one compile-time dispatching example (which uses both mklcpu and cuda backends), located in `example/<$domain>/run_time_dispatching` and `example/<$domain>/compile_time_dispatching` subfolders, respectively.
 
 To build examples, use cmake build option `-DBUILD_EXAMPLES=true`.  
 Compile_time_dispatching will be built if `-DBUILD_EXAMPLES=true` and cuda backend is enabled, because the compile-time dispatching example runs on both mklcpu and cuda backends.
-Run_time_dispatching will be built if `-DBUILD_EXAMPLES=true` and `-DBUILD_SHARED_LIBS=true`
+Run_time_dispatching will be built if `-DBUILD_EXAMPLES=true` and `-DBUILD_SHARED_LIBS=true`.
+All DFT examples require the mklgpu backend to be enabled.
 
 The example executable naming convention follows `example_<$domain>_<$routine>_<$backend>` for compile-time dispatching examples 
   or `example_<$domain>_<$routine>` for run-time dispatching examples. 
   E.g. `example_blas_gemm_usm_mklcpu_cublas `  `example_blas_gemm_usm`
 
-## Example outputs (blas, rng, lapack)
+## Example outputs (blas, rng, lapack, dft)
 
 ## blas
 
@@ -349,3 +351,58 @@ Random number generator example with uniform distribution ran OK on MKLCPU and C
 
 ```
 
+## dft
+
+Compile-time dispatching example with mklgpu backend
+
+```none
+$ SYCL_DEVICE_FILTER=gpu ./bin/example_dft_complex_fwd_buffer_mklgpu
+
+########################################################################
+# Complex out-of-place forward transform for Buffer API's example:
+#
+# Using APIs:
+#   Compile-time dispatch API
+#   Buffer forward complex out-of-place
+#
+# Using single precision (float) data type
+#
+# For Intel GPU with Intel MKLGPU backend.
+#
+# The environment variable SYCL_DEVICE_FILTER can be used to specify
+# SYCL device
+########################################################################
+
+Running DFT Complex forward out-of-place buffer example
+Using compile-time dispatch API with MKLGPU.
+Running with single precision real data type on:
+	GPU device :Intel(R) UHD Graphics 750 [0x4c8a]
+DFT Complex USM example ran OK on MKLGPU
+```
+
+Runtime dispatching example with both mklgpu backend
+
+```none
+SYCL_DEVICE_FILTER=gpu ./bin/example_dft_complex_fwd_buffer_mklgpu
+
+########################################################################
+# Complex out-of-place forward transform for Buffer API's example:
+#
+# Using APIs:
+#   Compile-time dispatch API
+#   Buffer forward complex out-of-place
+#
+# Using single precision (float) data type
+#
+# For Intel GPU with Intel MKLGPU backend.
+#
+# The environment variable SYCL_DEVICE_FILTER can be used to specify
+# SYCL device
+########################################################################
+
+Running DFT Complex forward out-of-place buffer example
+Using compile-time dispatch API with MKLGPU.
+Running with single precision real data type on:
+	GPU device :Intel(R) UHD Graphics 750 [0x4c8a]
+DFT Complex USM example ran OK on MKLGPU
+```

examples/dft/CMakeLists.txt

@@ -0,0 +1,25 @@
+#===============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+#
+# SPDX-License-Identifier: Apache-2.0
+#===============================================================================
+
+add_subdirectory(compile_time_dispatching)
+
+# runtime compilation is only possible with dynamic libraries
+if (BUILD_SHARED_LIBS)
+  add_subdirectory(run_time_dispatching)
+endif()

examples/dft/compile_time_dispatching/CMakeLists.txt

@@ -0,0 +1,43 @@
+#===============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+#
+# SPDX-License-Identifier: Apache-2.0
+#===============================================================================
+
+#Build object from all sources
+set(DFTI_CT_SOURCES "")
+if(ENABLE_MKLGPU_BACKEND)
+  list(APPEND DFTI_CT_SOURCES "complex_fwd_buffer_mklgpu")
+endif()
+
+foreach(dfti_ct_sources ${DFTI_CT_SOURCES})
+  add_executable(example_${domain}_${dfti_ct_sources} ${dfti_ct_sources}.cpp)
+  target_include_directories(example_${domain}_${dfti_ct_sources}
+      PUBLIC ${PROJECT_SOURCE_DIR}/examples/include
+      PUBLIC ${PROJECT_SOURCE_DIR}/include
+      PUBLIC ${CMAKE_BINARY_DIR}/bin
+  )
+  if(domain STREQUAL "dft" AND ENABLE_MKLGPU_BACKEND)
+    add_dependencies(example_${domain}_${dfti_ct_sources} onemkl_${domain}_mklgpu)
+    list(APPEND ONEMKL_LIBRARIES_${domain} onemkl_${domain}_mklgpu)
+  endif()
+  target_link_libraries(example_${domain}_${dfti_ct_sources} PUBLIC
+      ${ONEMKL_LIBRARIES_${domain}}
+      ONEMKL::SYCL::SYCL
+  )
+  # Register example as ctest
+ add_test(NAME ${domain}/EXAMPLE/CT/${dfti_ct_sources} COMMAND example_${domain}_${dfti_ct_sources})
+endforeach(dfti_ct_sources)

examples/dft/compile_time_dispatching/complex_fwd_buffer_mklgpu.cpp

@@ -0,0 +1,132 @@
+/*******************************************************************************
+* Copyright 2023 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions
+* and limitations under the License.
+*
+*
+* SPDX-License-Identifier: Apache-2.0
+*******************************************************************************/
+
+// STL includes
+#include <iostream>
+
+// oneMKL/SYCL includes
+#if __has_include(<sycl/sycl.hpp>)
+#include <sycl/sycl.hpp>
+#else
+#include <CL/sycl.hpp>
+#endif
+#include "oneapi/mkl.hpp"
+
+void run_example(const sycl::device& gpu_device) {
+    constexpr int N = 10;
+
+    // Catch asynchronous exceptions for cpu
+    auto gpu_error_handler = [&](sycl::exception_list exceptions) {
+        for (auto const& e : exceptions) {
+            try {
+                std::rethrow_exception(e);
+            }
+            catch (sycl::exception const& e) {
+                // Handle not dft related exceptions that happened during asynchronous call
+                std::cerr << "Caught asynchronous SYCL exception:" << std::endl;
+                std::cerr << "\t" << e.what() << std::endl;
+            }
+        }
+        std::exit(2);
+    };
+
+    sycl::queue gpu_queue(gpu_device, gpu_error_handler);
+
+    std::vector<std::complex<float>> input_data(N);
+    std::vector<std::complex<float>> output_data(N);
+
+    // enabling
+    // 1. create descriptors
+    oneapi::mkl::dft::descriptor<oneapi::mkl::dft::precision::SINGLE,
+                                 oneapi::mkl::dft::domain::COMPLEX>
+        desc(N);
+
+    // 2. variadic set_value
+    desc.set_value(oneapi::mkl::dft::config_param::PLACEMENT,
+                   oneapi::mkl::dft::config_value::NOT_INPLACE);
+    desc.set_value(oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
+                   static_cast<std::int64_t>(1));
+
+    // 3. commit_descriptor (compile_time MKLGPU)
+    desc.commit(oneapi::mkl::backend_selector<oneapi::mkl::backend::mklgpu>{ gpu_queue });
+
+    // 4. compute_forward / compute_backward (MKLGPU)
+    {
+        sycl::buffer<std::complex<float>> input_buffer(input_data.data(), sycl::range<1>(N));
+        sycl::buffer<std::complex<float>> output_buffer(output_data.data(), sycl::range<1>(N));
+        oneapi::mkl::dft::compute_forward<decltype(desc), std::complex<float>, std::complex<float>>(
+            desc, input_buffer, output_buffer);
+    }
+}
+
+//
+// Description of example setup, apis used and supported floating point type precisions
+//
+void print_example_banner() {
+    std::cout << "\n"
+                 "########################################################################\n"
+                 "# Complex out-of-place forward transform for Buffer API's example:\n"
+                 "#\n"
+                 "# Using APIs:\n"
+                 "#   Compile-time dispatch API\n"
+                 "#   Buffer forward complex out-of-place\n"
+                 "#\n"
+                 "# Using single precision (float) data type\n"
+                 "#\n"
+                 "# For Intel GPU with Intel MKLGPU backend.\n"
+                 "#\n"
+                 "# The environment variable SYCL_DEVICE_FILTER can be used to specify\n"
+                 "#SYCL device\n"
+                 "########################################################################\n"
+              << std::endl;
+}
+
+//
+// Main entry point for example.
+//
+int main(int argc, char** argv) {
+    print_example_banner();
+
+    try {
+        sycl::device gpu_device((sycl::gpu_selector_v));
+        std::cout << "Running DFT Complex forward out-of-place buffer example" << std::endl;
+        std::cout << "Using compile-time dispatch API with MKLGPU." << std::endl;
+        std::cout << "Running with single precision real data type on:" << std::endl;
+        std::cout << "\tGPU device :" << gpu_device.get_info<sycl::info::device::name>()
+                  << std::endl;
+
+        run_example(gpu_device);
+        std::cout << "DFT Complex USM example ran OK on MKLGPU" << std::endl;
+    }
+    catch (sycl::exception const& e) {
+        // Handle not dft related exceptions that happened during synchronous call
+        std::cerr << "Caught synchronous SYCL exception:" << std::endl;
+        std::cerr << "\t" << e.what() << std::endl;
+        std::cerr << "\tSYCL error code: " << e.code().value() << std::endl;
+        return 1;
+    }
+    catch (std::exception const& e) {
+        // Handle not SYCL related exceptions that happened during synchronous call
+        std::cerr << "Caught synchronous std::exception:" << std::endl;
+        std::cerr << "\t" << e.what() << std::endl;
+        return 1;
+    }
+
+    return 0;
+}

examples/dft/run_time_dispatching/CMakeLists.txt

@@ -0,0 +1,69 @@
+#===============================================================================
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+#
+# SPDX-License-Identifier: Apache-2.0
+#===============================================================================
+
+# NOTE: user needs to set env var SYCL_DEVICE_FILTER to use runtime example (no need to specify backend when building with CMake)
+
+# Build object from all example sources
+set(DFT_RT_SOURCES "")
+if(ENABLE_MKLGPU_BACKEND)
+  list(APPEND DFT_RT_SOURCES "real_fwd_usm")
+endif()
+
+# Set up for the right backend for run-time dispatching examples
+# If users build more than one backend (i.e. mklcpu and mklgpu, or mklcpu and CUDA), they may need to
+# overwrite SYCL_DEVICE_FILTER in their environment to run on the desired backend
+set(DEVICE_FILTERS "")
+if(ENABLE_MKLCPU_BACKEND)
+  list(APPEND DEVICE_FILTERS "cpu")
+endif()
+if(ENABLE_MKLGPU_BACKEND)
+  list(APPEND DEVICE_FILTERS "gpu")
+endif()
+
+message(STATUS "SYCL_DEVICE_FILTER will be set to the following value(s): [${DEVICE_FILTERS}] for run-time dispatching examples")
+
+foreach(dft_rt_sources ${DFT_RT_SOURCES})
+  add_executable(example_${domain}_${dft_rt_sources} ${dft_rt_sources}.cpp)
+  target_include_directories(example_${domain}_${dft_rt_sources}
+      PUBLIC ${PROJECT_SOURCE_DIR}/examples/include
+      PUBLIC ${PROJECT_SOURCE_DIR}/include
+      PUBLIC ${CMAKE_BINARY_DIR}/bin
+  )
+
+  add_dependencies(example_${domain}_${dft_rt_sources} onemkl)
+
+  if (USE_ADD_SYCL_TO_TARGET_INTEGRATION)
+    add_sycl_to_target(TARGET example_${domain}_${dft_rt_sources} SOURCES ${DFT_RT_SOURCES})
+  endif()
+
+  target_link_libraries(example_${domain}_${dft_rt_sources} PUBLIC
+      onemkl
+      ONEMKL::SYCL::SYCL
+      ${CMAKE_DL_LIBS}
+  )
+
+  # Register example as ctest
+  foreach(device_filter ${DEVICE_FILTERS})
+    add_test(NAME ${domain}/EXAMPLE/RT/${dft_rt_sources}/${device_filter} COMMAND example_${domain}_${dft_rt_sources})
+    set_property(TEST ${domain}/EXAMPLE/RT/${dft_rt_sources}/${device_filter} PROPERTY
+      ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:$ENV{LD_LIBRARY_PATH}
+      ENVIRONMENT SYCL_DEVICE_FILTER=${device_filter})
+  endforeach(device_filter)
+
+endforeach()