CMake: add support for DPCPP (AMReX-Codes#1016)

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

CMakeLists.txt

@@ -45,6 +45,20 @@ endif ()
 set( AMREX_PKG_VERSION "${_pkg_version}" CACHE INTERNAL "" )
 unset(_pkg_version)
 
+#
+# For the time being, set this option here.
+# We must do this because we need to provide a default for CMAKE_CXX_COMPILER
+#
+option( ENABLE_DPCPP  "Enable DPCPP support" OFF )
+if (ENABLE_DPCPP)
+   if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+      # Set default for CMAKE_CXX_COMPILER only if amrex
+      # is not a subproject
+      set(CMAKE_CXX_COMPILER dpcpp)
+   endif ()
+   message(STATUS "Enabling experimental support for dpc++")
+endif ()
+
 
 ########################################################################
 #

Tools/CMake/AMReXConfig.cmake.in

@@ -48,6 +48,7 @@ set(AMReX_@DIM@D_FOUND                  ON)
 set(AMReX_MPI_FOUND                 @ENABLE_MPI@)
 set(AMReX_OMP_FOUND                 @ENABLE_OMP@)
 set(AMReX_CUDA_FOUND                @ENABLE_CUDA@)
+set(AMReX_DPCPP_FOUND               @ENABLE_DPCPP@)
 set(AMReX_DP_FOUND                  @ENABLE_DP@)
 
 # Actual components selection

Tools/CMake/AMReX_Config.cmake

@@ -157,6 +157,60 @@ function (configure_amrex)
       endif()
    endif()
 
+   #
+   # Setup DPC++ -- for now simply add public compile flags
+   #
+   if (ENABLE_DPCPP)
+
+      target_compile_options( amrex
+         PUBLIC
+         $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-Wno-error=sycl-strict -fsycl -fsycl-unnamed-lambda>)
+
+      target_link_options(amrex PUBLIC -Wno-error=sycl-strict -fsycl -fsycl-unnamed-lambda -device-math-lib=fp32,fp64)
+
+      if (ENABLE_DPCPP_AOT)
+         message(FATAL_ERROR "\nAhead-of-time (AOT) compilation support not available yet.\nRe-configure with ENABLE_DPCPP_AOT=OFF.")
+
+         #
+         # TODO: remove comments to enable AOT support when the time comes
+         #
+         # if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
+         #    ## TODO: use file(READ)
+         #    execute_process( COMMAND cat /sys/devices/cpu/caps/pmu_name OUTPUT_VARIABLE _cpu_long_name )
+         # else ()
+         #    message(FATAL_ERROR "\nENABLE_DPCPP_AOT is not supported on ${CMAKE_SYSTEM_NAME}\n")
+         # endif ()
+
+         # string(STRIP "${_cpu_long_name}" _cpu_long_name)
+         # if (_cpu_long_name STREQUAL "skylake")
+         #    set(_cpu_short_name "skl")
+         # elseif (_cpu_long_name STREQUAL "kabylake")
+         #    set(_cpu_short_name "kbl")
+         # elseif (_cpu_long_name STREQUAL "cascadelake")
+         #    set(_cpu_short_name "cfl")
+         # else ()
+         #    message(FATAL_ERROR "\n Ahead-of-time compilation for CPU ${_cpu_long_name} is not yet supported\n"
+         #       "Maybe set ENABLE_DPCPP_AOT to OFF?\n")
+         # endif ()
+
+         # target_compile_options( amrex
+         #    PUBLIC
+         #    $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>,$<NOT:$<CONFIG:Debug>>>:
+         #    -fsycl-targets=spir64_gen-unknown-unknown-sycldevice -Xsycl-target-backend "-device ${_cpu_short_name}" >)
+         # target_link_options(amrex PUBLIC -fsycl-targets=spir64_gen-unknown-unknown-sycldevice -Xsycl-target-backend "-device ${_cpu_short_name}" )
+         # unset(_cpu_long_name)
+         # unset(_cpu_short_name)
+       else ()
+         if (ENABLE_DPCPP_SPLIT_KERNEL)
+            target_compile_options( amrex
+               PUBLIC
+               $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:Clang>>:-fsycl-device-code-split=per_kernel>)
+            target_link_options(amrex PUBLIC -fsycl-device-code-split=per_kernel)
+         endif ()
+      endif ()
+   endif ()
+
+
    #
    # Setup third-party profilers
    #

Tools/CMake/AMReX_Defines.cmake

@@ -66,6 +66,10 @@ function ( set_amrex_defines )
    # OpenMP -- This one has legacy definition only in Base/AMReX_omp_mod.F90
    add_amrex_define( AMREX_USE_OMP IF ENABLE_OMP )
 
+   # DPCPP
+   add_amrex_define( AMREX_USE_DPCPP NO_LEGACY IF ENABLE_DPCPP )
+   add_amrex_define( AMREX_USE_GPU NO_LEGACY IF ENABLE_DPCPP )
+
    # Precision
    if (NOT ENABLE_DP)
       add_amrex_define(AMREX_USE_FLOAT)

Tools/CMake/AMReX_Options.cmake

@@ -11,8 +11,8 @@ include(CMakeDependentOption)
 #
 # Define a macro to check the value of the inputs integer options
 #
-macro (print_option var)
-   message( STATUS "   ${var} = ${${var}}")
+macro (print_option _var)
+   message( STATUS "   ${_var} = ${${_var}}")
 endmacro ()
 
 #
@@ -90,15 +90,33 @@ endif ()
 print_option( DIM )
 
 #
-# Programming models  ========================================================
+# Parallel backends    ========================================================
 #
-option( ENABLE_MPI  "Enable MPI"  ON)
+
+# For the time being ENABLE_DPCPP is defined before project() is called
+# Check whether the C++ compiler is dpcpp
+print_option(ENABLE_DPCPP)
+if (ENABLE_DPCPP AND (NOT (CMAKE_CXX_COMPILER MATCHES "dpcpp") ) )
+   message(FATAL_ERROR "\nENABLE_DPCPP=${ENABLE_DPCPP} but CXX compiler is not dpcpp\n")
+endif ()
+
+cmake_dependent_option( ENABLE_DPCPP_AOT  "Enable DPCPP ahead-of-time compilation (WIP)"  OFF
+   "ENABLE_DPCPP" OFF)
+print_option( ENABLE_DPCPP_AOT )
+
+cmake_dependent_option( ENABLE_DPCPP_SPLIT_KERNEL "Enable DPCPP kernel splitting"  ON
+   "ENABLE_DPCPP" OFF)
+print_option(  ENABLE_DPCPP_SPLIT_KERNEL )
+
+cmake_dependent_option( ENABLE_MPI  "Enable MPI"  ON
+   "NOT ENABLE_DPCPP" OFF)
 print_option( ENABLE_MPI )
 
 option( ENABLE_OMP  "Enable OpenMP" OFF)
 print_option( ENABLE_OMP )
 
-option( ENABLE_CUDA "Enable GPU support via CUDA" OFF )
+cmake_dependent_option( ENABLE_CUDA "Enable GPU support via CUDA" OFF
+   "NOT ENABLE_DPCPP" OFF)
 print_option( ENABLE_CUDA )
 
 option( ENABLE_ACC  "Enable GPU support via OpenACC" OFF )