CMake: begin adding CUDA support

wazedxwxx · Jan 17, 2019 · 4a1c7e9 · 4a1c7e9
1 parent b2cf854
commit 4a1c7e9
Show file tree

Hide file tree

Showing 6 changed files with 241 additions and 97 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,6 +27,35 @@ include( AMReX_Utils )
 include( AMReX_Options )
 include( AMReX_Machines )
 
+#
+# Enable CUDA if requested
+#
+# CMake let you decide which host compiler to use
+# via the env variable CUDAHOSTCXX and the CMake
+# variable CMAKE_CUDA_HOST_COMPILER.
+# For the time being we force the CUDA host compiler
+# to be the C++ compiler. 
+# 
+if ( (CMAKE_CUDA_HOST_COMPILER) OR
+      NOT ("$ENV{CUDAHOSTCXX}" STREQUAL "") )
+   message(WARNING
+      "User-defined CUDA host compiler does not match C++ compiler:  overwriting user setting.")
+endif ()
+unset(ENV{CUDAHOSTCXX})
+set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+if (ENABLE_CUDA)
+   enable_language(CUDA)
+   print(CMAKE_CUDA_COMPILER_VERSION)
+   print(CMAKE_CUDA_COMPILER_ID)
+   print(CMAKE_CUDA_HOST_COMPILER)
+
+   # For now we require C++ to be the same as 
+   #find is deprecated for CUDA but currently
+   #it still provides some features not yet implemented
+   # natively. Remove this part when native support is complete
+   #find_package(CUDA 8.0 REQUIRED QUIET)
+endif ()
+
 #
 # Set CMAKE_<LANG>_FLAGS_<CONFIG> if not already defined
 # 
@@ -57,17 +86,6 @@ endif ()
 set ( AMREX_GIT_VERSION "${TMP}" CACHE INTERNAL "" )
 unset (TMP)
 
-#
-# Install Blitz and Algoim at configuration time
-# if needed.
-# This is only required if ENABLE_3D_NODAL_MLMG is ON
-# and external libraries paths are not given 
-#
-# if ( ENABLE_3D_NODAL_MLMG )
-#    include(AMReX_InstallExternalLibs) 
-# endif ()
-
-
 # 
 # Source files for all binaries and libraries found under src
 # 

diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt
@@ -5,7 +5,12 @@
 # the properties of this object, like setting the sources,
 # setting the compile definitions and so on
 #
-add_library ( amrex "" )
+add_library( amrex "" )
+
+#
+# List of directories with CUDA kernels
+#
+set(CUDA_DIRS Base AmrCore Particles)
 
 #
 # Next, we implement a wrapper to add sources and headers
@@ -15,32 +20,39 @@ add_library ( amrex "" )
 # Variable PUBLIC_HEADERS accumulate all the headers for
 # final installation
 # WARNING: this routine add automatically ${CMAKE_CURRENT_LIST_DIR}
-# to the input source file, so always add sources with the relative path
+# to the input source file, so always add sources using the path relative
 # to the directory where add_sources is called from
-set (PUBLIC_HEADERS)
-macro ( add_sources )
-   foreach ( item IN ITEMS ${ARGV} )
-      target_sources ( amrex PRIVATE ${CMAKE_CURRENT_LIST_DIR}/${item})
+set(PUBLIC_HEADERS)
+macro( add_sources )
+   foreach( item IN ITEMS ${ARGV} )
+      target_sources( amrex PRIVATE ${CMAKE_CURRENT_LIST_DIR}/${item})
+      get_filename_component( ext ${item} EXT )
+      get_filename_component( dirname ${CMAKE_CURRENT_LIST_DIR} NAME )
+
       # If it's a header, add it the public header group so it gets installed
-      get_filename_component ( ext ${item} EXT )
       if ( ( ${ext} STREQUAL ".H" ) OR ( ${ext} STREQUAL ".h" ) )
 	 # Do the following command to handle the case when ${item} is not the name
 	 # of the source file, but it's in the form <relative-path>/<file-name>
 	 # where <relative-path> is the path relative to the directory containing
 	 # the list file invoking add_sources
-	 get_filename_component ( dir ${CMAKE_CURRENT_LIST_DIR}/${item} DIRECTORY )
-	 list ( APPEND PUBLIC_HEADERS ${CMAKE_CURRENT_LIST_DIR}/${item} )
-	 target_include_directories ( amrex PUBLIC $<BUILD_INTERFACE:${dir}> )
-      endif()      
+	 get_filename_component( dir ${CMAKE_CURRENT_LIST_DIR}/${item} DIRECTORY )
+	 list( APPEND PUBLIC_HEADERS ${CMAKE_CURRENT_LIST_DIR}/${item} )
+	 target_include_directories( amrex PUBLIC $<BUILD_INTERFACE:${dir}> )
+      endif()
+
+      # If it's a cpp file located in CUDA_DIRS and ENABLE_CUDA is on
+      # then force cpp file to be compiled with CUDA compiler    
+      if ( ENABLE_CUDA AND ( ${ext} STREQUAL ".cpp") AND ("${dirname}" IN_LIST CUDA_DIRS ) )
+         set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/${item}
+            PROPERTIES LANGUAGE CUDA)
+        # set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/${item} PROPERTIES CUDA_SOURCE_PROPERTY_FORMAT OBJ) #LANGUAGE CUDA)
+      endif ()
    endforeach ()
-   unset (ext)
-   unset (dir)
-endmacro ()
+   unset(ext)
+   unset(dir)
+   unset(dirname)
+endmacro()
 
-#
-# Set some properties for target 'amrex', like
-# compile definitions and so on
-#
 
 # General configuration
 include ( AMReX_Config )

diff --git a/Tools/CMake/AMReX_Compilers.cmake b/Tools/CMake/AMReX_Compilers.cmake
@@ -121,6 +121,15 @@ function ( set_amrex_compiler_flags )
 	 >>>>
 	 )	  
    endif ()
+
+
+   #
+   # Set REQUIRED CUDA flags 
+   # 
+   target_compile_options ( amrex
+      PRIVATE
+      $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr --expt-extended-lambda>
+      )
 
    #
    # Floating-point exceptions flags only if enabled

diff --git a/Tools/CMake/AMReX_Config.cmake b/Tools/CMake/AMReX_Config.cmake
@@ -230,8 +230,14 @@ function (print_amrex_configuration_summary)
    message( STATUS "   Fortran defines          = ${AMREX_Fortran_DEFINES}")
    message( STATUS "   C++ compiler             = ${CMAKE_CXX_COMPILER}")
    message( STATUS "   Fortran compiler         = ${CMAKE_Fortran_COMPILER}")
+   if (ENABLE_CUDA)
+      message( STATUS "   CUDA compiler            = ${CMAKE_CUDA_COMPILER}")
+   endif ()
    message( STATUS "   C++ flags                = ${AMREX_CXX_FLAGS}")
    message( STATUS "   Fortran flags            = ${AMREX_Fortran_FLAGS}")
+   if (ENABLE_CUDA)
+      message( STATUS "   CUDA flags               = ${CMAKE_CUDA_FLAGS_${AMREX_BUILD_TYPE}} ${CMAKE_CUDA_FLAGS}")
+   endif ()
    message( STATUS "   C++ include paths        = ${AMREX_CXX_INCLUDE_PATHS}")  
    message( STATUS "   Fortran include paths    = ${AMREX_Fortran_INCLUDE_PATHS}")
    message( STATUS "   Link line                = ${AMREX_LINK_LINE}") 

diff --git a/Tools/CMake/AMReX_Defines.cmake b/Tools/CMake/AMReX_Defines.cmake
@@ -72,11 +72,7 @@ function ( set_amrex_defines )
 
    # OpenMP
    add_amrex_define( AMREX_USE_OMP IF ENABLE_OMP )
-
-   # CUDA
-   add_amrex_define( AMREX_USE_CUDA    IF ENABLE_CUDA )
-   add_amrex_define( AMREX_USE_GPU     IF ENABLE_CUDA )
-
+
    # Precision
    if (NOT ENABLE_DP)
       add_amrex_define(AMREX_USE_FLOAT)
@@ -97,8 +93,8 @@ function ( set_amrex_defines )
    endif ()
 
    #  Assertions
-   add_amrex_define( AMREX_USE_ASSERTION IF ENABLE_ASSERTIONS )
-   add_amrex_define( AMREX_USE_EB IF ENABLE_EB )
+   add_amrex_define( AMREX_USE_ASSERTION    IF ENABLE_ASSERTIONS )
+   add_amrex_define( AMREX_USE_EB NO_LEGACY IF ENABLE_EB )
    add_amrex_define( AMREX_USE_F_INTERFACES IF ENABLE_FORTRAN_INTERFACES ) 
    add_amrex_define( AMREX_NO_STRICT_PREFIX )
 
@@ -116,7 +112,8 @@ function ( set_amrex_defines )
    if ( ${CMAKE_C_COMPILER_ID} STREQUAL "GNU" ) 
 
       if ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8" )
-	 message ( WARNING " Your default GCC is version ${CMAKE_CXX_COMPILER_VERSION}.This might break during build. GCC>=4.8 is recommended.")
+	 message ( WARNING
+            " Your default GCC is version ${CMAKE_CXX_COMPILER_VERSION}.This might break during build. GCC>=4.8 is recommended.")
       endif ()
 
       string ( REPLACE "." ";" VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION})
@@ -134,26 +131,87 @@ function ( set_amrex_defines )
    # 
    # Fortran/C mangling scheme
    # 
-   include ( FortranCInterface )
-   include ( ${FortranCInterface_BINARY_DIR}/Output.cmake )
+   include( FortranCInterface )
+   include( ${FortranCInterface_BINARY_DIR}/Output.cmake )
 
-   set ( FORTLINK "" )
+   set( FORTLINK "" )
 
    if ( FortranCInterface_GLOBAL_SUFFIX STREQUAL "" )
-      set (FORTLINK "${FortranCInterface_GLOBAL_CASE}CASE" )
-      message (STATUS "Fortran name mangling scheme: ${FORTLINK} (no append underscore)")
+      set(FORTLINK "${FortranCInterface_GLOBAL_CASE}CASE" )
+      message(STATUS "Fortran name mangling scheme: ${FORTLINK} (no append underscore)")
    elseif ( (FortranCInterface_GLOBAL_SUFFIX STREQUAL "_")  AND
 	 ( FortranCInterface_GLOBAL_CASE STREQUAL "LOWER" ) )
-      set (FORTLINK "UNDERSCORE")
-      message (STATUS "Fortran name mangling scheme: ${FORTLINK} (lower case, append underscore)")
+      set(FORTLINK "UNDERSCORE")
+      message(STATUS "Fortran name mangling scheme: ${FORTLINK} (lower case, append underscore)")
    else ()
-      message (AUTHOR_WARNING "Fortran to C mangling not compatible with AMReX code")
+      message(AUTHOR_WARNING "Fortran to C mangling not compatible with AMReX code")
    endif ()
 
    add_amrex_define( AMREX_FORT_USE_${FORTLINK} )
 
    # SENSEI Insitu
    add_amrex_define( AMREX_USE_SENSEI_INSITU IF ENABLE_SENSEI_INSITU )
+
+   # 
+   # CUDA
+   #
+   add_amrex_define( AMREX_USE_CUDA NO_LEGACY IF ENABLE_CUDA )
+   add_amrex_define( AMREX_CUDA_MAX_THREADS=${CUDA_MAX_THREADS} NO_LEGACY
+      IF ENABLE_CUDA )
+
+   if (ENABLE_CUDA)
+
+      string( REPLACE "." ";" VERSION_LIST ${CMAKE_CUDA_COMPILER_VERSION})
+      list( GET VERSION_LIST 0 NVCC_VERSION_MAJOR )
+      list( GET VERSION_LIST 1 NVCC_VERSION_MINOR )
+
+      target_compile_definitions( amrex PUBLIC
+	 AMREX_NVCC_VERSION=${CMAKE_CUDA_COMPILER_VERSION}
+	 AMREX_NVCC_MAJOR_VERSION=${NVCC_VERSION_MAJOR}
+	 AMREX_NVCC_MINOR_VERSION=${NVCC_VERSION_MINOR} )
+
+   endif ()
+
+
+   # Fortran macros useful by application code only
+   # (they are not present in AMReX source code) 
+   if (ENABLE_CUDA AND ( ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "PGI" ) OR
+            ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "XL"  ) ) )
+
+      # THIS DOES NOT WORK
+      # target_compile_definitions( amrex PUBLIC
+      #    $<$<COMPILE_LANGUAGE:Fortran>:AMREX_LAUNCH=\"attributes(global)\";
+      #    AMREX_DEVICE=\"attributes(device)\";
+      #    AMREX_CUDA_FORT_GLOBAL=\"attributes(global)\";
+      #    AMREX_CUDA_FORT_DEVICE=\"attributes(device)\";
+      #    AMREX_CUDA_FORT_HOST=\"attributes(host)\">)
+   else ()
+      target_compile_definitions( amrex PUBLIC
+         $<$<COMPILE_LANGUAGE:Fortran>:AMREX_LAUNCH=\"\";
+         AMREX_DEVICE=\"\";
+         AMREX_CUDA_FORT_GLOBAL=\"\";
+         AMREX_CUDA_FORT_DEVICE=\"\";
+         AMREX_CUDA_FORT_HOST=\"\">)
+   endif ()
+
+   #
+   # OpenACC
+   #
+   add_amrex_define( AMREX_USE_ACC NO_LEGACY IF ENABLE_ACC ) 
+
+   #
+   # General setup for any GPUs 
+   #
+   if (ENABLE_CUDA OR ENABLE_ACC)
+      add_amrex_define( AMREX_USE_GPU  NO_LEGACY )
+      add_amrex_define( BL_COALESCE_FABS )
+
+      add_amrex_define( AMREX_GPUS_PER_SOCKET=${GPUS_PER_SOCKET}
+         NO_LEGACY IF GPUS_PER_SOCKET)
+
+      add_amrex_define( AMREX_GPUS_PER_NODE=${GPUS_PER_NODE}
+         NO_LEGACY IF GPUS_PER_NODE)         
+   endif ()
 
 endfunction ()