Use of Cray compilers + Nvidia GPU offloading on DeltaAI

[sbryngelson]

On DeltaAI, one can build and run MFC with Cray compilers on Hopper GPUs. This might be a route to an intermediate debugging without access to OLCF Frontier, but use of Cray compilers + offloading.

The modules are

module load craype-arm-grace libfabric/1.20.1 craype-network-ofi xpmem/2.9.6-1.1_20240511212657__g087dc11fc19d default nghttp2/1.62.0-dklrugx PrgEnv-cray/8.5.0 openssl/1.1.1l-fips-53w6fpf zlib-ng/2.2.1-p2pfju3 curl/8.7.1-qizcfnh ncurses/6.5-w7at37e cmake/3.30.2 bzip2/1.0.8-hcy2frn libmd/1.0.4-73szclq libbsd/0.12.2-4bpq5lv expat/2.6.2-6prv4ln readline/8.2-qvipktz gdbm/1.23-3ygypmw gettext/0.20.2-z2f5xqe libffi/3.4.6-rxyqhce libxcrypt/4.4.35-cvskaru zlib/1.2.13-lwy2f2t sqlite/3.46.0-uujvfbj util-linux-uuid/2.40.2-lwugcvc xz/5.2.3-yfszzud python/3.11.9

module load cpe/24.07 cce/18.0.0 cray-mpich/8.1.30 craype-accel-nvidia90 cpe-cuda/24.07 cray-fftw/3.3.10.8 perftools-base/24.07.0 cray-hdf5/1.14.3.1  cray-dsmml/0.3.0

the patch looks something like this:

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a5835bd2..f9b9b0d3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -154,11 +154,17 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
     add_compile_options(
+        "SHELL:-h acc"
+        "SHELL:-craype-verbose"
+        "SHELL:-haccel=nvidia90"
         "SHELL:-M 296,878,1391,1069,5025"
-        "SHELL:-h static" "SHELL:-h keepfiles"
+        "SHELL:-h dynamic"
+        "SHELL:-h keepfiles"
         "SHELL:-h acc_model=auto_async_none"
-        "SHELL: -h acc_model=no_fast_addr"
-        "SHELL: -h list=adm"
+        "SHELL:-h acc_model=no_fast_addr"
+        "SHELL:-v"
+        "SHELL:-m 3"
+        "SHELL:-Wx,-O0"
     )

     add_link_options("SHELL:-hkeepfiles")
@@ -444,13 +450,13 @@ function(MFC_SETUP_TARGET)

         if (ARGS_FFTW)
             if (MFC_OpenACC AND ARGS_OpenACC)
-                if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
-                    find_package(CUDAToolkit REQUIRED)
-                    target_link_libraries(${a_target} PRIVATE CUDA::cudart CUDA::cufft)
-                else()
-                    find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
-                    target_link_libraries(${a_target} PRIVATE hipfort::hipfft)
-                endif()
+                # if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
+                find_package(CUDAToolkit REQUIRED)
+                target_link_libraries(${a_target} PRIVATE CUDA::cudart CUDA::cufft)
+                # else()
+                #     find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
+                #     target_link_libraries(${a_target} PRIVATE hipfort::hipfft)
+                # endif()

@@ -513,14 +519,17 @@ function(MFC_SETUP_TARGET)
                     )
                 endif()
             elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
-                find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
-                target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
+                # find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
+                # target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
+                # target_compile_options(${a_target}
+                #     PRIVATE -O0
+                # )


-        if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
+        if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI" OR CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
             find_package(CUDAToolkit REQUIRED)
             target_link_libraries(${a_target} PRIVATE CUDA::nvToolsExt)
         endif()

and

diff --git a/toolchain/dependencies/CMakeLists.txt b/toolchain/dependencies/CMakeLists.txt
index 116e2887..df06cddb 100644
--- a/toolchain/dependencies/CMakeLists.txt
+++ b/toolchain/dependencies/CMakeLists.txt
@@ -100,20 +100,20 @@ endif()

 # HIPFORT
 if (MFC_HIPFORT)
-    if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
-        ExternalProject_Add(hipfort
-            GIT_REPOSITORY "https://github.com/ROCmSoftwarePlatform/hipfort"
-            GIT_TAG        rocm-6.0.2
-            GIT_SHALLOW    ON
-            GIT_PROGRESS   ON
-            CMAKE_ARGS     "-DHIPFORT_COMPILER=${CMAKE_Fortran_COMPILER}"
-                           "-DHIPFORT_AR=${CMAKE_AR}"
-                           "-DHIPFORT_RANLIB=${CMAKE_RANLIB}"
-                           "-DHIPFORT_COMPILER_FLAGS=-f free -e F -O0 -h ipa0"
-                           "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
-        )
-    else()
-        message(WARNING "The Fortran compiler vendor is not Cray so HIPFORT will not be built.")
-        add_custom_target(hipfort)
-    endif()
+    # if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+    #     ExternalProject_Add(hipfort
+    #         GIT_REPOSITORY "https://github.com/ROCmSoftwarePlatform/hipfort"
+    #         GIT_TAG        rocm-6.0.2
+    #         GIT_SHALLOW    ON
+    #         GIT_PROGRESS   ON
+    #         CMAKE_ARGS     "-DHIPFORT_COMPILER=${CMAKE_Fortran_COMPILER}"
+    #                        "-DHIPFORT_AR=${CMAKE_AR}"
+    #                        "-DHIPFORT_RANLIB=${CMAKE_RANLIB}"
+    #                        "-DHIPFORT_COMPILER_FLAGS=-f free -e F -O0 -h ipa0"
+    #                        "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
+    #     )
+    # else()
+    message(WARNING "The Fortran compiler vendor is not Cray so HIPFORT will not be built.")
+    add_custom_target(hipfort)
+    # endif()
 endif()

One also needs to remove the FFTW calls (haven't figured that out, but don't really care... fftw should be optional). Ideally, we would wrap the FFTW stuff in an #ifdef MFC_FFTW or something.

[sbryngelson]

Update: Fails at runtime due to ptx compilation error in the GPU case. Should just add CPU tests on Frontier, perhaps.

Update 2: This can be fixed via

-Wx,-O0 will pass -O0 through to PTX as. 
Unfortunately the optimize directive won't pass it, so you'll probably want to blanket apply it to everything.
It's going to really hurt performance, but should at least allow testing correctness.

Unfortunately, DeltaAI has a max time on processes, so the GH runner gets terminated after 7 days.