From 32d8a84510a6857a7a27e9b19571ccfae3c84a42 Mon Sep 17 00:00:00 2001 From: Victor Mateevitsi Date: Fri, 22 Nov 2024 10:28:08 -0600 Subject: [PATCH] Added SYCL and Aurora support (#1398) Co-authored-by: Victor Mateevitsi --- scripts/build_ascent/build_ascent.sh | 49 +++++++++++++++++---- scripts/build_ascent/build_ascent_aurora.sh | 21 +++++++++ scripts/build_ascent/build_ascent_sycl.sh | 21 +++++++++ 3 files changed, 82 insertions(+), 9 deletions(-) create mode 100755 scripts/build_ascent/build_ascent_aurora.sh create mode 100755 scripts/build_ascent/build_ascent_sycl.sh diff --git a/scripts/build_ascent/build_ascent.sh b/scripts/build_ascent/build_ascent.sh index 4eef315bf..76837d33b 100755 --- a/scripts/build_ascent/build_ascent.sh +++ b/scripts/build_ascent/build_ascent.sh @@ -24,6 +24,7 @@ set -eu -o pipefail # shared options enable_cuda="${enable_cuda:=OFF}" enable_hip="${enable_hip:=OFF}" +enable_sycl="${enable_sycl:=OFF}" enable_fortran="${enable_fortran:=OFF}" enable_python="${enable_python:=OFF}" enable_openmp="${enable_openmp:=OFF}" @@ -81,6 +82,14 @@ if [[ "$enable_hip" == "ON" ]]; then ROCM_PATH="${ROCM_PATH:=/opt/rocm/}" # NOTE: this script only builds kokkos when enable_hip=ON + build_kokkos="${build_kokkos:=true}" +elif [[ "$enable_sycl" == "ON" ]]; then + echo "*** configuring with SYCL support" + + CC=`which icx` + CXX=`which icpx` + FTN=`which ifx` + build_kokkos="${build_kokkos:=true}" else build_kokkos="${build_kokkos:=false}" @@ -519,7 +528,7 @@ kokkos_build_dir=$(ospath ${build_dir}/kokkos-${kokkos_version}) kokkos_install_dir=$(ospath ${install_dir}/kokkos-${kokkos_version}/) kokkos_tarball=$(ospath ${source_dir}/kokkos-${kokkos_version}.tar.gz) -if [[ "$enable_hip" == "ON" ]]; then +if [[ "$enable_hip" == "ON" ]] || [[ "$enable_sycl" == "ON" ]]; then # build only if install doesn't exist if [ ! -d ${kokkos_install_dir} ]; then if ${build_kokkos}; then @@ -529,6 +538,23 @@ if [ ! -d ${kokkos_src_dir} ]; then tar ${tar_extra_args} -xzf ${kokkos_tarball} -C ${source_dir} fi +kokkos_extra_cmake_args="" +if [[ "$enable_hip" == "ON" ]]; then + kokkos_extra_cmake_args="-DKokkos_ENABLE_HIP=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_VEGA90A=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_FLAGS=--amdgpu-target=${ROCM_ARCH}" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc" +fi + +if [[ "$enable_sycl" == "ON" ]]; then + kokkos_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_SYCL=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_INTEL_PVC=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_EXTENSIONS=OFF" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_STANDARD=17" +fi + # TODO: DKokkos_ARCH_VEGA90A needs to be controlled / mapped? echo "**** Configuring Kokkos ${kokkos_version}" @@ -536,14 +562,9 @@ cmake -S ${kokkos_src_dir} -B ${kokkos_build_dir} ${cmake_compiler_settings} \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose}\ -DCMAKE_BUILD_TYPE=${build_config} \ -DBUILD_SHARED_LIBS=${build_shared_libs} \ - -DKokkos_ARCH_VEGA90A=ON \ - -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc \ - -DKokkos_ENABLE_HIP=ON \ -DKokkos_ENABLE_SERIAL=ON \ - -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF \ -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir} \ - -DCMAKE_CXX_FLAGS="--amdgpu-target=${ROCM_ARCH}" \ - -DBUILD_TESTING=OFF \ + -DBUILD_TESTING=OFF "${kokkos_extra_cmake_args}" \ -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir} echo "**** Building Kokkos ${kokkos_version}" @@ -556,7 +577,7 @@ else echo "**** Skipping Kokkos build, install found at: ${kokkos_install_dir}" fi # build_kokkos -fi # if enable_hip +fi # if enable_hip || enable_sycl ################ # VTK-m @@ -598,6 +619,12 @@ if [[ "$enable_hip" == "ON" ]]; then vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DVTKm_ENABLE_KOKKOS_THRUST=OFF" fi +if [[ "$enable_sycl" == "ON" ]]; then + vtkm_extra_cmake_args="-DVTKm_ENABLE_KOKKOS=ON" + vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DCMAKE_PREFIX_PATH=${kokkos_install_dir}" + vtkm_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128" +fi + if [[ "$enable_mpicc" == "ON" ]]; then vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_C_COMPILER=${mpicc_exe}" vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_CXX_COMPILER=${mpicxx_exe}" @@ -617,7 +644,7 @@ cmake -S ${vtkm_src_dir} -B ${vtkm_build_dir} ${cmake_compiler_settings} \ -DVTKm_ENABLE_RENDERING=ON \ -DVTKm_ENABLE_TESTING=OFF\ -DBUILD_TESTING=OFF \ - -DVTKm_ENABLE_BENCHMARKS=OFF ${vtkm_extra_cmake_args} \ + -DVTKm_ENABLE_BENCHMARKS=OFF "${vtkm_extra_cmake_args}" \ -DCMAKE_INSTALL_PREFIX=${vtkm_install_dir} echo "**** Building VTK-m ${vtkm_version}" @@ -763,6 +790,10 @@ if [[ "$enable_hip" == "ON" ]]; then umpire_extra_cmake_args="${umpire_extra_cmake_args} -DROCM_PATH=${ROCM_PATH}" fi +if [[ "$enable_sycl" == "ON" ]]; then + umpire_extra_cmake_args="${umpire_extra_cmake_args} -DENABLE_SYCL=ON" +fi + # build only if install doesn't exist if [ ! -d ${umpire_install_dir} ]; then if ${build_umpire}; then diff --git a/scripts/build_ascent/build_ascent_aurora.sh b/scripts/build_ascent/build_ascent_aurora.sh new file mode 100755 index 000000000..245d47440 --- /dev/null +++ b/scripts/build_ascent/build_ascent_aurora.sh @@ -0,0 +1,21 @@ +#!/bin/bash -l + +export IGC_FunctionCloningThreshold=1 +export IGC_ControlInlineTinySize=100 +export IGC_OCLInlineThreshold=200 +export IGC_PartitionUnit=1 +export IGC_ForceOCLSIMDWidth=16 +export ZE_AFFINITY_MASK=0.0 + +# Proxies +export HTTP_PROXY=http://proxy.alcf.anl.gov:3128 +export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128 +export http_proxy=http://proxy.alcf.anl.gov:3128 +export https_proxy=http://proxy.alcf.anl.gov:3128 + +module reset +module use /soft/modulefiles +module load spack-pe-gcc cmake +module load oneapi/eng-compiler/2023.12.15.002 + +env enable_sycl=ON enable_mpi=ON enable_fortran=ON raja_enable_vectorization=OFF enable_tests=ON enable_verbose=ON ./build_ascent_sycl.sh diff --git a/scripts/build_ascent/build_ascent_sycl.sh b/scripts/build_ascent/build_ascent_sycl.sh new file mode 100755 index 000000000..4f97bef9e --- /dev/null +++ b/scripts/build_ascent/build_ascent_sycl.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +############################################################################## +# Demonstrates how to manually build Ascent and its dependencies, including: +# +# hdf5, conduit, vtk-m, mfem, raja, and umpire +# +# usage example: +# env enable_mpi=ON enable_openmp=ON ./build_ascent.sh +# +# +# Assumes: +# - cmake is in your path +# - selected compilers are in your path or set via env vars +# - [when enabled] MPI and Python (+numpy and mpi4py), are in your path +# +############################################################################## +set -eu -o pipefail + +# 2024-02-08 SYCL support is handled by our unified script +env enable_sycl=ON ./build_ascent.sh