build_ascent caliper + hip support (#1359)

* update rzvernal build recipe

* caliper build for rocm, fix for kokkos path

* update rzvernal recipe

* improve py module install path for pyenv case

scripts/build_ascent/build_ascent.sh

@@ -350,6 +350,11 @@ else
   echo "**** Skipping Python venv build, install found at: ${venv_install_dir}"
 fi # build_pyvenv
 
+if ${build_pyvenv}; then
+    venv_python_ver=`${venv_python_exe} -c "import sys;print('{0}.{1}'.format(sys.version_info.major, sys.version_info.minor))"`
+    venv_python_site_pkgs_dir=${venv_install_dir}/lib/python${venv_python_ver}/site-packages
+fi
+
 ################
 # Caliper
 ################
@@ -382,21 +387,24 @@ fi
 # -DWITH_CUPTI=ON -DWITH_NVTX=ON -DCUDA_TOOLKIT_ROOT_DIR={path} -DCUPTI_PREFIX={path}
 # -DWITH_ROCTRACER=ON -DWITH_ROCTX=ON -DROCM_PREFIX={path}
 
-caliper_windows_cmake_flags="-DCMAKE_CXX_STANDARD=17 -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=ON"
+caliper_windows_cmake_flags="-DCMAKE_CXX_STANDARD=17 -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=ON -DWITH_TOOLS=OFF"
 
 caliper_extra_cmake_args=""
 if [[ "$build_windows" == "ON" ]]; then
   caliper_extra_cmake_args="${caliper_windows_cmake_flags}"
 fi 
 
+if [[ "$enable_hip" == "ON" ]]; then
+  caliper_extra_cmake_args="${caliper_extra_cmake_args} -DWITH_ROCTRACER=ON -DWITH_ROCTX=ON -DROCM_PREFIX=${ROCM_PATH}"
+fi
+
 echo "**** Configuring Caliper ${caliper_version}"
 cmake -S ${caliper_src_dir} -B ${caliper_build_dir} ${cmake_compiler_settings} \
   -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose} \
   -DCMAKE_BUILD_TYPE=${build_config} \
   -DBUILD_SHARED_LIBS=${build_shared_libs} \
   -DCMAKE_INSTALL_PREFIX=${caliper_install_dir} \
-  -DWITH_TOOLS=OFF \
-  -DWITH_MPI=${enable_mpi} ${caliper_windows_cmake_flags}
+  -DWITH_MPI=${enable_mpi} ${caliper_extra_cmake_args}
 
 echo "**** Building Caliper ${caliper_version}"
 cmake --build ${caliper_build_dir} --config ${build_config} -j${build_jobs}
@@ -443,6 +451,7 @@ conduit_extra_cmake_opts=-DENABLE_PYTHON=${enable_python}
 if ${build_pyvenv}; then
   conduit_extra_cmake_opts="${conduit_extra_cmake_opts} -DPYTHON_EXECUTABLE=${venv_python_exe}"
   conduit_extra_cmake_opts="${conduit_extra_cmake_opts} -DSPHINX_EXECUTABLE=${venv_sphinx_exe}"
+  conduit_extra_cmake_opts="${conduit_extra_cmake_opts} -DPYTHON_MODULE_INSTALL_PREFIX=${venv_python_site_pkgs_dir}"
 fi
 
 if ${build_caliper}; then
@@ -483,7 +492,7 @@ fi # build_conduit
 #########################
 kokkos_version=3.7.02
 kokkos_src_dir=$(ospath ${source_dir}/kokkos-${kokkos_version})
-kokkos_build_dir=$(ospath ${build_dir}kokkos-${kokkos_version})
+kokkos_build_dir=$(ospath ${build_dir}/kokkos-${kokkos_version})
 kokkos_install_dir=$(ospath ${install_dir}/kokkos-${kokkos_version}/)
 kokkos_tarball=$(ospath ${source_dir}/kokkos-${kokkos_version}.tar.gz)
 
@@ -905,6 +914,7 @@ echo 'set(ENABLE_FORTRAN ' ${enable_fortran} ' CACHE BOOL "")' >> ${root_dir}/as
 echo 'set(ENABLE_PYTHON ' ${enable_python} ' CACHE BOOL "")' >> ${root_dir}/ascent-config.cmake
 if ${build_pyvenv}; then
 echo 'set(PYTHON_EXECUTABLE ' ${venv_python_exe} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
+echo 'set(PYTHON_MODULE_INSTALL_PREFIX ' ${venv_python_site_pkgs_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
 echo 'set(ENABLE_DOCS ON CACHE BOOL "")' >> ${root_dir}/ascent-config.cmake
 echo 'set(SPHINX_EXECUTABLE ' ${venv_sphinx_exe} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
 fi

scripts/build_ascent/build_ascent_hip_rzvernal.sh

@@ -1,19 +1,20 @@
 module load cmake/3.24.2
 module load craype-accel-amd-gfx90a
-module load rocm/5.2.3
-module load cray-mpich
+module load rocmcc/6.1.2-magic
 
 export MPICH_GPU_SUPPORT_ENABLED=1
 export ROCM_ARCH=gfx90a
-#export CC=$(which cc)
-#export CXX=$(which CC)
-#export FTN=$(which ftn)
 
-export CC=/opt/rocm-5.2.3/llvm/bin/amdclang
-export CXX=/opt/rocm-5.2.3/llvm/bin/amdclang++
-export CFLAGS="-I${ROCM_PATH}/include"
-export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
-export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64"
+export CC=/opt/rocm-6.1.2/bin/amdclang
+export CXX=/opt/rocm-6.1.2/bin/amdclang++
+#export CFLAGS="-I${ROCM_PATH}/include"
+#export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
+#export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64"
+
+export enable_python="${enable_python:=ON}"
+export build_caliper="${build_caliper:=true}"
+export build_pyvenv="${bbuild_pyvenv:=true}"
+
 ./build_ascent_hip.sh