Doc: Update Lassen (LLNL) (#3942)

* Doc: Update Lassen (LLNL) Update the modules and dependencies on Lassen (LLNL). * Switch to Clang 12 Avoid GCC 8.3 compiler bugs.
ECP-WarpX · May 27, 2023 · cfa9f50 · cfa9f50
1 parent 652c6a2
commit cfa9f50
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 23 deletions.
diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst
@@ -51,16 +51,22 @@ And since Lassen does not yet provide a module for them, install ADIOS2, BLAS++
    cmake -S src/c-blosc -B src/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/c-blosc-1.21.1
    cmake --build src/c-blosc-lassen-build --target install --parallel 16
 
+   # HDF5
+   git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5
+   rm -rf src/hdf5-lassen-build
+   cmake -S src/hdf5 -B src/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/hdf5-1.14.1.2
+   cmake --build src/hdf5-lassen-build --target install --parallel 16
+
    # ADIOS2
-   git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2
+   git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2
    rm -rf src/adios2-lassen-build
-   cmake -S src/adios2 -B src/adios2-lassen-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.7.1
+   cmake -S src/adios2 -B src/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.8.3
    cmake --build src/adios2-lassen-build --target install -j 16
 
    # BLAS++ (for PSATD+RZ)
    git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp
    rm -rf src/blaspp-lassen-build
-   cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=CUDA -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master
+   cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master
    cmake --build src/blaspp-lassen-build --target install --parallel 16
 
    # LAPACK++ (for PSATD+RZ)
@@ -74,15 +80,15 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman
 .. code-block:: bash
 
    cd $HOME/src/warpx
-   rm -rf build
 
-   cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON
-   cmake --build build -j 10
+   rm -rf build_lassen
+   cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_DIMS="1;2;RZ;3" -DWarpX_PSATD=ON
+   cmake --build build_lassen -j 10
 
 The other :ref:`general compile-time options <building-cmake>` apply as usual.
 
 **That's it!**
-A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run <running-cpp-lassen>` with a :ref:`3D example inputs file <usage-examples>`.
+WarpX executables for 1D, 2D, RZ and 3D are now in ``build_lassen/bin/`` and :ref:`can be run <running-cpp-lassen>` with the respective :ref:`example inputs files <usage-examples>`.
 Most people execute the binary directly or copy it out to a location in ``/p/gpfs1/$(whoami)``.
 
 
@@ -133,7 +139,7 @@ Known System Issues
 .. warning::
 
    Feb 17th, 2022 (INC0278922):
-   The implementation of AllGatherv in IBM's MPI optimization library "libcollectives" is broken and leads to HDF5 crashes for multi-node runs.
+   The implementation of ``AllGatherv`` in IBM's MPI optimization library "libcollectives" is broken and leads to HDF5 crashes for multi-node runs.
 
    Our batch script templates above `apply this work-around <https://github.com/ECP-WarpX/WarpX/pull/2874>`__ *before* the call to ``jsrun``, which avoids the broken routines from IBM and trades them for an OpenMPI implementation of collectives:
 

diff --git a/Docs/source/install/hpc/summit.rst b/Docs/source/install/hpc/summit.rst
@@ -53,7 +53,7 @@ For PSATD+RZ simulations, you will need to build BLAS++ and LAPACK++:
   # BLAS++ (for PSATD+RZ)
   git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp
   rm -rf src/blaspp-summit-build
-  cmake -S src/blaspp -B src/blaspp-summit-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/summit/blaspp-master
+  cmake -S src/blaspp -B src/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/summit/blaspp-master
   cmake --build src/blaspp-summit-build --target install --parallel 10
 
   # LAPACK++ (for PSATD+RZ)

diff --git a/Tools/machines/lassen-llnl/lassen.bsub b/Tools/machines/lassen-llnl/lassen.bsub
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2022 Axel Huebl
+# Copyright 2020-2023 Axel Huebl
 #
 # This file is part of WarpX.
 #

diff --git a/Tools/machines/lassen-llnl/lassen_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_warpx.profile.example
@@ -1,23 +1,21 @@
 # please set your project account
-#export proj=<yourProject>
+#export proj="<yourProjectNameHere>"  # edit this and comment in
 
 # required dependencies
-module load cmake/3.21.1
-module load gcc/8.3.1
-module load cuda/11.2.0
-
-# optional: for PSATD support
-module load fftw/3.3.8
+module load cmake/3.23.1
+module load clang/12.0.1-gcc-8.3.1
+module load cuda/12.0.0
 
 # optional: for QED lookup table generation support
 module load boost/1.70.0
 
 # optional: for openPMD support
-module load hdf5-parallel/1.12.2
 export CMAKE_PREFIX_PATH=$HOME/sw/lassen/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=$HOME/sw/lassen/adios2-2.7.1:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=$HOME/sw/lassen/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=$HOME/sw/lassen/adios2-2.8.3:$CMAKE_PREFIX_PATH
 export LD_LIBRARY_PATH=$HOME/sw/lassen/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=$HOME/sw/lassen/adios2-2.7.1/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$HOME/sw/lassen/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$HOME/sw/lassen/adios2-2.8.3/lib64:$LD_LIBRARY_PATH
 
 # optional: for PSATD in RZ geometry support
 export CMAKE_PREFIX_PATH=$HOME/sw/lassen/blaspp-master:$CMAKE_PREFIX_PATH
@@ -36,10 +34,11 @@ shopt -s direxpand
 
 # optimize CUDA compilation for V100
 export AMREX_CUDA_ARCH=7.0
+export CUDAARCHS=70
 
 # compiler environment hints
-export CC=$(which gcc)
-export CXX=$(which g++)
+export CC=$(which clang)
+export CXX=$(which clang++)
 export FC=$(which gfortran)
 export CUDACXX=$(which nvcc)
-export CUDAHOSTCXX=$(which g++)
+export CUDAHOSTCXX=$(which clang++)