diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 704b7ed239e..1c7cdbe86a9 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -51,16 +51,22 @@ And since Lassen does not yet provide a module for them, install ADIOS2, BLAS++ cmake -S src/c-blosc -B src/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/c-blosc-1.21.1 cmake --build src/c-blosc-lassen-build --target install --parallel 16 + # HDF5 + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 + rm -rf src/hdf5-lassen-build + cmake -S src/hdf5 -B src/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/hdf5-1.14.1.2 + cmake --build src/hdf5-lassen-build --target install --parallel 16 + # ADIOS2 - git clone -b v2.7.1 https://github.com/ornladios/ADIOS2.git src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2 rm -rf src/adios2-lassen-build - cmake -S src/adios2 -B src/adios2-lassen-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.7.1 + cmake -S src/adios2 -B src/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/adios2-2.8.3 cmake --build src/adios2-lassen-build --target install -j 16 # BLAS++ (for PSATD+RZ) git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp rm -rf src/blaspp-lassen-build - cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=CUDA -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master + cmake -S src/blaspp -B src/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DBLA_VENDOR=IBMESSL -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/lassen/blaspp-master cmake --build src/blaspp-lassen-build --target install --parallel 16 # LAPACK++ (for PSATD+RZ) @@ -74,15 +80,15 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman .. code-block:: bash cd $HOME/src/warpx - rm -rf build - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON - cmake --build build -j 10 + rm -rf build_lassen + cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_DIMS="1;2;RZ;3" -DWarpX_PSATD=ON + cmake --build build_lassen -j 10 The other :ref:`general compile-time options ` apply as usual. **That's it!** -A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. +WarpX executables for 1D, 2D, RZ and 3D are now in ``build_lassen/bin/`` and :ref:`can be run ` with the respective :ref:`example inputs files `. Most people execute the binary directly or copy it out to a location in ``/p/gpfs1/$(whoami)``. @@ -133,7 +139,7 @@ Known System Issues .. warning:: Feb 17th, 2022 (INC0278922): - The implementation of AllGatherv in IBM's MPI optimization library "libcollectives" is broken and leads to HDF5 crashes for multi-node runs. + The implementation of ``AllGatherv`` in IBM's MPI optimization library "libcollectives" is broken and leads to HDF5 crashes for multi-node runs. Our batch script templates above `apply this work-around `__ *before* the call to ``jsrun``, which avoids the broken routines from IBM and trades them for an OpenMPI implementation of collectives: diff --git a/Docs/source/install/hpc/summit.rst b/Docs/source/install/hpc/summit.rst index 6f9526a5e23..9eeb660bd96 100644 --- a/Docs/source/install/hpc/summit.rst +++ b/Docs/source/install/hpc/summit.rst @@ -53,7 +53,7 @@ For PSATD+RZ simulations, you will need to build BLAS++ and LAPACK++: # BLAS++ (for PSATD+RZ) git clone https://github.com/icl-utk-edu/blaspp.git src/blaspp rm -rf src/blaspp-summit-build - cmake -S src/blaspp -B src/blaspp-summit-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/summit/blaspp-master + cmake -S src/blaspp -B src/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=$HOME/sw/summit/blaspp-master cmake --build src/blaspp-summit-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) diff --git a/Tools/machines/lassen-llnl/lassen.bsub b/Tools/machines/lassen-llnl/lassen.bsub index 5196acfb49b..bb14b04d958 100644 --- a/Tools/machines/lassen-llnl/lassen.bsub +++ b/Tools/machines/lassen-llnl/lassen.bsub @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2020-2022 Axel Huebl +# Copyright 2020-2023 Axel Huebl # # This file is part of WarpX. # diff --git a/Tools/machines/lassen-llnl/lassen_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_warpx.profile.example index 61e8f971689..fb605b53ec2 100644 --- a/Tools/machines/lassen-llnl/lassen_warpx.profile.example +++ b/Tools/machines/lassen-llnl/lassen_warpx.profile.example @@ -1,23 +1,21 @@ # please set your project account -#export proj= +#export proj="" # edit this and comment in # required dependencies -module load cmake/3.21.1 -module load gcc/8.3.1 -module load cuda/11.2.0 - -# optional: for PSATD support -module load fftw/3.3.8 +module load cmake/3.23.1 +module load clang/12.0.1-gcc-8.3.1 +module load cuda/12.0.0 # optional: for QED lookup table generation support module load boost/1.70.0 # optional: for openPMD support -module load hdf5-parallel/1.12.2 export CMAKE_PREFIX_PATH=$HOME/sw/lassen/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lassen/adios2-2.7.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/lassen/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/lassen/adios2-2.8.3:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=$HOME/sw/lassen/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lassen/adios2-2.7.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/lassen/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/lassen/adios2-2.8.3/lib64:$LD_LIBRARY_PATH # optional: for PSATD in RZ geometry support export CMAKE_PREFIX_PATH=$HOME/sw/lassen/blaspp-master:$CMAKE_PREFIX_PATH @@ -36,10 +34,11 @@ shopt -s direxpand # optimize CUDA compilation for V100 export AMREX_CUDA_ARCH=7.0 +export CUDAARCHS=70 # compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) +export CC=$(which clang) +export CXX=$(which clang++) export FC=$(which gfortran) export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=$(which g++) +export CUDAHOSTCXX=$(which clang++)