Merge pull request CompFUSE#319 from PDoakORNL/new_fixes

Fix Frontier build
PDoakORNL · Jan 16, 2024 · 017bae8 · 017bae8
2 parents b60e876 + f33e4dc
commit 017bae8
Show file tree

Hide file tree

Showing 13 changed files with 40 additions and 25 deletions.
diff --git a/build-aux/frontier_build.sh b/build-aux/frontier_build.sh
@@ -1,16 +1,15 @@
 
-#export CXXFLAGS="-I/opt/cray/pe/papi/6.0.0.17/include"
-#export LDFLAGS="-L/opt/cray/pe/papi/6.0.0.17/lib -lpapi"
-#-DDCA_PROFILER=PAPI
+export FFTW_PATH=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/fftw-3.3.10-tajdtzkealhold4bmpuq7wiwzurnclr4
+export MAGMA_ROOT=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/magma-2.7.2-gbjcrprqdw7y5uplm5upmqbi65zqwubb
+export OPENBLAS_ROOT=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/openblas-0.3.25-t62dxdtaqba6lzrwoy4uddswlprgma6n
+export HDF5_ROOT=/lustre/orion/world-shared/cph102/epd/spack/opt/spack/linux-sles15-zen3/gcc-12.2.0/hdf5-1.14.3-3so3g5x2roywum3edvjun7jbhwisei6p
+export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/opt/rocm-5.7.0/hip/bin:${HDF5_ROOT}
+export PATH=/sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/cmake-3.23.2-4r4mpiba7cwdw2hlakh5i7tchi64s3qd/bin:${PATH}
 
-FFTW_PATH=/sw/frontier/spack-envs/base/opt/cray-sles15-zen3/gcc-11.2.0/fftw-3.3.10-njtwalw5xctv6e3fshucadlgu32jb4k6
-MAGMA_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/magma-2.7.2-c5m5kzaz7irix5hk5zzf3mrwwlij43is
-OPENBLAS_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/openblas-0.3.25-scaywvuh5zsm5u7smg54plj2oyf7nekv
-HDF5_ROOT=/lustre/orion/world-shared/cph102/epd/gcc-11.2.0/hdf5-1.14.3-mjg4476nl2cfrmk5qmcrmvo7ryg2bdwd
 cmake -DDCA_WITH_CUDA=off -DDCA_WITH_HIP=ON \
       -DFFTW_ROOT=$FFTW_PATH \
       -DDCA_FIX_BROKEN_MPICH=ON \
-      -DROCM_ROOT=${OLCF_ROCM_ROOT} \
+      -DROCM_ROOT=${ROCM_PATH} \
       -DMAGMA_ROOT=${MAGMA_ROOT} \
       -DLAPACK_ROOT=${OPENBLAS_ROOT} \
       -DBLAS_ROOT=${OPENBLAS_ROOT} \
@@ -20,7 +19,8 @@ cmake -DDCA_WITH_CUDA=off -DDCA_WITH_HIP=ON \
       -DAMDGPU_TARGETS=gfx90a \
       -DCMAKE_C_COMPILER=mpicc \
       -DCMAKE_CXX_COMPILER=mpic++ \
+      -DCMAKE_HIP_COMPILER=/opt/rocm-5.7.0/llvm/bin/clang++ \
       -DCMAKE_INSTALL_PREFIX=$INST \
-      -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH} ${HDF5_ROOT}" \
+      -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \
       -GNinja \
       ..
diff --git a/build-aux/frontier_load_modules.sh b/build-aux/frontier_load_modules.sh
@@ -7,12 +7,8 @@
 # Usage: source frontier_load_modules.sh
 
 module reset
-module load gcc/11.2.0
-module load openblas
-module load hdf5
-module load rocm/5.4.0
-module load fftw
-module load cmake
+module load gcc/12.2.0
+module load rocm/5.7.0
 module load ninja
 
 # After 2 weeks of digging through opaque linking and runtime errors,

diff --git a/cmake/dca_external_libs.cmake b/cmake/dca_external_libs.cmake
@@ -42,7 +42,7 @@ list(APPEND DCA_EXTERNAL_LIBS ${BLAS_LIBRARIES})
 
 if (NOT HDF5_LIBRARIES)
   message("Trying to find HDF5 library")
-  set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE true)
+  set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE false)
   set(HDF5_PREFER_PARALLEL false)
   find_package(HDF5 REQUIRED COMPONENTS C CXX)
   message("HDF5: ${HDF5_FOUND} ${HDF5_LIBRARIES}")

diff --git a/include/dca/parallel/mpi_concurrency/mpi_type_map.hpp b/include/dca/parallel/mpi_concurrency/mpi_type_map.hpp
@@ -18,6 +18,7 @@
 
 #include <complex>
 #include <cstdlib>
+#include <cstdint>
 #include <type_traits>
 #include "dca_mpi.h"
 

diff --git a/include/dca/phys/dca_loop/dca_loop.hpp b/include/dca/phys/dca_loop/dca_loop.hpp
@@ -236,7 +236,6 @@ void DcaLoop<ParametersType, DDT, MCIntegratorType, DIST>::initialize() {
   }
   else if (parameters.get_initial_self_energy() != "zero") {
 #ifdef DCA_HAVE_ADIOS2
-    io::IOType iotype = io::extensionToIOType(autoresume_filename);
     if (io::extensionToIOType(parameters.get_initial_self_energy()) == io::IOType::ADIOS2)
       MOMS.initializeSigma(concurrency.get_adios(), parameters.get_initial_self_energy());
     else

diff --git a/include/dca/phys/dca_step/cluster_solver/ctint/accumulator/ctint_accumulator.hpp b/include/dca/phys/dca_step/cluster_solver/ctint/accumulator/ctint_accumulator.hpp
@@ -40,8 +40,8 @@ template <class Parameters, linalg::DeviceType device,
 class CtintAccumulator : public MC_accumulator_data<typename Parameters::Scalar> {
 public:
   constexpr static ClusterSolverId solver_id{ClusterSolverId::CT_INT};
-  using Real = typename dca::config::McOptions::MC_REAL;
-  using Scalar = typename dca::util::ScalarSelect<Real,Parameters::complex_g0>::type;
+  using Real = typename Parameters::Real;
+  using Scalar = typename Parameters::Scalar;
   using Base = MC_accumulator_data<Scalar>;
   using this_type = CtintAccumulator<Parameters, device, DIST>;
   using Base::accumulated_phase_;

diff --git a/include/dca/phys/dca_step/cluster_solver/ctint/ctint_cluster_solver.hpp b/include/dca/phys/dca_step/cluster_solver/ctint/ctint_cluster_solver.hpp
@@ -521,7 +521,8 @@ auto CtintClusterSolver<device_t, Parameters, use_submatrix, DIST>::local_G_k_w(
   SpGreensFunction M;
   math::transform::FunctionTransform<RDmn, KDmn>::execute(M_r, M);
 
-  const double sign = accumulator_.get_accumulated_sign();
+  // This phase can be a long long if we're dealing with real Scalars and immense iterations.
+  const long double sign = accumulator_.get_accumulated_phase();
 
   M /= sign;
   SpGreensFunction G_k_w("G_k_w");

diff --git a/test/integration/statistical_tests/real_materials/CMakeLists.txt b/test/integration/statistical_tests/real_materials/CMakeLists.txt
@@ -4,7 +4,7 @@ if(DCA_WITH_TESTS_STOCHASTIC)
   #  leave it to break the build if its not maintained.  I found it completely inoperable after 4 years of not being built.
   add_executable(NiO_fullDCA NiO_fullDCA.cpp)
   target_link_libraries(NiO_fullDCA ${DCA_LIBS} ${DCA_THREADING_LIBS})
-  target_include_directories(NiO_fullDCA PRIVATE ${DCA_INCLUDE_DIRS})
+  target_include_directories(NiO_fullDCA PRIVATE ${DCA_INCLUDE_DIRS} ${PROJECT_SOURCE_DIR})
   target_compile_definitions(NiO_fullDCA PRIVATE TEST_DIRECTORY="${CMAKE_CURRENT_SOURCE_DIR}/")
   target_compile_definitions(NiO_fullDCA PRIVATE DCA_SOURCE_DIR="${PROJECT_SOURCE_DIR}")
 

diff --git a/test/integration/statistical_tests/real_materials/NiO_fullDCA.cpp b/test/integration/statistical_tests/real_materials/NiO_fullDCA.cpp
@@ -22,6 +22,14 @@
 #include "dca/util/git_version.hpp"
 #include "dca/distribution/dist_types.hpp"
 #include "dca_loop_wrapper.hpp"
+
+#include "test/mock_mcconfig.hpp"
+namespace dca {
+namespace config {
+using McOptions = MockMcOptions<double>;
+}  // namespace config
+}  // namespace dca
+
 #include "NiO_setup.hpp"
 
 constexpr auto SOLVER = dca::ClusterSolverId::CT_INT;
@@ -51,7 +59,7 @@ struct MakeOutput;
 
 template <class DCACALC, class Parameters>
 struct MakeOutput<DCACALC, Parameters, dca::ClusterSolverId::CT_INT>{
-  void operator()(DCACALC& calculation, Parameters& parameters, Concurrency& concurrency) {
+  void operator()(DCACALC& calculation, Parameters& parameters [[maybe_unused]], Concurrency& concurrency) {
 
     const int n_frequencies = 10;
     auto G_k_cut = dca::math::util::bandDiagonal(

diff --git a/test/integration/statistical_tests/real_materials/NiO_setup.hpp b/test/integration/statistical_tests/real_materials/NiO_setup.hpp
@@ -50,19 +50,21 @@ constexpr DeviceType default_device = GPU;
 constexpr DeviceType default_device = CPU;
 #endif  // DCA_HAVE_CUDA
 
+using Scalar = double;
+
 const std::string test_directory =
     DCA_SOURCE_DIR "/test/integration/statistical_tests/real_materials/";
 
 using Model = dca::phys::models::TightBindingModel<dca::phys::models::material_lattice<
-    dca::phys::models::NiO_unsymmetric, dca::phys::domains::no_symmetry<3>>>;
+  dca::phys::models::Material::NiO_unsymmetric, dca::phys::domains::no_symmetry<3>>>;
 using RandomNumberGenerator = dca::math::random::StdRandomWrapper<std::ranlux48_base>;
 
 using dca::ClusterSolverId;
 
 template <class Concurrency, ClusterSolverId name>
 using TestParameters =
     dca::phys::params::Parameters<Concurrency, Threading, dca::profiling::NullProfiler, Model,
-                                  RandomNumberGenerator, name>;
+                                  RandomNumberGenerator, name, dca::NumericalTraits<dca::util::RealAlias<Scalar>, Scalar>>;
 
 template <ClusterSolverId name, class Concurrency>
 using DcaData = dca::phys::DcaData<TestParameters<Concurrency, name>>;

diff --git a/..._tests/real_materials/NIO_stattest_gs.cpp → ..._tests/real_materials/NiO_stattest_gs.cpp b/..._tests/real_materials/NIO_stattest_gs.cpp → ..._tests/real_materials/NiO_stattest_gs.cpp
diff --git a/...integration/statistical_tests/square_lattice/ctint_square_lattice_validation_stattest.cpp b/...integration/statistical_tests/square_lattice/ctint_square_lattice_validation_stattest.cpp
@@ -17,6 +17,14 @@
 
 #include "dca/math/statistical_testing/statistical_testing.hpp"
 #include "dca/config/profiler.hpp"
+
+#include "test/mock_mcconfig.hpp"
+namespace dca {
+namespace config {
+using McOptions = MockMcOptions<double>;
+}  // namespace config
+}  // namespace dca
+
 #include "test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp"
 
 dca::testing::DcaMpiTestEnvironment* dca_test_env;

diff --git a/test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp b/test/integration/statistical_tests/square_lattice/square_lattice_setup.hpp
@@ -62,7 +62,7 @@ using dca::ClusterSolverId;
 template <ClusterSolverId CS_NAME = ClusterSolverId::CT_AUX>
 using ParametersType =
     dca::phys::params::Parameters<dca::testing::DcaMpiTestEnvironment::ConcurrencyType, Threading,
-                                  dca::profiling::NullProfiler, Model, RandomNumberGenerator, CS_NAME>;
+                                  dca::profiling::NullProfiler, Model, RandomNumberGenerator, CS_NAME, dca::NumericalTraits<dca::util::RealAlias<Scalar>, Scalar>>;
 
 template <ClusterSolverId name>
 using DcaData = dca::phys::DcaData<ParametersType<name>>;