Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@toxa81
toxa81 committed Jan 14, 2019
1 parent c8c5c27 commit 4dba801
Show file tree
Hide file tree
Showing 7 changed files with 60 additions and 97 deletions.
0 Makefile → Makefile.osx
View file
File renamed without changes.
99 changes: 28 additions & 71 deletions SDDK/communicator.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
#include <algorithm>
#include <cstring>
#include <map>
#include <GPU/acc.hpp>

namespace sddk {

Expand Down Expand Up @@ -434,14 +433,11 @@ class Communicator

inline void barrier() const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Barrier");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Barrier");
#endif
assert(mpi_comm() != MPI_COMM_NULL);
CALL_MPI(MPI_Barrier, (mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T, mpi_op_t mpi_op__ = mpi_op_t::sum>
Expand Down Expand Up @@ -500,27 +496,21 @@ class Communicator
template <typename T, mpi_op_t mpi_op__ = mpi_op_t::sum>
inline void iallreduce(T* buffer__, int count__, MPI_Request* req__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Iallreduce");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Iallreduce");
#endif
CALL_MPI(MPI_Iallreduce, (MPI_IN_PLACE, buffer__, count__, mpi_type_wrapper<T>::kind(),
mpi_op_wrapper<mpi_op__>::kind(), mpi_comm(), req__));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

/// Perform buffer broadcast.
template <typename T>
inline void bcast(T* buffer__, int count__, int root__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Bcast");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Bcast");
#endif
CALL_MPI(MPI_Bcast, (buffer__, count__, mpi_type_wrapper<T>::kind(), root__, mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

inline void bcast(std::string& str__, int root__) const
Expand All @@ -543,29 +533,23 @@ class Communicator
template <typename T>
void allgather(T* buffer__, int const* recvcounts__, int const* displs__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Allgatherv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Allgatherv");
#endif
CALL_MPI(MPI_Allgatherv, (MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, buffer__, recvcounts__, displs__,
mpi_type_wrapper<T>::kind(), mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

/// Out-of-place MPI_Allgatherv.
template <typename T>
void
allgather(T* const sendbuf__, int sendcount__, T* recvbuf__, int const* recvcounts__, int const* displs__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Allgatherv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Allgatherv");
#endif
CALL_MPI(MPI_Allgatherv, (sendbuf__, sendcount__, mpi_type_wrapper<T>::kind(), recvbuf__, recvcounts__,
displs__, mpi_type_wrapper<T>::kind(), mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T>
Expand Down Expand Up @@ -614,53 +598,41 @@ class Communicator
template <typename T>
void send(T const* buffer__, int count__, int dest__, int tag__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Send");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Send");
#endif
CALL_MPI(MPI_Send, (buffer__, count__, mpi_type_wrapper<T>::kind(), dest__, tag__, mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T>
Request isend(T const* buffer__, int count__, int dest__, int tag__) const
{
Request req;
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Isend");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Isend");
#endif
CALL_MPI(MPI_Isend, (buffer__, count__, mpi_type_wrapper<T>::kind(), dest__, tag__, mpi_comm(), &req.handler()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
return std::move(req);
}

template <typename T>
void recv(T* buffer__, int count__, int source__, int tag__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Recv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Recv");
#endif
CALL_MPI(MPI_Recv,
(buffer__, count__, mpi_type_wrapper<T>::kind(), source__, tag__, mpi_comm(), MPI_STATUS_IGNORE));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T>
Request irecv(T* buffer__, int count__, int source__, int tag__) const
{
Request req;
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Irecv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Irecv");
#endif
CALL_MPI(MPI_Irecv, (buffer__, count__, mpi_type_wrapper<T>::kind(), source__, tag__, mpi_comm(), &req.handler()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
return std::move(req);
}

Expand All @@ -669,23 +641,20 @@ class Communicator
{
int sendcount = recvcounts__[rank()];

#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Gatherv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Gatherv");
#endif
CALL_MPI(MPI_Gatherv, (sendbuf__, sendcount, mpi_type_wrapper<T>::kind(), recvbuf__, recvcounts__, displs__,
mpi_type_wrapper<T>::kind(), root__, mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

/// Gather data on a given rank.
template <typename T>
void gather(T const* sendbuf__, T* recvbuf__, int offset__, int count__, int root__) const
{

#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Gatherv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Gatherv");
#endif
std::vector<int> v(size() * 2);
v[2 * rank()] = count__;
Expand All @@ -703,36 +672,27 @@ class Communicator
}
CALL_MPI(MPI_Gatherv, (sendbuf__, count__, mpi_type_wrapper<T>::kind(), recvbuf__, counts.data(),
offsets.data(), mpi_type_wrapper<T>::kind(), root__, mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T>
void scatter(T const* sendbuf__, T* recvbuf__, int const* sendcounts__, int const* displs__, int root__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Scatterv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Scatterv");
#endif
int recvcount = sendcounts__[rank()];
CALL_MPI(MPI_Scatterv, (sendbuf__, sendcounts__, displs__, mpi_type_wrapper<T>::kind(), recvbuf__, recvcount,
mpi_type_wrapper<T>::kind(), root__, mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T>
void alltoall(T const* sendbuf__, int sendcounts__, T* recvbuf__, int recvcounts__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Alltoall");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Alltoall");
#endif
CALL_MPI(MPI_Alltoall, (sendbuf__, sendcounts__, mpi_type_wrapper<T>::kind(), recvbuf__, recvcounts__,
mpi_type_wrapper<T>::kind(), mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

template <typename T>
Expand All @@ -743,14 +703,11 @@ class Communicator
int const* recvcounts__,
int const* rdispls__) const
{
#if defined(__GPU_NVTX_MPI)
acc::begin_range_marker("MPI_Alltoallv");
#if defined(__PROFILE_MPI)
PROFILE("MPI_Alltoallv");
#endif
CALL_MPI(MPI_Alltoallv, (sendbuf__, sendcounts__, sdispls__, mpi_type_wrapper<T>::kind(), recvbuf__,
recvcounts__, rdispls__, mpi_type_wrapper<T>::kind(), mpi_comm()));
#if defined(__GPU_NVTX_MPI)
acc::end_range_marker();
#endif
}

//==alltoall_descriptor map_alltoall(std::vector<int> local_sizes_in, std::vector<int> local_sizes_out) const
Expand Down
14 changes: 9 additions & 5 deletions SDDK/gvec.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,14 @@
#include <numeric>
#include <map>
#include <iostream>
#include <assert.h>
#include "memory.hpp"
#include "fft3d_grid.hpp"
#include "geometry3d.hpp"
#include "serializer.hpp"
//#include "constants.hpp"
#include "splindex.hpp"
#include "utils/utils.hpp"
#include "utils/profiler.hpp"

using namespace geometry3d;

Expand Down Expand Up @@ -611,8 +615,8 @@ class Gvec
/// Return the volume of the real space unit cell that corresponds to the reciprocal lattice of G-vectors.
inline double omega() const
{
double const twopi = 6.28318530717958647692528676656;
return std::pow(twopi, 3) / std::abs(lattice_vectors().det());
double const twopi_pow3 = 248.050213442398561403810520537;
return twopi_pow3 / std::abs(lattice_vectors().det());
}

/// Return the total number of G-vectors within the cutoff.
Expand Down Expand Up @@ -1162,15 +1166,15 @@ class Gvec_partition
return gvec_;
}

void gather_pw_fft(double_complex* f_pw_local__, double_complex* f_pw_fft__) const
void gather_pw_fft(std::complex<double>* f_pw_local__, std::complex<double>* f_pw_fft__) const
{
int rank = gvec().comm().rank();
/* collect scattered PW coefficients */
comm_ortho_fft().allgather(f_pw_local__, gvec().gvec_count(rank), f_pw_fft__,
gvec_fft_slab().counts.data(), gvec_fft_slab().offsets.data());
}

void gather_pw_global(double_complex* f_pw_fft__, double_complex* f_pw_global__) const
void gather_pw_global(std::complex<double>* f_pw_fft__, std::complex<double>* f_pw_global__) const
{
for (int ig = 0; ig < gvec().count(); ig++) {
/* position inside fft buffer */
Expand Down
1 change: 0 additions & 1 deletion SDDK/sddk.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ namespace sddk {

}

#include "profiler.hpp"
#include "communicator.hpp"
#include "mpi_grid.hpp"
#include "blacs_grid.hpp"
Expand Down
24 changes: 10 additions & 14 deletions test_fft_correctness_1.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ int test_fft(cmd_args& args, device_t pu__)

for (int ig = 0; ig < gvec.num_gvec(); ig++) {
auto v = gvec.gvec(ig);
//if (Communicator::world().rank() == 0) {
// printf("ig: %6i, gvec: %4i %4i %4i ", ig, v[0], v[1], v[2]);
//}
f.zero();
f[ig] = 1.0;
/* load local set of PW coefficients */
Expand Down Expand Up @@ -91,29 +88,28 @@ int run_test(cmd_args& args)

int main(int argn, char **argv)
{
std::cout << "aaaa" << "\n";
cmd_args args;
args.register_key("--cutoff=", "{double} cutoff radius in G-space");

args.parse_args(argn, argv);
std::cout << "bbb" << "\n";
if (args.exist("help")) {
printf("Usage: %s [options]\n", argv[0]);
args.print_help();
return 0;
}
std::cout << "ccc" << "\n";

sirius::initialize(true);
std::cout << "eeee" << "\n";
printf("running %-30s : ", argv[0]);
if (Communicator::world().rank() == 0) {
printf("running %-30s : ", argv[0]);
}
int result = run_test(args);
//if (result) {
// printf("\x1b[31m" "Failed" "\x1b[0m" "\n");
//} else {
// printf("\x1b[32m" "OK" "\x1b[0m" "\n");
//}
std::cout << "ddd" << "\n";
if (Communicator::world().rank() == 0) {
if (result) {
printf("\x1b[31m" "Failed" "\x1b[0m" "\n");
} else {
printf("\x1b[32m" "OK" "\x1b[0m" "\n");
}
}
sirius::finalize();

return result;
Expand Down
Binary file modified utils/a.out
View file
Binary file not shown.
19 changes: 13 additions & 6 deletions SDDK/profiler.hpp → utils/profiler.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,14 @@
#define __PROFILER_HPP__

#include <string>
#include "communicator.hpp"
#include "../utils/timer.hpp"
#include "timer.hpp"

#define __PROFILE
#define __PROFILE_TIME
//#define __PROFILE_STACK
//#define __PROFILE_FUNC

namespace sddk {
namespace utils {

/// Simple profiler and function call tracker.
class profiler
Expand Down Expand Up @@ -87,7 +86,11 @@ class profiler
for (int i = 0; i < tab; i++) {
printf(" ");
}
printf("[rank%04i] + %s\n", Communicator::world().rank(), label_.c_str());
#if defined(MPI_VERSION)
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
printf("[rank%04i] + %s\n", rank, label_.c_str());
#endif
#endif

#if defined(__PROFILE_TIME)
Expand All @@ -109,7 +112,11 @@ class profiler
for (int i = 0; i < tab; i++) {
printf(" ");
}
printf("[rank%04i] - %s\n", Communicator::world().rank(), label_.c_str());
#if defined(MPI_VERSION)
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
printf("[rank%04i] - %s\n", rank, label_.c_str());
#endif
#endif

#ifdef __PROFILE_STACK
Expand Down Expand Up @@ -143,7 +150,7 @@ class profiler
#endif

#ifdef __PROFILE
#define PROFILE(name) sddk::profiler profiler__(__function_name__, __FILE__, __LINE__, name);
#define PROFILE(name) utils::profiler profiler__(__function_name__, __FILE__, __LINE__, name);
#else
#define PROFILE(...)
#endif
Expand Down

0 comments on commit 4dba801

Please sign in to comment.