Skip to content

Commit

Permalink
Update HDF5 particle write and benchmark codes (AMReX-Codes#1035)
Browse files Browse the repository at this point in the history 
* Update HDF5 related code for better write performance

* Update Makefile for HDF5Benchmark

* Update GNUmakefile

* Update inputs

* Use correct MPI datatype from AMReX

Co-authored-by: Houjun Tang <htang4@lbl.gov>
  • Loading branch information
@houjun
houjun and houjun authored Jun 18, 2020
1 parent dfa064a commit 4b4719a
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 74 deletions.
124 changes: 76 additions & 48 deletions Src/Base/AMReX_PlotFileUtil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@

#ifdef AMREX_USE_HDF5
#include "hdf5.h"

#ifdef AMREX_USE_HDF5_ASYNC
#include "h5_vol_external_async_native.h"
#endif

#endif

namespace amrex {
Expand Down Expand Up @@ -453,7 +458,6 @@ EB_WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels,

#endif


#ifdef AMREX_USE_HDF5
static int CreateWriteHDF5AttrDouble(hid_t loc, const char *name, hsize_t n, const double *data)
{
Expand Down Expand Up @@ -753,12 +757,14 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
std::string filename(plotfilename + ".h5");

// Write out root level metadata
hid_t fapl, fid, grp;
hid_t fapl, dxpl, fid, grp;

if(ParallelDescriptor::IOProcessor()) {
// Have only one rank to create and write metadata (header)
fapl = H5Pcreate (H5P_FILE_ACCESS);
H5Pset_fapl_mpio(fapl, MPI_COMM_SELF, MPI_INFO_NULL);
H5Pset_coll_metadata_write(fapl, true);
H5Pset_all_coll_metadata_ops(fapl, true);

// Create the HDF5 file
fid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
Expand All @@ -778,16 +784,62 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,

ParallelDescriptor::Barrier();

hid_t babox_id;
babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int));
if (1 == AMREX_SPACEDIM) {
H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT);
}
else if (2 == AMREX_SPACEDIM) {
H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT);
}
else if (3 == AMREX_SPACEDIM) {
H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT);
}

hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int));
if (1 == AMREX_SPACEDIM) {
H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT);
}
else if (2 == AMREX_SPACEDIM) {
H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT);
}
else if (3 == AMREX_SPACEDIM) {
H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT);
}

fapl = H5Pcreate (H5P_FILE_ACCESS);
H5Pset_fapl_mpio(fapl, ParallelDescriptor::Communicator(), MPI_INFO_NULL);
int alignment = 16 * 1024 * 1024;
H5Pset_alignment(fapl, alignment, alignment);
H5Pset_coll_metadata_write(fapl, true);
H5Pset_all_coll_metadata_ops(fapl, true);

dxpl = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT);

// Only use async for writing actual data
#ifdef AMREX_USE_HDF5_ASYNC
H5Pset_vol_async(fapl);
H5Pset_dxpl_async(dxpl, true);
#endif

// All process open the file
fid = H5Fopen(filename.c_str(), H5F_ACC_RDWR, fapl);
if (fid < 0)
FileOpenFailed(filename.c_str());

H5Pclose(fapl);

// Write data for each level
char level_name[32];
for (int level = 0; level <= finest_level; ++level) {
Expand All @@ -798,6 +850,20 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
continue;
}

/* const MultiFab* data; */
/* std::unique_ptr<MultiFab> mf_tmp; */
/* if (mf[level]->nGrow() > 0) { */
/* mf_tmp.reset(new MultiFab(mf[level]->boxArray(), */
/* mf[level]->DistributionMap(), */
/* mf[level]->nComp(), 0, MFInfo(), */
/* mf[level]->Factory())); */
/* MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0); */
/* data = mf_tmp.get(); */
/* } else { */
/* data = mf[level]; */
/* } */
/* VisMF::Write(*data, MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix)); */

// Get the boxes assigned to all ranks and calculate their offsets and sizes
Vector<int> procMap = mf[level]->DistributionMap().ProcessorMap();
const BoxArray& grids = mf[level]->boxArray();
Expand All @@ -814,27 +880,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
flatdims[0] = grids.size();
boxdataspace = H5Screate_simple(1, flatdims, NULL);

hid_t babox_id;
babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int));
if (1 == AMREX_SPACEDIM) {
H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT);
}
else if (2 == AMREX_SPACEDIM) {
H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT);
}
else if (3 == AMREX_SPACEDIM) {
H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT);
}


boxdataset = H5Dcreate(grp, bdsname.c_str(), babox_id, boxdataspace, H5P_DEFAULT, H5P_DEFAULT,H5P_DEFAULT);

// Create a boxarray sorted by rank
Expand Down Expand Up @@ -862,20 +908,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
offsetdataspace = H5Screate_simple(1, oflatdims, NULL);
offsetdataset = H5Dcreate(grp, odsname.c_str(), H5T_NATIVE_LLONG, offsetdataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);

hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int));
if (1 == AMREX_SPACEDIM) {
H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT);
}
else if (2 == AMREX_SPACEDIM) {
H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT);
}
else if (3 == AMREX_SPACEDIM) {
H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT);
H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT);
}

hsize_t centerdims[1];
centerdims[0] = sortedGrids.size() ;
centerdataspace = H5Screate_simple(1, centerdims, NULL);
Expand Down Expand Up @@ -906,9 +938,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
totalOffset += procBufferSize[proc];
}

hid_t dxpl = H5Pcreate(H5P_DATASET_XFER);
ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT);

if(ParallelDescriptor::IOProcessor()) {
int vbCount(0);
Vector<int> vbox(sortedGrids.size() * 2 * AMREX_SPACEDIM);
Expand Down Expand Up @@ -949,7 +978,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);

Vector<Real> a_buffer(procBufferSize[myProc], -1.0);
Long dataCount(0);
long dataCount(0);
for(MFIter mfi(*mf[level]); mfi.isValid(); ++mfi) {
const Box &vbox = mfi.validbox();
const Real *dataPtr = (*mf[level])[mfi].dataPtr();
Expand All @@ -969,9 +998,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
BL_PROFILE_VAR_STOP(h5dwg);


H5Tclose(center_id);
H5Tclose(babox_id);
H5Pclose(dxpl);
H5Sclose(memdataspace);
H5Sclose(dataspace);
H5Dclose(dataset);
Expand All @@ -987,6 +1013,10 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename,
H5Gclose(grp);
} // For group

H5Tclose(center_id);
H5Tclose(babox_id);
H5Pclose(fapl);
H5Pclose(dxpl);
H5Fclose(fid);

/* double total_write_end_time(ParallelDescriptor::second()); */
Expand Down Expand Up @@ -1017,7 +1047,5 @@ WriteSingleLevelPlotfileHDF5 (const std::string& plotfilename,
level_steps, ref_ratio, versionName, levelPrefix, mfPrefix, extra_dirs);
}



#endif
}
53 changes: 43 additions & 10 deletions Src/Particle/AMReX_ParticleHDF5.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
#ifdef AMREX_USE_HDF5
#include <hdf5.h>

#ifdef AMREX_USE_HDF5_ASYNC
#include "h5_vol_external_async_native.h"
#endif

template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
void
ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
Expand Down Expand Up @@ -302,7 +306,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
int set_stripe = 0;
char setstripe[1024];
int stripe_count = 128;
int stripe_size = 32;
int stripe_size = 1;
char *stripe_count_str = getenv("HDF5_STRIPE_COUNT");
char *stripe_size_str = getenv("HDF5_STRIPE_SIZE");
if (stripe_count_str) {
Expand Down Expand Up @@ -354,7 +358,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
ParallelDescriptor::ReduceIntMax(maxnextid, IOProcNumber);
}

hid_t fapl, fid, grp;
hid_t fapl, dxpl, fid, grp;
int status;

hid_t comp_dtype = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int));
Expand Down Expand Up @@ -392,6 +396,10 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
// Have only one rank to create and write metadata (header)
fapl = H5Pcreate (H5P_FILE_ACCESS);
H5Pset_fapl_mpio(fapl, MPI_COMM_SELF, MPI_INFO_NULL);
int alignment = 1 * 1024 * 1024;
H5Pset_alignment(fapl, alignment, alignment);
H5Pset_coll_metadata_write(fapl, true);
H5Pset_all_coll_metadata_ops(fapl, true);

// Create the HDF5 file
fid = H5Fcreate(HDF5FileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
Expand Down Expand Up @@ -517,6 +525,14 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>

fapl = H5Pcreate (H5P_FILE_ACCESS);
H5Pset_fapl_mpio(fapl, ParallelDescriptor::Communicator(), MPI_INFO_NULL);
int alignment = 1 * 1024 * 1024;
H5Pset_alignment(fapl, alignment, alignment);
H5Pset_coll_metadata_write(fapl, true);
H5Pset_all_coll_metadata_ops(fapl, true);

#ifdef AMREX_USE_HDF5_ASYNC
H5Pset_vol_async(fapl);
#endif

// All process open the file
fid = H5Fopen(HDF5FileName.c_str(), H5F_ACC_RDWR, fapl);
Expand Down Expand Up @@ -590,6 +606,13 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>

std::map<int, Vector<int> > tile_map;

hid_t dxpl;
dxpl = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT);
#ifdef AMREX_USE_HDF5_ASYNC
H5Pset_dxpl_async(dxpl, true);
#endif

for (const auto& kv : m_particles[lev])
{
const int grid = kv.first.first;
Expand Down Expand Up @@ -644,13 +667,13 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
}

// Collect the number of mf and total size of mf from each rank
MPI_Allgather(&my_mfi_cnt, 1, MPI_INT, &(all_mfi_cnt[0]), 1, MPI_INT, ParallelDescriptor::Communicator());
MPI_Allgather(&my_mfi_cnt, 1, ParallelDescriptor::Mpi_typemap<int>::type(), &(all_mfi_cnt[0]), 1, ParallelDescriptor::Mpi_typemap<int>::type(), ParallelDescriptor::Communicator());
for (int i = 0; i < ParallelDescriptor::NProcs(); i++)
total_mfi += all_mfi_cnt[i];

// Create the int data
MPI_Allgather(&my_mfi_int_total_size, 1, Mpi_typemap<ULong>::type(),
&(all_mfi_int_total_size[0]), 1, Mpi_typemap<ULong>::type(), ParallelDescriptor::Communicator());
MPI_Allgather(&my_mfi_int_total_size, 1, ParallelDescriptor::Mpi_typemap<ULong>::type(),
&(all_mfi_int_total_size[0]), 1, ParallelDescriptor::Mpi_typemap<ULong>::type(), ParallelDescriptor::Communicator());

for (int i = 0; i < ParallelDescriptor::NProcs(); i++)
total_int_size += all_mfi_int_total_size[i];
Expand All @@ -667,8 +690,8 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>


// Create the real data
MPI_Allgather(&my_mfi_real_total_size, 1, Mpi_typemap<ULong>::type(),
&(all_mfi_real_total_size[0]), 1, Mpi_typemap<ULong>::type(), ParallelDescriptor::Communicator());
MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap<ULong>::type(),
&(all_mfi_real_total_size[0]), 1, ParallelDescriptor::Mpi_typemap<ULong>::type(), ParallelDescriptor::Communicator());

for (int i = 0; i < ParallelDescriptor::NProcs(); i++)
total_real_size += all_mfi_real_total_size[i];
Expand Down Expand Up @@ -727,7 +750,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
/* my_int_offset << ", my_int_count = " << my_int_count << ", total_int_size = " << total_int_size << '\n'; */
int_dset_space = H5Screate_simple(1, &total_int_size, NULL);
H5Sselect_hyperslab (int_dset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL);
H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, H5P_DEFAULT, istuff.dataPtr());
H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl, istuff.dataPtr());
H5Sclose(int_mem_space);
H5Sclose(int_dset_space);

Expand Down Expand Up @@ -767,7 +790,7 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
/* my_real_offset << ", my_real_count = " << my_real_count << ", total_real_size = " << total_real_size << '\n'; */
real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
H5Sselect_hyperslab (real_dset_space, H5S_SELECT_SET, &my_real_offset, NULL, &my_real_count, NULL);
H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, H5P_DEFAULT, rstuff.dataPtr());
H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl, rstuff.dataPtr());
H5Sclose(real_mem_space);
H5Sclose(real_dset_space);

Expand All @@ -792,7 +815,15 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
/* my_int_offset << ", my_int_count = " << my_int_count << ", total_mfi = " << total_mfi << '\n'; */
H5Sselect_hyperslab (offset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL);

H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, H5P_DEFAULT, &(my_nparticles[0]));
hid_t dxpl_col = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
#ifdef AMREX_USE_HDF5_ASYNC
H5Pset_dxpl_async(dxpl_col, true);
#endif
H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0]));

H5Pclose(dxpl);
H5Pclose(dxpl_col);

H5Sclose(int_mem_space);
H5Dclose(offset_id);
Expand Down Expand Up @@ -1314,6 +1345,8 @@ ParticleContainer<NStructReal, NStructInt, NArrayReal, NArrayInt>
Gpu::streamSynchronize();
}



#endif

#endif
25 changes: 17 additions & 8 deletions Tests/HDF5Benchmark/GNUmakefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,6 @@ TINY_PROFILE = TRUE

EBASE = main

# HDF5_HOME = /home/khl7265/.local
USE_HDF5 = TRUE
ifeq ($(USE_HDF5), TRUE)
DEFINES += -DAMREX_USE_HDF5
INCLUDE_LOCATIONS += $(HDF5_HOME)/include
LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib
endif

include $(AMREX_HOME)/Tools/GNUMake/Make.defs

include ./Make.package
Expand All @@ -34,3 +26,20 @@ include $(AMREX_HOME)/Src/Particle/Make.package

include $(AMREX_HOME)/Tools/GNUMake/Make.rules

USE_HDF5 = TRUE
HDF5_HOME = $(HOME)/hdf5/develop_build/hdf5
ifeq ($(USE_HDF5), TRUE)
DEFINES += -DAMREX_USE_HDF5
INCLUDE_LOCATIONS += $(HDF5_HOME)/include
LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib
endif

# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://bitbucket.hdfgroup.org/projects/HDF5VOL/repos/async/browse
USE_HDF5_ASYNC = FALSE
ABT_HOME = $(HOME)/cori/argobots/install
ASYNC_HOME = $(HOME)/hdf5vol/async/src
ifeq ($(USE_HDF5_ASYNC), TRUE)
DEFINES += -DAMREX_USE_HDF5_ASYNC -DAMREX_MPI_THREAD_MULTIPLE
INCLUDE_LOCATIONS += $(ABT_HOME)/include $(ASYNC_HOME)
LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME) -lh5async -labt
endif
Loading

0 comments on commit 4b4719a

Please sign in to comment.