From 4b4719ab7be2d5b7c8669df32e69b2226c66544c Mon Sep 17 00:00:00 2001 From: Houjun Date: Thu, 18 Jun 2020 15:53:05 -0700 Subject: [PATCH] Update HDF5 particle write and benchmark codes (#1035) * Update HDF5 related code for better write performance * Update Makefile for HDF5Benchmark * Update GNUmakefile * Update inputs * Use correct MPI datatype from AMReX Co-authored-by: Houjun Tang --- Src/Base/AMReX_PlotFileUtil.cpp | 124 ++++++++++++++++++------------ Src/Particle/AMReX_ParticleHDF5.H | 53 ++++++++++--- Tests/HDF5Benchmark/GNUmakefile | 25 ++++-- Tests/HDF5Benchmark/inputs | 11 ++- Tests/HDF5Benchmark/main.cpp | 66 ++++++++++++++-- 5 files changed, 205 insertions(+), 74 deletions(-) diff --git a/Src/Base/AMReX_PlotFileUtil.cpp b/Src/Base/AMReX_PlotFileUtil.cpp index 5da89d4158e..969628e7360 100644 --- a/Src/Base/AMReX_PlotFileUtil.cpp +++ b/Src/Base/AMReX_PlotFileUtil.cpp @@ -12,6 +12,11 @@ #ifdef AMREX_USE_HDF5 #include "hdf5.h" + +#ifdef AMREX_USE_HDF5_ASYNC +#include "h5_vol_external_async_native.h" +#endif + #endif namespace amrex { @@ -453,7 +458,6 @@ EB_WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels, #endif - #ifdef AMREX_USE_HDF5 static int CreateWriteHDF5AttrDouble(hid_t loc, const char *name, hsize_t n, const double *data) { @@ -753,12 +757,14 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, std::string filename(plotfilename + ".h5"); // Write out root level metadata - hid_t fapl, fid, grp; + hid_t fapl, dxpl, fid, grp; if(ParallelDescriptor::IOProcessor()) { // Have only one rank to create and write metadata (header) fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, MPI_COMM_SELF, MPI_INFO_NULL); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); // Create the HDF5 file fid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl); @@ -778,16 +784,62 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, ParallelDescriptor::Barrier(); + hid_t babox_id; + babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); + if (1 == AMREX_SPACEDIM) { + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT); + } + else if (2 == AMREX_SPACEDIM) { + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT); + } + else if (3 == AMREX_SPACEDIM) { + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT); + } + + hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int)); + if (1 == AMREX_SPACEDIM) { + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + } + else if (2 == AMREX_SPACEDIM) { + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); + } + else if (3 == AMREX_SPACEDIM) { + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT); + } + fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, ParallelDescriptor::Communicator(), MPI_INFO_NULL); + int alignment = 16 * 1024 * 1024; + H5Pset_alignment(fapl, alignment, alignment); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); + + dxpl = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); + + // Only use async for writing actual data + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_vol_async(fapl); + H5Pset_dxpl_async(dxpl, true); + #endif // All process open the file fid = H5Fopen(filename.c_str(), H5F_ACC_RDWR, fapl); if (fid < 0) FileOpenFailed(filename.c_str()); - H5Pclose(fapl); - // Write data for each level char level_name[32]; for (int level = 0; level <= finest_level; ++level) { @@ -798,6 +850,20 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, continue; } + /* const MultiFab* data; */ + /* std::unique_ptr mf_tmp; */ + /* if (mf[level]->nGrow() > 0) { */ + /* mf_tmp.reset(new MultiFab(mf[level]->boxArray(), */ + /* mf[level]->DistributionMap(), */ + /* mf[level]->nComp(), 0, MFInfo(), */ + /* mf[level]->Factory())); */ + /* MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0); */ + /* data = mf_tmp.get(); */ + /* } else { */ + /* data = mf[level]; */ + /* } */ + /* VisMF::Write(*data, MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix)); */ + // Get the boxes assigned to all ranks and calculate their offsets and sizes Vector procMap = mf[level]->DistributionMap().ProcessorMap(); const BoxArray& grids = mf[level]->boxArray(); @@ -814,27 +880,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, flatdims[0] = grids.size(); boxdataspace = H5Screate_simple(1, flatdims, NULL); - hid_t babox_id; - babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); - if (1 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT); - } - else if (2 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT); - } - else if (3 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT); - } - + boxdataset = H5Dcreate(grp, bdsname.c_str(), babox_id, boxdataspace, H5P_DEFAULT, H5P_DEFAULT,H5P_DEFAULT); // Create a boxarray sorted by rank @@ -862,20 +908,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, offsetdataspace = H5Screate_simple(1, oflatdims, NULL); offsetdataset = H5Dcreate(grp, odsname.c_str(), H5T_NATIVE_LLONG, offsetdataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int)); - if (1 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - } - else if (2 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); - } - else if (3 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT); - } - hsize_t centerdims[1]; centerdims[0] = sortedGrids.size() ; centerdataspace = H5Screate_simple(1, centerdims, NULL); @@ -906,9 +938,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, totalOffset += procBufferSize[proc]; } - hid_t dxpl = H5Pcreate(H5P_DATASET_XFER); - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); - if(ParallelDescriptor::IOProcessor()) { int vbCount(0); Vector vbox(sortedGrids.size() * 2 * AMREX_SPACEDIM); @@ -949,7 +978,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL); Vector a_buffer(procBufferSize[myProc], -1.0); - Long dataCount(0); + long dataCount(0); for(MFIter mfi(*mf[level]); mfi.isValid(); ++mfi) { const Box &vbox = mfi.validbox(); const Real *dataPtr = (*mf[level])[mfi].dataPtr(); @@ -969,9 +998,6 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, BL_PROFILE_VAR_STOP(h5dwg); - H5Tclose(center_id); - H5Tclose(babox_id); - H5Pclose(dxpl); H5Sclose(memdataspace); H5Sclose(dataspace); H5Dclose(dataset); @@ -987,6 +1013,10 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, H5Gclose(grp); } // For group + H5Tclose(center_id); + H5Tclose(babox_id); + H5Pclose(fapl); + H5Pclose(dxpl); H5Fclose(fid); /* double total_write_end_time(ParallelDescriptor::second()); */ @@ -1017,7 +1047,5 @@ WriteSingleLevelPlotfileHDF5 (const std::string& plotfilename, level_steps, ref_ratio, versionName, levelPrefix, mfPrefix, extra_dirs); } - - #endif } diff --git a/Src/Particle/AMReX_ParticleHDF5.H b/Src/Particle/AMReX_ParticleHDF5.H index 630b0b5bf24..528f4fc35a6 100644 --- a/Src/Particle/AMReX_ParticleHDF5.H +++ b/Src/Particle/AMReX_ParticleHDF5.H @@ -4,6 +4,10 @@ #ifdef AMREX_USE_HDF5 #include +#ifdef AMREX_USE_HDF5_ASYNC +#include "h5_vol_external_async_native.h" +#endif + template void ParticleContainer @@ -302,7 +306,7 @@ ParticleContainer int set_stripe = 0; char setstripe[1024]; int stripe_count = 128; - int stripe_size = 32; + int stripe_size = 1; char *stripe_count_str = getenv("HDF5_STRIPE_COUNT"); char *stripe_size_str = getenv("HDF5_STRIPE_SIZE"); if (stripe_count_str) { @@ -354,7 +358,7 @@ ParticleContainer ParallelDescriptor::ReduceIntMax(maxnextid, IOProcNumber); } - hid_t fapl, fid, grp; + hid_t fapl, dxpl, fid, grp; int status; hid_t comp_dtype = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); @@ -392,6 +396,10 @@ ParticleContainer // Have only one rank to create and write metadata (header) fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, MPI_COMM_SELF, MPI_INFO_NULL); + int alignment = 1 * 1024 * 1024; + H5Pset_alignment(fapl, alignment, alignment); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); // Create the HDF5 file fid = H5Fcreate(HDF5FileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl); @@ -517,6 +525,14 @@ ParticleContainer fapl = H5Pcreate (H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl, ParallelDescriptor::Communicator(), MPI_INFO_NULL); + int alignment = 1 * 1024 * 1024; + H5Pset_alignment(fapl, alignment, alignment); + H5Pset_coll_metadata_write(fapl, true); + H5Pset_all_coll_metadata_ops(fapl, true); + + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_vol_async(fapl); + #endif // All process open the file fid = H5Fopen(HDF5FileName.c_str(), H5F_ACC_RDWR, fapl); @@ -590,6 +606,13 @@ ParticleContainer std::map > tile_map; + hid_t dxpl; + dxpl = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_dxpl_async(dxpl, true); + #endif + for (const auto& kv : m_particles[lev]) { const int grid = kv.first.first; @@ -644,13 +667,13 @@ ParticleContainer } // Collect the number of mf and total size of mf from each rank - MPI_Allgather(&my_mfi_cnt, 1, MPI_INT, &(all_mfi_cnt[0]), 1, MPI_INT, ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_cnt, 1, ParallelDescriptor::Mpi_typemap::type(), &(all_mfi_cnt[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_mfi += all_mfi_cnt[i]; // Create the int data - MPI_Allgather(&my_mfi_int_total_size, 1, Mpi_typemap::type(), - &(all_mfi_int_total_size[0]), 1, Mpi_typemap::type(), ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_int_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), + &(all_mfi_int_total_size[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_int_size += all_mfi_int_total_size[i]; @@ -667,8 +690,8 @@ ParticleContainer // Create the real data - MPI_Allgather(&my_mfi_real_total_size, 1, Mpi_typemap::type(), - &(all_mfi_real_total_size[0]), 1, Mpi_typemap::type(), ParallelDescriptor::Communicator()); + MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), + &(all_mfi_real_total_size[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_real_size += all_mfi_real_total_size[i]; @@ -727,7 +750,7 @@ ParticleContainer /* my_int_offset << ", my_int_count = " << my_int_count << ", total_int_size = " << total_int_size << '\n'; */ int_dset_space = H5Screate_simple(1, &total_int_size, NULL); H5Sselect_hyperslab (int_dset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL); - H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, H5P_DEFAULT, istuff.dataPtr()); + H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl, istuff.dataPtr()); H5Sclose(int_mem_space); H5Sclose(int_dset_space); @@ -767,7 +790,7 @@ ParticleContainer /* my_real_offset << ", my_real_count = " << my_real_count << ", total_real_size = " << total_real_size << '\n'; */ real_dset_space = H5Screate_simple(1, &total_real_size, NULL); H5Sselect_hyperslab (real_dset_space, H5S_SELECT_SET, &my_real_offset, NULL, &my_real_count, NULL); - H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, H5P_DEFAULT, rstuff.dataPtr()); + H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl, rstuff.dataPtr()); H5Sclose(real_mem_space); H5Sclose(real_dset_space); @@ -792,7 +815,15 @@ ParticleContainer /* my_int_offset << ", my_int_count = " << my_int_count << ", total_mfi = " << total_mfi << '\n'; */ H5Sselect_hyperslab (offset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL); - H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, H5P_DEFAULT, &(my_nparticles[0])); + hid_t dxpl_col = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE); + #ifdef AMREX_USE_HDF5_ASYNC + H5Pset_dxpl_async(dxpl_col, true); + #endif + H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0])); + + H5Pclose(dxpl); + H5Pclose(dxpl_col); H5Sclose(int_mem_space); H5Dclose(offset_id); @@ -1314,6 +1345,8 @@ ParticleContainer Gpu::streamSynchronize(); } + + #endif #endif diff --git a/Tests/HDF5Benchmark/GNUmakefile b/Tests/HDF5Benchmark/GNUmakefile index 0524489fe45..509351d8214 100644 --- a/Tests/HDF5Benchmark/GNUmakefile +++ b/Tests/HDF5Benchmark/GNUmakefile @@ -18,14 +18,6 @@ TINY_PROFILE = TRUE EBASE = main -# HDF5_HOME = /home/khl7265/.local -USE_HDF5 = TRUE -ifeq ($(USE_HDF5), TRUE) -DEFINES += -DAMREX_USE_HDF5 -INCLUDE_LOCATIONS += $(HDF5_HOME)/include -LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib -endif - include $(AMREX_HOME)/Tools/GNUMake/Make.defs include ./Make.package @@ -34,3 +26,20 @@ include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Tools/GNUMake/Make.rules +USE_HDF5 = TRUE +HDF5_HOME = $(HOME)/hdf5/develop_build/hdf5 +ifeq ($(USE_HDF5), TRUE) +DEFINES += -DAMREX_USE_HDF5 +INCLUDE_LOCATIONS += $(HDF5_HOME)/include +LIBRARIES += -lhdf5 -lz -ldl -L$(HDF5_HOME)/lib +endif + +# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://bitbucket.hdfgroup.org/projects/HDF5VOL/repos/async/browse +USE_HDF5_ASYNC = FALSE +ABT_HOME = $(HOME)/cori/argobots/install +ASYNC_HOME = $(HOME)/hdf5vol/async/src +ifeq ($(USE_HDF5_ASYNC), TRUE) +DEFINES += -DAMREX_USE_HDF5_ASYNC -DAMREX_MPI_THREAD_MULTIPLE +INCLUDE_LOCATIONS += $(ABT_HOME)/include $(ASYNC_HOME) +LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME) -lh5async -labt +endif diff --git a/Tests/HDF5Benchmark/inputs b/Tests/HDF5Benchmark/inputs index c83791eb79f..1bbead63d1c 100644 --- a/Tests/HDF5Benchmark/inputs +++ b/Tests/HDF5Benchmark/inputs @@ -14,5 +14,14 @@ ncomp = 6 # Number of particles per cell nppc = 2 +# Number of plot files to write +nplotfile = 5 + +# Number of plot files to write +nparticlefile = 5 + +# Time to sleep before each write +sleeptime = 2 + # Whether to check the correctness of Checkpoint / Restart -restart_check = 1 \ No newline at end of file +restart_check = 1 diff --git a/Tests/HDF5Benchmark/main.cpp b/Tests/HDF5Benchmark/main.cpp index 189ea997aa1..621c9b39a26 100644 --- a/Tests/HDF5Benchmark/main.cpp +++ b/Tests/HDF5Benchmark/main.cpp @@ -3,6 +3,11 @@ #include #include +#include +#ifdef AMREX_USE_HDF5_ASYNC +#include "h5_vol_external_async_native.h" +#endif + using namespace amrex; int main(int argc, char* argv[]) @@ -11,7 +16,7 @@ int main(int argc, char* argv[]) { const int nghost = 0; int ncells, max_grid_size, ncomp, nlevs, nppc; - int restart_check = 0; + int restart_check = 0, nplotfile = 1, nparticlefile = 1, sleeptime = 0; ParmParse pp; pp.get("ncells", ncells); @@ -19,6 +24,9 @@ int main(int argc, char* argv[]) pp.get("ncomp", ncomp); pp.get("nlevs", nlevs); pp.get("nppc", nppc); + pp.query("nplotfile", nplotfile); + pp.query("nparticlefile", nparticlefile); + pp.query("sleeptime", sleeptime); pp.query("restart_check", restart_check); AMREX_ALWAYS_ASSERT(nlevs < 2); // relax this later @@ -110,12 +118,36 @@ int main(int argc, char* argv[]) Vector level_steps(nlevs, 0); - WriteMultiLevelPlotfile("plt00000", nlevs, amrex::GetVecOfConstPtrs(mf), - varnames, geom, time, level_steps, ref_ratio); + char fname[128]; + for (int ts = 0; ts < nplotfile; ts++) { + sprintf(fname, "plt%05d", ts); + + // Fake computation + if (ts > 0 && sleeptime > 0) { + if (ParallelDescriptor::IOProcessor()) { + std::cout << "Sleep for " << sleeptime << " seconds." << std::endl; + fflush(stdout); + } + sleep(sleeptime); + } + + if (ParallelDescriptor::IOProcessor()) + std::cout << "Writing plot file [" << fname << "]" << std::endl; #ifdef AMREX_USE_HDF5 - WriteMultiLevelPlotfileHDF5("plt00000", nlevs, amrex::GetVecOfConstPtrs(mf), + WriteMultiLevelPlotfileHDF5(fname, nlevs, amrex::GetVecOfConstPtrs(mf), + varnames, geom, time, level_steps, ref_ratio); +#else + WriteMultiLevelPlotfile(fname, nlevs, amrex::GetVecOfConstPtrs(mf), varnames, geom, time, level_steps, ref_ratio); #endif + } + +#ifdef AMREX_USE_HDF5_ASYNC + // Complete all previous async writes + H5VLasync_waitall(); +#endif + + /* ParallelDescriptor::Barrier(); */ Vector particle_realnames; for (int i = 0; i < NStructReal + NArrayReal; ++i) @@ -129,11 +161,30 @@ int main(int argc, char* argv[]) particle_intnames.push_back("particle_int_component_" + std::to_string(i)); } + for (int ts = 0; ts < nparticlefile; ts++) { + sprintf(fname, "plt%05d", ts); + + // Fake computation + if (ts > 0 && sleeptime > 0) { + if (ParallelDescriptor::IOProcessor()) { + std::cout << "Sleep for " << sleeptime << " seconds." << std::endl; + fflush(stdout); + } + sleep(sleeptime); + } + #ifdef AMREX_USE_HDF5 - myPC.CheckpointHDF5("plt00000", "particle0", false, particle_realnames, particle_intnames); + myPC.CheckpointHDF5(fname, "particle0", false, particle_realnames, particle_intnames); #else - myPC.Checkpoint("plt00000", "particle0", false, particle_realnames, particle_intnames); - /* myPC.WriteAsciiFile("particle0_ascii"); */ + myPC.Checkpoint(fname, "particle0", false, particle_realnames, particle_intnames); + /* myPC.WriteAsciiFile("particle0_ascii"); */ +#endif + } + +#ifdef AMREX_USE_HDF5_ASYNC + // Complete all previous async writes + H5VLasync_waitall(); + /* ParallelDescriptor::Barrier(); */ #endif if (restart_check) @@ -169,5 +220,6 @@ int main(int argc, char* argv[]) } } + amrex::Finalize(); }