Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

openPMD: ionizationLevel #1622

Merged
merged 2 commits into from
Jan 20, 2021
Merged

openPMD: ionizationLevel #1622

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b880658
openPMD: ionizationLevel
ax3l Jan 12, 2021
d132f05
openPMD Record Repetitions: Use Lambdas
ax3l Jan 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions Source/Diagnostics/WarpXOpenPMD.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,15 @@ private:
* @param[in] currSpecies The openPMD species
* @param[in] write_real_comp The real attribute ids, from WarpX
* @param[in] real_comp_names The real attribute names, from WarpX
* @param[in] write_int_comp The int attribute ids, from WarpX
* @param[in] int_comp_names The int attribute names, from WarpX
* @param[in] np Number of particles
*/
void SetupRealProperties(openPMD::ParticleSpecies& currSpecies,
void SetupSoAProperties(openPMD::ParticleSpecies& currSpecies,
const amrex::Vector<int>& write_real_comp,
const amrex::Vector<std::string>& real_comp_names,
const amrex::Vector<int>& write_int_comp,
const amrex::Vector<std::string>& int_comp_names,
unsigned long long np) const;

/** This function saves the values of the entries for particle properties
Expand All @@ -147,12 +151,16 @@ private:
* @param[in] offset offset to start saving the particle iterator contents
* @param[in] write_real_comp The real attribute ids, from WarpX
* @param[in] real_comp_names The real attribute names, from WarpX
* @param[in] write_int_comp The int attribute ids, from WarpX
* @param[in] int_comp_names The int attribute names, from WarpX
*/
void SaveRealProperty(WarpXParIter& pti, //int, int,
void SaveParticleProperties(WarpXParIter& pti, //int, int,
openPMD::ParticleSpecies& currSpecies,
unsigned long long offset,
const amrex::Vector<int>& write_real_comp,
const amrex::Vector<std::string>& real_comp_names) const;
const amrex::Vector<std::string>& real_comp_names,
const amrex::Vector<int>& write_int_comp,
const amrex::Vector<std::string>& int_comp_names) const;

/** This function saves the plot file
*
Expand Down Expand Up @@ -180,8 +188,11 @@ private:
int m_MPIRank = 0;
int m_MPISize = 1;

int m_NumSoARealAttributes = PIdx::nattribs; //! WarpX' addition particle attributes in SoA
int m_NumAoSRealAttributes = 0; //! WarpX definition: no additional attributes in particle AoS
int m_NumSoARealAttributes = PIdx::nattribs; //! WarpX' additional real particle attributes in SoA
int m_NumAoSRealAttributes = 0; //! WarpX definition: no additional real attributes in particle AoS

//int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA
int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS

bool m_OneFilePerTS = true; //! write in openPMD fileBased manner for individual time steps
std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5
Expand Down
151 changes: 102 additions & 49 deletions Source/Diagnostics/WarpXOpenPMD.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ namespace detail
{
std::string record_name = fullName;
std::string component_name = openPMD::RecordComponent::SCALAR;
std::size_t startComp = fullName.find_last_of("_");

// we use "_" as separator in names to group vector records
std::size_t startComp = fullName.find_last_of("_");
if( startComp != std::string::npos ) { // non-scalar
record_name = fullName.substr(0, startComp);
component_name = fullName.substr(startComp + 1u);
Expand Down Expand Up @@ -327,7 +328,7 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& part
real_names.push_back("theta");
#endif
if(pc->DoFieldIonization()){
int_names.push_back("ionization_level");
int_names.push_back("ionizationLevel");
// int_flags specifies, for each integer attribs, whether it is
// dumped as particle record in a plotfile. So far, ionization_level is the only
// integer attribs, and it is automatically dumped as particle record
Expand Down Expand Up @@ -364,10 +365,6 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
const amrex::Vector<std::string>& int_comp_names) const
{
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized");
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(write_int_comp.size() == 0u,
"openPMD: Particle integer components not implemented!");
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(int_comp_names.size() == 0u,
"openPMD: Particle integer components not implemented!");

WarpXParticleCounter counter(pc);

Expand Down Expand Up @@ -413,7 +410,7 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
// define positions & offsets
//
SetupPos(pc, currSpecies, counter.GetTotalNumParticles());
SetupRealProperties(currSpecies, write_real_comp, real_comp_names, counter.GetTotalNumParticles());
SetupSoAProperties(currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, counter.GetTotalNumParticles());

// open files from all processors, in case some will not contribute below
m_Series->flush();
Expand Down Expand Up @@ -494,10 +491,11 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64});
}
// save "extra" particle properties in AoS and SoA
SaveRealProperty(pti,
SaveParticleProperties(pti,
currSpecies,
offset,
write_real_comp, real_comp_names);
write_real_comp, real_comp_names,
write_int_comp, int_comp_names);

offset += numParticleOnTile64;
}
Expand All @@ -506,58 +504,96 @@ WarpXOpenPMDPlot::DumpToFile (WarpXParticleContainer* pc,
}

void
WarpXOpenPMDPlot::SetupRealProperties(openPMD::ParticleSpecies& currSpecies,
WarpXOpenPMDPlot::SetupSoAProperties(openPMD::ParticleSpecies& currSpecies,
const amrex::Vector<int>& write_real_comp,
const amrex::Vector<std::string>& real_comp_names,
const amrex::Vector<int>& write_int_comp,
const amrex::Vector<std::string>& int_comp_names,
unsigned long long np) const
{
auto particlesLineup = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np});
auto dtype_real = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np});
auto dtype_int = openPMD::Dataset(openPMD::determineDatatype<int>(), {np});

//
// the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names
// so using the min to be safe.
//
auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size());
for (int i = 0; i < real_counter; ++i) {
if (write_real_comp[i]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[i]);

//
// the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names
// so using the min to be safe.
//
auto counter = std::min(write_real_comp.size(), real_comp_names.size());
for (int i = 0; i < counter; ++i)
if (write_real_comp[i]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[i]);

auto particleVarComp = currSpecies[record_name][component_name];
particleVarComp.resetDataset(particlesLineup);
auto particleVarComp = currSpecies[record_name][component_name];
particleVarComp.resetDataset(dtype_real);
}
}
auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size());
for (int i = 0; i < int_counter; ++i) {
if (write_int_comp[i]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(int_comp_names[i]);

auto particleVarComp = currSpecies[record_name][component_name];
particleVarComp.resetDataset(dtype_int);
}
}

std::set< std::string > addedRecords; // add meta-data per record only once
for (auto idx=0; idx<m_NumSoARealAttributes; idx++) {
auto ii = m_NumAoSRealAttributes + idx;
if (write_real_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[ii]);
auto currRecord = currSpecies[record_name];

// meta data for ED-PIC extension
bool newRecord = false;
std::tie(std::ignore, newRecord) = addedRecords.insert(record_name);
if( newRecord ) {
currRecord.setUnitDimension( detail::getUnitDimension(record_name) );
currRecord.setAttribute( "macroWeighted", 0u );
if( record_name == "momentum" )
currRecord.setAttribute( "weightingPower", 1.0 );
else
currRecord.setAttribute( "weightingPower", 0.0 );
}
std::set< std::string > addedRecords; // add meta-data per record only once
for (auto idx=0; idx<m_NumSoARealAttributes; idx++) {
auto ii = m_NumAoSRealAttributes + idx; // jump over AoS names
if (write_real_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(real_comp_names[ii]);
auto currRecord = currSpecies[record_name];

// meta data for ED-PIC extension
bool newRecord = false;
std::tie(std::ignore, newRecord) = addedRecords.insert(record_name);
if( newRecord ) {
currRecord.setUnitDimension( detail::getUnitDimension(record_name) );
currRecord.setAttribute( "macroWeighted", 0u );
if( record_name == "momentum" )
currRecord.setAttribute( "weightingPower", 1.0 );
else
currRecord.setAttribute( "weightingPower", 0.0 );
}
}
}
for (auto idx=0; idx<int_counter; idx++) {
auto ii = m_NumAoSIntAttributes + idx; // jump over AoS names
if (write_int_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(int_comp_names[ii]);
auto currRecord = currSpecies[record_name];

// meta data for ED-PIC extension
bool newRecord = false;
std::tie(std::ignore, newRecord) = addedRecords.insert(record_name);
if( newRecord ) {
currRecord.setUnitDimension( detail::getUnitDimension(record_name) );
currRecord.setAttribute( "macroWeighted", 0u );
if( record_name == "momentum" )
currRecord.setAttribute( "weightingPower", 1.0 );
else
currRecord.setAttribute( "weightingPower", 0.0 );
}
}
}
}
}

void
WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
WarpXOpenPMDPlot::SaveParticleProperties(WarpXParIter& pti,
openPMD::ParticleSpecies& currSpecies,
unsigned long long const offset,
amrex::Vector<int> const& write_real_comp,
amrex::Vector<std::string> const& real_comp_names) const
amrex::Vector<std::string> const& real_comp_names,
amrex::Vector<int> const& write_int_comp,
amrex::Vector<std::string> const& int_comp_names) const

{
int numOutputReal = 0;
Expand All @@ -572,7 +608,7 @@ WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile
auto const& soa = pti.GetStructOfArrays();

// properties are saved separately
// first we concatinate the AoS into contiguous arrays
{
for( auto idx=0; idx<m_NumAoSRealAttributes; idx++ ) {
if( write_real_comp[idx] ) {
Expand All @@ -596,6 +632,7 @@ WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
}
}

// here we the save the SoA properties (real)
{
for (auto idx=0; idx<m_NumSoARealAttributes; idx++) {
auto ii = m_NumAoSRealAttributes + idx;
Expand All @@ -611,10 +648,26 @@ WarpXOpenPMDPlot::SaveRealProperty(WarpXParIter& pti,
}
}
}
// and now SoA int properties
{
auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size());
for (auto idx=0; idx<int_counter; idx++) {
auto ii = m_NumAoSIntAttributes + idx; // jump over AoS names
if (write_int_comp[ii]) {
// handle scalar and non-scalar records by name
std::string record_name, component_name;
std::tie(record_name, component_name) = detail::name2openPMD(int_comp_names[ii]);
auto& currRecord = currSpecies[record_name];
auto& currRecordComp = currRecord[component_name];

currRecordComp.storeChunk(openPMD::shareRaw(soa.GetIntData(idx)),
{offset}, {numParticleOnTile64});
}
}
}
}



void
WarpXOpenPMDPlot::SetupPos(WarpXParticleContainer* pc,
openPMD::ParticleSpecies& currSpecies,
Expand Down