Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Variable-Based iteration layout #855

Merged
merged 11 commits into from
Apr 23, 2021
Prev Previous commit
Next Next commit
Specifiy more precisely when to re-read attributes
  • Loading branch information
franzpoeschel committed Apr 22, 2021
commit 1b322d39a9e1c3ed047b4650ad1a0e3a9da817d6
3 changes: 0 additions & 3 deletions include/openPMD/Series.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -364,9 +364,6 @@ class SeriesImpl : public AttributableImpl
* Note on re-parsing of a Series:
* If init == false, the parsing process will seek for new
* Iterations/Records/Record Components etc.
* Re-parsing of objects that have already been parsed is not implemented
* as of yet. Such a facility will be required upon implementing things such
* as resizable datasets.
*/
void readGorVBased( bool init = true );
void readBase();
Expand Down
19 changes: 18 additions & 1 deletion include/openPMD/backend/Attributable.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,24 @@ class AttributableImpl
std::vector< std::string > myPath() const;

void flushAttributes();
void readAttributes( bool reread = false );
enum ReadMode {
/**
* Don't read an attribute from the backend if it has been previously
* read.
*/
IgnoreExisting,
/**
* Read all the attributes that the backend has to offer and override
* if it has been read previously.
*/
OverrideExisting,
/**
* Remove all attributes that have been read previously and read
* everything that the backend currently has to offer.
*/
FullyReread
};
void readAttributes( ReadMode );

/** Retrieve the value of a floating point Attribute of user-defined precision with ensured type-safety.
*
Expand Down
6 changes: 3 additions & 3 deletions src/Iteration.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ void Iteration::read_impl( std::string const & groupPath )
pOpen.path = s->meshesPath();
IOHandler()->enqueue(IOTask(&meshes, pOpen));

meshes.readAttributes();
meshes.readAttributes( ReadMode::FullyReread );

auto map = meshes.eraseStaleEntries();

Expand Down Expand Up @@ -510,7 +510,7 @@ void Iteration::read_impl( std::string const & groupPath )
pOpen.path = s->particlesPath();
IOHandler()->enqueue(IOTask(&particles, pOpen));

particles.readAttributes();
particles.readAttributes( ReadMode::FullyReread );

/* obtain all particle species */
pList.paths->clear();
Expand All @@ -532,7 +532,7 @@ void Iteration::read_impl( std::string const & groupPath )
particles.dirty() = false;
}

readAttributes();
readAttributes( ReadMode::FullyReread );
}

AdvanceStatus
Expand Down
2 changes: 1 addition & 1 deletion src/Mesh.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ Mesh::read()

readBase();

readAttributes();
readAttributes( ReadMode::FullyReread );
}
} // openPMD

Expand Down
2 changes: 1 addition & 1 deletion src/ParticleSpecies.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ ParticleSpecies::read()
}
}

readAttributes();
readAttributes( ReadMode::FullyReread );
}

namespace
Expand Down
2 changes: 1 addition & 1 deletion src/Record.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ Record::read()

readBase();

readAttributes();
readAttributes( ReadMode::FullyReread );
}

template <>
Expand Down
2 changes: 1 addition & 1 deletion src/RecordComponent.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ RecordComponent::readBase()
else
throw std::runtime_error("Unexpected Attribute datatype for 'unitSI'");

readAttributes();
readAttributes( ReadMode::FullyReread );
}

bool
Expand Down
8 changes: 4 additions & 4 deletions src/Series.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -948,8 +948,8 @@ void SeriesImpl::readOneIterationFileBased( std::string const & filePath )
throw std::runtime_error("Unknown openPMD version - " + version);
IOHandler()->enqueue(IOTask(&series.iterations, pOpen));

readAttributes();
series.iterations.readAttributes();
readAttributes( ReadMode::IgnoreExisting );
series.iterations.readAttributes(ReadMode::OverrideExisting );
}

void
Expand Down Expand Up @@ -1017,11 +1017,11 @@ SeriesImpl::readGorVBased( bool do_init )
throw std::runtime_error("Unknown openPMD version - " + version);
IOHandler()->enqueue(IOTask(&series.iterations, pOpen));

readAttributes();
readAttributes( ReadMode::IgnoreExisting );
/*
* __step__ changes over steps, so reread that.
*/
series.iterations.readAttributes( /* reread = */ true );
series.iterations.readAttributes( ReadMode::OverrideExisting );
/* obtain all paths inside the basepath (i.e. all iterations) */
Parameter< Operation::LIST_PATHS > pList;
IOHandler()->enqueue(IOTask(&series.iterations, pList));
Expand Down
33 changes: 21 additions & 12 deletions src/backend/Attributable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,30 +186,39 @@ AttributableImpl::flushAttributes()
}

void
AttributableImpl::readAttributes( bool reread )
AttributableImpl::readAttributes( ReadMode mode )
{
auto & attri = get();
Parameter< Operation::LIST_ATTS > aList;
IOHandler()->enqueue(IOTask(this, aList));
IOHandler()->flush();
std::vector< std::string > written_attributes = attributes();

/* std::set_difference requires sorted ranges */
std::sort(aList.attributes->begin(), aList.attributes->end());
std::sort(written_attributes.begin(), written_attributes.end());

std::set< std::string > tmpAttributes;
if( reread )
{
tmpAttributes = std::set< std::string >(
aList.attributes->begin(),
aList.attributes->end() );
}
else
switch( mode )
{
/* std::set_difference requires sorted ranges */
std::sort(aList.attributes->begin(), aList.attributes->end());
std::sort(written_attributes.begin(), written_attributes.end());

case ReadMode::IgnoreExisting:
// reread: aList - written_attributes
std::set_difference(
aList.attributes->begin(), aList.attributes->end(),
written_attributes.begin(), written_attributes.end(),
std::inserter(tmpAttributes, tmpAttributes.begin()));
break;
case ReadMode::OverrideExisting:
tmpAttributes = std::set< std::string >(
aList.attributes->begin(),
aList.attributes->end() );
break;
case ReadMode::FullyReread:
attri.m_attributes.clear();
tmpAttributes = std::set< std::string >(
aList.attributes->begin(),
aList.attributes->end() );
break;
}

using DT = Datatype;
Expand Down
2 changes: 1 addition & 1 deletion src/backend/PatchRecordComponent.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ PatchRecordComponent::read()
else
throw std::runtime_error("Unexpected Attribute datatype for 'unitSI'");

readAttributes(); // this will set dirty() = false
readAttributes( ReadMode::FullyReread ); // this will set dirty() = false
}

bool
Expand Down