Random-access for variable-based encoding (i.e. use SetStepSelection for ADIOS2 steps)

CMakeLists.txt

@@ -787,6 +787,10 @@ if(openPMD_BUILD_TESTING)
                 test/Files_SerialIO/close_and_reopen_test.cpp
                 test/Files_SerialIO/filebased_write_test.cpp
             )
+        elseif(${test_name} STREQUAL "ParallelIO")
+            list(APPEND ${out_list}
+                test/Files_ParallelIO/read_variablebased_randomaccess.cpp
+            )
         endif()
     endmacro()
 

include/openPMD/Datatype.hpp

@@ -513,7 +513,7 @@ inline bool isFloatingPoint(Datatype d)
  * @param d Datatype to test
  * @return true if complex floating point, otherwise false
  */
-inline bool isComplexFloatingPoint(Datatype d)
+constexpr inline bool isComplexFloatingPoint(Datatype d)
 {
     using DT = Datatype;
 
@@ -554,7 +554,7 @@ inline bool isFloatingPoint()
  * @return true if complex floating point, otherwise false
  */
 template <typename T>
-inline bool isComplexFloatingPoint()
+constexpr inline bool isComplexFloatingPoint()
 {
     Datatype dtype = determineDatatype<T>();
 

include/openPMD/IO/ADIOS/ADIOS2File.hpp

@@ -79,7 +79,8 @@ struct DatasetReader
         BufferedGet &bp,
         adios2::IO &IO,
         adios2::Engine &engine,
-        std::string const &fileName);
+        std::string const &fileName,
+        std::optional<size_t> stepSelection);
 
     static constexpr char const *errorMsg = "ADIOS2: readDataset()";
 };
@@ -412,6 +413,8 @@ class ADIOS2File
     StreamStatus streamStatus = StreamStatus::OutsideOfStep;
 
     size_t currentStep();
+    void setStepSelection(size_t);
+    [[nodiscard]] std::optional<size_t> stepSelection() const;
 
 private:
     ADIOS2IOHandlerImpl *m_impl;
@@ -422,6 +425,7 @@ class ADIOS2File
      * implement this manually.
      */
     size_t m_currentStep = 0;
+    bool useStepSelection = false;
 
     /*
      * ADIOS2 does not give direct access to its internal attribute and

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -190,6 +190,9 @@ class ADIOS2IOHandlerImpl
 
     void readAttribute(Writable *, Parameter<Operation::READ_ATT> &) override;
 
+    void readAttributeAllsteps(
+        Writable *, Parameter<Operation::READ_ATT_ALLSTEPS> &) override;
+
     void listPaths(Writable *, Parameter<Operation::LIST_PATHS> &) override;
 
     void
@@ -431,7 +434,8 @@ class ADIOS2IOHandlerImpl
         Offset const &offset,
         Extent const &extent,
         adios2::IO &IO,
-        std::string const &varName)
+        std::string const &varName,
+        std::optional<size_t> stepSelection)
     {
         {
             auto requiredType = adios2::GetType<T>();
@@ -458,6 +462,10 @@ class ADIOS2IOHandlerImpl
             throw std::runtime_error(
                 "[ADIOS2] Internal error: Failed opening ADIOS2 variable.");
         }
+        if (stepSelection.has_value())
+        {
+            var.SetStepSelection({*stepSelection, 1});
+        }
         // TODO leave this check to ADIOS?
         adios2::Dims shape = var.Shape();
         auto actualDim = shape.size();
@@ -533,11 +541,8 @@ namespace detail
     struct AttributeReader
     {
         template <typename T>
-        static Datatype call(
-            ADIOS2IOHandlerImpl &,
-            adios2::IO &IO,
-            std::string name,
-            Attribute::resource &resource);
+        static Datatype
+        call(adios2::IO &IO, std::string name, Attribute::resource &resource);
 
         template <int n, typename... Params>
         static Datatype call(Params &&...);
@@ -562,7 +567,8 @@ namespace detail
             ADIOS2IOHandlerImpl *impl,
             InvalidatableFile const &,
             std::string const &varName,
-            Parameter<Operation::OPEN_DATASET> &parameters);
+            Parameter<Operation::OPEN_DATASET> &parameters,
+            std::optional<size_t> stepSelection);
 
         static constexpr char const *errorMsg = "ADIOS2: openDataset()";
     };

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -360,6 +360,10 @@ class AbstractIOHandlerImpl
      */
     virtual void
     readAttribute(Writable *, Parameter<Operation::READ_ATT> &) = 0;
+    /** COLLECTIVE!
+     */
+    virtual void readAttributeAllsteps(
+        Writable *, Parameter<Operation::READ_ATT_ALLSTEPS> &);
     /** List all paths/sub-groups inside a group, non-recursively.
      *
      * The operation should fail if the Writable was not marked written.

include/openPMD/IO/IOTask.hpp

@@ -26,6 +26,7 @@
 #include "openPMD/Streaming.hpp"
 #include "openPMD/auxiliary/Export.hpp"
 #include "openPMD/auxiliary/Memory.hpp"
+#include "openPMD/auxiliary/TypeTraits.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Attribute.hpp"
 #include "openPMD/backend/ParsePreference.hpp"
@@ -72,6 +73,7 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
     DELETE_ATT,
     WRITE_ATT,
     READ_ATT,
+    READ_ATT_ALLSTEPS,
     LIST_ATTS,
 
     ADVANCE,
@@ -602,6 +604,36 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::READ_ATT>
         std::make_shared<Attribute::resource>();
 };
 
+template <>
+struct OPENPMDAPI_EXPORT Parameter<Operation::READ_ATT_ALLSTEPS>
+    : public AbstractParameter
+{
+    Parameter() = default;
+    Parameter(Parameter &&) = default;
+    Parameter(Parameter const &) = default;
+    Parameter &operator=(Parameter &&) = default;
+    Parameter &operator=(Parameter const &) = default;
+
+    std::unique_ptr<AbstractParameter> to_heap() && override
+    {
+        return std::unique_ptr<AbstractParameter>(
+            new Parameter<Operation::READ_ATT_ALLSTEPS>(std::move(*this)));
+    }
+
+    std::string name = "";
+    std::shared_ptr<Datatype> dtype = std::make_shared<Datatype>();
+
+    struct to_vector_type
+    {
+        template <typename T>
+        using type = std::vector<T>;
+    };
+    using result_type = typename auxiliary::detail::
+        map_variant<to_vector_type, Attribute::resource>::type;
+
+    std::shared_ptr<result_type> resource = std::make_shared<result_type>();
+};
+
 template <>
 struct OPENPMDAPI_EXPORT Parameter<Operation::LIST_ATTS>
     : public AbstractParameter
@@ -638,8 +670,13 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::ADVANCE>
             new Parameter<Operation::ADVANCE>(std::move(*this)));
     }
 
+    struct StepSelection
+    {
+        size_t step;
+    };
+
     //! input parameter
-    AdvanceMode mode;
+    std::variant<AdvanceMode, StepSelection> mode;
     bool isThisStepMandatory = false;
     //! output parameter
     std::shared_ptr<AdvanceStatus> status =

include/openPMD/Iteration.hpp

@@ -52,6 +52,23 @@ namespace internal
                               propagated to the backend */
     };
 
+    namespace BeginStepTypes
+    {
+        struct DontBeginStep
+        {};
+        struct BeginStepSynchronously
+        {};
+        struct BeginStepRandomAccess
+        {
+            size_t step;
+        };
+    } // namespace BeginStepTypes
+
+    using BeginStep = std::variant<
+        BeginStepTypes::DontBeginStep,
+        BeginStepTypes::BeginStepSynchronously,
+        BeginStepTypes::BeginStepRandomAccess>;
+
     struct DeferredParseAccess
     {
         /**
@@ -69,7 +86,7 @@ namespace internal
          * (Group- and variable-based parsing shares the same code logic.)
          */
         bool fileBased = false;
-        bool beginStep = false;
+        BeginStep beginStep = BeginStepTypes::DontBeginStep{};
     };
 
     class IterationData : public AttributableData
@@ -305,7 +322,8 @@ class Iteration : public Attributable
         std::string const &filePath,
         std::string const &groupPath,
         bool beginStep);
-    void readGorVBased(std::string const &groupPath, bool beginStep);
+    void readGorVBased(
+        std::string const &groupPath, internal::BeginStep const &beginStep);
     void read_impl(std::string const &groupPath);
     void readMeshes(std::string const &meshesPath);
     void readParticles(std::string const &particlesPath);

include/openPMD/Series.hpp

@@ -118,6 +118,10 @@ namespace internal
          * snapshot attribute.
          */
         std::set<IterationIndex_t> m_currentlyActiveIterations;
+        /**
+         * For reading: In which IO step do I need to look for an Iteration?
+         */
+        std::unordered_map<IterationIndex_t, size_t> m_snapshotToStep;
         /**
          * This map contains the filenames of those Iterations which were found
          * on the file system upon opening the Series for reading in file-based
@@ -969,12 +973,16 @@ OPENPMD_private
      * Returns the current content of the /data/snapshot attribute.
      * (We could also add this to the public API some time)
      */
-    std::optional<std::vector<IterationIndex_t>> currentSnapshot() const;
+    std::optional<std::vector<IterationIndex_t>> currentSnapshot();
 
     AbstractIOHandler *runDeferredInitialization();
 
     AbstractIOHandler *IOHandler();
     AbstractIOHandler const *IOHandler() const;
+
+    std::optional<std::vector<std::vector<IterationIndex_t>>>
+    preparseSnapshots();
+    [[nodiscard]] bool randomAccessSteps() const;
 }; // Series
 
 using SnapshotWorkflow = Series::SnapshotWorkflow;

include/openPMD/auxiliary/Mpi.hpp

@@ -47,10 +47,38 @@ namespace
         {
             return MPI_CHAR;
         }
+        else if constexpr (std::is_same_v<T_decay, signed char>)
+        {
+            return MPI_SIGNED_CHAR;
+        }
+        else if constexpr (std::is_same_v<T_decay, unsigned char>)
+        {
+            return MPI_UNSIGNED_CHAR;
+        }
+        else if constexpr (std::is_same_v<T_decay, short>)
+        {
+            return MPI_SHORT;
+        }
+        else if constexpr (std::is_same_v<T_decay, short unsigned>)
+        {
+            return MPI_UNSIGNED_SHORT;
+        }
         else if constexpr (std::is_same_v<T_decay, unsigned>)
         {
             return MPI_UNSIGNED;
         }
+        else if constexpr (std::is_same_v<T_decay, int>)
+        {
+            return MPI_INT;
+        }
+        else if constexpr (std::is_same_v<T_decay, long>)
+        {
+            return MPI_LONG;
+        }
+        else if constexpr (std::is_same_v<T_decay, long long>)
+        {
+            return MPI_LONG_LONG;
+        }
         else if constexpr (std::is_same_v<T_decay, unsigned long>)
         {
             return MPI_UNSIGNED_LONG;
@@ -59,6 +87,18 @@ namespace
         {
             return MPI_UNSIGNED_LONG_LONG;
         }
+        else if constexpr (std::is_same_v<T_decay, float>)
+        {
+            return MPI_FLOAT;
+        }
+        else if constexpr (std::is_same_v<T_decay, double>)
+        {
+            return MPI_DOUBLE;
+        }
+        else if constexpr (std::is_same_v<T_decay, long double>)
+        {
+            return MPI_LONG_DOUBLE;
+        }
         else
         {
             static_assert(

include/openPMD/backend/Attribute.hpp

@@ -310,6 +310,25 @@ namespace detail
 #else
     }
 #endif
+
+    template <typename T, typename U>
+    auto doConvertOptional(T const *pv) -> std::optional<U>
+    {
+        auto eitherValueOrError = doConvert<T, U>(pv);
+        return std::visit(
+            [](auto &containedValue) -> std::optional<U> {
+                using Res = std::decay_t<decltype(containedValue)>;
+                if constexpr (std::is_same_v<Res, std::runtime_error>)
+                {
+                    return std::nullopt;
+                }
+                else
+                {
+                    return {std::move(containedValue)};
+                }
+            },
+            eitherValueOrError);
+    }
 } // namespace detail
 
 template <typename U>
@@ -339,25 +358,12 @@ U Attribute::get() const
 template <typename U>
 std::optional<U> Attribute::getOptional() const
 {
-    auto eitherValueOrError = std::visit(
-        [](auto &&containedValue) -> std::variant<U, std::runtime_error> {
-            using containedType = std::decay_t<decltype(containedValue)>;
-            return detail::doConvert<containedType, U>(&containedValue);
-        },
-        Variant::getResource());
     return std::visit(
         [](auto &&containedValue) -> std::optional<U> {
-            using T = std::decay_t<decltype(containedValue)>;
-            if constexpr (std::is_same_v<T, std::runtime_error>)
-            {
-                return std::nullopt;
-            }
-            else
-            {
-                return {std::move(containedValue)};
-            }
+            using containedType = std::decay_t<decltype(containedValue)>;
+            return detail::doConvertOptional<containedType, U>(&containedValue);
         },
-        std::move(eitherValueOrError));
+        Variant::getResource());
 }
 } // namespace openPMD