Eliminate "extent" field from datasets

openPMD · Nov 28, 2018 · b18a1e3 · b18a1e3
1 parent 22606a8
commit b18a1e3
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 51 deletions.
diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp
@@ -327,6 +327,11 @@ namespace openPMD
         // essentially: m_i = \prod_{j=0}^{i-1} extent_j
         static Extent getMultiplicators( Extent const & extent );
 
+        static nlohmann::json initializeNDArray( Extent const & extent );
+
+        static Extent getExtent( nlohmann::json & j );
+
+
         // remove single '/' in the beginning and end of a string
         static std::string removeSlashes( std::string );
 

diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp
@@ -213,10 +213,9 @@ namespace openPMD
                 name
             );
             auto & dset = jsonVal[name];
-            Extent extent = parameter.extent;
-            dset["extent"] = extent;
             dset["datatype"] = datatypeToString( parameter.dtype );
             dset["object_type"] = "dataset";
+            dset["data"] = initializeNDArray( parameter.extent );
             writable->written = true;
             m_dirty.emplace( file );
         }
@@ -237,27 +236,26 @@ namespace openPMD
 
         try
         {
-            size_t currentdim = 0;
-            auto it = j["extent"].begin( );
-            for( ; currentdim <
-                   parameters.extent
-                       .size( ) && it != j["extent"].end( );
-                it++, currentdim++ )
+            auto datasetExtent = getExtent( j["data"] );
+            VERIFY_ALWAYS( datasetExtent.size( ) ==
+                           parameters.extent
+                               .size( ),
+                "Cannot change dimensionality of a dataset" )
+            for( size_t currentdim = 0;
+                currentdim <
+                parameters.extent
+                    .size( );
+                currentdim++ )
             {
-                VERIFY_ALWAYS( it.value( )
-                                   .get< Extent::value_type >( ) <=
+                VERIFY_ALWAYS( datasetExtent[currentdim] <=
                                parameters.extent[currentdim],
                     "Cannot shrink the extent of a dataset" )
             }
-            VERIFY_ALWAYS( currentdim ==
-                           parameters.extent
-                               .size( ) && it == j["extent"].end( ),
-                "Cannot change dimensionality of a dataset" )
         } catch( json::basic_json::type_error & e )
         {
             throw std::runtime_error( "The specified location contains no valid dataset" );
         }
-        j["extent"] = parameters.extent;
+        j["data"] = initializeNDArray( parameters.extent );
         writable->written = true;
 
     }
@@ -348,18 +346,7 @@ namespace openPMD
 
         *parameters.dtype =
             Datatype( stringToDatatype( datasetJson["datatype"].get< std::string >( ) ) );
-        parameters.extent
-            ->clear( );
-        for( auto it = datasetJson["extent"].begin( );
-            it != datasetJson["extent"].end( );
-            it++ )
-        {
-            parameters.extent
-                ->push_back(
-                    it.value( )
-                        .get< Extent::value_type >( )
-                );
-        }
+        *parameters.extent = getExtent( datasetJson["data"] );
         writable->written = true;
 
     }
@@ -418,12 +405,10 @@ namespace openPMD
         ),
             "Paths passed for deletion should be relative, the given path is absolute (starts with '/')" )
         auto file = refreshFileFromParent( writable );
-        auto
-            filepos =
-            setAndGetFilePosition(
-                writable,
-                false
-            );
+        auto filepos = setAndGetFilePosition(
+            writable,
+            false
+        );
         auto path = removeSlashes( parameters.path );
         VERIFY( !path.empty( ),
             "No path passed for deletion." )
@@ -520,12 +505,10 @@ namespace openPMD
             return;
         }
 
-        auto
-            filepos =
-            setAndGetFilePosition(
-                writable,
-                false
-            );
+        auto filepos = setAndGetFilePosition(
+            writable,
+            false
+        );
 
         auto file = refreshFileFromParent( writable );
         auto dataset = removeSlashes( parameters.name );
@@ -931,6 +914,37 @@ namespace openPMD
     }
 
 
+    nlohmann::json JSONIOHandlerImpl::initializeNDArray( Extent const & extent )
+    {
+        // idea: begin from the innermost shale and copy the result into the
+        // outer shales
+        nlohmann::json accum; // null
+        for( auto it = extent.rbegin( ); it != extent.rend( ); it++ )
+        {
+            nlohmann::json old = accum;
+            accum = nlohmann::json {};
+            for( Extent::value_type i = 0; i < *it; i++ )
+            {
+                accum[i] = old; // copy boi
+            }
+        }
+        return accum;
+    }
+
+
+    Extent JSONIOHandlerImpl::getExtent( nlohmann::json & j )
+    {
+        Extent res;
+        nlohmann::json * ptr = &j;
+        while( ptr->is_array( ) )
+        {
+            res.push_back( ptr->size( ) );
+            ptr = &( *ptr )[0];
+        }
+        return res;
+    }
+
+
     std::string JSONIOHandlerImpl::removeSlashes( std::string s )
     {
         if( auxiliary::starts_with(
@@ -1244,23 +1258,22 @@ namespace openPMD
 
         try
         {
-            unsigned int dimension = 0;
-            auto it = j["extent"].begin( );
-            for( ; dimension <
-                   parameters.extent
-                       .size( ) && it != j["extent"].end( ); it++, dimension++ )
+            auto datasetExtent = getExtent( j["data"] );
+            VERIFY_ALWAYS( datasetExtent.size( ) ==
+                           parameters.extent
+                               .size( ),
+                "Read/Write request does not fit the dataset's dimension" );
+            for( unsigned int dimension = 0;
+                dimension <
+                parameters.extent
+                    .size( );
+                dimension++ )
             {
                 VERIFY_ALWAYS( parameters.offset[dimension] +
                                parameters.extent[dimension] <=
-                               it.value( )
-                                   .get< Extent::value_type >( ),
+                               datasetExtent[dimension],
                     "Read/Write request exceeds the dataset's size" );
             }
-            VERIFY_ALWAYS( parameters.extent
-                               .size( ) == dimension &&
-                           it == j["extent"].end( ),
-                "Read/Write request does not fit the dataset's dimension" );
-
             Datatype
                 dt = stringToDatatype( j["datatype"].get< std::string >( ) );
             VERIFY_ALWAYS( dt == parameters.dtype,