diff --git a/CMakeLists.txt b/CMakeLists.txt index 82b361298f..7a3ede5f49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -293,7 +293,9 @@ set(IO_SOURCE src/IO/AbstractIOHandlerHelper.cpp src/IO/IOTask.cpp src/IO/HDF5/HDF5IOHandler.cpp - src/IO/HDF5/ParallelHDF5IOHandler.cpp) + src/IO/HDF5/ParallelHDF5IOHandler.cpp + src/IO/JSON/JSONIOHandler.cpp + src/IO/JSON/JSONIOHandlerImpl.cpp) set(IO_ADIOS1_SEQUENTIAL_SOURCE src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp diff --git a/README.md b/README.md index 88a1b80e9f..ab2f8c4d0e 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Shipped internally in `share/openPMD/thirdParty/`: * [NLohmann-JSON](https://github.com/nlohmann/json) 3.4.0+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) Optional I/O backends: -* [JSON](https://en.wikipedia.org/wiki/JSON) (*not yet implemented*) +* [JSON](https://en.wikipedia.org/wiki/JSON) * [HDF5](https://support.hdfgroup.org/HDF5) 1.8.13+ * [ADIOS1](https://www.olcf.ornl.gov/center-projects/adios) 1.13.1+ * [ADIOS2](https://github.com/ornladios/ADIOS2) 2.1+ (*not yet implemented*) @@ -175,7 +175,7 @@ CMake controls options with prefixed `-D`, e.g. `-DopenPMD_USE_MPI=OFF`: | CMake Option | Values | Description | |------------------------------|------------------|------------------------------------------------------------------------------| | `openPMD_USE_MPI` | **AUTO**/ON/OFF | Enable MPI support | -| `openPMD_USE_JSON` | **AUTO**/ON/OFF | Enable support for JSON 1 | +| `openPMD_USE_JSON` | **AUTO**/ON/OFF | Enable support for JSON | | `openPMD_USE_HDF5` | **AUTO**/ON/OFF | Enable support for HDF5 | | `openPMD_USE_ADIOS1` | **AUTO**/ON/OFF | Enable support for ADIOS1 | | `openPMD_USE_ADIOS2` | AUTO/ON/**OFF** | Enable support for ADIOS2 1 | diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst new file mode 100644 index 0000000000..bccedc5f17 --- /dev/null +++ b/docs/source/backends/json.rst @@ -0,0 +1,82 @@ +.. _backends-json: + +JSON Backend +============ + +openPMD supports writing to and reading from JSON files. +For this, the installed copy of openPMD must have been built with support for the JSON backend. +To build openPMD with support for JSON, use the CMake option ``-DopenPMD_USE_JSON=ON``. +For further information, check out the :ref:`installation guide `, +:ref:`build dependencies ` and the :ref:`build options `. + + +JSON File Format +---------------- +A JSON file uses the file ending ``.json``. The JSON backend is chosen by creating +a ``Series`` object with a filename that has this file ending. + +The top-level JSON object is a group representing the openPMD root group ``"/"``. +Any **openPMD group** is represented in JSON as a JSON object with three keys: + + * ``attributes``: Attributes associated with the group. + * ``subgroups``: A JSON array of groups that appear below the current group. + * ``datasets``: A JSON array of datasets contained in the group. + +Any of these keys may point to ``null`` or not be present, +thus representing an empty array / object. + +Additionally to the three mentioned keys, the top-level group stores information about +the byte widths specific to the writing platform behind the key ``platform_byte_widths``. +Will be overwritten every time that a JSON value is stored to disk, hence this information +is only available about the last platform writing the JSON value. + +Any **openPMD dataset** is a JSON object with four keys: + + * ``attributes``: Attributes associated with the dataset. May be ``null`` or not present if no attributes are associated with the dataset. + * ``datatype``: A string describing the type of the stored data. + * ``extent``: A JSON array describing the extent of the dataset in every dimension. + * ``data`` A nested array storing the actual data in row-major manner. + The data needs to be consistent with the fields ``datatype`` and ``extent``. + +**Attributes** are stored as a JSON object with a key for each attribute. +Every such attribute is itself a JSON object with two keys: + + * ``datatype``: A string describing the type of the value. + * ``value``: The actual value of type ``datatype``. + +Restrictions +------------ +For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are +equivalent to those of the `JSON library by Niels Lohmann `_ +used by the openPMD backend. + +Numerical values, integral as well as floating point, are supported up to a length of +64 bits. +Since JSON does not support special floating point values (i.e. NaN, Infinity, -Infinity), +those values are rendered as ``null``. + +Instructing openPMD to write values of a datatype that is too wide for the JSON +backend does *not* result in an error: + * If casting the value to the widest supported datatype of the same category (integer or floating point) + is possible without data loss, the cast is performed and the value is written. + As an example, on a platform with ``sizeof(double) == 8``, writing the value + ``static_cast(std::numeric_limits::max())`` will work as expected + since it can be cast back to ``double``. + * Otherwise, a ``null`` value is written. + +Upon reading ``null`` when expecting a floating point number, a NaN value will be +returned. Take notice that a NaN value returned from the deserialization process +may have originally been +/-Infinity or beyond the supported value range. + +Upon reading ``null`` when expecting any other datatype, the JSON backend will +propagate the exception thrown by Niels Lohmann's library. + +A parallel (i.e. MPI) implementation is *not* available. + +Example +------- +The example code in the :ref:`usage section ` will produce the following JSON serialization +when picking the JSON backend: + +.. literalinclude:: json_example.json + diff --git a/docs/source/backends/json_example.json b/docs/source/backends/json_example.json new file mode 100644 index 0000000000..ca4e9ea88a --- /dev/null +++ b/docs/source/backends/json_example.json @@ -0,0 +1,154 @@ +{ + "attributes": { + "basePath": { + "datatype": "STRING", + "value": "/data/%T/" + }, + "iterationEncoding": { + "datatype": "STRING", + "value": "groupBased" + }, + "iterationFormat": { + "datatype": "STRING", + "value": "/data/%T/" + }, + "meshesPath": { + "datatype": "STRING", + "value": "meshes/" + }, + "openPMD": { + "datatype": "STRING", + "value": "1.1.0" + }, + "openPMDextension": { + "datatype": "UINT", + "value": 0 + } + }, + "platform_byte_widths": { + "BOOL": 1, + "CHAR": 1, + "DOUBLE": 8, + "FLOAT": 4, + "INT": 4, + "LONG": 8, + "LONGLONG": 8, + "LONG_DOUBLE": 16, + "SHORT": 2, + "UCHAR": 1, + "UINT": 4, + "ULONG": 8, + "ULONGLONG": 8, + "USHORT": 2 + }, + "subgroups": { + "data": { + "subgroups": { + "1": { + "attributes": { + "dt": { + "datatype": "DOUBLE", + "value": 1 + }, + "time": { + "datatype": "DOUBLE", + "value": 0 + }, + "timeUnitSI": { + "datatype": "DOUBLE", + "value": 1 + } + }, + "subgroups": { + "meshes": { + "datasets": { + "rho": { + "attributes": { + "axisLabels": { + "datatype": "VEC_STRING", + "value": [ + "x" + ] + }, + "dataOrder": { + "datatype": "STRING", + "value": "C" + }, + "geometry": { + "datatype": "STRING", + "value": "cartesian" + }, + "gridGlobalOffset": { + "datatype": "VEC_DOUBLE", + "value": [ + 0 + ] + }, + "gridSpacing": { + "datatype": "VEC_DOUBLE", + "value": [ + 1 + ] + }, + "gridUnitSI": { + "datatype": "DOUBLE", + "value": 1 + }, + "position": { + "datatype": "VEC_DOUBLE", + "value": [ + 0 + ] + }, + "timeOffset": { + "datatype": "FLOAT", + "value": 0 + }, + "unitDimension": { + "datatype": "ARR_DBL_7", + "value": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + "unitSI": { + "datatype": "DOUBLE", + "value": 1 + } + }, + "data": [ + [ + 0, + 1, + 2 + ], + [ + 3, + 4, + 5 + ], + [ + 6, + 7, + 8 + ] + ], + "datatype": "DOUBLE", + "extent": [ + 3, + 3 + ] + } + } + } + } + } + } + } + } +} diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index bcae829343..e016a38ef4 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -15,7 +15,7 @@ CMake controls options with prefixed ``-D``, e.g. ``-DopenPMD_USE_MPI=OFF``: CMake Option Values Description ============================== =============== ======================================================================== ``openPMD_USE_MPI`` **AUTO**/ON/OFF Enable MPI support -``openPMD_USE_JSON`` **AUTO**/ON/OFF Enable support for JSON :sup:`1` +``openPMD_USE_JSON`` **AUTO**/ON/OFF Enable support for JSON ``openPMD_USE_HDF5`` **AUTO**/ON/OFF Enable support for HDF5 ``openPMD_USE_ADIOS1`` **AUTO**/ON/OFF Enable support for ADIOS1 ``openPMD_USE_ADIOS2`` AUTO/ON/**OFF** Enable support for ADIOS2 :sup:`1` diff --git a/docs/source/index.rst b/docs/source/index.rst index 1d56bc50de..7b907c34a6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -83,3 +83,13 @@ Development dev/sphinx dev/doxygen dev/release + +******** +Backends +******** +.. toctree:: + :caption: BACKENDS + :maxdepth: 1 + :hidden: + + backends/json diff --git a/docs/source/usage/firststeps.rst b/docs/source/usage/firststeps.rst index be53d92d60..a799ca56ae 100644 --- a/docs/source/usage/firststeps.rst +++ b/docs/source/usage/firststeps.rst @@ -1,4 +1,4 @@ -.. usage-firststeps: +.. _usage-firststeps: First Steps =========== diff --git a/docs/source/usage/parallel.rst b/docs/source/usage/parallel.rst index 17c2c881bf..c98e362929 100644 --- a/docs/source/usage/parallel.rst +++ b/docs/source/usage/parallel.rst @@ -1,4 +1,4 @@ -.. usage-parallel: +.. _usage-parallel: Parallel API ============ diff --git a/docs/source/usage/serial.rst b/docs/source/usage/serial.rst index fc92894d5d..fe77c2d446 100644 --- a/docs/source/usage/serial.rst +++ b/docs/source/usage/serial.rst @@ -1,4 +1,4 @@ -.. usage-serial: +.. _usage-serial: Serial API ========== diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 887aab2fe5..c0c19a2e0d 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -28,7 +28,8 @@ #include #include #include - +#include +#include namespace openPMD { @@ -518,6 +519,179 @@ isSame( openPMD::Datatype const d, openPMD::Datatype const e ) return false; } +/** + * Generalizes switching over an openPMD datatype. + * + * Will call the functor passed + * to it using the C++ internal datatype corresponding to the openPMD datatype + * as template parameter for the templated (). + * + * @tparam ReturnType The functor's return type. + * @tparam Action The functor's type. + * @tparam Args The functors argument types. + * @param dt The openPMD datatype. + * @param action The functor. + * @param args The functor's arguments. + * @return The return value of the functor, when calling its () with + * the passed arguments and the template parameter type corresponding to the + * openPMD type. + */ + +#if _MSC_VER && !__INTEL_COMPILER +#define OPENPMD_TEMPLATE_OPERATOR operator +#else +#define OPENPMD_TEMPLATE_OPERATOR template operator +#endif +template< + typename ReturnType = void, + typename Action, + typename ...Args +> +ReturnType switchType( + Datatype dt, + Action action, + Args && ...args +) { + using fun = decltype(&Action::OPENPMD_TEMPLATE_OPERATOR() < int >); + static std::map< + Datatype, + fun + > funs { + { + Datatype::CHAR , + &Action::OPENPMD_TEMPLATE_OPERATOR() < char > }, + { + Datatype::UCHAR , + &Action::OPENPMD_TEMPLATE_OPERATOR() < unsigned char > }, + { + Datatype::SHORT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < short > }, + { + Datatype::INT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < int > }, + { + Datatype::LONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < long > }, + { + Datatype::LONGLONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < long long > }, + { + Datatype::USHORT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < unsigned short > }, + { + Datatype::UINT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < unsigned int > }, + { + Datatype::ULONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < unsigned long > }, + { + Datatype::ULONGLONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < unsigned long long > }, + { + Datatype::FLOAT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < float > }, + { + Datatype::DOUBLE , + &Action::OPENPMD_TEMPLATE_OPERATOR() < double > }, + { + Datatype::LONG_DOUBLE , + &Action::OPENPMD_TEMPLATE_OPERATOR() < long double > }, + { + Datatype::STRING , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::string > }, + { + Datatype::VEC_CHAR , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< char>> + }, + { + Datatype::VEC_SHORT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< short>> + }, + { + Datatype::VEC_INT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< int>> + }, + { + Datatype::VEC_LONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< long>> + }, + { + Datatype::VEC_LONGLONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< long long>> + }, + { + Datatype::VEC_UCHAR , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< unsigned char>> + }, + { + Datatype::VEC_USHORT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< unsigned short>> + }, + { + Datatype::VEC_UINT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< unsigned int>> + }, + { + Datatype::VEC_ULONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< unsigned long>> + }, + { + Datatype::VEC_ULONGLONG , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< unsigned long long>> + }, + { + Datatype::VEC_FLOAT , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< float>> + }, + { + Datatype::VEC_DOUBLE , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< double>> + }, + { + Datatype::VEC_LONG_DOUBLE , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< long double>> + }, + { + Datatype::VEC_STRING , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::vector< std::string>> + }, + { + Datatype::ARR_DBL_7 , + &Action::OPENPMD_TEMPLATE_OPERATOR() < std::array< + double, + 7>> + }, + { + Datatype::BOOL , + &Action::OPENPMD_TEMPLATE_OPERATOR() < bool > }, + { + Datatype::DATATYPE , + &Action::OPENPMD_TEMPLATE_OPERATOR() < 1000 > }, + { + Datatype::UNDEFINED , + &Action::OPENPMD_TEMPLATE_OPERATOR() < 0 > } + }; + auto it = funs.find( dt ); + if( it != funs.end( ) ) + { + return ( ( action ).* + ( it->second ) )( std::forward< Args >( args )... ); + } + else + { + throw std::runtime_error( + "Internal error: Encountered unknown datatype (switchType) ->" + + std::to_string( static_cast(dt) ) + ); + } +} + +#undef OPENPMD_TEMPLATE_OPERATOR + +std::string datatypeToString( Datatype dt ); + +Datatype stringToDatatype( std::string s ); + void warnWrongDtype(std::string const& key, Datatype store, diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 4d48e315da..3b80a85327 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -30,6 +30,7 @@ enum class Format HDF5, ADIOS1, ADIOS2, + JSON, DUMMY }; //Format } // openPMD diff --git a/include/openPMD/IO/JSON/JSONFilePosition.hpp b/include/openPMD/IO/JSON/JSONFilePosition.hpp new file mode 100644 index 0000000000..e96c01ed5a --- /dev/null +++ b/include/openPMD/IO/JSON/JSONFilePosition.hpp @@ -0,0 +1,53 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + + +#include "openPMD/IO/AbstractFilePosition.hpp" + +#if openPMD_HAVE_JSON +#include +#endif + +namespace openPMD +{ + + + struct JSONFilePosition : + public AbstractFilePosition +#if openPMD_HAVE_JSON + { + using json = nlohmann::json; + + + JSONFilePosition( json::json_pointer ptr = json::json_pointer( ) ) : + id( ptr ) + {} + + + json::json_pointer id; + }; +#else + {}; +#endif + +} // openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp new file mode 100644 index 0000000000..a6d0ba3ffd --- /dev/null +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -0,0 +1,47 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" + + +namespace openPMD +{ + class JSONIOHandler : + public AbstractIOHandler + { + public: + JSONIOHandler( + std::string path, + AccessType at + ); + + virtual ~JSONIOHandler( ); + + std::future< void > flush( ) override; + + private: + JSONIOHandlerImpl m_impl; + }; +} // openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp new file mode 100644 index 0000000000..ab6f01828f --- /dev/null +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -0,0 +1,566 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + + +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/AccessType.hpp" +#include "openPMD/IO/JSON/JSONFilePosition.hpp" + +#include +#include +#include +#include +#include +#include + + +#if openPMD_HAVE_JSON + + +#include + + +#endif + + +namespace openPMD +{ + // Wrapper around a shared pointer to: + // * a filename + // * and a boolean indicating whether the file still exists + // The wrapper adds no extra information, but some commodity functions. + // Invariant for JSONIOHandlerImpl: + // For any valid filename, there is at any time at most one + // such shared pointer (wrapper) in the HandlerImpl's data structures + // (counting by pointer equality) + // This means, that a file can be invalidated (i.e. deleted or overwritten) + // by simply searching for one instance of the file e.g. in m_files and + // invalidating this instance + // A new instance may hence only be created after making sure that there are + // no valid instances in the data structures. + struct File + { + explicit File( std::string s ) : + fileState { std::make_shared< FileState >( s ) } + {} + + + File( ) = default; + + + struct FileState + { + explicit FileState( std::string s ) : + name { std::move( s ) } + {} + + + std::string name; + bool valid = true; + }; + + std::shared_ptr< FileState > fileState; + + + void invalidate( ) + { + fileState->valid = false; + } + + + bool valid( ) const + { + return fileState->valid; + } + + + File & operator=( std::string s ) + { + if( fileState ) + { + fileState->name = s; + } + else + { + fileState = std::make_shared< FileState >( s ); + } + return *this; + } + + + bool operator==( + File const & f + ) const + { + return this->fileState == f.fileState; + } + + + std::string & operator*( ) const + { + return fileState->name; + } + + + std::string * operator->( ) const + { + return &fileState->name; + } + + + explicit operator bool( ) const + { + return fileState.operator bool( ); + } + }; +} + +namespace std +{ + template< > + struct hash< openPMD::File > + { + typedef openPMD::File argument_type; + typedef std::size_t result_type; + + + result_type operator()( argument_type const & s ) const noexcept + { + return std::hash< shared_ptr< openPMD::File::FileState>> {}( s.fileState ); + } + }; +} + +namespace openPMD +{ +#if openPMD_HAVE_JSON + + class JSONIOHandlerImpl : + public AbstractIOHandlerImpl + { + using json = nlohmann::json; + + public: + explicit JSONIOHandlerImpl( AbstractIOHandler * ); + + ~JSONIOHandlerImpl( ) override; + + void createFile( + Writable *, + Parameter< Operation::CREATE_FILE > const & + ) override; + + void createPath( + Writable *, + Parameter< Operation::CREATE_PATH > const & + ) override; + + void createDataset( + Writable *, + Parameter< Operation::CREATE_DATASET > const & + ) override; + + void extendDataset( + Writable *, + Parameter< Operation::EXTEND_DATASET > const & + ) override; + + void openFile( + Writable *, + Parameter< Operation::OPEN_FILE > const & + ) override; + + void openPath( + Writable *, + Parameter< Operation::OPEN_PATH > const & + ) override; + + void openDataset( + Writable *, + Parameter< Operation::OPEN_DATASET > & + ) override; + + void deleteFile( + Writable *, + Parameter< Operation::DELETE_FILE > const & + ) override; + + void deletePath( + Writable *, + Parameter< Operation::DELETE_PATH > const & + ) override; + + void deleteDataset( + Writable *, + Parameter< Operation::DELETE_DATASET > const & + ) override; + + void deleteAttribute( + Writable *, + Parameter< Operation::DELETE_ATT > const & + ) override; + + void writeDataset( + Writable *, + Parameter< Operation::WRITE_DATASET > const & + ) override; + + void writeAttribute( + Writable *, + Parameter< Operation::WRITE_ATT > const & + ) override; + + void readDataset( + Writable *, + Parameter< Operation::READ_DATASET > & + ) override; + + void readAttribute( + Writable *, + Parameter< Operation::READ_ATT > & + ) override; + + void listPaths( + Writable *, + Parameter< Operation::LIST_PATHS > & + ) override; + + void listDatasets( + Writable *, + Parameter< Operation::LIST_DATASETS > & + ) override; + + void listAttributes( + Writable *, + Parameter< Operation::LIST_ATTS > & + ) override; + + std::future< void > flush( ) override; + + + private: + + using FILEHANDLE = std::fstream; + + // map each Writable to its associated file + // contains only the filename, without the OS path + std::unordered_map< + Writable *, + File + > m_files; + + std::unordered_map< + File, + std::shared_ptr< nlohmann::json >> m_jsonVals; + + // files that have logically, but not physically been written to + std::unordered_set< File > m_dirty; + + + // HELPER FUNCTIONS + + + // will use the IOHandler to retrieve the correct directory + // shared pointer to circumvent the fact that c++ pre 17 does + // not enforce (only allow) copy elision in return statements + std::shared_ptr< FILEHANDLE > getFilehandle( + File, + AccessType accessType + ); //, AccessType accessType=this->m_handler->accessType); + + // full operating system path of the given file + std::string fullPath( File ); + + std::string fullPath( std::string ); + + // from a path specification /a/b/c, remove the last + // "folder" (i.e. modify the string to equal /a/b) + static void parentDir( std::string & ); + + // Fileposition is assumed to have already been set, + // get it in string form + static std::string filepositionOf( Writable * w ); + + // Execute visitor on each pair of positions in the json value + // and the flattened multidimensional array. + // Used for writing from the data to JSON and for reading back into + // the array from JSON + template< + typename T, + typename Visitor + > + static void syncMultidimensionalJson( + nlohmann::json & j, + Offset const & offset, + Extent const & extent, + Extent const & multiplicator, + Visitor visitor, + T * data, + size_t currentdim = 0 + ); + + // multiplicators: an array [m_0,...,m_n] s.t. + // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n] + // (m_n = 1) + // essentially: m_i = \prod_{j=0}^{i-1} extent_j + static Extent getMultiplicators( Extent const & extent ); + + // remove single '/' in the beginning and end of a string + static std::string removeSlashes( std::string ); + + template< typename KeyT > + static bool hasKey( + nlohmann::json &, + KeyT && key + ); + + // make sure that the given path exists in proper form in + // the passed json value + static void ensurePath( + nlohmann::json * json, + std::string path + ); + + // In order not to insert the same file name into the data structures + // with a new pointer (e.g. when reopening), search for a possibly + // existing old pointer. Construct a new pointer only upon failure. + // The bool is true iff the pointer has been newly-created. + // The iterator is an iterator for m_files + std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + > getPossiblyExisting( + std::string file + ); + + // get the json value representing the whole file, possibly reading + // from disk + std::shared_ptr< nlohmann::json > obtainJsonContents( File ); + + // get the json value at the writable's fileposition + nlohmann::json & obtainJsonContents( Writable * writable ); + + // write to disk the json contents associated with the file + // remove from m_dirty if unsetDirty == true + void putJsonContents( + File, + bool unsetDirty = true + ); + + // figure out the file position of the writable + // (preferring the parent's file position) and extend it + // by extend. return the modified file position. + std::shared_ptr< JSONFilePosition > setAndGetFilePosition( + Writable *, + std::string extend + ); + + // figure out the file position of the writable + // (preferring the parent's file position) + // only modify the writable's fileposition when specified + std::shared_ptr< JSONFilePosition > setAndGetFilePosition( + Writable *, + bool write = true + ); + + // get the writable's containing file + // if the parent is associated with another file, + // associate the writable with that file and return it + File refreshFileFromParent( Writable * writable ); + + void associateWithFile( + Writable * writable, + File + ); + + + // check whether the json reference contains a valid dataset + template< typename Param > + void verifyDataset( + Param const & parameters, + nlohmann::json & + ); + + static nlohmann::json platformSpecifics( ); + + struct DatasetWriter + { + template< typename T > + void operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ); + + template< int n > + void operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ); + + }; + + struct DatasetReader + { + template< typename T > + void operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ); + + template< int n > + void operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ); + }; + + struct AttributeWriter + { + template< typename T > + void operator()( + nlohmann::json &, + Attribute::resource const & + ); + + template< int n > + void operator()( + nlohmann::json &, + Attribute::resource const & + ); + + }; + + struct AttributeReader + { + template< typename T > + void operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ); + + template< int n > + void operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ); + + }; + + template< typename T > + struct CppToJSON + { + nlohmann::json operator()( T const & ); + }; + + template< typename T > + struct CppToJSON< std::vector< T>> + { + nlohmann::json operator()( std::vector< T > const & ); + }; + + template< typename T, int n > + struct CppToJSON< + std::array< + T, + n>> + { + nlohmann::json operator()( + std::array< + T, + n + > const & + ); + }; + + template< + typename T, + typename Enable = T + > + struct JsonToCpp + { + T operator()( nlohmann::json const & ); + }; + + template< typename T > + struct JsonToCpp< std::vector< T > > + { + std::vector< T > operator()( nlohmann::json const & ); + }; + + template< typename T, int n > + struct JsonToCpp< + std::array< + T, + n + > + > + { + std::array< + T, + n + > operator()( nlohmann::json const & ); + }; + + template< typename T > + struct JsonToCpp< + T, + typename std::enable_if< + std::is_floating_point< + T + >::value + >::type + > + { + T operator()( nlohmann::json const & ); + }; + }; + +#else + + class JSONIOHandlerImpl + { + public: + JSONIOHandlerImpl( openPMD::AbstractIOHandler * ) + {}; + + + ~JSONIOHandlerImpl( ) + {}; + + + std::future< void > flush( ) + { + return std::future< void >( ); + } + }; + +#endif + + +} // openPMD diff --git a/include/openPMD/auxiliary/Filesystem.hpp b/include/openPMD/auxiliary/Filesystem.hpp index ad7c23e207..dd6ac6d3b4 100644 --- a/include/openPMD/auxiliary/Filesystem.hpp +++ b/include/openPMD/auxiliary/Filesystem.hpp @@ -87,5 +87,27 @@ remove_directory(std::string const& path); */ bool remove_file(std::string const& path); + +// /** Opens the file identified by the given path. +// * +// * @param path Absolute or relative path to the file to create. +// * @param accessType The access type. +// * If it is `CREATE` a new file is created and opened in write mode. +// * If it is `READ_WRITE`, an existing file is opened in read-write mode. +// * If it is `READ_ONLY`, an existing file is opened in read-only mode. +// * @return A file handle if the file was successfully opened or created, +// * a nullpointer otherwise. +// */ +// FILEHANDLE +// open_file(std::string const& path, AccessType); + +// /** Closes the file identified by the file handle. +// * +// * @param fileHandle The filehandle of the file to close. +// * @return True if closing was successful, false otherwise. +// */ +// bool +// close_file(FILEHANDLE); + } // auxiliary } // openPMD diff --git a/include/openPMD/auxiliary/Memory.hpp b/include/openPMD/auxiliary/Memory.hpp index 5fd6333f1e..b92e292bbd 100644 --- a/include/openPMD/auxiliary/Memory.hpp +++ b/include/openPMD/auxiliary/Memory.hpp @@ -24,8 +24,10 @@ #include "openPMD/Datatype.hpp" #include +#include #include #include +#include namespace openPMD diff --git a/include/openPMD/auxiliary/StringManip.hpp b/include/openPMD/auxiliary/StringManip.hpp index 78c7db6ac8..a3b278180b 100644 --- a/include/openPMD/auxiliary/StringManip.hpp +++ b/include/openPMD/auxiliary/StringManip.hpp @@ -104,6 +104,26 @@ replace_last(std::string s, return s; } +inline std::string +replace_all_nonrecursively(std::string s, + std::string const& target, + std::string const& replacement) +{ + std::string::size_type pos = 0; + auto tsize = target.size(); + auto rsize = replacement.size(); + while (true) + { + pos = s.find(target, pos); + if (pos == std::string::npos) + break; + s.replace(pos, tsize, replacement); + pos += rsize; + } + s.shrink_to_fit(); + return s; +} + inline std::string replace_all(std::string s, std::string const& target, diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index af905d31d8..9c1b2627c9 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -69,6 +69,7 @@ class Writable friend class ADIOS2IOHandlerImpl; friend class HDF5IOHandlerImpl; friend class ParallelHDF5IOHandlerImpl; + friend class JSONIOHandlerImpl; friend struct test::TestHelper; friend std::string concrete_h5_file_position(Writable*); friend std::string concrete_bp1_file_position(Writable*); diff --git a/src/Datatype.cpp b/src/Datatype.cpp index 2a4ffc4bca..e4ef8f849b 100644 --- a/src/Datatype.cpp +++ b/src/Datatype.cpp @@ -22,6 +22,8 @@ #include #include +#include +#include namespace openPMD @@ -144,3 +146,161 @@ std::operator<<(std::ostream& os, openPMD::Datatype d) return os; } + +namespace openPMD +{ + Datatype stringToDatatype( std::string s ) + { + static std::unordered_map< + std::string, + Datatype + > m { + { + "CHAR", + Datatype::CHAR + }, + { + "UCHAR", + Datatype::UCHAR + }, + { + "SHORT", + Datatype::SHORT + }, + { + "INT", + Datatype::INT + }, + { + "LONG", + Datatype::LONG + }, + { + "LONGLONG", + Datatype::LONGLONG + }, + { + "USHORT", + Datatype::USHORT + }, + { + "UINT", + Datatype::UINT + }, + { + "ULONG", + Datatype::ULONG + }, + { + "ULONGLONG", + Datatype::ULONGLONG + }, + { + "FLOAT", + Datatype::FLOAT + }, + { + "DOUBLE", + Datatype::DOUBLE + }, + { + "LONG_DOUBLE", + Datatype::LONG_DOUBLE + }, + { + "STRING", + Datatype::STRING + }, + { + "VEC_CHAR", + Datatype::VEC_CHAR + }, + { + "VEC_SHORT", + Datatype::VEC_SHORT + }, + { + "VEC_INT", + Datatype::VEC_INT + }, + { + "VEC_LONG", + Datatype::VEC_LONG + }, + { + "VEC_LONGLONG", + Datatype::VEC_LONGLONG + }, + { + "VEC_UCHAR", + Datatype::VEC_UCHAR + }, + { + "VEC_USHORT", + Datatype::VEC_USHORT + }, + { + "VEC_UINT", + Datatype::VEC_UINT + }, + { + "VEC_ULONG", + Datatype::VEC_ULONG + }, + { + "VEC_ULONGLONG", + Datatype::VEC_ULONGLONG + }, + { + "VEC_FLOAT", + Datatype::VEC_FLOAT + }, + { + "VEC_DOUBLE", + Datatype::VEC_DOUBLE + }, + { + "VEC_LONG_DOUBLE", + Datatype::VEC_LONG_DOUBLE + }, + { + "VEC_STRING", + Datatype::VEC_STRING + }, + { + "ARR_DBL_7", + Datatype::ARR_DBL_7 + }, + { + "BOOL", + Datatype::BOOL + }, + { + "DATATYPE", + Datatype::DATATYPE + }, + { + "UNDEFINED", + Datatype::UNDEFINED + } + }; + auto it = m.find( s ); + if( it != m.end( ) ) + { + return it->second; + } + else + { + throw std::runtime_error( "Unknown datatype in string deserialization." ); + } + } + + + std::string datatypeToString( openPMD::Datatype dt ) + { + std::stringbuf buf; + std::ostream os(&buf); + os << dt; + return buf.str(); + } +} diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 926bafd448..ab358c2bfa 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -24,6 +24,7 @@ #include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" #include "openPMD/IO/HDF5/HDF5IOHandler.hpp" #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" +#include "openPMD/IO/JSON/JSONIOHandler.hpp" namespace openPMD @@ -65,6 +66,8 @@ namespace openPMD return std::make_shared< ADIOS1IOHandler >(path, accessType); case Format::ADIOS2: throw std::runtime_error("ADIOS2 backend not yet implemented"); + case Format::JSON: + return std::make_shared< JSONIOHandler >(path, accessType); default: return std::make_shared< DummyIOHandler >(path, accessType); } diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp new file mode 100644 index 0000000000..dde4b2df3b --- /dev/null +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -0,0 +1,47 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/JSON/JSONIOHandler.hpp" + + +namespace openPMD +{ + JSONIOHandler::~JSONIOHandler( ) + {} + + + JSONIOHandler::JSONIOHandler( + std::string path, + AccessType at + ) : + AbstractIOHandler { + path, + at + }, + m_impl { JSONIOHandlerImpl { this } } + {} + + + std::future< void > JSONIOHandler::flush( ) + { + return m_impl.flush( ); + } +} // openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp new file mode 100644 index 0000000000..83af3d55af --- /dev/null +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -0,0 +1,1543 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Memory.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Writable.hpp" +#include "openPMD/Datatype.hpp" +#include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" + + + +namespace openPMD +{ +#if openPMD_USE_VERIFY +# define VERIFY( CONDITION, TEXT ) { if(!(CONDITION)) throw std::runtime_error((TEXT)); } +#else +# define VERIFY( CONDITION, TEXT ) do{ (void)sizeof(CONDITION); } while( 0 ); +#endif + +#define VERIFY_ALWAYS( CONDITION, TEXT ) { if(!(CONDITION)) throw std::runtime_error((TEXT)); } + +#if openPMD_HAVE_JSON + + + JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler * handler ) : + AbstractIOHandlerImpl( handler ) + {} + + + JSONIOHandlerImpl::~JSONIOHandlerImpl( ) + { + flush( ); + } + + + std::future< void > JSONIOHandlerImpl::flush( ) + { + + AbstractIOHandlerImpl::flush( ); + for( auto const & file: m_dirty ) + { + putJsonContents( + file, + false + ); + } + m_dirty.clear( ); + return std::future< void >( ); + } + + + void JSONIOHandlerImpl::createFile( + Writable * writable, + Parameter< Operation::CREATE_FILE > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Creating a file in read-only mode is not possible." ); + + if( !writable->written ) + { + std::string name = parameters.name; + if( !auxiliary::ends_with( + name, + ".json" + ) ) + { + name += ".json"; + } + + auto res_pair = getPossiblyExisting( name ); + File shared_name = File( name ); + VERIFY_ALWAYS( !( m_handler->accessType == AccessType::READ_WRITE && + ( !std::get< 2 >( res_pair ) || + auxiliary::file_exists( fullPath( std::get< 0 >( res_pair ) ) ) ) ), + "Can only overwrite existing file in CREATE mode." ); + + if( !std::get< 2 >( res_pair ) ) + { + auto file = std::get< 0 >( res_pair ); + m_dirty.erase( file ); + m_jsonVals.erase( file ); + file.invalidate( ); + } + + std::string const dir( m_handler->directory ); + if( !auxiliary::directory_exists( dir ) ) + { + auto success = auxiliary::create_directories( dir ); + VERIFY( success, + "Could not create directory." ); + } + + associateWithFile( + writable, + shared_name + ); + this->m_dirty + .emplace( shared_name ); + // make sure to overwrite! + this->m_jsonVals[shared_name] = + std::make_shared< nlohmann::json >( ); + + + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( ); + } + } + + + void JSONIOHandlerImpl::createPath( + Writable * writable, + Parameter< Operation::CREATE_PATH > const & parameter + ) + { + std::string path = parameter.path; + /* Sanitize: + * The JSON API does not like to have slashes in the end. + */ + if( auxiliary::ends_with( + path, + "/" + ) ) + { + path = auxiliary::replace_last( + path, + "/", + "" + ); + } + auto expandedPath = auxiliary::replace_all_nonrecursively( + path, + "/", + "/subgroups/" + ); + + auto file = refreshFileFromParent( writable ); + + auto * jsonVal = &*obtainJsonContents( file ); + if( !auxiliary::starts_with( + path, + "/" + ) ) + { // path is relative + auto filepos = setAndGetFilePosition( + writable, + false + ); + + jsonVal = &( *jsonVal )[filepos->id]; + expandedPath = + filepos->id + .to_string( ) + "/subgroups/" + expandedPath; + } + + ensurePath( + jsonVal, + path + ); + + + m_dirty.emplace( file ); + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( nlohmann::json::json_pointer( expandedPath ) ); + } + + + void JSONIOHandlerImpl::createDataset( + Writable * writable, + Parameter< Operation::CREATE_DATASET > const & parameter + ) + { + if( m_handler->accessType == AccessType::READ_ONLY ) + { + throw std::runtime_error( "Creating a dataset in a file opened as read only is not possible." ); + } + if( !writable->written ) + { + /* Sanitize name */ + std::string name = removeSlashes( parameter.name ); + + auto file = refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto & jsonVal = obtainJsonContents( writable ); + if( jsonVal["datasets"].empty( ) ) + { + jsonVal["datasets"] = nlohmann::json::object( ); + } + setAndGetFilePosition( + writable, + "datasets/" + name + ); + auto & dset = jsonVal["datasets"][name]; + Extent extent = parameter.extent; + dset["extent"] = extent; + dset["datatype"] = datatypeToString( parameter.dtype ); + writable->written = true; + m_dirty.emplace( file ); + } + } + + + void JSONIOHandlerImpl::extendDataset( + Writable * writable, + Parameter< Operation::EXTEND_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot extend a dataset in read-only mode." ) + refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto name = removeSlashes( parameters.name ); + auto & jsonLoc = obtainJsonContents( writable )["datasets"]; + VERIFY_ALWAYS( hasKey( + jsonLoc, + name + ), + "The specified location contains no dataset" ) + auto & j = jsonLoc[name]; + + try + { + size_t currentdim = 0; + auto it = j["extent"].begin( ); + for( ; currentdim < + parameters.extent + .size( ) && it != j["extent"].end( ); + it++, currentdim++ ) + { + VERIFY_ALWAYS( it.value( ) + .get< Extent::value_type >( ) <= + parameters.extent[currentdim], + "Cannot shrink the extent of a dataset" ) + } + VERIFY_ALWAYS( currentdim == + parameters.extent + .size( ) && it == j["extent"].end( ), + "Cannot change dimensionality of a dataset" ) + } catch( json::basic_json::type_error & e ) + { + throw std::runtime_error( "The specified location contains no valid dataset" ); + } + j["extent"] = parameters.extent; + writable->written = true; + + } + + + void JSONIOHandlerImpl::openFile( + Writable * writable, + Parameter< Operation::OPEN_FILE > const & parameter + ) + { + if( !auxiliary::directory_exists( m_handler->directory ) ) + { + throw no_such_file_error( + "Supplied directory is not valid: " + m_handler->directory + ); + } + + std::string name = parameter.name; + if( !auxiliary::ends_with( + name, + ".json" + ) ) + { + name += ".json"; + } + + auto file = std::get< 0 >( getPossiblyExisting( name ) ); + + associateWithFile( + writable, + file + ); + + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( ); + } + + + void JSONIOHandlerImpl::openPath( + Writable * writable, + Parameter< Operation::OPEN_PATH > const & parameters + ) + { + auto file = refreshFileFromParent( writable ); + + nlohmann::json * j = &obtainJsonContents( writable->parent ); + std::string path; + auto extended = "/subgroups/" + auxiliary::replace_all_nonrecursively( + removeSlashes( parameters.path ), + "/", + "/subgroups/" + ); + + path = + parameters.path + .empty( ) + ? filepositionOf( writable->parent ) + : filepositionOf( writable->parent ) + extended; + + if( writable->abstractFilePosition ) + { + *setAndGetFilePosition( + writable, + false + ) = JSONFilePosition( json::json_pointer( path ) ); + } + else + { + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( json::json_pointer( path ) ); + } + + ensurePath( + j, + removeSlashes( parameters.path ) + ); + + writable->written = true; + } + + + void JSONIOHandlerImpl::openDataset( + Writable * writable, + Parameter< Operation::OPEN_DATASET > & parameters + ) + { + refreshFileFromParent( writable ); + auto containingGroup = setAndGetFilePosition( + writable->parent, + false + ); + auto name = removeSlashes( parameters.name ); + auto + & datasetJson = + obtainJsonContents( writable->parent )["datasets"][name]; + writable->abstractFilePosition = std::make_shared< JSONFilePosition >( + nlohmann::json::json_pointer( + containingGroup->id + .to_string( ) + "/datasets/" + name + ) + ); + + *parameters.dtype = + Datatype( stringToDatatype( datasetJson["datatype"].get< std::string >( ) ) ); + parameters.extent + ->clear( ); + for( auto it = datasetJson["extent"].begin( ); + it != datasetJson["extent"].end( ); + it++ ) + { + parameters.extent + ->push_back( + it.value( ) + .get< Extent::value_type >( ) + ); + } + writable->written = true; + + } + + + void JSONIOHandlerImpl::deleteFile( + Writable * writable, + Parameter< Operation::DELETE_FILE > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete files in read-only mode" ) + + if( !writable->written ) + { + return; + } + + auto filename = auxiliary::ends_with( + parameters.name, + ".json" + ) ? parameters.name : parameters.name + ".json"; + + auto tuple = getPossiblyExisting( filename ); + if( !std::get< 2 >( tuple ) ) + { + // file is already in the system + auto file = std::get< 0 >( tuple ); + m_dirty.erase( file ); + m_jsonVals.erase( file ); + file.invalidate( ); + } + + std::remove( fullPath( filename ).c_str( ) ); + + writable->written = false; + } + + + void JSONIOHandlerImpl::deletePath( + Writable * writable, + Parameter< Operation::DELETE_PATH > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete paths in read-only mode" ) + + if( !writable->written ) + { + return; + } + + VERIFY_ALWAYS( !auxiliary::starts_with( + parameters.path, + '/' + ), + "Paths passed for deletion should be relative, the given path is absolute (starts with '/')" ) + auto file = refreshFileFromParent( writable ); + auto filepos = setAndGetFilePosition( + writable, + false + ); + auto path = removeSlashes( parameters.path ); + VERIFY( !path.empty( ), + "No path passed for deletion." ) + nlohmann::json * j; + if( path == "." ) + { + auto + s = + filepos->id + .to_string( ); + if( s == "/" ) + { + throw std::runtime_error( "Cannot delete the root group" ); + } + + auto i = s.rfind( '/' ); + path = s; + path.replace( + 0, + i + 1, + "" + ); + // path should now be equal to the name of the current group + // go up one group + + // go to parent directory (need to invoke twice, since + // current == parent[subgroups][currentname] + // parent exists since we have verified that the current + // directory is != root + parentDir( s ); + parentDir( s ); + j = + &( *obtainJsonContents( file ) )[nlohmann::json::json_pointer( s )]; + } + else + { + if( auxiliary::starts_with( + path, + "./" + ) ) + { + path = auxiliary::replace_first( + path, + "./", + "" + ); + } + j = &obtainJsonContents( writable ); + } + nlohmann::json * lastPointer = j; + bool needToDelete = true; + auto splitPath = auxiliary::split( + path, + "/" + ); + // be careful not to create the group by accident + // the loop will execute at least once + for( auto folder: splitPath ) + { + auto & s = ( *j )["subgroups"]; + auto it = s.find( folder ); + if( it == s.end( ) ) + { + needToDelete = false; + break; + } + else + { + lastPointer = j; + j = &it.value( ); + } + } + if( needToDelete ) + { + ( *lastPointer )["subgroups"].erase( + splitPath[splitPath.size( ) - 1] + ); + } + + putJsonContents( file ); + writable->abstractFilePosition + .reset( ); + writable->written = false; + } + + + void JSONIOHandlerImpl::deleteDataset( + Writable * writable, + Parameter< Operation::DELETE_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete datasets in read-only mode" ) + + if( !writable->written ) + { + return; + } + + auto filepos = setAndGetFilePosition( + writable, + false + ); + + auto file = refreshFileFromParent( writable ); + auto dataset = removeSlashes( parameters.name ); + nlohmann::json * parent; + if( dataset == "." ) + { + auto + s = + filepos->id + .to_string( ); + if( s.empty( ) ) + { + throw std::runtime_error( "Invalid position for a dataset in the JSON file." ); + } + dataset = s; + auto i = dataset.rfind( '/' ); + dataset.replace( + 0, + i + 1, + "" + ); + + parentDir( s ); + parent = + &( *obtainJsonContents( file ) )[nlohmann::json::json_pointer( s )]; + } + else + { + parent = &obtainJsonContents( writable )["datasets"]; + } + parent->erase( dataset ); + putJsonContents( file ); + writable->written = false; + writable->abstractFilePosition + .reset( ); + } + + + void JSONIOHandlerImpl::deleteAttribute( + Writable * writable, + Parameter< Operation::DELETE_ATT > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete attributes in read-only mode" ) + if( !writable->written ) + { + return; + } + setAndGetFilePosition( writable ); + auto file = refreshFileFromParent( writable ); + auto & j = obtainJsonContents( writable ); + j.erase( parameters.name ); + putJsonContents( file ); + } + + + void JSONIOHandlerImpl::writeDataset( + Writable * writable, + Parameter< Operation::WRITE_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot write data in read-only mode." ); + + auto pos = setAndGetFilePosition( writable ); + auto file = refreshFileFromParent( writable ); + auto & j = obtainJsonContents( writable ); + + verifyDataset( + parameters, + j + ); + + + DatasetWriter dw; + switchType( + parameters.dtype, + dw, + j, + parameters + ); + + writable->written = true; + putJsonContents( file ); + } + + + void JSONIOHandlerImpl::writeAttribute( + Writable * writable, + Parameter< Operation::WRITE_ATT > const & parameter + ) + { + if( m_handler->accessType == AccessType::READ_ONLY ) + { + throw std::runtime_error( "Creating a dataset in a file opened as read only is not possible." ); + } + + /* Sanitize name */ + std::string name = removeSlashes( parameter.name ); + + auto file = refreshFileFromParent( writable ); + auto jsonVal = obtainJsonContents( file ); + auto filePosition = setAndGetFilePosition( writable ); + if( ( *jsonVal )[filePosition->id]["attributes"].empty( ) ) + { + ( *jsonVal )[filePosition->id]["attributes"] = + nlohmann::json::object( ); + } + nlohmann::json value; + AttributeWriter aw; + switchType( + parameter.dtype, + aw, + value, + parameter.resource + ); + ( *jsonVal )[filePosition->id]["attributes"][parameter.name] = { + { + "datatype", + datatypeToString( parameter.dtype ) + }, + { + "value", + value + } + }; + writable->written = true; + m_dirty.emplace( file ); + } + + + void JSONIOHandlerImpl::readDataset( + Writable * writable, + Parameter< Operation::READ_DATASET > & parameters + ) + { + refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable ); + verifyDataset( + parameters, + j + ); + + try + { + DatasetReader dr; + switchType( + parameters.dtype, + dr, + j["data"], + parameters + ); + } catch( json::basic_json::type_error & ) + { + throw std::runtime_error( "The given path does not contain a valid dataset." ); + } + } + + + void JSONIOHandlerImpl::readAttribute( + Writable * writable, + Parameter< Operation::READ_ATT > & parameters + ) + { + // VERIFY_ALWAYS( writable->written, + // "Attributes have to be written before reading." ) + // TODO ??? + refreshFileFromParent( writable ); + auto name = removeSlashes( parameters.name ); + auto & jsonLoc = obtainJsonContents( writable )["attributes"]; + setAndGetFilePosition( writable ); + VERIFY_ALWAYS( hasKey( + jsonLoc, + name + ), + "No such attribute in the given location." ) + auto & j = jsonLoc[name]; + try + { + *parameters.dtype = + Datatype( stringToDatatype( j["datatype"].get< std::string >( ) ) ); + AttributeReader ar; + switchType( + *parameters.dtype, + ar, + j["value"], + parameters + ); + } catch( json::type_error & ) + { + throw std::runtime_error( "The given location does not contain a properly formatted attribute" ); + } + } + + + void JSONIOHandlerImpl::listPaths( + Writable * writable, + Parameter< Operation::LIST_PATHS > & parameters + ) + { + // VERIFY_ALWAYS( writable->written, + // "Values have to be written before reading a directory" ); + // TODO ??? + auto & j = obtainJsonContents( writable )["subgroups"]; + setAndGetFilePosition( writable ); + refreshFileFromParent( writable ); + parameters.paths + ->clear( ); + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.paths + ->push_back( it.key( ) ); + } + } + + + void JSONIOHandlerImpl::listDatasets( + Writable * writable, + Parameter< Operation::LIST_DATASETS > & parameters + ) + { + // VERIFY_ALWAYS( writable->written, + // "Datasets have to be written before reading." ) + // TODO ??? + refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable )["datasets"]; + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.datasets + ->push_back( it.key( ) ); + } + } + + + void JSONIOHandlerImpl::listAttributes( + Writable * writable, + Parameter< Operation::LIST_ATTS > & parameters + ) + { + // VERIFY_ALWAYS( writable->written, + // "Attributes have to be written before reading." ) + // TODO ??? + refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable )["attributes"]; + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.attributes + ->push_back( it.key( ) ); + } + } + + + std::shared_ptr< JSONIOHandlerImpl::FILEHANDLE > + JSONIOHandlerImpl::getFilehandle( + File fileName, + AccessType accessType + ) + { + VERIFY_ALWAYS( fileName.valid( ), + "Tried opening a file that has been overwritten or deleted." ) + auto path = fullPath( std::move( fileName ) ); + auto fs = std::make_shared< std::fstream >( ); + switch( accessType ) + { + case AccessType::CREATE: + case AccessType::READ_WRITE: + fs->open( + path, + std::ios_base::out | std::ios_base::trunc + ); + break; + case AccessType::READ_ONLY: + fs->open( + path, + std::ios_base::in + ); + break; + } + VERIFY( fs->good( ), + "Failed opening a file" ); + return fs; + } + + + std::string JSONIOHandlerImpl::fullPath( File fileName ) + { + return fullPath( *fileName ); + } + + + std::string JSONIOHandlerImpl::fullPath( std::string fileName ) + { + if( auxiliary::ends_with( + m_handler->directory, + "/" + ) ) + { + return m_handler->directory + fileName; + } + else + { + return m_handler->directory + "/" + fileName; + } + } + + + void JSONIOHandlerImpl::parentDir( std::string & s ) + { + auto i = s.rfind( '/' ); + if( i != std::string::npos ) + { + s.replace( + i, + s.size( ) - i, + "" + ); + s.shrink_to_fit( ); + } + } + + + std::string JSONIOHandlerImpl::filepositionOf( Writable * writable ) + { + return std::dynamic_pointer_cast< JSONFilePosition >( writable->abstractFilePosition )->id + .to_string( ); + } + + + template< + typename T, + typename Visitor + > + void JSONIOHandlerImpl::syncMultidimensionalJson( + nlohmann::json & j, + Offset const & offset, + Extent const & extent, + Extent const & multiplicator, + Visitor visitor, + T * data, + size_t currentdim + ) + { + // Offset only relevant for JSON, the array data is contiguous + auto off = offset[currentdim]; + // maybe rewrite iteratively, using a stack that stores for each level the + // current iteration value i + + if( currentdim == offset.size( ) - 1 ) + { + for( std::size_t i = 0; i < extent[currentdim]; ++i ) + { + visitor( + j[i + off], + data[i] + ); + } + } + else + { + for( std::size_t i = 0; i < extent[currentdim]; ++i ) + { + syncMultidimensionalJson< + T, + Visitor + >( + j[i + off], + offset, + extent, + multiplicator, + visitor, + data + i * multiplicator[currentdim], + currentdim + 1 + ); + } + } + } + + + // multiplicators: an array [m_0,...,m_n] s.t. + // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n] + // (m_n = 1) + Extent JSONIOHandlerImpl::getMultiplicators( Extent const & extent ) + { + Extent res( extent ); + Extent::value_type n = 1; + size_t i = extent.size( ); + do + { + --i; + res[i] = n; + n *= extent[i]; + } + while( i > 0 ); + return res; + } + + + std::string JSONIOHandlerImpl::removeSlashes( std::string s ) + { + if( auxiliary::starts_with( + s, + '/' + ) ) + { + s = auxiliary::replace_first( + s, + "/", + "" + ); + } + if( auxiliary::ends_with( + s, + '/' + ) ) + { + s = auxiliary::replace_last( + s, + "/", + "" + ); + } + return s; + } + + + template< typename KeyT > + bool JSONIOHandlerImpl::hasKey( + nlohmann::json & j, + KeyT && key + ) + { + return j.find( std::forward< KeyT >( key ) ) != j.end( ); + } + + + void JSONIOHandlerImpl::ensurePath( + nlohmann::json * jsonp, + std::string path + ) + { + auto groups = auxiliary::split( + path, + "/" + ); + for( std::string & group: groups ) + { + // Enforce a JSON object + // the library will otherwise create a list if the group + // is parseable as an int + if( ( *jsonp )["subgroups"].empty( ) ) + { + ( *jsonp )["subgroups"] = nlohmann::json::object( ); + } + if( ( *jsonp )["subgroups"][group].empty( ) ) + { + ( *jsonp )["subgroups"][group] = nlohmann::json::object( ); + } + jsonp = &( *jsonp )["subgroups"][group]; + } + } + + + std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + > JSONIOHandlerImpl::getPossiblyExisting( std::string file ) + { + + auto it = std::find_if( + m_files.begin( ), + m_files.end( ), + [file]( + std::unordered_map< + Writable *, + File + >::value_type const & entry + ) + { + return *entry.second == file && + entry.second + .valid( ); + } + ); + + bool newlyCreated; + File name; + if( it == m_files.end( ) ) + { + name = file; + newlyCreated = true; + } + else + { + name = it->second; + newlyCreated = false; + } + return std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + >( + std::move( name ), + it, + newlyCreated + ); + } + + + std::shared_ptr< nlohmann::json > + JSONIOHandlerImpl::obtainJsonContents( File file ) + { + VERIFY_ALWAYS( file.valid( ), + "File has been overwritten or deleted before reading" ); + auto it = m_jsonVals.find( file ); + if( it != m_jsonVals.end( ) ) + { + return it->second; + } + // read from file + auto fh = getFilehandle( + file, + AccessType::READ_ONLY + ); + std::shared_ptr< nlohmann::json > + res = std::make_shared< nlohmann::json >( ); + *fh >> *res; + VERIFY( fh->good( ), + "Failed reading from a file." ); + m_jsonVals.emplace( + file, + res + ); + return res; + } + + + nlohmann::json & + JSONIOHandlerImpl::obtainJsonContents( Writable * writable ) + { + auto file = refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( + writable, + false + ); + return ( *obtainJsonContents( file ) )[filePosition->id]; + } + + + void JSONIOHandlerImpl::putJsonContents( + File filename, + bool unsetDirty // = true + ) + { + VERIFY_ALWAYS( filename.valid( ), + "File has been overwritten/deleted before writing" ); + auto it = m_jsonVals.find( filename ); + if( it != m_jsonVals.end( ) ) + { + auto fh = getFilehandle( + filename, + AccessType::CREATE + ); + ( *it->second )["platform_byte_widths"] = platformSpecifics( ); + *fh << *it->second << std::endl; + VERIFY( fh->good( ), + "Failed writing data to disk." ) + m_jsonVals.erase( it ); + if( unsetDirty ) + { + m_dirty.erase( filename ); + } + } + + } + + + std::shared_ptr< JSONFilePosition > + JSONIOHandlerImpl::setAndGetFilePosition( + Writable * writable, + std::string extend + ) + { + std::string path; + if( writable->abstractFilePosition ) + { + // do NOT reuse the old pointer, we want to change the file position + // only for the writable! + path = filepositionOf( writable ) + "/" + extend; + } + else if( writable->parent ) + { + path = filepositionOf( writable->parent ) + "/" + extend; + } + else + { // we are root + path = extend; + if( !auxiliary::starts_with( + path, + "/" + ) ) + { + path = "/" + path; + } + } + auto + res = + std::make_shared< JSONFilePosition >( json::json_pointer( path ) ); + + writable->abstractFilePosition = res; + + return res; + } + + + std::shared_ptr< JSONFilePosition > + JSONIOHandlerImpl::setAndGetFilePosition( + Writable * writable, + bool write + ) + { + std::shared_ptr< AbstractFilePosition > res; + + + if( writable->abstractFilePosition ) + { + res = writable->abstractFilePosition; + } + else if( writable->parent ) + { + res = + writable->parent + ->abstractFilePosition; + } + else + { // we are root + res = std::make_shared< JSONFilePosition >( ); + } + if( write ) + { + writable->abstractFilePosition = res; + } + return std::dynamic_pointer_cast< JSONFilePosition >( res ); + } + + + File JSONIOHandlerImpl::refreshFileFromParent( Writable * writable ) + { + if( writable->parent ) + { + auto + file = + m_files.find( writable->parent ) + ->second; + associateWithFile( + writable, + file + ); + return file; + } + else + { + return m_files.find( writable ) + ->second; + } + } + + + void JSONIOHandlerImpl::associateWithFile( + Writable * writable, + File file + ) + { + // make sure to overwrite + m_files[writable] = std::move( file ); + } + + + template< typename Param > + void JSONIOHandlerImpl::verifyDataset( + Param const & parameters, + nlohmann::json & j + ) + { + VERIFY_ALWAYS( !j.empty( ), + "Specified dataset does not exist." ); + + try + { + unsigned int dimension = 0; + auto it = j["extent"].begin( ); + for( ; dimension < + parameters.extent + .size( ) && it != j["extent"].end( ); + it++, dimension++ ) + { + VERIFY_ALWAYS( parameters.offset[dimension] + + parameters.extent[dimension] <= + it.value( ) + .get< Extent::value_type >( ), + "Read/Write request exceeds the dataset's size" ); + } + VERIFY_ALWAYS( parameters.extent + .size( ) == dimension && + it == j["extent"].end( ), + "Read/Write request does not fit the dataset's dimension" ); + + Datatype + dt = stringToDatatype( j["datatype"].get< std::string >( ) ); + VERIFY_ALWAYS( dt == parameters.dtype, + "Read/Write request does not fit the dataset's type" ); + } catch( json::basic_json::type_error & e ) + { + throw std::runtime_error( "The given path does not contain a valid dataset." ); + } + } + + + nlohmann::json JSONIOHandlerImpl::platformSpecifics( ) + { + nlohmann::json res; + static Datatype datatypes[] = { + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::BOOL + }; + for( auto it = std::begin( datatypes ); + it != std::end( datatypes ); + it++ ) + { + res[datatypeToString( *it )] = toBytes( *it ); + } + return res; + } + + + template< typename T > + void JSONIOHandlerImpl::DatasetWriter::operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ) + { + CppToJSON< T > ctj; + syncMultidimensionalJson( + json["data"], + parameters.offset, + parameters.extent, + getMultiplicators( parameters.extent ), + [&ctj]( + nlohmann::json & j, + T const & data + ) + { + j = ctj( data ); + }, + static_cast(parameters.data + .get( )) + ); + } + + + template< int n > + void JSONIOHandlerImpl::DatasetWriter::operator()( + nlohmann::json &, + const Parameter< Operation::WRITE_DATASET > & + ) + { + throw std::runtime_error( "Unknown datatype given for writing." ); + } + + + template< typename T > + void JSONIOHandlerImpl::DatasetReader::operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ) + { + JsonToCpp< + T + > jtc; + syncMultidimensionalJson( + json, + parameters.offset, + parameters.extent, + getMultiplicators( parameters.extent ), + [&jtc]( + nlohmann::json & j, + T & data + ) + { + data = jtc( j ); + }, + static_cast(parameters.data + .get( )) + ); + } + + + template< int n > + void JSONIOHandlerImpl::DatasetReader::operator()( + nlohmann::json &, + Parameter< Operation::READ_DATASET > & + ) + { + throw std::runtime_error( "Unknown datatype while reading a dataset." ); + } + + + template< typename T > + void JSONIOHandlerImpl::AttributeWriter::operator()( + nlohmann::json & value, + Attribute::resource const & resource + ) + { + CppToJSON< T > ctj; + value = ctj( variantSrc::get< T >( resource ) ); + } + + + template< int n > + void JSONIOHandlerImpl::AttributeWriter::operator()( + nlohmann::json &, + Attribute::resource const & + ) + { + throw std::runtime_error( "Unknown datatype in attribute writing." ); + } + + + template< int n > + void JSONIOHandlerImpl::AttributeReader::operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ) + { + throw std::runtime_error( "Unknown datatype while reading attribute." ); + } + + + template< typename T > + void JSONIOHandlerImpl::AttributeReader::operator()( + nlohmann::json & json, + Parameter< Operation::READ_ATT > & parameters + ) + { + JsonToCpp< + T + > jtc; + *parameters.resource = jtc( + json + ); + } + + + template< typename T > + nlohmann::json + JSONIOHandlerImpl::CppToJSON< T >::operator()( const T & val ) + { + return nlohmann::json( val ); + } + + + template< typename T > + nlohmann::json + JSONIOHandlerImpl::CppToJSON< std::vector< T > >::operator()( const std::vector< T > & v ) + { + nlohmann::json j; + CppToJSON< T > ctj; + for( auto a: v ) + { + j.push_back( ctj( a ) ); + } + return j; + } + + + template< typename T, int n > + nlohmann::json JSONIOHandlerImpl::CppToJSON< + std::array< + T, + n + > + >::operator()( + const std::array< + T, + n + > & v + ) + { + nlohmann::json j; + CppToJSON< T > ctj; + for( auto a: v ) + { + j.push_back( ctj( a ) ); + } + return j; + } + + + template< + typename T, + typename Dummy + > + T JSONIOHandlerImpl::JsonToCpp< + T, + Dummy + >::operator()( nlohmann::json const & json ) + { return json.get< T >( ); } + + + template< typename T > + std::vector< T > + JSONIOHandlerImpl::JsonToCpp< std::vector< T > >::operator()( nlohmann::json const & json ) + { + std::vector< T > v; + JsonToCpp< T > jtp; + for( auto & j: json ) + { + v.push_back( jtp( j ) ); + } + return v; + } + + + template< typename T, int n > + std::array< + T, + n + > JSONIOHandlerImpl::JsonToCpp< + std::array< + T, + n + > + >::operator()( nlohmann::json const & json ) + { + std::array< + T, + n + > a; + JsonToCpp< T > jtp; + size_t i = 0; + for( auto & j: json ) + { + a[i] = jtp( j ); + i++; + } + return a; + } + + + template< + typename T + > + T JSONIOHandlerImpl::JsonToCpp< + T, + typename std::enable_if< + std::is_floating_point< + T + >::value + >::type + >::operator()( nlohmann::json const & j ) + { + try + { + return j.get< T >( ); + } catch( ... ) + { + return std::numeric_limits< T >::quiet_NaN( ); + } + } + + +#endif + + +} // openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp.autosave b/src/IO/JSON/JSONIOHandlerImpl.cpp.autosave new file mode 100644 index 0000000000..b1a3e8d2d3 --- /dev/null +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp.autosave @@ -0,0 +1,1539 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Memory.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Writable.hpp" +#include "openPMD/Datatype.hpp" +#include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" + + + +namespace openPMD +{ +#if openPMD_USE_VERIFY +# define VERIFY( CONDITION, TEXT ) { if(!(CONDITION)) throw std::runtime_error((TEXT)); } +#else +# define VERIFY( CONDITION, TEXT ) do{ (void)sizeof(CONDITION); } while( 0 ); +#endif + +#define VERIFY_ALWAYS( CONDITION, TEXT ) { if(!(CONDITION)) throw std::runtime_error((TEXT)); } + +#if openPMD_HAVE_JSON + + + JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler * handler ) : + AbstractIOHandlerImpl( handler ) + {} + + + JSONIOHandlerImpl::~JSONIOHandlerImpl( ) + { + flush( ); + } + + + std::future< void > JSONIOHandlerImpl::flush( ) + { + + AbstractIOHandlerImpl::flush( ); + for( auto const & file: m_dirty ) + { + putJsonContents( + file, + false + ); + } + m_dirty.clear( ); + return std::future< void >( ); + } + + + void JSONIOHandlerImpl::createFile( + Writable * writable, + Parameter< Operation::CREATE_FILE > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Creating a file in read-only mode is not possible." ); + + if( !writable->written ) + { + std::string name = parameters.name; + if( !auxiliary::ends_with( + name, + ".json" + ) ) + { + name += ".json"; + } + + auto res_pair = getPossiblyExisting( name ); + File shared_name = File( name ); + VERIFY_ALWAYS( !( m_handler->accessType == AccessType::READ_WRITE && + ( !std::get< 2 >( res_pair ) || + auxiliary::file_exists( fullPath( std::get< 0 >( res_pair ) ) ) ) ), + "Can only overwrite existing file in CREATE mode." ); + + if( !std::get< 2 >( res_pair ) ) + { + auto file = std::get< 0 >( res_pair ); + m_dirty.erase( file ); + m_jsonVals.erase( file ); + file.invalidate( ); + } + + std::string const dir( m_handler->directory ); + if( !auxiliary::directory_exists( dir ) ) + { + auto success = auxiliary::create_directories( dir ); + VERIFY( success, + "Could not create directory." ); + } + + associateWithFile( + writable, + shared_name + ); + this->m_dirty + .emplace( shared_name ); + // make sure to overwrite! + this->m_jsonVals[shared_name] = + std::make_shared< nlohmann::json >( ); + + + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( ); + } + } + + + void JSONIOHandlerImpl::createPath( + Writable * writable, + Parameter< Operation::CREATE_PATH > const & parameter + ) + { + std::string path = parameter.path; + /* Sanitize: + * The JSON API does not like to have slashes in the end. + */ + if( auxiliary::ends_with( + path, + "/" + ) ) + { + path = auxiliary::replace_last( + path, + "/", + "" + ); + } + auto expandedPath = auxiliary::replace_all_nonrecursively( + path, + "/", + "/subgroups/" + ); + + auto file = refreshFileFromParent( writable ); + + auto * jsonVal = &*obtainJsonContents( file ); + if( !auxiliary::starts_with( + path, + "/" + ) ) + { // path is relative + auto filepos = setAndGetFilePosition( + writable, + false + ); + + jsonVal = &( *jsonVal )[filepos->id]; + expandedPath = + filepos->id + .to_string( ) + "/subgroups/" + expandedPath; + } + + ensurePath( + jsonVal, + path + ); + + + m_dirty.emplace( file ); + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( nlohmann::json::json_pointer( expandedPath ) ); + } + + + void JSONIOHandlerImpl::createDataset( + Writable * writable, + Parameter< Operation::CREATE_DATASET > const & parameter + ) + { + if( m_handler->accessType == AccessType::READ_ONLY ) + { + throw std::runtime_error( "Creating a dataset in a file opened as read only is not possible." ); + } + if( !writable->written ) + { + /* Sanitize name */ + std::string name = removeSlashes( parameter.name ); + + auto file = refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto & jsonVal = obtainJsonContents( writable ); + if( jsonVal["datasets"].empty( ) ) + { + jsonVal["datasets"] = nlohmann::json::object( ); + } + setAndGetFilePosition( + writable, + "datasets/" + name + ); + auto & dset = jsonVal["datasets"][name]; + Extent extent = parameter.extent; + dset["extent"] = extent; + dset["datatype"] = datatypeToString( parameter.dtype ); + writable->written = true; + m_dirty.emplace( file ); + } + } + + + void JSONIOHandlerImpl::extendDataset( + Writable * writable, + Parameter< Operation::EXTEND_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot extend a dataset in read-only mode." ) + refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto name = removeSlashes( parameters.name ); + auto & jsonLoc = obtainJsonContents( writable )["datasets"]; + VERIFY_ALWAYS( hasKey( + jsonLoc, + name + ), + "The specified location contains no dataset" ) + auto & j = jsonLoc[name]; + + try + { + size_t currentdim = 0; + auto it = j["extent"].begin( ); + for( ; currentdim < + parameters.extent + .size( ) && it != j["extent"].end( ); + it++, currentdim++ ) + { + VERIFY_ALWAYS( it.value( ) + .get< Extent::value_type >( ) <= + parameters.extent[currentdim], + "Cannot shrink the extent of a dataset" ) + } + VERIFY_ALWAYS( currentdim == + parameters.extent + .size( ) && it == j["extent"].end( ), + "Cannot change dimensionality of a dataset" ) + } catch( json::basic_json::type_error & e ) + { + throw std::runtime_error( "The specified location contains no valid dataset" ); + } + j["extent"] = parameters.extent; + writable->written = true; + + } + + + void JSONIOHandlerImpl::openFile( + Writable * writable, + Parameter< Operation::OPEN_FILE > const & parameter + ) + { + if( !auxiliary::directory_exists( m_handler->directory ) ) + { + throw no_such_file_error( + "Supplied directory is not valid: " + m_handler->directory + ); + } + + std::string name = parameter.name; + if( !auxiliary::ends_with( + name, + ".json" + ) ) + { + name += ".json"; + } + + auto file = std::get< 0 >( getPossiblyExisting( name ) ); + + associateWithFile( + writable, + file + ); + + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( ); + } + + + void JSONIOHandlerImpl::openPath( + Writable * writable, + Parameter< Operation::OPEN_PATH > const & parameters + ) + { + auto file = refreshFileFromParent( writable ); + + nlohmann::json * j = &obtainJsonContents( writable->parent ); + std::string path; + auto extended = "/subgroups/" + auxiliary::replace_all_nonrecursively( + removeSlashes( parameters.path ), + "/", + "/subgroups/" + ); + + path = + parameters.path + .empty( ) + ? filepositionOf( writable->parent ) + : filepositionOf( writable->parent ) + extended; + + if( writable->abstractFilePosition ) + { + *setAndGetFilePosition( + writable, + false + ) = JSONFilePosition( json::json_pointer( path ) ); + } + else + { + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( json::json_pointer( path ) ); + } + + ensurePath( + j, + removeSlashes( parameters.path ) + ); + + writable->written = true; + } + + + void JSONIOHandlerImpl::openDataset( + Writable * writable, + Parameter< Operation::OPEN_DATASET > & parameters + ) + { + refreshFileFromParent( writable ); + auto containingGroup = setAndGetFilePosition( + writable->parent, + false + ); + auto name = removeSlashes( parameters.name ); + auto + & datasetJson = + obtainJsonContents( writable->parent )["datasets"][name]; + writable->abstractFilePosition = std::make_shared< JSONFilePosition >( + nlohmann::json::json_pointer( + containingGroup->id + .to_string( ) + "/datasets/" + name + ) + ); + + *parameters.dtype = + Datatype( stringToDatatype( datasetJson["datatype"].get< std::string >( ) ) ); + parameters.extent + ->clear( ); + for( auto it = datasetJson["extent"].begin( ); + it != datasetJson["extent"].end( ); + it++ ) + { + parameters.extent + ->push_back( + it.value( ) + .get< Extent::value_type >( ) + ); + } + writable->written = true; + + } + + + void JSONIOHandlerImpl::deleteFile( + Writable * writable, + Parameter< Operation::DELETE_FILE > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete files in read-only mode" ) + + if( !writable->written ) + { + return; + } + + auto filename = auxiliary::ends_with( + parameters.name, + ".json" + ) ? parameters.name : parameters.name + ".json"; + + auto tuple = getPossiblyExisting( filename ); + if( !std::get< 2 >( tuple ) ) + { + // file is already in the system + auto file = std::get< 0 >( tuple ); + m_dirty.erase( file ); + m_jsonVals.erase( file ); + file.invalidate( ); + } + + std::remove( fullPath( filename ).c_str( ) ); + + writable->written = false; + } + + + void JSONIOHandlerImpl::deletePath( + Writable * writable, + Parameter< Operation::DELETE_PATH > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete paths in read-only mode" ) + + if( !writable->written ) + { + return; + } + + VERIFY_ALWAYS( !auxiliary::starts_with( + parameters.path, + '/' + ), + "Paths passed for deletion should be relative, the given path is absolute (starts with '/')" ) + auto file = refreshFileFromParent( writable ); + auto filepos = setAndGetFilePosition( + writable, + false + ); + auto path = removeSlashes( parameters.path ); + VERIFY( !path.empty( ), + "No path passed for deletion." ) + nlohmann::json * j; + if( path == "." ) + { + auto + s = + filepos->id + .to_string( ); + if( s == "/" ) + { + throw std::runtime_error( "Cannot delete the root group" ); + } + + auto i = s.rfind( '/' ); + path = s; + path.replace( + 0, + i + 1, + "" + ); + // path should now be equal to the name of the current group + // go up one group + + // go to parent directory (need to invoke twice, since + // current == parent[subgroups][currentname] + // parent exists since we have verified that the current + // directory is != root + parentDir( s ); + parentDir( s ); + j = + &( *obtainJsonContents( file ) )[nlohmann::json::json_pointer( s )]; + } + else + { + if( auxiliary::starts_with( + path, + "./" + ) ) + { + path = auxiliary::replace_first( + path, + "./", + "" + ); + } + j = &obtainJsonContents( writable ); + } + nlohmann::json * lastPointer = j; + bool needToDelete = true; + auto splitPath = auxiliary::split( + path, + "/" + ); + // be careful not to create the group by accident + // the loop will execute at least once + for( auto folder: splitPath ) + { + auto & s = ( *j )["subgroups"]; + auto it = s.find( folder ); + if( it == s.end( ) ) + { + needToDelete = false; + break; + } + else + { + lastPointer = j; + j = &it.value( ); + } + } + if( needToDelete ) + { + ( *lastPointer )["subgroups"].erase( + splitPath[splitPath.size( ) - 1] + ); + } + + putJsonContents( file ); + writable->abstractFilePosition + .reset( ); + writable->written = false; + } + + + void JSONIOHandlerImpl::deleteDataset( + Writable * writable, + Parameter< Operation::DELETE_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete datasets in read-only mode" ) + + if( !writable->written ) + { + return; + } + + auto filepos = setAndGetFilePosition( + writable, + false + ); + + auto file = refreshFileFromParent( writable ); + auto dataset = removeSlashes( parameters.name ); + nlohmann::json * parent; + if( dataset == "." ) + { + auto + s = + filepos->id + .to_string( ); + if( s.empty( ) ) + { + throw std::runtime_error( "Invalid position for a dataset in the JSON file." ); + } + dataset = s; + auto i = dataset.rfind( '/' ); + dataset.replace( + 0, + i + 1, + "" + ); + + parentDir( s ); + parent = + &( *obtainJsonContents( file ) )[nlohmann::json::json_pointer( s )]; + } + else + { + parent = &obtainJsonContents( writable )["datasets"]; + } + parent->erase( dataset ); + putJsonContents( file ); + writable->written = false; + writable->abstractFilePosition + .reset( ); + } + + + void JSONIOHandlerImpl::deleteAttribute( + Writable * writable, + Parameter< Operation::DELETE_ATT > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete attributes in read-only mode" ) + if( !writable->written ) + { + return; + } + setAndGetFilePosition( writable ); + auto file = refreshFileFromParent( writable ); + auto & j = obtainJsonContents( writable ); + j.erase( parameters.name ); + putJsonContents( file ); + } + + + void JSONIOHandlerImpl::writeDataset( + Writable * writable, + Parameter< Operation::WRITE_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot write data in read-only mode." ); + + auto pos = setAndGetFilePosition( writable ); + auto file = refreshFileFromParent( writable ); + auto & j = obtainJsonContents( writable ); + + verifyDataset( + parameters, + j + ); + + + DatasetWriter dw; + switchType( + parameters.dtype, + dw, + j, + parameters + ); + + writable->written = true; + putJsonContents( file ); + } + + + void JSONIOHandlerImpl::writeAttribute( + Writable * writable, + Parameter< Operation::WRITE_ATT > const & parameter + ) + { + if( m_handler->accessType == AccessType::READ_ONLY ) + { + throw std::runtime_error( "Creating a dataset in a file opened as read only is not possible." ); + } + + /* Sanitize name */ + std::string name = removeSlashes( parameter.name ); + + auto file = refreshFileFromParent( writable ); + auto jsonVal = obtainJsonContents( file ); + auto filePosition = setAndGetFilePosition( writable ); + if( ( *jsonVal )[filePosition->id]["attributes"].empty( ) ) + { + ( *jsonVal )[filePosition->id]["attributes"] = + nlohmann::json::object( ); + } + nlohmann::json value; + AttributeWriter aw; + switchType( + parameter.dtype, + aw, + value, + parameter.resource + ); + ( *jsonVal )[filePosition->id]["attributes"][parameter.name] = { + { + "datatype", + datatypeToString( parameter.dtype ) + }, + { + "value", + value + } + }; + writable->written = true; + m_dirty.emplace( file ); + } + + + void JSONIOHandlerImpl::readDataset( + Writable * writable, + Parameter< Operation::READ_DATASET > & parameters + ) + { + refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable ); + verifyDataset( + parameters, + j + ); + + try + { + DatasetReader dr; + switchType( + parameters.dtype, + dr, + j["data"], + parameters + ); + } catch( json::basic_json::type_error & ) + { + throw std::runtime_error( "The given path does not contain a valid dataset." ); + } + } + + + void JSONIOHandlerImpl::readAttribute( + Writable * writable, + Parameter< Operation::READ_ATT > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Attributes have to be written before reading." ) + refreshFileFromParent( writable ); + auto name = removeSlashes( parameters.name ); + auto & jsonLoc = obtainJsonContents( writable )["attributes"]; + setAndGetFilePosition( writable ); + VERIFY_ALWAYS( hasKey( + jsonLoc, + name + ), + "No such attribute in the given location." ) + auto & j = jsonLoc[name]; + try + { + *parameters.dtype = + Datatype( stringToDatatype( j["datatype"].get< std::string >( ) ) ); + AttributeReader ar; + switchType( + *parameters.dtype, + ar, + j["value"], + parameters + ); + } catch( json::type_error & ) + { + throw std::runtime_error( "The given location does not contain a properly formatted attribute" ); + } + } + + + void JSONIOHandlerImpl::listPaths( + Writable * writable, + Parameter< Operation::LIST_PATHS > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Values have to be written before reading a directory" ); + auto & j = obtainJsonContents( writable )["subgroups"]; + setAndGetFilePosition( writable ); + refreshFileFromParent( writable ); + parameters.paths + ->clear( ); + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.paths + ->push_back( it.key( ) ); + } + } + + + void JSONIOHandlerImpl::listDatasets( + Writable * writable, + Parameter< Operation::LIST_DATASETS > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Datasets have to be written before reading." ) + refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable )["datasets"]; + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.datasets + ->push_back( it.key( ) ); + } + } + + + void JSONIOHandlerImpl::listAttributes( + Writable * writable, + Parameter< Operation::LIST_ATTS > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Attributes have to be written before reading." ) + refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable )["attributes"]; + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.attributes + ->push_back( it.key( ) ); + } + } + + + std::shared_ptr< JSONIOHandlerImpl::FILEHANDLE > + JSONIOHandlerImpl::getFilehandle( + File fileName, + AccessType accessType + ) + { + VERIFY_ALWAYS( fileName.valid( ), + "Tried opening a file that has been overwritten or deleted." ) + auto path = fullPath( std::move( fileName ) ); + auto fs = std::make_shared< std::fstream >( ); + switch( accessType ) + { + case AccessType::CREATE: + case AccessType::READ_WRITE: + fs->open( + path, + std::ios_base::out | std::ios_base::trunc + ); + break; + case AccessType::READ_ONLY: + fs->open( + path, + std::ios_base::in + ); + break; + } + VERIFY( fs->good( ), + "Failed opening a file" ); + return fs; + } + + + std::string JSONIOHandlerImpl::fullPath( File fileName ) + { + return fullPath( *fileName ); + } + + + std::string JSONIOHandlerImpl::fullPath( std::string fileName ) + { + if( auxiliary::ends_with( + m_handler->directory, + "/" + ) ) + { + return m_handler->directory + fileName; + } + else + { + return m_handler->directory + "/" + fileName; + } + } + + + void JSONIOHandlerImpl::parentDir( std::string & s ) + { + auto i = s.rfind( '/' ); + if( i != std::string::npos ) + { + s.replace( + i, + s.size( ) - i, + "" + ); + s.shrink_to_fit( ); + } + } + + + std::string JSONIOHandlerImpl::filepositionOf( Writable * writable ) + { + return std::dynamic_pointer_cast< JSONFilePosition >( writable->abstractFilePosition )->id + .to_string( ); + } + + + template< + typename T, + typename Visitor + > + void JSONIOHandlerImpl::syncMultidimensionalJson( + nlohmann::json & j, + Offset const & offset, + Extent const & extent, + Extent const & multiplicator, + Visitor visitor, + T * data, + size_t currentdim + ) + { + // Offset only relevant for JSON, the array data is contiguous + auto off = offset[currentdim]; + // maybe rewrite iteratively, using a stack that stores for each level the + // current iteration value i + + if( currentdim == offset.size( ) - 1 ) + { + for( std::size_t i = 0; i < extent[currentdim]; ++i ) + { + visitor( + j[i + off], + data[i] + ); + } + } + else + { + for( std::size_t i = 0; i < extent[currentdim]; ++i ) + { + syncMultidimensionalJson< + T, + Visitor + >( + j[i + off], + offset, + extent, + multiplicator, + visitor, + data + i * multiplicator[currentdim], + currentdim + 1 + ); + } + } + } + + + // multiplicators: an array [m_0,...,m_n] s.t. + // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n] + // (m_n = 1) + Extent JSONIOHandlerImpl::getMultiplicators( Extent const & extent ) + { + Extent res( extent ); + Extent::value_type n = 1; + size_t i = extent.size( ); + do + { + --i; + res[i] = n; + n *= extent[i]; + } + while( i > 0 ); + return res; + } + + + std::string JSONIOHandlerImpl::removeSlashes( std::string s ) + { + if( auxiliary::starts_with( + s, + '/' + ) ) + { + s = auxiliary::replace_first( + s, + "/", + "" + ); + } + if( auxiliary::ends_with( + s, + '/' + ) ) + { + s = auxiliary::replace_last( + s, + "/", + "" + ); + } + return s; + } + + + template< typename KeyT > + bool JSONIOHandlerImpl::hasKey( + nlohmann::json & j, + KeyT && key + ) + { + return j.find( std::forward< KeyT >( key ) ) != j.end( ); + } + + + void JSONIOHandlerImpl::ensurePath( + nlohmann::json * jsonp, + std::string path + ) + { + auto groups = auxiliary::split( + path, + "/" + ); + for( std::string & group: groups ) + { + // Enforce a JSON object + // the library will otherwise create a list if the group + // is parseable as an int + if( ( *jsonp )["subgroups"].empty( ) ) + { + ( *jsonp )["subgroups"] = nlohmann::json::object( ); + } + if( ( *jsonp )["subgroups"][group].empty( ) ) + { + ( *jsonp )["subgroups"][group] = nlohmann::json::object( ); + } + jsonp = &( *jsonp )["subgroups"][group]; + } + } + + + std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + > JSONIOHandlerImpl::getPossiblyExisting( std::string file ) + { + + auto it = std::find_if( + m_files.begin( ), + m_files.end( ), + [file]( + std::unordered_map< + Writable *, + File + >::value_type const & entry + ) + { + return *entry.second == file && + entry.second + .valid( ); + } + ); + + bool newlyCreated; + File name; + if( it == m_files.end( ) ) + { + name = file; + newlyCreated = true; + } + else + { + name = it->second; + newlyCreated = false; + } + return std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + >( + std::move( name ), + it, + newlyCreated + ); + } + + + std::shared_ptr< nlohmann::json > + JSONIOHandlerImpl::obtainJsonContents( File file ) + { + VERIFY_ALWAYS( file.valid( ), + "File has been overwritten or deleted before reading" ); + auto it = m_jsonVals.find( file ); + if( it != m_jsonVals.end( ) ) + { + return it->second; + } + // read from file + auto fh = getFilehandle( + file, + AccessType::READ_ONLY + ); + std::shared_ptr< nlohmann::json > + res = std::make_shared< nlohmann::json >( ); + *fh >> *res; + VERIFY( fh->good( ), + "Failed reading from a file." ); + m_jsonVals.emplace( + file, + res + ); + return res; + } + + + nlohmann::json & + JSONIOHandlerImpl::obtainJsonContents( Writable * writable ) + { + auto file = refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( + writable, + false + ); + return ( *obtainJsonContents( file ) )[filePosition->id]; + } + + + void JSONIOHandlerImpl::putJsonContents( + File filename, + bool unsetDirty // = true + ) + { + VERIFY_ALWAYS( filename.valid( ), + "File has been overwritten/deleted before writing" ); + auto it = m_jsonVals.find( filename ); + if( it != m_jsonVals.end( ) ) + { + auto fh = getFilehandle( + filename, + AccessType::CREATE + ); + ( *it->second )["platform_byte_widths"] = platformSpecifics( ); + *fh << *it->second << std::endl; + VERIFY( fh->good( ), + "Failed writing data to disk." ) + m_jsonVals.erase( it ); + if( unsetDirty ) + { + m_dirty.erase( filename ); + } + } + + } + + + std::shared_ptr< JSONFilePosition > + JSONIOHandlerImpl::setAndGetFilePosition( + Writable * writable, + std::string extend + ) + { + std::string path; + if( writable->abstractFilePosition ) + { + // do NOT reuse the old pointer, we want to change the file position + // only for the writable! + path = filepositionOf( writable ) + "/" + extend; + } + else if( writable->parent ) + { + path = filepositionOf( writable->parent ) + "/" + extend; + } + else + { // we are root + path = extend; + if( !auxiliary::starts_with( + path, + "/" + ) ) + { + path = "/" + path; + } + } + auto + res = + std::make_shared< JSONFilePosition >( json::json_pointer( path ) ); + + writable->abstractFilePosition = res; + + return res; + } + + + std::shared_ptr< JSONFilePosition > + JSONIOHandlerImpl::setAndGetFilePosition( + Writable * writable, + bool write + ) + { + std::shared_ptr< AbstractFilePosition > res; + + + if( writable->abstractFilePosition ) + { + res = writable->abstractFilePosition; + } + else if( writable->parent ) + { + res = + writable->parent + ->abstractFilePosition; + } + else + { // we are root + res = std::make_shared< JSONFilePosition >( ); + } + if( write ) + { + writable->abstractFilePosition = res; + } + return std::dynamic_pointer_cast< JSONFilePosition >( res ); + } + + + File JSONIOHandlerImpl::refreshFileFromParent( Writable * writable ) + { + if( writable->parent ) + { + auto + file = + m_files.find( writable->parent ) + ->second; + associateWithFile( + writable, + file + ); + return file; + } + else + { + return m_files.find( writable ) + ->second; + } + } + + + void JSONIOHandlerImpl::associateWithFile( + Writable * writable, + File file + ) + { + // make sure to overwrite + m_files[writable] = std::move( file ); + } + + + template< typename Param > + void JSONIOHandlerImpl::verifyDataset( + Param const & parameters, + nlohmann::json & j + ) + { + VERIFY_ALWAYS( !j.empty( ), + "Specified dataset does not exist." ); + + try + { + unsigned int dimension = 0; + auto it = j["extent"].begin( ); + for( ; dimension < + parameters.extent + .size( ) && it != j["extent"].end( ); + it++, dimension++ ) + { + VERIFY_ALWAYS( parameters.offset[dimension] + + parameters.extent[dimension] <= + it.value( ) + .get< Extent::value_type >( ), + "Read/Write request exceeds the dataset's size" ); + } + VERIFY_ALWAYS( parameters.extent + .size( ) == dimension && + it == j["extent"].end( ), + "Read/Write request does not fit the dataset's dimension" ); + + Datatype + dt = stringToDatatype( j["datatype"].get< std::string >( ) ); + VERIFY_ALWAYS( dt == parameters.dtype, + "Read/Write request does not fit the dataset's type" ); + } catch( json::basic_json::type_error & e ) + { + throw std::runtime_error( "The given path does not contain a valid dataset." ); + } + } + + + nlohmann::json JSONIOHandlerImpl::platformSpecifics( ) + { + nlohmann::json res; + static Datatype datatypes[] = { + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::BOOL + }; + for( auto it = std::begin( datatypes ); + it != std::end( datatypes ); + it++ ) + { + res[datatypeToString( *it )] = toBytes( *it ); + } + return res; + } + + + template< typename T > + void JSONIOHandlerImpl::DatasetWriter::operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ) + { + CppToJSON< T > ctj; + syncMultidimensionalJson( + json["data"], + parameters.offset, + parameters.extent, + getMultiplicators( parameters.extent ), + [&ctj]( + nlohmann::json & j, + T const & data + ) + { + j = ctj( data ); + }, + static_cast(parameters.data + .get( )) + ); + } + + + template< int n > + void JSONIOHandlerImpl::DatasetWriter::operator()( + nlohmann::json &, + const Parameter< Operation::WRITE_DATASET > & + ) + { + throw std::runtime_error( "Unknown datatype given for writing." ); + } + + + template< typename T > + void JSONIOHandlerImpl::DatasetReader::operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ) + { + JsonToCpp< + T + > jtc; + syncMultidimensionalJson( + json, + parameters.offset, + parameters.extent, + getMultiplicators( parameters.extent ), + [&jtc]( + nlohmann::json & j, + T & data + ) + { + data = jtc( j ); + }, + static_cast(parameters.data + .get( )) + ); + } + + + template< int n > + void JSONIOHandlerImpl::DatasetReader::operator()( + nlohmann::json &, + Parameter< Operation::READ_DATASET > & + ) + { + throw std::runtime_error( "Unknown datatype while reading a dataset." ); + } + + + template< typename T > + void JSONIOHandlerImpl::AttributeWriter::operator()( + nlohmann::json & value, + Attribute::resource const & resource + ) + { + CppToJSON< T > ctj; + value = ctj( variantSrc::get< T >( resource ) ); + } + + + template< int n > + void JSONIOHandlerImpl::AttributeWriter::operator()( + nlohmann::json &, + Attribute::resource const & + ) + { + throw std::runtime_error( "Unknown datatype in attribute writing." ); + } + + + template< int n > + void JSONIOHandlerImpl::AttributeReader::operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ) + { + throw std::runtime_error( "Unknown datatype while reading attribute." ); + } + + + template< typename T > + void JSONIOHandlerImpl::AttributeReader::operator()( + nlohmann::json & json, + Parameter< Operation::READ_ATT > & parameters + ) + { + JsonToCpp< + T + > jtc; + *parameters.resource = jtc( + json + ); + } + + + template< typename T > + nlohmann::json + JSONIOHandlerImpl::CppToJSON< T >::operator()( const T & val ) + { + return nlohmann::json( val ); + } + + + template< typename T > + nlohmann::json + JSONIOHandlerImpl::CppToJSON< std::vector< T > >::operator()( const std::vector< T > & v ) + { + nlohmann::json j; + CppToJSON< T > ctj; + for( auto a: v ) + { + j.push_back( ctj( a ) ); + } + return j; + } + + + template< typename T, int n > + nlohmann::json JSONIOHandlerImpl::CppToJSON< + std::array< + T, + n + > + >::operator()( + const std::array< + T, + n + > & v + ) + { + nlohmann::json j; + CppToJSON< T > ctj; + for( auto a: v ) + { + j.push_back( ctj( a ) ); + } + return j; + } + + + template< + typename T, + typename Dummy + > + T JSONIOHandlerImpl::JsonToCpp< + T, + Dummy + >::operator()( nlohmann::json const & json ) + { return json.get< T >( ); } + + + template< typename T > + std::vector< T > + JSONIOHandlerImpl::JsonToCpp< std::vector< T > >::operator()( nlohmann::json const & json ) + { + std::vector< T > v; + JsonToCpp< T > jtp; + for( auto & j: json ) + { + v.push_back( jtp( j ) ); + } + return v; + } + + + template< typename T, int n > + std::array< + T, + n + > JSONIOHandlerImpl::JsonToCpp< + std::array< + T, + n + > + >::operator()( nlohmann::json const & json ) + { + std::array< + T, + n + > a; + JsonToCpp< T > jtp; + size_t i = 0; + for( auto & j: json ) + { + a[i] = jtp( j ); + i++; + } + return a; + } + + + template< + typename T + > + T JSONIOHandlerImpl::JsonToCpp< + T, + typename std::enable_if< + std::is_floating_point< + T + >::value + >::type + >::operator()( nlohmann::json const & j ) + { + try + { + return j.get< T >( ); + } catch( ... ) + { + return std::numeric_limits< T >::quiet_NaN( ); + } + } + + +#endif + + +} // openPMD diff --git a/src/Series.cpp b/src/Series.cpp index b84a2560cf..161a5508a8 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -862,6 +862,8 @@ determineFormat(std::string const& filename) return Format::HDF5; if( auxiliary::ends_with(filename, ".bp") ) return Format::ADIOS1; + if( auxiliary::ends_with(filename, ".json") ) + return Format::JSON; if( std::string::npos != filename.find('.') /* extension is provided */ ) throw std::runtime_error("Unknown file format. Did you append a valid filename extension?"); @@ -879,6 +881,8 @@ suffix(Format f) case Format::ADIOS1: case Format::ADIOS2: return ".bp"; + case Format::JSON: + return ".json"; default: return ""; } @@ -892,6 +896,7 @@ cleanFilename(std::string const& filename, Format f) case Format::HDF5: case Format::ADIOS1: case Format::ADIOS2: + case Format::JSON: return auxiliary::replace_last(filename, suffix(f), ""); default: return filename; @@ -952,6 +957,16 @@ matcher(std::string const& prefix, int padding, std::string const& postfix, Form nameReg += + ")" + postfix + ".bp$"; return buildMatcher(nameReg); } + case Format::JSON: + { + std::string nameReg = "^" + prefix + "([[:digit:]]"; + if( padding != 0 ) + nameReg += "{" + std::to_string(padding) + "}"; + else + nameReg += "+"; + nameReg += + ")" + postfix + ".json$"; + return buildMatcher(nameReg); + } default: return [](std::string const&) -> std::tuple< bool, int > { return std::tuple< bool, int >{false, 0}; }; } diff --git a/src/auxiliary/Filesystem.cpp b/src/auxiliary/Filesystem.cpp index 39c5500293..11f2d2b90c 100644 --- a/src/auxiliary/Filesystem.cpp +++ b/src/auxiliary/Filesystem.cpp @@ -28,6 +28,7 @@ # include # include # include +# include #endif #include @@ -160,6 +161,7 @@ remove_file( std::string const& path ) #else return (0 == remove(path.c_str())); #endif -} +} + } // auxiliary } // openPMD diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 73fff3e11e..b2135262bc 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2296,3 +2296,475 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") REQUIRE(true); } #endif + +#if openPMD_HAVE_JSON + +TEST_CASE( "json_write_test", "[serial][json]") +{ + Series o = Series("../samples/serial_write.json", AccessType::CREATE); + + ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + + std::vector< double > position_global(4); + double pos{0.}; + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_1(new double); + e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_1 = position_global[i]; + e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); + } + + std::vector< uint64_t > positionOffset_global(4); + uint64_t posOff{0}; + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); + e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *positionOffset_local_1 = positionOffset_global[i]; + e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); + } + + ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_2(new double); + e_2["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_2), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_2 = position_global[i]; + e_2["position"]["x"].storeChunk(position_local_2, {i}, {1}); + } + + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); + e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *positionOffset_local_2 = positionOffset_global[i]; + e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); + } + + o.flush(); + + ParticleSpecies& e_3 = o.iterations[3].particles["e"]; + + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_3(new double); + e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_3 = position_global[i]; + e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); + } + + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); + e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *positionOffset_local_3 = positionOffset_global[i]; + e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); + } + + o.flush(); +} + +TEST_CASE( "json_fileBased_write_empty_test", "[serial][json]" ) +{ + if( auxiliary::directory_exists("../samples/subdir") ) + auxiliary::remove_directory("../samples/subdir"); + + Dataset dset = Dataset(Datatype::DOUBLE, {2}); + { + Series o = Series("../samples/subdir/serial_fileBased_write%T.json", AccessType::CREATE); + + ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + e_1["position"][RecordComponent::SCALAR].resetDataset(dset); + e_1["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[1].setTime(1.f); + ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + e_2["position"][RecordComponent::SCALAR].resetDataset(dset); + e_2["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[2].setTime(2.f); + ParticleSpecies& e_3 = o.iterations[3].particles["e"]; + e_3["position"][RecordComponent::SCALAR].resetDataset(dset); + e_3["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[3].setTime(3.f); + } + + REQUIRE(auxiliary::directory_exists("../samples/subdir")); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write1.json")); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write2.json")); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write3.json")); + + { + Series o = Series("../samples/subdir/serial_fileBased_write%T.json", AccessType::READ_ONLY); + + REQUIRE(o.iterations.size() == 3); + REQUIRE(o.iterations.count(1) == 1); + REQUIRE(o.iterations.count(2) == 1); + REQUIRE(o.iterations.count(3) == 1); + + REQUIRE(o.iterations[1].time< float >() == 1.f); + REQUIRE(o.iterations[2].time< float >() == 2.f); + REQUIRE(o.iterations[3].time< float >() == 3.f); + + REQUIRE(o.iterations[1].particles.size() == 1); + REQUIRE(o.iterations[1].particles.count("e") == 1); + REQUIRE(o.iterations[2].particles.size() == 1); + REQUIRE(o.iterations[2].particles.count("e") == 1); + REQUIRE(o.iterations[3].particles.size() == 1); + REQUIRE(o.iterations[3].particles.count("e") == 1); + + REQUIRE(o.iterations[1].particles["e"].size() == 2); + REQUIRE(o.iterations[1].particles["e"].count("position") == 1); + REQUIRE(o.iterations[1].particles["e"].count("positionOffset") == 1); + REQUIRE(o.iterations[2].particles["e"].size() == 2); + REQUIRE(o.iterations[2].particles["e"].count("position") == 1); + REQUIRE(o.iterations[2].particles["e"].count("positionOffset") == 1); + REQUIRE(o.iterations[3].particles["e"].size() == 2); + REQUIRE(o.iterations[3].particles["e"].count("position") == 1); + REQUIRE(o.iterations[3].particles["e"].count("positionOffset") == 1); + + REQUIRE(o.iterations[1].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[1].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[1].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[1].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + REQUIRE(o.iterations[2].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[2].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[2].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[2].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + REQUIRE(o.iterations[3].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[3].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[3].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[3].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + } + + { + Series o = Series("../samples/subdir/serial_fileBased_write%T.json", AccessType::READ_WRITE); + ParticleSpecies& e_4 = o.iterations[4].particles["e"]; + e_4["position"][RecordComponent::SCALAR].resetDataset(dset); + e_4["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[4].setTime(4.f); + } + + { + Series o = Series("../samples/subdir/serial_fileBased_write%T.json", AccessType::READ_ONLY); + + REQUIRE(o.iterations.size() == 4); + REQUIRE(o.iterations.count(4) == 1); + + REQUIRE(o.iterations[4].time< float >() == 4.f); + + REQUIRE(o.iterations[4].particles.size() == 1); + REQUIRE(o.iterations[4].particles.count("e") == 1); + + REQUIRE(o.iterations[4].particles["e"].size() == 2); + REQUIRE(o.iterations[4].particles["e"].count("position") == 1); + REQUIRE(o.iterations[4].particles["e"].count("positionOffset") == 1); + + REQUIRE(o.iterations[4].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[4].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[4].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[4].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + } +} + +TEST_CASE( "json_fileBased_write_test", "[serial][json]" ) +{ + { + Series o = Series("../samples/serial_fileBased_write%T.json", AccessType::CREATE); + + ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + + std::vector< double > position_global(4); + double pos{0.}; + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_1(new double); + e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); + std::vector< uint64_t > positionOffset_global(4); + uint64_t posOff{0}; + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); + e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_1 = position_global[i]; + e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); + *positionOffset_local_1 = positionOffset_global[i]; + e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); + o.flush(); + } + + o.iterations[1].setTime(1.f); + + ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + e_2["position"]["x"].resetDataset(Dataset(determineDatatype(), {4})); + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); + e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + double const position_local_2 = position_global.at(i); + e_2["position"]["x"].storeChunk(shareRaw(&position_local_2), {i}, {1}); + *positionOffset_local_2 = positionOffset_global[i]; + e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); + o.flush(); + } + + o.iterations[2].setTime(2.f); + + ParticleSpecies& e_3 = o.iterations[3].particles["e"]; + + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_3(new double); + e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); + e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_3 = position_global[i]; + e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); + *positionOffset_local_3 = positionOffset_global[i]; + e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); + o.flush(); + } + + o.setOpenPMDextension(1); + o.iterations[3].setTime(3.f); + } + REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write1.json")); + REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write2.json")); + REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write3.json")); + + { + Series o = Series("../samples/serial_fileBased_write%T.json", AccessType::READ_ONLY); + + REQUIRE(o.iterations.size() == 3); + REQUIRE(o.iterations.count(1) == 1); + REQUIRE(o.iterations.count(2) == 1); + REQUIRE(o.iterations.count(3) == 1); + + REQUIRE(o.iterations.at(1).time< float >() == 1.f); + REQUIRE(o.iterations.at(2).time< float >() == 2.f); + REQUIRE(o.iterations.at(3).time< float >() == 3.f); + + REQUIRE(o.basePath() == "/data/%T/"); + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "serial_fileBased_write%T"); + REQUIRE(o.openPMD() == "1.1.0"); + REQUIRE(o.openPMDextension() == 1u); + REQUIRE(o.particlesPath() == "particles/"); + REQUIRE_FALSE(o.containsAttribute("meshesPath")); + REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error); + + for( uint64_t i = 1; i <= 3; ++i ) + { + Iteration iteration = o.iterations.at(i); + + REQUIRE(iteration.particles.size() == 1); + REQUIRE(iteration.particles.count("e") == 1); + + ParticleSpecies& species = iteration.particles.at("e"); + + REQUIRE(species.size() == 2); + REQUIRE(species.count("position") == 1); + REQUIRE(species.count("positionOffset") == 1); + + REQUIRE(species.at("position").size() == 1); + REQUIRE(species.at("position").count("x") == 1); + REQUIRE(species.at("position").at("x").getDatatype() == Datatype::DOUBLE); + REQUIRE(species.at("position").at("x").getDimensionality() == 1); + REQUIRE(species.at("position").at("x").getExtent() == Extent{4}); + REQUIRE(species.at("positionOffset").size() == 1); + REQUIRE(species.at("positionOffset").count("x") == 1); +#if !defined(_MSC_VER) + REQUIRE(species.at("positionOffset").at("x").getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(species.at("positionOffset").at("x").getDatatype(), determineDatatype< uint64_t >())); + REQUIRE(species.at("positionOffset").at("x").getDimensionality() == 1); + REQUIRE(species.at("positionOffset").at("x").getExtent() == Extent{4}); + + auto position = species.at("position").at("x").loadChunk< double >({0}, {4}); + auto position_raw = position.get(); + auto positionOffset = species.at("positionOffset").at("x").loadChunk< uint64_t >({0}, {4}); + auto positionOffset_raw = positionOffset.get(); + o.flush(); + for( uint64_t j = 0; j < 4; ++j ) + { + REQUIRE(position_raw[j] == static_cast< double >(j + (i-1)*4)); + REQUIRE(positionOffset_raw[j] == j + (i-1)*4); + } + } + } +} + +TEST_CASE( "json_deletion_test", "[serial][hdf5]" ) +{ + Series o = Series("../samples/serial_deletion.json", AccessType::CREATE); + + + o.setAttribute("removed", + "this attribute will be removed after being written to disk"); + o.flush(); + + o.deleteAttribute("removed"); + o.flush(); + + ParticleSpecies& e = o.iterations[1].particles["e"]; + auto dset = Dataset(Datatype::DOUBLE, {1}); + e["position"][RecordComponent::SCALAR].resetDataset(dset); + e["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + e.erase("deletion"); + o.flush(); + + e["deletion_scalar"][RecordComponent::SCALAR].resetDataset(dset); + o.flush(); + + e["deletion_scalar"].erase(RecordComponent::SCALAR); + e.erase("deletion_scalar"); + o.flush(); + + double value = 0.; + e["deletion_scalar_constant"][RecordComponent::SCALAR].resetDataset(dset); + e["deletion_scalar_constant"][RecordComponent::SCALAR].makeConstant(value); + o.flush(); + + e["deletion_scalar_constant"].erase(RecordComponent::SCALAR); + e.erase("deletion_scalar_constant"); + o.flush(); +} + +TEST_CASE( "json_dtype_test", "[serial][json]" ) +{ + { + Series s = Series("../samples/dtype_test.json", AccessType::CREATE); + + char c = 'c'; + s.setAttribute("char", c); + unsigned char uc = 'u'; + s.setAttribute("uchar", uc); + int16_t i16 = 16; + s.setAttribute("int16", i16); + int32_t i32 = 32; + s.setAttribute("int32", i32); + int64_t i64 = 64; + s.setAttribute("int64", i64); + uint16_t u16 = 16u; + s.setAttribute("uint16", u16); + uint32_t u32 = 32u; + s.setAttribute("uint32", u32); + uint64_t u64 = 64u; + s.setAttribute("uint64", u64); + float f = 16.e10f; + s.setAttribute("float", f); + double d = 1.e64; + s.setAttribute("double", d); + // json does not capture the complete value range of 128bit floats + long double l = 1.e10L; + s.setAttribute("longdouble", l); + std::string str = "string"; + s.setAttribute("string", str); + s.setAttribute("vecChar", std::vector< char >({'c', 'h', 'a', 'r'})); + s.setAttribute("vecInt16", std::vector< int16_t >({32766, 32767})); + s.setAttribute("vecInt32", std::vector< int32_t >({2147483646, 2147483647})); + s.setAttribute("vecInt64", std::vector< int64_t >({9223372036854775806, 9223372036854775807})); + s.setAttribute("vecUchar", std::vector< char >({'u', 'c', 'h', 'a', 'r'})); + s.setAttribute("vecUint16", std::vector< uint16_t >({65534u, 65535u})); + s.setAttribute("vecUint32", std::vector< uint32_t >({4294967294u, 4294967295u})); + s.setAttribute("vecUint64", std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); + s.setAttribute("vecFloat", std::vector< float >({0.f, 3.40282e+38f})); + s.setAttribute("vecDouble", std::vector< double >({0., 1.79769e+308})); + // json does not capture the complete value range of 128bit floats + s.setAttribute("vecLongdouble", std::vector< long double >({0.L, -2355.L})); + s.setAttribute("vecString", std::vector< std::string >({"vector", "of", "strings"})); + s.setAttribute("bool", true); + short ss = 16; + s.setAttribute("short", ss); + int si = 32; + s.setAttribute("int", si); + long sl = 64; + s.setAttribute("long", sl); + long long sll = 128; + s.setAttribute("longlong", sll); + unsigned short us = 16u; + s.setAttribute("ushort", us); + unsigned int ui = 32u; + s.setAttribute("uint", ui); + unsigned long ul = 64u; + s.setAttribute("ulong", ul); + unsigned long long ull = 128u; + s.setAttribute("ulonglong", ull); + } + + Series s = Series("../samples/dtype_test.json", AccessType::READ_ONLY); + + REQUIRE(s.getAttribute("char").get< char >() == 'c'); + REQUIRE(s.getAttribute("uchar").get< unsigned char >() == 'u'); + REQUIRE(s.getAttribute("int16").get< int16_t >() == 16); + REQUIRE(s.getAttribute("int32").get< int32_t >() == 32); + REQUIRE(getCast< int64_t >(s.getAttribute("int64")) == 64); + REQUIRE(s.getAttribute("uint16").get< uint16_t >() == 16u); + REQUIRE(s.getAttribute("uint32").get< uint32_t >() == 32u); + REQUIRE(getCast< uint64_t >(s.getAttribute("uint64")) == 64u); + REQUIRE(s.getAttribute("float").get< float >() == 16.e10f); + REQUIRE(s.getAttribute("double").get< double >() == 1.e64); +#if !defined(_MSC_VER) + REQUIRE(s.getAttribute("longdouble").get< long double >() == 1.e10L); +#endif + REQUIRE(s.getAttribute("string").get< std::string >() == "string"); + REQUIRE(s.getAttribute("vecChar").get< std::vector< char > >() == std::vector< char >({'c', 'h', 'a', 'r'})); + REQUIRE(s.getAttribute("vecInt16").get< std::vector< int16_t > >() == std::vector< int16_t >({32766, 32767})); + REQUIRE(s.getAttribute("vecInt32").get< std::vector< int32_t > >() == std::vector< int32_t >({2147483646, 2147483647})); + REQUIRE(getCast< std::vector< int64_t > >(s.getAttribute("vecInt64")) == std::vector< int64_t >({9223372036854775806, 9223372036854775807})); + REQUIRE(s.getAttribute("vecUchar").get< std::vector< char > >() == std::vector< char >({'u', 'c', 'h', 'a', 'r'})); + REQUIRE(s.getAttribute("vecUint16").get< std::vector< uint16_t > >() == std::vector< uint16_t >({65534u, 65535u})); + REQUIRE(s.getAttribute("vecUint32").get< std::vector< uint32_t > >() == std::vector< uint32_t >({4294967294u, 4294967295u})); + REQUIRE(getCast< std::vector< uint64_t > >(s.getAttribute("vecUint64")) == std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); + REQUIRE(s.getAttribute("vecFloat").get< std::vector< float > >() == std::vector< float >({0.f, 3.40282e+38f})); + REQUIRE(s.getAttribute("vecDouble").get< std::vector< double > >() == std::vector< double >({0., 1.79769e+308})); +#if !defined(_MSC_VER) + REQUIRE(s.getAttribute("vecLongdouble").get< std::vector< long double > >() == std::vector< long double >({0.L, -2355.L})); +#endif + REQUIRE(s.getAttribute("vecString").get< std::vector< std::string > >() == std::vector< std::string >({"vector", "of", "strings"})); + REQUIRE(s.getAttribute("bool").get< bool >() == true); + + // same implementation types (not necessary aliases) detection + REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); + REQUIRE(s.getAttribute("int").dtype == Datatype::INT); + REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); + REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); + REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); + REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); + REQUIRE(s.getAttribute("ulonglong").dtype == Datatype::ULONGLONG); +} + +#endif