Basic RESTVOL support

.devcontainer/Dockerfile

@@ -15,18 +15,20 @@ RUN apt-get update && \
     libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev ca-certificates \
     # tools
+    gdb libeigen3-dev openssh-client\
     telnet \
     && rm -rf /var/lib/apt/lists/*
 
-# # Build REST VOL
-# # https://github.com/HDFGroup/vol-rest
+# Build REST VOL
+# https://github.com/HDFGroup/vol-rest
 WORKDIR /HDFGroup
 RUN git clone https://github.com/HDFGroup/vol-rest
 WORKDIR /HDFGroup/vol-rest
 RUN git checkout ${REST_VOL_COMMIT}
 RUN mkdir build && cd build && \
-    cmake -DHDF5_VOL_REST_ENABLE_CURL_DEBUG=ON -DHDF5_VOL_REST_ENABLE_DEBUG=ON -DCMAKE_INSTALL_PREFIX=/usr/local/vol-rest -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DHDF5_VOL_REST_ENABLE_EXAMPLES=OFF .. && \
+    cmake -G Ninja -DHDF5_VOL_REST_ENABLE_CURL_DEBUG=ON -DHDF5_VOL_REST_ENABLE_DEBUG=ON -DCMAKE_INSTALL_PREFIX=/usr/local/vol-rest -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DHDF5_VOL_REST_ENABLE_EXAMPLES=OFF .. && \
     cmake --build . && cmake --install .
+RUN ln -s /usr/local/vol-rest/lib/libhdf5_vol_rest.so  /usr/local/vol-rest/lib/librest_vol.so
 
 USER kothar
 # We need pyenv to use older version of python for HSDS
@@ -37,6 +39,7 @@ RUN pyenv install 3.9.21 && pyenv global 3.9.21
 RUN pip install hsds h5py h5pyd --break-system-packages
 
 # Setup env variables for HSDS / REST VOL
+ENV HDF5_VOL_CONNECTOR_DEBUG=1
 ENV HDF5_PLUGIN_PATH=/usr/local/vol-rest/lib
 ENV HDF5_VOL_CONNECTOR=REST
 ENV HSDS_USERNAME=admin
@@ -48,7 +51,6 @@ ENV HS_USERNAME=admin
 ENV HS_PASSWORD=admin
 ENV HS_ENDPOINT=http://localhost:5101
 
-
 # Create `~/launch-hsds.sh script file
 RUN mkdir -p /home/kothar/.bin && cat <<'EOF' > /home/kothar/.bin/start-hsds
 #!/bin/bash

CMakeLists.txt

@@ -62,7 +62,8 @@ endif()
 option(HIGHFIVE_UNIT_TESTS "Compile unit-tests" ${HIGHFIVE_EXTRAS_DEFAULT})
 option(HIGHFIVE_EXAMPLES "Compile examples" ${HIGHFIVE_EXTRAS_DEFAULT})
 option(HIGHFIVE_BUILD_DOCS "Build documentation" ${HIGHFIVE_EXTRAS_DEFAULT})
-
+option(HIGHFIVE_ENABLE_RESTVOL "Enable rest-vol API" OFF)
+set(RESTVOL_ROOT_DIR "" CACHE PATH "Path to the REST VOL installation")
 option(HIGHFIVE_TEST_SPAN "Enable testing std::span, requires C++20" ${HIGHFIVE_TEST_SPAN_DEFAULT})
 option(HIGHFIVE_TEST_BOOST "Enable testing Boost features" OFF)
 option(HIGHFIVE_TEST_BOOST_SPAN "Additionally, enable testing `boost::span`" OFF)
@@ -83,7 +84,11 @@ mark_as_advanced(HIGHFIVE_SANITIZER)
 
 # Check compiler cxx_std requirements
 # -----------------------------------
-set(HIGHFIVE_CXX_STANDARD_DEFAULT 14)
+if(HIGHFIVE_ENABLE_RESTVOL)
+    set(HIGHFIVE_CXX_STANDARD_DEFAULT 17)
+else()
+    set(HIGHFIVE_CXX_STANDARD_DEFAULT 14)
+endif()
 
 if(NOT DEFINED CMAKE_CXX_STANDARD)
     set(CMAKE_CXX_STANDARD ${HIGHFIVE_CXX_STANDARD_DEFAULT})
@@ -173,7 +178,7 @@ export(
 # Preparing local building (tests, examples)
 # ------------------------------------------
 
-if(HIGHFIVE_EXAMPLES OR HIGHFIVE_UNIT_TESTS)
+if(HIGHFIVE_EXAMPLES OR HIGHFIVE_UNIT_TESTS OR HIGHFIVE_ENABLE_RESTVOL)
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/HighFiveWarnings.cmake)
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/HighFiveFlags.cmake)
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/HighFiveOptionalDependencies.cmake)

cmake/HighFiveOptionalDependencies.cmake

@@ -47,6 +47,37 @@ if(NOT TARGET HighFiveSpanDependency)
   endif()
 endif()
 
+if(NOT TARGET HighFiveRestVol)
+  add_library(HighFiveRestVol INTERFACE)
+  if(HIGHFIVE_ENABLE_RESTVOL)
+    if(RESTVOL_ROOT_DIR)
+      set(_RESTVOL_SEARCH_PATHS "${RESTVOL_ROOT_DIR}/lib")
+      set(_RESTVOL_INCLUDE_HINT "${RESTVOL_ROOT_DIR}/include")
+    else()
+      set(_RESTVOL_SEARCH_PATHS /usr/local/vol-rest/lib /usr/lib /usr/local/lib)
+      set(_RESTVOL_INCLUDE_HINT /usr/local/vol-rest/include /usr/include)
+    endif()
+    find_library(RESTVOL_LIBRARY
+      NAMES rest_vol
+      PATHS ${_RESTVOL_SEARCH_PATHS}
+      NO_DEFAULT_PATH
+    )
+    find_path(RESTVOL_INCLUDE_DIR
+      NAMES rest_vol_public.h
+      PATHS ${_RESTVOL_INCLUDE_HINT}
+      NO_DEFAULT_PATH
+    )
+    if(NOT RESTVOL_LIBRARY OR NOT RESTVOL_INCLUDE_DIR)
+      message(FATAL_ERROR "Could not find REST VOL library or headers. Set -DRESTVOL_ROOT_DIR=/path/to/vol-rest")
+    else()
+      message(STATUS "REST VOL library and headers: ${RESTVOL_LIBRARY} -- ${RESTVOL_INCLUDE_DIR}")
+    endif()
+    target_link_libraries(HighFiveRestVol INTERFACE "${RESTVOL_LIBRARY}")
+    target_include_directories(HighFiveRestVol INTERFACE "${RESTVOL_INCLUDE_DIR}")
+    target_compile_definitions(HighFiveRestVol INTERFACE HIGHFIVE_USE_RESTVOL=1)
+  endif()
+endif()
+
 if(NOT TARGET HighFiveOptionalDependencies)
   add_library(HighFiveOptionalDependencies INTERFACE)
   target_link_libraries(HighFiveOptionalDependencies INTERFACE
@@ -55,5 +86,6 @@ if(NOT TARGET HighFiveOptionalDependencies)
     HighFiveXTensorDependency
     HighFiveOpenCVDependency
     HighFiveSpanDependency
+    HighFiveRestVol
   )
 endif()

include/highfive/H5Exception.hpp

@@ -159,6 +159,15 @@ class ReferenceException: public Exception {
     ReferenceException(const std::string& err_msg)
         : Exception(err_msg) {}
 };
+
+///
+/// \brief Exception specific to HighFive Slice interface
+///
+class SliceException: public Exception {
+  public:
+    SliceException(const std::string& err_msg)
+        : Exception(err_msg) {}
+};
 }  // namespace HighFive
 
 #include "bits/H5Exception_misc.hpp"

include/highfive/H5File.hpp

@@ -113,7 +113,11 @@ class File: public Object, public NodeTraits<File>, public AnnotateTraits<File>
 
     /// \brief Get the list of properties for creation of this file
     FileCreateProps getCreatePropertyList() const {
+#if defined(HIGHFIVE_USE_RESTVOL)
+        throw FileException("H5Fget_create_plist is not supported with REST VOL.");
+#else
         return details::get_plist<FileCreateProps>(*this, H5Fget_create_plist);
+#endif
     }
 
     /// \brief Get the list of properties for accession of this file

include/highfive/H5Object.hpp

@@ -108,7 +108,7 @@ class ObjectInfo {
     /// \brief Retrieve the address of the object (within its file)
     /// \deprecated Deprecated since HighFive 2.2. Soon supporting VOL tokens
     H5_DEPRECATED("Deprecated since HighFive 2.2. Soon supporting VOL tokens")
-    haddr_t getAddress() const noexcept;
+    haddr_t getAddress() const HIGHFIVE_NOEXCEPT_IF_NOT_RESTVOL;
 
     /// \brief Retrieve the number of references to this object
     size_t getRefCount() const noexcept;
@@ -120,7 +120,9 @@ class ObjectInfo {
     time_t getModificationTime() const noexcept;
 
   protected:
-#if (H5Oget_info_vers < 3)
+#if defined(HIGHFIVE_USE_RESTVOL)
+    H5O_info2_t raw_info;
+#elif (H5Oget_info_vers < 3)
     H5O_info_t raw_info;
 #else
     // Use compat H5O_info1_t while getAddress() is supported (deprecated)

include/highfive/bits/H5Annotate_traits_misc.hpp

@@ -37,19 +37,50 @@ template <typename Derivate>
 template <typename Type>
 inline Attribute AnnotateTraits<Derivate>::createAttribute(const std::string& attribute_name,
                                                            const DataSpace& space) {
-    return createAttribute(attribute_name, space, create_and_check_datatype<Type>());
+#if defined(HIGHFIVE_USE_RESTVOL)
+    if constexpr (std::is_same_v<Type, std::string>) {
+        throw AttributeException("Variable length strings not supported with REST VOL.");
+    } else {
+#endif
+        return createAttribute(attribute_name, space, create_and_check_datatype<Type>());
+#if defined(HIGHFIVE_USE_RESTVOL)
+    }
+#endif
 }
 
 template <typename Derivate>
 template <typename T>
 inline Attribute AnnotateTraits<Derivate>::createAttribute(const std::string& attribute_name,
                                                            const T& data) {
-    Attribute att =
-        createAttribute(attribute_name,
-                        DataSpace::From(data),
-                        create_and_check_datatype<typename details::inspector<T>::base_type>());
-    att.write(data);
-    return att;
+    auto dataspace = DataSpace::From(data);
+#if defined(HIGHFIVE_USE_RESTVOL)
+    if constexpr (std::is_same_v<T, std::string>) {
+        auto datatype = FixedLengthStringType(data.size() + 1, StringPadding::NullTerminated);
+        Attribute att = createAttribute(attribute_name, dataspace, datatype);
+        att.write(data);
+        return att;
+    } else if constexpr (std::is_same_v<typename details::inspector<T>::base_type, std::string>) {
+        auto string_length = std::max_element(data.begin(),
+                                              data.end(),
+                                              [](const std::string& a, const std::string& b) {
+                                                  return a.size() < b.size();
+                                              })
+                                 ->size();
+        auto datatype = FixedLengthStringType(string_length + 1, StringPadding::NullPadded);
+        Attribute att = createAttribute(attribute_name, dataspace, datatype);
+        att.write(data);
+        return att;
+    } else {
+#endif
+        Attribute att =
+            createAttribute(attribute_name,
+                            dataspace,
+                            create_and_check_datatype<typename details::inspector<T>::base_type>());
+        att.write(data);
+        return att;
+#if defined(HIGHFIVE_USE_RESTVOL)
+    }
+#endif
 }
 
 template <typename Derivate>

include/highfive/bits/H5File_misc.hpp

@@ -7,7 +7,9 @@
  *
  */
 #pragma once
-
+#if defined(HIGHFIVE_USE_RESTVOL)
+#include <filesystem>
+#endif
 #include <string>
 
 #include <H5Fpublic.h>
@@ -48,6 +50,11 @@ inline File::File(const std::string& filename,
                   AccessMode access_mode,
                   const FileCreateProps& fileCreateProps,
                   const FileAccessProps& fileAccessProps) {
+#if defined(HIGHFIVE_USE_RESTVOL)
+    if (!std::filesystem::path(filename).is_absolute()) {
+        throw FileException("The provided path must be absolute when working with REST VOL.");
+    }
+#endif
     unsigned openFlags = convert_open_flag(access_mode);
 
     unsigned createMode = openFlags & (H5F_ACC_TRUNC | H5F_ACC_EXCL);

include/highfive/bits/H5Node_traits_misc.hpp

@@ -32,6 +32,26 @@
 
 namespace HighFive {
 
+#if defined(HIGHFIVE_USE_RESTVOL)
+template <typename Derivate>
+void create_parent_group(NodeTraits<Derivate>* node,
+                         const std::string& path,
+                         const DataSetAccessProps& accessProps) {
+    auto pos = path.rfind('/');
+    if (pos != std::string::npos && pos != 0) {
+        const std::string subpath = path.substr(0, pos);
+        create_parent_group(node, subpath, accessProps);
+        htri_t err = detail::h5l_exists(static_cast<Derivate*>(node)->getId(),
+                                        subpath.c_str(),
+                                        accessProps.getId());
+        if (err < 0) {
+            throw DataSetException("");
+        } else if (err == 0) {
+            node->createGroup(subpath);
+        }
+    }
+}
+#endif
 
 template <typename Derivate>
 inline DataSet NodeTraits<Derivate>::createDataSet(const std::string& dataset_name,
@@ -40,6 +60,11 @@ inline DataSet NodeTraits<Derivate>::createDataSet(const std::string& dataset_na
                                                    const DataSetCreateProps& createProps,
                                                    const DataSetAccessProps& accessProps,
                                                    bool parents) {
+#if defined(HIGHFIVE_USE_RESTVOL)
+    if (parents) {
+        create_parent_group(this, dataset_name, accessProps);
+    }
+#endif
     LinkCreateProps lcpl;
     lcpl.add(CreateIntermediateGroup(parents));
     return DataSet(detail::h5d_create2(static_cast<Derivate*>(this)->getId(),
@@ -69,15 +94,40 @@ inline DataSet NodeTraits<Derivate>::createDataSet(const std::string& dataset_na
                                                    const DataSetCreateProps& createProps,
                                                    const DataSetAccessProps& accessProps,
                                                    bool parents) {
-    DataSet ds =
-        createDataSet(dataset_name,
-                      DataSpace::From(data),
-                      create_and_check_datatype<typename details::inspector<T>::base_type>(),
-                      createProps,
-                      accessProps,
-                      parents);
-    ds.write(data);
-    return ds;
+    auto dataspace = DataSpace::From(data);
+#if defined(HIGHFIVE_USE_RESTVOL)
+    if constexpr (std::is_same_v<T, std::string>) {
+        auto datatype = FixedLengthStringType(data.size() + 1, StringPadding::NullTerminated);
+        DataSet ds =
+            createDataSet(dataset_name, dataspace, datatype, createProps, accessProps, parents);
+        ds.write(data);
+        return ds;
+    } else if constexpr (std::is_same_v<typename details::inspector<T>::base_type, std::string>) {
+        auto string_length = std::max_element(data.begin(),
+                                              data.end(),
+                                              [](const std::string& a, const std::string& b) {
+                                                  return a.size() < b.size();
+                                              })
+                                 ->size();
+        auto datatype = FixedLengthStringType(string_length + 1, StringPadding::NullPadded);
+        DataSet ds =
+            createDataSet(dataset_name, dataspace, datatype, createProps, accessProps, parents);
+        ds.write(data);
+        return ds;
+    } else {
+#endif
+        DataSet ds =
+            createDataSet(dataset_name,
+                          dataspace,
+                          create_and_check_datatype<typename details::inspector<T>::base_type>(),
+                          createProps,
+                          accessProps,
+                          parents);
+        ds.write(data);
+        return ds;
+#if defined(HIGHFIVE_USE_RESTVOL)
+    }
+#endif
 }
 
 template <typename Derivate>

include/highfive/bits/H5Object_misc.hpp

@@ -90,7 +90,9 @@ inline ObjectType Object::getType() const {
 
 inline ObjectInfo Object::getInfo() const {
     ObjectInfo info;
-#if (H5Oget_info_vers < 3)
+#if defined(HIGHFIVE_USE_RESTVOL)
+    if (H5Oget_info3(_hid, &info.raw_info, H5O_INFO_BASIC | H5O_INFO_TIME) < 0) {
+#elif (H5Oget_info_vers < 3)
     if (H5Oget_info(_hid, &info.raw_info) < 0) {
 #else
     if (H5Oget_info1(_hid, &info.raw_info) < 0) {
@@ -100,8 +102,12 @@ inline ObjectInfo Object::getInfo() const {
     return info;
 }
 
-inline haddr_t ObjectInfo::getAddress() const noexcept {
+inline haddr_t ObjectInfo::getAddress() const HIGHFIVE_NOEXCEPT_IF_NOT_RESTVOL {
+#if defined(HIGHFIVE_USE_RESTVOL)
+    throw ObjectException("Unable to obtain addr info with REST VOL.");
+#else
     return raw_info.addr;
+#endif
 }
 inline size_t ObjectInfo::getRefCount() const noexcept {
     return raw_info.rc;

include/highfive/bits/H5Path_traits_misc.hpp

@@ -22,8 +22,10 @@ inline PathTraits<Derivate>::PathTraits() {
                   "PathTraits can only be applied to Group, DataSet and Attribute");
     const auto& obj = static_cast<const Derivate&>(*this);
     if (obj.isValid()) {
+#if !defined(HIGHFIVE_USE_RESTVOL)
         const hid_t file_id = detail::h5i_get_file_id<PropertyException>(obj.getId());
         _file_obj.reset(new File(file_id));
+#endif
     }
 }
 
@@ -37,6 +39,9 @@ inline std::string PathTraits<Derivate>::getPath() const {
 template <typename Derivate>
 inline File& PathTraits<Derivate>::getFile() const {
     const auto& obj = static_cast<const Derivate&>(*this);
+#if defined(HIGHFIVE_USE_RESTVOL)
+    throw ObjectException("`PathTraits::getFile` is not supported with REST VOL.");
+#endif
     if (!obj.isValid()) {
         throw ObjectException("Invalid call to `PathTraits::getFile` for invalid object");
     }