How to correctly open a different HDF5 file inside a VOL's dataset_read callback?

[yzanhua]

Describe the issue
When developing Log VOL, we encounter a problem that we cannot open a different file (say exist.h5) inside the read call back that handles a read request to a file, say test.h5. An error message complaining about not a VOL connector ID will occur. We'd like to know what is the correct way to open a different HDF5 file inside a VOL's dataset_read callback?

Reproduce the issue
We can create a minimal VOL to reproduce the issue by modifying the existing HDF5 pass_through VOL. Only H5VL_pass_through_dataset_read is modified.

Click here to see modified H5VL_pass_through_dataset_read.

static herr_t
H5VL_pass_through_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t mem_space_id[],
                               hid_t file_space_id[], hid_t plist_id, void *buf[], void **req)
{
    void  *obj_local;        /* Local buffer for obj */
    void **obj = &obj_local; /* Array of object pointers */
    size_t i;                /* Local index variable */
    herr_t ret_value;
    {  // extra codes to reproduce the error
        printf("******* PASS THROUGH VOL DATASET Read begin\n");

        void* lib_state = NULL, *file_ptr = NULL;
        hid_t fapl_id = -1, native_vol_id = -1;
        herr_t err = 0;

        // start a new lib state
        err = H5VLretrieve_lib_state (&lib_state); MY_CHECK_ERR(err)
        err = H5VLstart_lib_state (); MY_CHECK_ERR(err)
        err = H5VLrestore_lib_state (lib_state); MY_CHECK_ERR(err)

        // set up fapl_id to use native vol
        native_vol_id = H5VLget_connector_id_by_name(H5VL_NATIVE_NAME); MY_CHECK_ERR(native_vol_id)
        fapl_id = H5Pcreate(H5P_FILE_ACCESS); MY_CHECK_ERR(fapl_id)
        err = H5Pset_vol(fapl_id, native_vol_id, NULL); MY_CHECK_ERR(err)

        // open the file using fapl_id
        file_ptr = H5VLfile_open("exist.h5", H5F_ACC_RDWR, fapl_id, H5P_DATASET_XFER_DEFAULT, NULL);
        if (file_ptr == NULL) {printf("******* failed at file_open, line:%d\n", __LINE__ );}

        // close the file
        if (file_ptr != NULL) {
            err = H5VLfile_close(file_ptr, native_vol_id, H5P_DATASET_XFER_DEFAULT, NULL); MY_CHECK_ERR(err)
        }
        // close fapl_id
        if (fapl_id >= 0) {
            err = H5Pclose(fapl_id); MY_CHECK_ERR(err)
        }

        // finish the lib state
        err = H5VLfinish_lib_state (); MY_CHECK_ERR(err)
        err = H5VLrestore_lib_state (lib_state); MY_CHECK_ERR(err)
        err = H5VLfree_lib_state (lib_state); MY_CHECK_ERR(err)
    } // end extra codes to reproduce the error. Codes below are not modified
    

    /* Allocate obj array if necessary */
    if (count > 1)
        if (NULL == (obj = (void **)malloc(count * sizeof(void *))))
            return -1;

    /* Build obj array */
    for (i = 0; i < count; i++) {
        /* Get the object */
        obj[i] = ((H5VL_pass_through_t *)dset[i])->under_object;

        /* Make sure the class matches */
        if (((H5VL_pass_through_t *)dset[i])->under_vol_id != ((H5VL_pass_through_t *)dset[0])->under_vol_id)
            return -1;
    }

    ret_value = H5VLdataset_read(count, obj, ((H5VL_pass_through_t *)dset[0])->under_vol_id, mem_type_id,
                                 mem_space_id, file_space_id, plist_id, buf, req);

    /* Check for async request */
    if (req && *req)
        *req = H5VL_pass_through_new_obj(*req, ((H5VL_pass_through_t *)dset[0])->under_vol_id);

    /* Free memory */
    if (obj != &obj_local)
        free(obj);

    return ret_value;
} /* end H5VL_pass_through_dataset_read() */

Modified codes are surrounded by bracket { // extra codes to reproduce the error. MY_CHECK_ERR is defined as

#define MY_CHECK_ERR(A) { \
    if ((A) < 0) {printf("******* failed at line:%d\n", __LINE__ );} \
}

Click here to see an application that triggers the error.

#include <stdio.h>
#include <stdlib.h>
#include "hdf5.h"

int preprocess();
int create_then_close_file(const char* filename);

#define CHECK_ERR(A) { \
    if((A) < 0) {\
        printf("Error at %s:%d\n", __func__, __LINE__); \
        nerrs ++; \
        goto fn_exit; \
    } \
}

int main(int argc, char *argv[])
{
    int err = 0, nerrs=0;
    hid_t native_vol_id = -1, passthru_vol_id = -1;
    hid_t file_id = -1, dset_id = -1, fspace_id = -1, mspace_id = -1, fapl_id = -1;
    H5VL_pass_through_info_t passthru_info;
    hsize_t dims[3] = {4, 6, 24};
    int buf[24] = {0};

    // the following function creates 2 non-empty HDF5 files:
    //      exist.h5, test.h5
    err = preprocess();
    CHECK_ERR(err);

    native_vol_id = H5VLget_connector_id_by_name(H5VL_NATIVE_NAME);
    CHECK_ERR(native_vol_id);
    passthru_vol_id = H5VL_pass_through_register();
    CHECK_ERR(passthru_vol_id);
    passthru_info.under_vol_id   = native_vol_id;
    passthru_info.under_vol_info = NULL;

    // open test.h5 with modified pass_through VOL
    fapl_id = H5Pcreate(H5P_FILE_ACCESS);
    CHECK_ERR(fapl_id);
    err = H5Pset_vol(fapl_id, passthru_vol_id, &passthru_info);
    CHECK_ERR(err);
    file_id = H5Fopen("test.h5", H5F_ACC_RDWR, fapl_id);
    CHECK_ERR(file_id);

    // read dataset "/dset" from test.h5
    dset_id = H5Dopen2(file_id, "/dset", H5P_DEFAULT);
    CHECK_ERR(dset_id);
    fspace_id = H5Screate_simple (2, dims, dims);
    CHECK_ERR(fspace_id);

    mspace_id = H5Screate_simple (1, &dims[2], &dims[2]);
    CHECK_ERR(mspace_id);
    err = H5Dread(dset_id, H5T_NATIVE_INT, mspace_id, fspace_id, H5P_DEFAULT, buf);
    CHECK_ERR (err)

    for (int i = 0; i < 24; i++) {
        printf("%d ", buf[i]);
    }
    printf("\n");


fn_exit:;
    if (dset_id >= 0) {
        H5Dclose(dset_id);
    }
    if (fspace_id >= 0) {
        H5Sclose(fspace_id);
    }
    if (mspace_id >= 0) {
        H5Sclose(mspace_id);
    }
    if (file_id >= 0) {
        H5Fclose(file_id);
    }
    if (fapl_id >= 0) {
        H5Pclose(fapl_id);
    }
    if (native_vol_id >= 0) {
        H5VLclose(native_vol_id);
    }
    if (passthru_vol_id >= 0) {
        H5VLclose(passthru_vol_id);
    }
    return 0;
}

// This function creates 2 HDF5 files
int preprocess() {
    int err = 0, nerrs = 0, rank = 0;

    err = create_then_close_file("exist.h5");
    CHECK_ERR(err);

    err = create_then_close_file("test.h5");
    CHECK_ERR(err);

fn_exit:;
    return nerrs;
}

int create_then_close_file(const char* filename) {
    hid_t file_id = -1;
    hid_t dataset_id = -1, dataspace_id = -1;
    hsize_t dims[2] = {4, 6};
    int nerrs = 0;
    herr_t err;
    int data[4][6] = {{1, 2, 3, 4, 5, 6},
                      {7, 8, 9, 10, 11, 12},
                      {13, 14, 15, 16, 17, 18},
                      {19, 20, 21, 22, 23, 24}};

    file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
    CHECK_ERR(file_id);

    dataspace_id = H5Screate_simple(2, dims, NULL);
    CHECK_ERR(dataspace_id);

    dataset_id = H5Dcreate2(file_id, "/dset", H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
    CHECK_ERR(dataset_id);

    err = H5Dwrite(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data);
    CHECK_ERR(err);

    err = H5Dclose(dataset_id);
    CHECK_ERR(err);

    err = H5Sclose(dataspace_id);
    CHECK_ERR(err);

    err = H5Fclose(file_id);
    CHECK_ERR(err);

fn_exit:;
    return nerrs;
}

Click here to see a simple makefile

HDF5_DIR=${HOME}/hdf5/test

all:
	gcc -g -O0 -o test test.c -I${HDF5_DIR}/include -L${HDF5_DIR}/lib -lhdf5 -lhdf5_hl

run:
	LD_LIBRARY_PATH=${HDF5_DIR}/lib \
	HDF5_PLUGIN_PATH=${HDF5_DIR}/lib \
	./test

clean:
	rm -rf test *.h5 core core.*

Click here to see the error message.

******* PASS THROUGH VOL DATASET Read begin
HDF5-DIAG: Error detected in HDF5 (1.14.0) thread 0:
  #000: H5VLcallback.c line 3906 in H5VLfile_open(): unable to open file
    major: Virtual Object Layer
    minor: Can't open object
  #001: H5VLcallback.c line 3675 in H5VL__file_open(): open failed
    major: Virtual Object Layer
    minor: Can't open object
  #002: H5VLnative_file.c line 128 in H5VL__native_file_open(): unable to open file
    major: File accessibility
    minor: Unable to open file
  #003: H5Fint.c line 1884 in H5F_open(): unable to initialize file structure
    major: File accessibility
    minor: Unable to open file
  #004: H5Fint.c line 1291 in H5F__new(): can't cache VOL connector info
    major: File accessibility
    minor: Unable to initialize object
  #005: H5Fint.c line 291 in H5F__set_vol_conn(): not a VOL connector ID
    major: File accessibility
    minor: Inappropriate type
******* failed at file_open, line:1230
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24

Additional context
We thought the problem was related to those lib_state calls so we also tried calling those functions at different places but with no luck (e.g calling H5VLrestore_lib_state right before H5VLfile_open).

Platform

• HDF5 version: 1.14.0
• Compiler and version: gcc version 11.3.0 (Ubuntu 11.3.0-1ubuntu1~22.04)
• Build system: automake (GNU automake) 1.16.5
• Any configure options you specified:

  ./configure \
  --prefix=${HOME}/hdf5/${HDF5_VER} \
  CC=gcc \
  CXX=g++ \
  --enable-build-mode=production

• MPI library and version: The issue exists with and without MPI.