memory backed files


#1

I am having trouble to set up an image into a memory backed file. I’d like to set up a memory file, copy some data into it and then extract the a file image which will be set in another memory backed file (this last step seems to be the issue). Below code works but it is extracting the image from the disk file and I’d like to use the memory one. The only way I found to make this work is to relaxing the version bounding to the earliest (H5F_LIBVER_EARLIEST).
Am I doing something wrong or not allowed? Is this a bug? I am using the C API and need to keep it for now.

HDF5 version is 1.12.1

herr_t copy_group( hid_t groupId, const char* name, H5L_info_t const* info, void* operatorData )
{
if (!operatorData) {
return -1;
}
hid_t targetFileId = ((hid_t)operatorData);
if (targetFileId <= 0) {
return -1;
}

H5O_info_t objInfo;
H5Oget_info(groupId, &objInfo, H5O_INFO_ALL);
if (objInfo.type == H5O_TYPE_GROUP) {
    return H5Ocopy(groupId,name,targetFileId,name,H5P_DEFAULT,H5P_DEFAULT);
}
return 0; // do nothing if not a group

}

void workflow()
{
size_t memory_increments = 10485760; //10 MB
std::pair<hid_t,hid_t> fapl_id_disk (-1,-1);
std::pair<hid_t,hid_t> fapl_id_memory1 (-1,-1);
std::pair<hid_t,hid_t> fapl_id_memory2 (-1,-1);
unsigned char * file_image_memory;
size_t file_image_memory_size;
unsigned char * file_image_disk;
size_t file_image_disk_size;
herr_t ret;

auto convert_h5_array = [&](hid_t fapl_id, unsigned char **buffer, size_t &buffer_size)
{
    herr_t err = -1;
    *buffer = nullptr;
    buffer_size = 0;

    if(fapl_id < 0)
        return err;

    H5Fflush(fapl_id,H5F_SCOPE_GLOBAL);

    //getting a copy of the file_image
    int64_t sbyteCount = H5Fget_file_image(fapl_id, NULL,0);
    if(sbyteCount < 0)
        return err;

    size_t size       = (size_t)sbyteCount;
    unsigned char *file_image = (unsigned char *)malloc(size);
    if(file_image){
        err = H5Fget_file_image(fapl_id, file_image, size);
        if((size_t)err == size){
            *buffer = (unsigned char *)malloc(size);
            buffer_size = size;
            memcpy(*buffer, file_image,size);
            return err;
        }
    }
    return herr_t(-1);
};


auto close_h5 =[&](std::pair<hid_t,hid_t> &file_apl_id)
{
    herr_t err = H5Pclose(file_apl_id.second);
    assert(err >= 0);
    file_apl_id.second = -1;

    err = H5Fclose(file_apl_id.first);
    assert(err >= 0);
    file_apl_id.first = -1;
};


// open a good hdf5 file from the disk
fapl_id_disk.second = H5Pcreate(H5P_FILE_ACCESS);
assert(fapl_id_disk.second >= 0);

ret = H5Pset_fclose_degree(fapl_id_disk.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
assert(ret >= 0);

fapl_id_disk.first = H5Fopen("file_image_core_test.h5", H5F_ACC_RDONLY, fapl_id_disk.second);
assert(fapl_id_disk.first >= 0);





// create file memory backed file
fapl_id_memory1.second = H5Pcreate(H5P_FILE_ACCESS);
assert(fapl_id_memory1.second >= 0);

ret = H5Pset_fclose_degree(fapl_id_memory1.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
assert(ret >= 0);

ret = H5Pset_libver_bounds(fapl_id_memory1.second, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
assert(ret >= 0);

/* Set up the core VFD */
ret = H5Pset_fapl_core(fapl_id_memory1.second, memory_increments, false);
assert(ret >= 0);

fapl_id_memory1.first = H5Fcreate("dne.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id_memory1.second);
assert(fapl_id_memory1.first >= 0);






// copy from disk file to memory file
hid_t rootGroupId, memRootGroupId;
rootGroupId = H5Gopen(fapl_id_disk.first,"/",H5P_DEFAULT);
assert(rootGroupId > 0);
memRootGroupId = H5Gopen(fapl_id_memory1.first,"/",H5P_DEFAULT);
assert(memRootGroupId > 0);
H5Literate(rootGroupId,H5_INDEX_NAME,H5_ITER_NATIVE,NULL,copy_group,(void*)&memRootGroupId);


// assumption is that both file images (disk and memory) must be identical
// images are not identical and only when extracting from disk file the second memory file is created successfully
convert_h5_array(fapl_id_memory1.first, &file_image_memory, file_image_memory_size);
convert_h5_array(fapl_id_disk.first, &file_image_disk, file_image_disk_size);

// assert(file_image_disk_size == file_image_memory_size);
// assert(0 == memcmp(file_image_disk, file_image_memory, file_image_disk_size));

close_h5(fapl_id_disk);
close_h5(fapl_id_memory1);



/* Create the second memory backed file */
fapl_id_memory2.second = H5Pcreate(H5P_FILE_ACCESS);
assert(fapl_id_memory2.second >= 0);

ret = H5Pset_fclose_degree(fapl_id_memory2.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
assert(ret >= 0);

ret = H5Pset_libver_bounds(fapl_id_memory2.second, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
assert(ret >= 0);

/* Set up the core VFD */
ret = H5Pset_fapl_core(fapl_id_memory2.second, memory_increments, false);
assert(ret >= 0);

/* Set file image in plist */

// ret = H5Pset_file_image(fapl_id_memory2.second, file_image_memory, file_image_memory_size); // this doesn’t work, why??!!
** ret = H5Pset_file_image(fapl_id_memory2.second, file_image_disk, file_image_disk_size);**
assert(ret >= 0);

/* Test open with file image */
fapl_id_memory2.first = H5Fopen("dne2.h5", H5F_ACC_RDONLY, fapl_id_memory2.second);
assert(fapl_id_memory2.first >= 0);

close_h5(fapl_id_memory2);


/* Release resources */
free(file_image_memory);

}


#2

If I understand your question correctly, you should be able to do what you are describing.

Please take a look at RFC and the test
test/file_image.c.

As usual, a standalone program to demonstrate the issue would be extremely helpful.


#3

Hi, thanks for answering. I pasted a full standalone program but some how it was broken up in several chunks which are in order. I don’t know why it happened but anyway, you just need to copy, paste, and run it. As per the example you referred, it does not seem to be doing what I want to which is copying a image file from a memory backed file into another memory backed file. The example does not have any call to H5Pset_fapl_core which set up the VFD.


#4

Just reformatted the example…

herr_t copy_group( hid_t groupId, const char* name, H5L_info_t const* info, void* operatorData )
{
  if (!operatorData) {
    return -1;
  }

  hid_t targetFileId = ((hid_t)operatorData);
  if (targetFileId <= 0) {
    return -1;
  }

  H5O_info_t objInfo;
  H5Oget_info(groupId, &objInfo, H5O_INFO_ALL);
  if (objInfo.type == H5O_TYPE_GROUP) {
    return H5Ocopy(groupId,name,targetFileId,name,H5P_DEFAULT,H5P_DEFAULT);
  }
  return 0; // do nothing if not a group
}

void workflow()
{
  size_t memory_increments = 10485760; //10 MB
  std::pair<hid_t,hid_t> fapl_id_disk (-1,-1);
  std::pair<hid_t,hid_t> fapl_id_memory1 (-1,-1);
  std::pair<hid_t,hid_t> fapl_id_memory2 (-1,-1);
  unsigned char * file_image_memory;
  size_t file_image_memory_size;
  unsigned char * file_image_disk;
  size_t file_image_disk_size;
  herr_t ret;

  auto convert_h5_array = [&](hid_t fapl_id, unsigned char **buffer, size_t &buffer_size)
                          {
                            herr_t err = -1;
                            *buffer = nullptr;
                            buffer_size = 0;

                            if(fapl_id < 0)
                              return err;

                            H5Fflush(fapl_id,H5F_SCOPE_GLOBAL);

                            //getting a copy of the file_image
                            int64_t sbyteCount = H5Fget_file_image(fapl_id, NULL,0);
                            if(sbyteCount < 0)
                              return err;

                            size_t size       = (size_t)sbyteCount;
                            unsigned char *file_image = (unsigned char *)malloc(size);
                            if(file_image){
                              err = H5Fget_file_image(fapl_id, file_image, size);
                              if((size_t)err == size){
                                *buffer = (unsigned char *)malloc(size);
                                buffer_size = size;
                                memcpy(*buffer, file_image,size);
                                return err;
                              }
                            }
                            return herr_t(-1);
                          };


  auto close_h5 =[&](std::pair<hid_t,hid_t> &file_apl_id)
                 {
                   herr_t err = H5Pclose(file_apl_id.second);
                   assert(err >= 0);
                   file_apl_id.second = -1;

                   err = H5Fclose(file_apl_id.first);
                   assert(err >= 0);
                   file_apl_id.first = -1;
                 };


  // open a good hdf5 file from the disk
  fapl_id_disk.second = H5Pcreate(H5P_FILE_ACCESS);
  assert(fapl_id_disk.second >= 0);

  ret = H5Pset_fclose_degree(fapl_id_disk.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
  assert(ret >= 0);

  fapl_id_disk.first = H5Fopen("file_image_core_test.h5", H5F_ACC_RDONLY, fapl_id_disk.second);
  assert(fapl_id_disk.first >= 0);

  // create file memory backed file
  fapl_id_memory1.second = H5Pcreate(H5P_FILE_ACCESS);
  assert(fapl_id_memory1.second >= 0);

  ret = H5Pset_fclose_degree(fapl_id_memory1.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
  assert(ret >= 0);

  ret = H5Pset_libver_bounds(fapl_id_memory1.second, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
  assert(ret >= 0);

  /* Set up the core VFD */
  ret = H5Pset_fapl_core(fapl_id_memory1.second, memory_increments, false);
  assert(ret >= 0);

  fapl_id_memory1.first = H5Fcreate("dne.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id_memory1.second);
  assert(fapl_id_memory1.first >= 0);

  // copy from disk file to memory file
  hid_t rootGroupId, memRootGroupId;
  rootGroupId = H5Gopen(fapl_id_disk.first,"/",H5P_DEFAULT);
  assert(rootGroupId > 0);
  memRootGroupId = H5Gopen(fapl_id_memory1.first,"/",H5P_DEFAULT);
  assert(memRootGroupId > 0);
  H5Literate(rootGroupId,H5_INDEX_NAME,H5_ITER_NATIVE,NULL,copy_group,(void*)&memRootGroupId);

  // assumption is that both file images (disk and memory) must be identical
  // images are not identical and only when extracting from disk file the second memory file is created successfully
  convert_h5_array(fapl_id_memory1.first, &file_image_memory, file_image_memory_size);
  convert_h5_array(fapl_id_disk.first, &file_image_disk, file_image_disk_size);
  // assert(file_image_disk_size == file_image_memory_size);
  // assert(0 == memcmp(file_image_disk, file_image_memory, file_image_disk_size));

  close_h5(fapl_id_disk);
  close_h5(fapl_id_memory1);

  /* Create the second memory backed file */
  fapl_id_memory2.second = H5Pcreate(H5P_FILE_ACCESS);
  assert(fapl_id_memory2.second >= 0);

  ret = H5Pset_fclose_degree(fapl_id_memory2.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
  assert(ret >= 0);

  ret = H5Pset_libver_bounds(fapl_id_memory2.second, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
  assert(ret >= 0);

  /* Set up the core VFD */
  ret = H5Pset_fapl_core(fapl_id_memory2.second, memory_increments, false);
  assert(ret >= 0);

  /* Set file image in plist */
  // ret = H5Pset_file_image(fapl_id_memory2.second, file_image_memory, file_image_memory_size); // this doesn’t work, why??!!
  ret = H5Pset_file_image(fapl_id_memory2.second, file_image_disk, file_image_disk_size);
  assert(ret >= 0);

  /* Test open with file image */
  fapl_id_memory2.first = H5Fopen("dne2.h5", H5F_ACC_RDONLY, fapl_id_memory2.second);
  assert(fapl_id_memory2.first >= 0);

  close_h5(fapl_id_memory2);

  /* Release resources */
  free(file_image_memory);
}


#5

Thanks! Much appreciated. Hopefully, someone will provide a solution for my case.


#6

Hello,

Sorry! Maybe I am missing something, but I think test_get_file_image test does everything you need (or can be extended to do what you need).

To summarize: One can create HDF5 file in memory, modify it and then provide access to the file (it is in memory!) for further modifications/usage to another process (if on a shared memory system) or to the same process.

The programming model of getting access to an HDF5 file created with the core driver (i.e., HDF5 file in memory) is the following (bold indicates important steps not to be missed):

  1. Create or open an HDF5 file using core driver
  2. Perform all required modifications
  3. Flush the file
  4. Get the size of the file
  5. Allocate memory buffer of that size
  6. Load the file image into the image buffer
  7. Close all objects and the HDF5 file
  8. Create file access property using core driver and set up image file using pointer to the image buffer
  9. Open the image file (it is in memory) using file access property

Lines 785-807 show steps 1-7
Lines 938-950 show steps 8-9 to access the content of the initial file.

You may also want to look at the H5LTopen_file_image function that is in the high-level HDF5 library (and in RFC) to open file using a pointer to the image buffer and core driver.


#7

I think the problem is that your call

frees the underlying memory buffer file_image_memory. By the time you get to

that content is long gone.

G.


#8

Thanks for replying. I believe you are mistaken, unless I misunderstood the H5Fget_file_image method which I supposed returns a copy of the underlying memory buffer. If I am right, calling close_h5 has no impact on file_image_memory (which is a newly created copy). Also, the code doesn’t failed as is and using HDF5-1.12 if you change all calls to H5Pset_libver_bounds to use H5F_LIBVER_EARLIEST like this H5Pset_libver_bounds(fapl_id_memory2.second, H5F_LIBVER_EARLIEST , H5F_LIBVER_LATEST);


#9

Thanks for replying.
I have already went over the example you pointed out several times also I triple checked my example (posted here) on the steps you enumerated. So, unless I am overlooking something I’d say the code is right and would be expected to work. Actually, it works as is when using HDF5-1.8.17 (minor some library calls that have changed from on version to the new one). Did you run my example? I believe that could be bring some light into the issue.


#10

I am, and not for the first or last time :wink:

Going back to your original post, you are suggesting that somehow H5Pset_libver_bounds(..., H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); is a deal-breaker. This would be odd indeed because it has an effect only when creating new objects or, in your case, copying objects to a new file. It has no effect on reading existing objects, provided we are using a fixed major version of the library.

Are you using a shared memory segment or are you sending the file image over a socket to a different host? Do the participating processes use the same major library version (e.g., 1.8.x or 1.10.x)?

Let’s try to write a simple C reproducer! What kinds of objects are you copying? Compared to the earliest file format spec., there were updates for example for group and chunked dataset representations or attribute storage, etc., for which, when copied, the layout in the destination file might be different (“upgraded”) from the source file.

Best, G.


#11

Thanks for answering.

Here it is the code that reproduces the issue. There is no buffer copy over network, everything is running on a single machine (Red Hat Enterprise Linux Server 7.9). File used to replicate the issue (file_image_core_test.h5) can be found on the HDF5-1.12.1 repo test folder. I would like to stress the below code works fine as is when using HDF5-1.8.17 but breaks on the second H5Fopen call when using HDF5-1.12.1. To make the code work again I need to change the first H5F_LIBVER_LATEST by H5F_LIBVER_EARLIEST on H5Pset_libver_bounds(..., H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST); which introduces a humongous slow down.
The exact same issue happens on HDF5-1.12.2.
I would appreciate any help I can get on this.

#include <string.h>
#include <iostream>
#include <assert.h>

#include "hdf5.h"
#include "H5LTpublic.h"

#define USE_HDF5_1_12_1 // uncomment here when using HDF5-1.12.1


herr_t copy_group( hid_t groupId, const char* name, H5L_info_t const* info, void* operatorData )
{
    if (!operatorData) {
        return -1;
    }
    hid_t targetFileId = *((hid_t*)operatorData);
    if (targetFileId <= 0) {
        return -1;
    }

    H5O_info_t objInfo;
    #ifdef  USE_HDF5_1_12_1
        H5Oget_info(groupId, &objInfo, H5O_INFO_ALL);
    #else
        H5Oget_info(groupId, &objInfo);
    #endif
    if (objInfo.type == H5O_TYPE_GROUP) {
        return H5Ocopy(groupId,name,targetFileId,name,H5P_DEFAULT,H5P_DEFAULT);
    }
    return 0; // do nothing if not a group
}



int main()
{
    std::string                 fName = "file_image_core_test.h5";
    size_t                      memory_increments = 10485760; //10 MB
    std::pair<hid_t,hid_t>      fapl_id_disk (-1,-1);
    std::pair<hid_t,hid_t>      fapl_id_memory1 (-1,-1);
    std::pair<hid_t,hid_t>      fapl_id_memory2 (-1,-1);
    unsigned char *             file_image_memory;
    size_t                      file_image_memory_size;
    unsigned char *             file_image_disk;
    size_t                      file_image_disk_size;
    herr_t                      ret;


    auto convert_h5_array = [&](hid_t fapl_id, unsigned char **buffer, size_t &buffer_size)
    {
        herr_t err = -1;
        *buffer = nullptr;
        buffer_size = 0;

        if(fapl_id < 0)
            return err;

        H5Fflush(fapl_id,H5F_SCOPE_GLOBAL);

        //getting a copy of the file_image
        int64_t sbyteCount = H5Fget_file_image(fapl_id, NULL,0);
        if(sbyteCount < 0)
            return err;

        size_t size       = (size_t)sbyteCount;
        unsigned char *file_image = (unsigned char *)malloc(size);
        if(file_image){
            err = H5Fget_file_image(fapl_id, file_image, size);
            if((size_t)err == size){
                *buffer = (unsigned char *)malloc(size);
                buffer_size = size;
                memcpy(*buffer, file_image,size);
                return err;
            }
        }
        return herr_t(-1);
    };


    auto close_h5 =[&](std::pair<hid_t,hid_t> &file_apl_id)
    {
        herr_t err = H5Pclose(file_apl_id.second);
        assert(err >= 0);
        file_apl_id.second = -1;

        err = H5Fclose(file_apl_id.first);
        assert(err >= 0);
        file_apl_id.first = -1;
    };


    // open a good hdf5 file from the disk
    fapl_id_disk.second = H5Pcreate(H5P_FILE_ACCESS);
    assert(fapl_id_disk.second >= 0);

    ret = H5Pset_fclose_degree(fapl_id_disk.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
    assert(ret >= 0);

    fapl_id_disk.first = H5Fopen(fName.c_str(), H5F_ACC_RDONLY, fapl_id_disk.second);
    assert(fapl_id_disk.first >= 0);





    // create file memory backed file
    fapl_id_memory1.second = H5Pcreate(H5P_FILE_ACCESS);
    assert(fapl_id_memory1.second >= 0);

    ret = H5Pset_fclose_degree(fapl_id_memory1.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
    assert(ret >= 0);

    ret = H5Pset_libver_bounds(fapl_id_memory1.second, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
    assert(ret >= 0);

    /* Set up the core VFD */
    ret = H5Pset_fapl_core(fapl_id_memory1.second, memory_increments, false);
    assert(ret >= 0);

    fapl_id_memory1.first = H5Fcreate("dne.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id_memory1.second);
    assert(fapl_id_memory1.first >= 0);






    // copy from disk file to memory file
    hid_t rootGroupId, memRootGroupId;
    rootGroupId = H5Gopen(fapl_id_disk.first,"/",H5P_DEFAULT);
    assert(rootGroupId > 0);
    memRootGroupId = H5Gopen(fapl_id_memory1.first,"/",H5P_DEFAULT);
    assert(memRootGroupId > 0);
    H5Literate(rootGroupId,H5_INDEX_NAME,H5_ITER_NATIVE,NULL,copy_group,(void*)&memRootGroupId);


    // ============= ASSUMPTION:that both file images (disk and memory) must be identical
    // Images are not identical regardless HDF5 version (used 1.8.17 and 1.12.1)
    // The second memory file (fapl_id_memory2) is created successfully ONLY when extracting from disk file (HDF5 1.12.1)
    convert_h5_array(fapl_id_memory1.first, &file_image_memory, file_image_memory_size);
    convert_h5_array(fapl_id_disk.first, &file_image_disk, file_image_disk_size);
    /* assert(file_image_disk_size == file_image_memory_size); */
    /* assert(0 == memcmp(file_image_disk, file_image_memory, file_image_disk_size)); */

    close_h5(fapl_id_disk);
    close_h5(fapl_id_memory1);



    /* Create the second memory backed file */
    fapl_id_memory2.second = H5Pcreate(H5P_FILE_ACCESS);
    assert(fapl_id_memory2.second >= 0);

    ret = H5Pset_fclose_degree(fapl_id_memory2.second, H5F_CLOSE_STRONG); //needed for close file and all open objects
    assert(ret >= 0);

    ret = H5Pset_libver_bounds(fapl_id_memory2.second, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
    assert(ret >= 0);

    /* Set up the core VFD */
    ret = H5Pset_fapl_core(fapl_id_memory2.second, memory_increments, false);
    assert(ret >= 0);

    /* Set file image in plist */
    ret = H5Pset_file_image(fapl_id_memory2.second, file_image_memory, file_image_memory_size); // this doesn't work on HDF5-1.12.1 but works on 1.8.17, why??!!
    assert(ret >= 0);

    /* Test open with file image */
    fapl_id_memory2.first = H5Fopen("dne2.h5", H5F_ACC_RDONLY, fapl_id_memory2.second);
    assert(fapl_id_memory2.first >= 0);

    close_h5(fapl_id_memory2);


    /* Release resources */
    free(file_image_memory);
    free(file_image_disk);

    return 0;
}

#12

Missing free? (Nothing to do w/ the problem…)

G.


#13

I missed one, thanks. I really hope you (or someone else) are able to help me with this.


#14

OK, I’m trying to come up w/ a simpler reproducer. Here’s a snippet of C that creates an “HDF5 buffer” and we use that to initialize another in-core HDF5 file. I’m not creating or copying any objects, which may or may not have anything to do w/ the problem. What am I missing?

#include "hdf5.h"

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>

int get_h5_buffer(void** buf, size_t* len)
{
  int retval = EXIT_SUCCESS;
  *buf = NULL;
  *len = 0;

  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
  H5Pset_fapl_core(fapl, 4096, 0);
  hid_t file = H5Fcreate("foo.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
  H5Fflush(file, H5F_SCOPE_LOCAL);

  ssize_t slen = H5Fget_file_image(file, NULL, 0);
  *len = (size_t) slen;
  *buf = H5allocate_memory(*len, 0);
  H5Fget_file_image(file, *buf, *len);

  H5Pclose(fapl);
  H5Fclose(file);

  assert(*buf != NULL && *len > 0);
  return retval;
}

int main()
{
  int retval = EXIT_SUCCESS;

  // create an "HDF5 buffer"
  void* buf = NULL;
  size_t len = 0;
  get_h5_buffer(&buf, &len);
  printf("len = %lu\n", len);

  // use the buffer to initalize a new file
  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_fapl_core(fapl, 8192, 1);
  H5Pset_file_image(fapl, buf, len);
  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);

  hid_t file = H5Fcreate("bar.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);

  H5Fclose(file);
  H5Pclose(fapl);

  H5free_memory(buf);

  return retval;
}

This works fine w/ the current development branch (1.13).

G.file_image.c (1.2 KB)


#15

Hi, thanks for looking into it. I’m not sure your snippet reproduces the issue, for instance, when you initialize the new file using the buffer, your are using H5Fcreate but I’m using H5Fopen (maybe that is what I’m doing wrong), also there you are not starting from reading a file from disk. Maybe a you could start from my snippet and remove unnecessary parts as you see them. Thanks again for looking at this issue


#16

You are right, it should be H5Fopen (not H5Fcreate).

OK, here’s a slightly modified version. Replace INP_FNAME with your favorite HDF5 file (name) & don’t call it foo.h5! This version will just create a copy of your favorite called OUT_FNAME.

#include "hdf5.h"

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>

// CHANGE THIS!
#define INP_FNAME "my.h5"
#define OUT_FNAME "foo.h5"

int get_h5_buffer(void** buf, size_t* len)
{
  int retval = EXIT_SUCCESS;
  *buf = NULL;
  *len = 0;

  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
  H5Pset_fapl_core(fapl, 4096, 0);

  hid_t file = H5Fopen(INP_FNAME, H5F_ACC_RDONLY, fapl);

  ssize_t slen = H5Fget_file_image(file, NULL, 0);
  *len = (size_t) slen;
  *buf = H5allocate_memory(*len, 0);
  H5Fget_file_image(file, *buf, *len);

  H5Pclose(fapl);
  H5Fclose(file);

  assert(*buf != NULL && *len > 0);
  return retval;
}

int main()
{
  int retval = EXIT_SUCCESS;

  // create an "HDF5 buffer"
  void* buf = NULL;
  size_t len = 0;
  get_h5_buffer(&buf, &len);
  printf("len = %lu\n", len);

  // use the buffer to initalize a new file
  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_fapl_core(fapl, 8192, 1);
  assert(H5Pset_file_image(fapl, buf, len) >= 0);
  H5free_memory(buf);
  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);

  hid_t file = H5Fopen(OUT_FNAME, H5F_ACC_RDWR, fapl);

  H5Fclose(file);
  H5Pclose(fapl);

  return retval;
}

Works fine w/ develop.

G.

file_image1.c (1.2 KB)


#17

Hi again! Your code works. So it does not reproduce the issue nor my workflow. Now, why my code does not work? Did you give it a try? I see you load a file from disk and copy its content into a buffer then you use the same buffer to create a memory backed file. My workflow is different from yours and I believe that is key. My workflow is:

  • read from disk, I’ll call it F1
  • created a memory backed file, I’ll call it M1
  • copy F1 into M1
  • extract a buffer (image file) from M1, I’ll call it BF
  • created a memory backed file using the buffer (BF), I’ll call it M2 <===== my code fails here!!!

Thanks for your help!


#18

(Unless you are only interested in a subset of F1, I don’t understand why you need to copy stuff. It’s in the buffer already.)

How about this one?

#include "hdf5.h"

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>

// CHANGE THIS!
#define INP_FNAME "my.h5"
#define OUT_FNAME1 "foo.h5"
#define OUT_FNAME2 "bar.h5"

int get_h5_buffer(hid_t file, void** buf, size_t* len)
{
  int retval = EXIT_SUCCESS;
  assert(file != H5I_INVALID_HID);
  *buf = NULL;
  *len = 0;

  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_fapl_core(fapl, 4096, 0);

  ssize_t slen = H5Fget_file_image(file, NULL, 0);
  *len = (size_t) slen;
  *buf = H5allocate_memory(*len, 0);
  H5Fget_file_image(file, *buf, *len);
  H5Pclose(fapl);

  assert(*buf != NULL && *len > 0);
  return retval;
}

int main()
{
  int retval = EXIT_SUCCESS;

  // create an "HDF5 buffer"
  hid_t file = H5Fopen(INP_FNAME, H5F_ACC_RDONLY, H5P_DEFAULT);
  void* buf = NULL;
  size_t len = 0;
  get_h5_buffer(file, &buf, &len);
  printf("len = %lu\n", len);
  H5Fclose(file);

  // use the buffer to initalize a new file
  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
  H5Pset_fapl_core(fapl, 8192, 1);
  assert(H5Pset_file_image(fapl, buf, len) >= 0);
  H5free_memory(buf);

  file = H5Fopen(OUT_FNAME1, H5F_ACC_RDWR, fapl);
  H5Fflush(file, H5F_SCOPE_LOCAL);
  get_h5_buffer(file, &buf, &len);
  printf("len = %lu\n", len);
  H5Fclose(file);

  // and another one
  assert(H5Pset_file_image(fapl, buf, len) >= 0);
  H5free_memory(buf);
  file = H5Fopen(OUT_FNAME2, H5F_ACC_RDWR, fapl);
  H5Fclose(file);
  H5Pclose(fapl);

  return retval;
}

Works fine w/ develop and bar.h5 is created from foo.h5's buffer.

G.

file_image2.c (1.5 KB)


#19

I’m interested in M1, my application changes it (adding, removing and/or modifying data) but this is not relevant and that is why I omitted from my snippet. I came up with the bare minimum that reproduces my problem without sacrificing my workflow.


#20

What is the error message that you are seeing? G.