H5Dwrite fails to write extendible dataset


#1

I want to write a function that will append a buffer to an extendible HDF5 dataset. The buffer will always have the same size (NXN) during the program execution. Following this brilliant answer https://stackoverflow.com/questions/15379399/writing-appending-arrays-of-float-to-the-only-dataset-in-hdf5-file-in-c I have written the following code.

MAIN


#include <iostream>
#include <H5Cpp.h>
#include "Myclass.hpp"

const char saveFilePath[] = "test.h5";

int main()
{  

  const hsize_t nrows = 2;
  const hsize_t ncols = 2;
  const hsize_t nghost = 1;

  Myclass hfield(ncols,nrows,nghost); // the dimension of the buffer is 4X4
  hfield(0,0) = 1;
  hfield(1,0) = 2;
  hfield(0,1) = 3;
  hfield(1,1) = 4;

  hid_t file = H5Fcreate(saveFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
  std::cout << "- File created" << std::endl;

  write_to_extendible_H5("test.h5", hfield);
  write_to_extendible_H5("test.h5", hfield);

  return 0;
}

FUNCTION

void write_to_extendible_H5(const char* FILENAME, Myclass& hfield)
{
  hsize_t ndims = 2;
  hsize_t nrows = hfield.getrows() + 2*hfield.getnghost();
  hsize_t ncols = hfield.getcols() + 2*hfield.getnghost();

  /* Create a memory dataspace to indicate the 
     size of our buffer to be written in memory. */

  hsize_t mbuff_dims[ndims];
  mbuff_dims[0] = nrows;
  mbuff_dims[1] = ncols;
  hid_t mem_space = H5Screate_simple(ndims, mbuff_dims, NULL);
  std::cout << "- Memory dataspace created" << std::endl;
 
  /* Open the file. */
  
  hid_t file = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT);

  /* Check if there is a dataset. */
  
  if ( !H5Lexists(file,"dset1",H5P_DEFAULT) )
    {
      /* Dataset does not exist. Create it and
	 write the first buffer. */
      
      // Create a 2D dataspace.
      
      hsize_t dims[ndims] = {nrows, ncols};
      hsize_t max_dims[ndims] = {H5S_UNLIMITED, ncols};
      hid_t file_space = H5Screate_simple(ndims, dims, max_dims);
      std::cout << "- Dataspace created" << std::endl;
      
      // Then create a dataset creation property list.  
	      
      hid_t plist = H5Pcreate(H5P_DATASET_CREATE);
      H5Pset_layout(plist, H5D_CHUNKED);
      hsize_t chunk_dims[ndims] = {nrows, ncols};
      H5Pset_chunk(plist, ndims, chunk_dims);
      std::cout << "- Property list created" << std::endl;
      
      // Create the dataset.
      
      hid_t dset = H5Dcreate(file, "dset1", H5T_NATIVE_DOUBLE,
			     file_space, H5P_DEFAULT, plist, H5P_DEFAULT);
      std::cout << "- Dataset 'dset1' created" << std::endl;
      
      /* Close resources. */

      H5Pclose(plist);
      H5Sclose(file_space);

      /* Write the first buffer */
      
      // Select hyperslab on file dataset.
      
      file_space = H5Dget_space(dset);
      hsize_t start[2] = {0, 0};
      hsize_t count[2] = {nrows, ncols};
      H5Sselect_hyperslab(file_space, H5S_SELECT_SET, start,
			  NULL, count, NULL);
      std::cout << "- First hyperslab selected" << std::endl;
      
      /* Write buffer to dataset. */
      
      H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space,
	       H5P_DEFAULT, hfield.getmemory());
      std::cout << "- First buffer written" << std::endl;
      
      /*We can now close the file dataspace. */
      
      H5Sclose(file_space);      
    }
  else
    {
      /* Dataset already exists. Extend it and write 
	 the next buffer. */

      // Open the dataset and get the dimensions of the existing dataset.
      
      hid_t dset = H5Dopen(file,"dset1",H5P_DEFAULT);

      hid_t file_space = H5Dget_space(dset);
      hsize_t dims[ndims];
      H5Sget_simple_extent_dims(file_space, dims, NULL);
      std::cout << "- The dataset dimensions before extension are:" << std::endl;
      std::cout << "No of rows: " << dims[0] << std::endl;
      std::cout << "No of cols: " << dims[1] << std::endl;
      
      // Extend the dimensions.

      dims[0] += nrows;
      dims[1] = ncols;
      H5Dset_extent(dset, dims);
      std::cout << "- Dataset extended" << std::endl;
      
      // Select hyperslab
      
      hsize_t start[2] = {dims[0]-nrows, 0};
      hsize_t count[2] = {nrows, ncols};
      H5Sselect_hyperslab(file_space, H5S_SELECT_SET, start,
			  NULL, count, NULL);
      std::cout << "- Next hyperslab selected" << std::endl;
	  
      // Write buffer
      
      H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space,
       	       H5P_DEFAULT, hfield.getmemory());
      std::cout << "- Next buffer written" << std::endl;
      
      /*We can now close the file dataspace. */
      
      H5Sclose(file_space);      
    } 
}

The output from h5dump is:

HDF5 "test.h5" {
GROUP "/" {
   DATASET "dset1" {
      DATATYPE  H5T_IEEE_F64LE
      DATASPACE  SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, 4 ) }
      DATA {
      (0,0): 0, 0, 0, 0,
      (1,0): 0, 1, 2, 0,
      (2,0): 0, 3, 4, 0,
      (3,0): 0, 0, 0, 0,
      (4,0): 0, 0, 0, 0,
      (5,0): 0, 0, 0, 0,
      (6,0): 0, 0, 0, 0,
      (7,0): 0, 0, 0, 0
      }
   }
}
}

And the console gives:

- File created
- Memory dataspace created
- Dataspace created
- Property list created
- Dataset 'dset1' created
- First hyperslab selected
- First buffer written
- Memory dataspace created
- The dataset dimensions before extension are:
No of rows: 4
No of cols: 4
- Dataset extended
- Next hyperslab selected
HDF5-DIAG: Error detected in HDF5 (1.12.0) thread 0:
  #000: H5Dio.c line 314 in H5Dwrite(): can't write data
    major: Dataset
    minor: Write failed
  #001: H5VLcallback.c line 2186 in H5VL_dataset_write(): dataset write failed
    major: Virtual Object Layer
    minor: Write failed
  #002: H5VLcallback.c line 2152 in H5VL__dataset_write(): dataset write failed
    major: Virtual Object Layer
    minor: Write failed
  #003: H5VLnative_dataset.c line 203 in H5VL__native_dataset_write(): could not get a validated dataspace from file_space_id
    major: Invalid arguments to routine
    minor: Bad value
  #004: H5S.c line 279 in H5S_get_validated_dataspace(): selection + offset not within extent
    major: Dataspace
    minor: Out of range
- Next buffer written

Can you please tell me why is the second buffer not written?


#2

Is this question so obvious? Don’t understand why nobody cares to answer…

UPDATE:

Anyway, I found the answer. After extending the dataset with H5Dset_extent(dset, dims); one must set file_space = H5Dget_space(dset);


#3

High throughout low latency event processing is not quite obvious, here are my slides on the topic and here is the link to h5::append operator that makes it happen.

If you are interested in H5CPP, an idiomatic approach to HDF5 and C++ compiler assisted persistence please check out the H5CPP booth at the upcoming and MeetingC++ online Tool Fair and I will answer your questions in person.

H5CPP is a new approach to C++ persistence, with linguistic features much similar to python , but without the performance trade offs. It is the result of the collaboration between Gerd Heber HDFGroup, The leadership of the HDFGroup and my tiny research/consulting company.

It is in the economics, in order to answer your question someone has to put work into it – which requires motivation. In your case, even if I solved your problem – with your approach and the library you are using; I would gain little, if any.
If you are interested in my work, which solves the exact problem – with angles covered you may have missed; would change the outcome: your input could improve on H5CPP, benefiting others.

best wishes: steven