Hello,
Can anyone tell me where I am going wrong? I have a dataset that is too big to fit into ram, so I need to read a file, processes it and write the output in sections. I have previously been doing this with fixed width records, but this has proven too inflexible for my needs, hence I thought I would try to use HDF5.
I have created a little test program below, where I simply try to select each element in the file one at a time using hyperslab, and attempt to write to that -- however this is causing problems (see end of message) when I try to do this.
I am new to the HDF5 API, and may not fully understand what I am doing.
Thanks.
···
====
#include <iostream>
#include <hdf5.h>
using namespace std;
int main()
{
hid_t fileId,vlenDataTypeId,dataTypeId,dataSetId,
dataSpaceId,pListId;
//Try to create the output file
fileId = H5Fcreate("det.hdf5",H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT);
if(!fileId)
{
cerr << "Error opening file" << endl;
return 1;
}
//Create a variable length data type to store TOI info
vlenDataTypeId = H5Tvlen_create(H5T_NATIVE_FLOAT);
const unsigned int HIT_OFFSET_X=0;
const unsigned int HIT_OFFSET_Y=HIT_OFFSET_X+4;
const unsigned int HIT_OFFSET_TOIARRAY=HIT_OFFSET_Y+4;
const unsigned int HIT_OFFSET_VOLTAGE=HIT_OFFSET_TOIARRAY+sizeof(hvl_t);
const unsigned int HIT_OFFSET_PULSENUM=HIT_OFFSET_VOLTAGE+4;
const unsigned int HIT_OFFSET_EVENTTYPE=HIT_OFFSET_PULSENUM+8;
const unsigned int HIT_SIZE=HIT_OFFSET_EVENTTYPE+2;
dataTypeId = H5Tcreate(H5T_COMPOUND,HIT_SIZE);
if(!dataTypeId)
cerr << "Some kind of error with datatypeID" << endl;
if(H5Tinsert(dataTypeId,"x",HIT_OFFSET_X,H5T_NATIVE_FLOAT))
{
cerr << "Err inserting x into type" << endl;
return 1;
}
if(H5Tinsert(dataTypeId,"y",HIT_OFFSET_Y,H5T_NATIVE_FLOAT))
{
cerr << "Err inserting y into type" << endl;
return 1;
}
if(H5Tinsert(dataTypeId,"toiArray",HIT_OFFSET_TOIARRAY,vlenDataTypeId))
{
cerr << "Err inserting toiArray into type" << endl;
return 1;
}
if(H5Tinsert(dataTypeId,"voltage",HIT_OFFSET_VOLTAGE,H5T_NATIVE_FLOAT))
{
cerr << "Err inserting voltage into type" << endl;
return 1;
}
if(H5Tinsert(dataTypeId,"pulseNum",HIT_OFFSET_PULSENUM,H5T_NATIVE_ULLONG))
{
cerr << "Err inserting pulseNum into type" << endl;
return 1;
}
if(H5Tinsert(dataTypeId,"eventType",HIT_OFFSET_EVENTTYPE,H5T_NATIVE_SHORT))
{
cerr << "Err inserting eventType into type" << endl;
return 1;
}
hsize_t columnDim=5;
dataSpaceId=H5Screate_simple(1,&columnDim,NULL);
pListId = H5Pcreate(H5P_LINK_CREATE); //The example has this as H5P_DATASET_CREATE, but HDF libs doesn't like it
dataSetId = H5Dcreate2(fileId,"/hitdata",dataTypeId,dataSpaceId,pListId,NULL,NULL);
//Create the input array
char *buffer = new char[HIT_SIZE];
float *timeData=new float;
for(unsigned int ui=0;ui<5; ui++)
{
*timeData=1.2345f;
*((float *)(buffer+HIT_OFFSET_X)) = 0;
*((float *)(buffer+HIT_OFFSET_Y)) = 1;
((hvl_t*)(buffer+HIT_OFFSET_TOIARRAY))->len = 1;
((hvl_t*)(buffer+HIT_OFFSET_TOIARRAY))->p= timeData;
*((float *)(buffer+HIT_OFFSET_VOLTAGE)) = 2;
*((unsigned long long*)(buffer+HIT_OFFSET_PULSENUM)) =3;
*((unsigned short*)(buffer+HIT_OFFSET_EVENTTYPE)) = 4;
hsize_t pos,count;
pos = ui+1;
count=1;
if( H5Sselect_hyperslab(dataSpaceId,H5S_SELECT_SET, &pos,NULL, &count, NULL) < 0)
{
cerr << "Hyperslab selection problem" << endl;
return 1;
}
cerr << "Writing data: " << endl;
if(H5Dwrite(dataSetId,dataTypeId,H5S_ALL,H5S_ALL,H5P_DEFAULT, buffer))
{
cerr << "Error writing data" << endl;
return 1;
}
}
delete timeData;
delete[] buffer;
H5Pclose(pListId);
H5Dclose(dataSetId);
H5Sclose(dataSpaceId);
H5Tclose(vlenDataTypeId);
H5Tclose(dataTypeId);
H5Fclose(fileId);
return 0;
}
./main
Writing data:
HDF5-DIAG: Error detected in HDF5 (1.8.4) thread 0:
#000: H5Dio.c line 266 in H5Dwrite(): can't write data
major: Dataset
minor: Write failed
#001: H5Dio.c line 578 in H5D_write(): can't write data
major: Dataset
minor: Write failed
#002: H5Dcontig.c line 557 in H5D_contig_write(): contiguous write failed
major: Dataset
minor: Write failed
#003: H5Dscatgath.c line 677 in H5D_scatgath_write(): datatype conversion failed
major: Dataset
minor: Can't convert datatypes
#004: H5T.c line 4704 in H5T_convert(): data type conversion failed
major: Attribute
minor: Unable to encode value
#005: H5Tconv.c line 2470 in H5T_conv_struct_opt(): unable to convert compound datatype member
major: Datatype
minor: Unable to initialize object
#006: H5T.c line 4704 in H5T_convert(): data type conversion failed
major: Attribute
minor: Unable to encode value
#007: H5Tconv.c line 3140 in H5T_conv_vlen(): can't write VL data
major: Datatype
minor: Write failed
#008: H5Tvlen.c line 1015 in H5T_vlen_disk_write(): Unable to write VL information
major: Datatype
minor: Write failed
#009: H5HG.c line 624 in H5HG_insert(): unable to allocate a global heap collection
major: Heap
minor: Unable to initialize object
#010: H5HG.c line 182 in H5HG_create(): unable to allocate file space for global heap
major: Heap
minor: Unable to initialize object
#011: H5MF.c line 488 in H5MF_alloc(): allocation failed from aggr/vfd
major: Virtual File Layer
minor: Can't allocate space
#012: H5MFaggr.c line 114 in H5MF_aggr_vfd_alloc(): can't allocate metadata
major: Resource unavailable
minor: Can't allocate space
#013: H5MFaggr.c line 219 in H5MF_aggr_alloc(): 'normal' file space allocation request will overlap into 'temporary' file space
major: Resource unavailable
minor: Out of range
Error writing data
