More general reader

The code below illustrates how to use the HDF5DotNet library to traverse the group/dataset structure of a file, and then how to read data from a dataset. The code presented assumes only a two-level scheme of groups and associated datasets. It also assumes 16-bit integer data. I am studying how to get the exact numeric data type from a file but like most of the things commonly needed, this does not seem to be easy to do with the HDF5 libraries at this time.

// HDF5 Generic File Access in C#
// Copyright (C) 2011 Morris Maynard
// All rights reserved
// Please give credit to the author when using this code.
using HDF5DotNet;

namespace HDF5
{
       public class H5Dataset
       {
              public H5GroupId id;
              public String Name;
              public ulong lIndex;
              public long[] dims;
              public H5Dataset(H5GroupId _idg, String strN, ulong _idx)
              { id = _idg; Name = strN; lIndex = _idx; dims = null; }
       }

       public class H5Group
       {
              public H5GroupId id;
              public H5FileId idFile;
              public String Name;
              public ulong lIndex;
              public List<H5Dataset> datasets;
              public H5Group(H5GroupId _id, String strN, ulong _idx) { id = _id; Name = strN; lIndex = _idx; datasets = null; }
       }

       // HDF File Reader
       // Usage example:
                     //String fileName = textBox1.Text;
                     //H5Group[] groups = HDF5.H5Reader.GetGroups(fileName);
                     //long[] dims = HDF5.H5Reader.GetDims(H5Group group, HDF5.H5Dataset set);
                     //Array arr = HDF5.H5Reader.ReadFrame(groups[0], nFrame: 0);
                     //Console.Write((Int16[,])arr)[0,1]);
       public class H5Reader
       {
              public static H5Group[] GetGroups(String fileName)
              {
                     H5FileId idFile = H5F.open(fileName, H5F.OpenMode.ACC_RDONLY);
                     H5GroupId idGroup = H5G.open(idFile, "/");
                     List<H5Group> groups = new List<H5Group>();
                     ulong lIndex = 0;
                     H5L.iterate(idGroup, H5IndexType.NAME, H5IterationOrder.INCREASING, ref lIndex, (H5GroupId iGroup, string gName, H5LinkInfo info, object objData) =>
                     {
                           H5Group newGroup = new H5Group(iGroup, gName, lIndex);
                           groups.Add(newGroup);
                           return H5IterationResult.SUCCESS;
                     }, null);
                     foreach (H5Group g in groups)
                           GetDatasets(g, idFile);

                     return groups.ToArray();
              }

              public static void GetDatasets(H5Group group, H5LocId idFileOrGroup)
              {
                     H5GroupId idGroup = null;
                     idGroup = H5G.open(idFileOrGroup, group.Name);

                     ulong lIndex = 0;
                     // Going to get all of the subgroups of this group (no recursion - 1 level)
                     // Assuming here that subgroups are the same as datasets
                     List<H5Dataset> datasets = new List<H5Dataset>();
                     H5L.iterate(idGroup, H5IndexType.NAME, H5IterationOrder.INCREASING, ref lIndex, (H5GroupId iGroup, string gName, H5LinkInfo info, object objData) =>
                     {
                           H5Dataset newDataset = new H5Dataset(iGroup, gName, lIndex);
                           newDataset.dims = GetDims(group, newDataset);
                           datasets.Add(newDataset);
                           return H5IterationResult.SUCCESS;
                     }, null);

                     group.datasets = datasets;
                     return; // all but intial call only accessible via linked-list
              }

              public static long[] GetDims(H5Group group, HDF5.H5Dataset set)
              {
                     H5DataSetId idDataSet =
                           H5D.open(group.id, group.Name + "/" + set.Name);
                     // Open Data Set in group
                     H5DataSpaceId idFileSpace = H5D.getSpace(idDataSet);
                     // Get Data Space Id for Data Set in file.
                     int nRank = H5S.getSimpleExtentNDims(idFileSpace);
                     // Get rank of data from data space
                     long[] dims = H5S.getSimpleExtentDims(idFileSpace);
                     return dims;
              }

              // don't use this one before the group is all set up
              public static long[] GetDims(H5Group group, int idxDataset)
              {
                     H5Dataset set = group.datasets[idxDataset];
                     return GetDims(group, group.datasets[idxDataset]);
              }

              public static Array ReadFrame(H5Group group, int dataset = 0, int nFrame = 0)
              {
                     H5DataSetId idDataSet =
                           H5D.open(group.id, group.Name + "/" + group.datasets[dataset].Name);
                     // Open Data Set in group
                     H5DataSpaceId idFileSpace = H5D.getSpace(idDataSet);
                     // Get Data Space Id for Data Set in file.
                     int nRank = H5S.getSimpleExtentNDims(idFileSpace);
                     // Get rank of data from data space
                     long[] dims = H5S.getSimpleExtentDims(idFileSpace);
                     long[] start, stride, count;
                     long nFrames, nx, ny;
                     if (nRank > 2) // dims are z y x here
                     {
                           nFrames = dims[0]; nx = dims[1]; ny = dims[2];
                           start = new long[] { nFrame, 0, 0 };
                           stride = new long[] { 1, 1, 1 };
                           count = new long[] { 1, nx, ny };
                     }
                     else // assume nRank == 2 !!
                     {
                           nFrames = 1; nx = dims[0]; ny = dims[1];
                           start = new long[] { nFrame };
                           stride = new long[] { 1 };
                           count = new long[] { 1 };
                     }

                     // setup counts and offsets to just read 1st "slice" of data
                     H5DataSpaceId idMemSpace = H5S.create_simple(nRank, count);
                     H5S.selectHyperslab(idMemSpace, H5S.SelectOperator.SET, new long[] { 0, 0, 0}, count);
                     idFileSpace = H5D.getSpace(idDataSet);
                     // select hyperslab in filespace
                     H5S.selectHyperslab(idFileSpace, H5S.SelectOperator.SET, start, count);

                     // ************* Assuming data is 16-bit integer here!!! **********
                     // Set up memory to read into
                     Int16[,] rdata = new Int16[nx, ny];
                     H5Array<Int16> udata = new H5Array<Int16>(rdata);

                     H5D.read(idDataSet, new H5DataTypeId(H5T.H5Type.NATIVE_SHORT), idMemSpace, idFileSpace,
                           new H5PropertyListId(new H5P.Template()), new H5Array<short>(rdata));
                     return rdata;
              }
       }
}

Morris, how are you? I'm not sure what you mean by the 'exact numeric data
type'.

In your example, you are using H5T_NATIVE_SHORT (in memory) which, using

the H5T API (H5Tget_size, H5Tget_sign etc.), can be parsed out as:

Name Value

---- -----

SignType 2

BitOffset 0

Class INTEGER

MSBitPadding Zero

Precision 16

LSBitPadding Zero

ByteOrder LE

Size 2

Yes, not all of these functions have been wrapped as part of HDF5DotNet, if
that's

your question. (Typically, only functions asked and paid for by our
customers have been

wrapped.)

Best, G.

···

From: hdf-forum-bounces@hdfgroup.org [mailto:hdf-forum-bounces@hdfgroup.org]
On Behalf Of Morris Maynard
Sent: Friday, August 19, 2011 3:59 PM
To: hdf-forum@hdfgroup.org
Subject: [Hdf-forum] More general reader

The code below illustrates how to use the HDF5DotNet library to traverse the
group/dataset structure of a file, and then how to read data from a dataset.
The code presented assumes only a two-level scheme of groups and associated
datasets. It also assumes 16-bit integer data. I am studying how to get the
exact numeric data type from a file but like most of the things commonly
needed, this does not seem to be easy to do with the HDF5 libraries at this
time.

// HDF5 Generic File Access in C#

// Copyright (C) 2011 Morris Maynard

// All rights reserved

// Please give credit to the author when using this code.

using HDF5DotNet;

namespace HDF5

{

       public class H5Dataset

       {

              public H5GroupId id;

              public String Name;

              public ulong lIndex;

              public long[] dims;

              public H5Dataset(H5GroupId _idg, String strN, ulong _idx)

              { id = _idg; Name = strN; lIndex = _idx; dims = null; }

       }

       public class H5Group

       {

              public H5GroupId id;

              public H5FileId idFile;

              public String Name;

              public ulong lIndex;

              public List<H5Dataset> datasets;

              public H5Group(H5GroupId _id, String strN, ulong _idx) { id =
_id; Name = strN; lIndex = _idx; datasets = null; }

       }

       // HDF File Reader

       // Usage example:

                     //String fileName = textBox1.Text;

                     //H5Group[] groups = HDF5.H5Reader.GetGroups(fileName);

                     //long[] dims = HDF5.H5Reader.GetDims(H5Group group,
HDF5.H5Dataset set);

                     //Array arr = HDF5.H5Reader.ReadFrame(groups[0],
nFrame: 0);

                     //Console.Write((Int16[,])arr)[0,1]);

       public class H5Reader

       {

              public static H5Group[] GetGroups(String fileName)

              {

                     H5FileId idFile = H5F.open(fileName,
H5F.OpenMode.ACC_RDONLY);

                     H5GroupId idGroup = H5G.open(idFile, "/");

                     List<H5Group> groups = new List<H5Group>();

                     ulong lIndex = 0;

                     H5L.iterate(idGroup, H5IndexType.NAME,
H5IterationOrder.INCREASING, ref lIndex, (H5GroupId iGroup, string gName,
H5LinkInfo info, object objData) =>

                     {

                           H5Group newGroup = new H5Group(iGroup, gName,
lIndex);

                           groups.Add(newGroup);

                           return H5IterationResult.SUCCESS;

                     }, null);

                     foreach (H5Group g in groups)

                           GetDatasets(g, idFile);

                     return groups.ToArray();

              }

              public static void GetDatasets(H5Group group, H5LocId
idFileOrGroup)

              {

                     H5GroupId idGroup = null;

                     idGroup = H5G.open(idFileOrGroup, group.Name);

                     ulong lIndex = 0;

                     // Going to get all of the subgroups of this group (no
recursion - 1 level)

                     // Assuming here that subgroups are the same as
datasets

                     List<H5Dataset> datasets = new List<H5Dataset>();

                     H5L.iterate(idGroup, H5IndexType.NAME,
H5IterationOrder.INCREASING, ref lIndex, (H5GroupId iGroup, string gName,
H5LinkInfo info, object objData) =>

                     {

                           H5Dataset newDataset = new H5Dataset(iGroup,
gName, lIndex);

                           newDataset.dims = GetDims(group, newDataset);

                           datasets.Add(newDataset);

                           return H5IterationResult.SUCCESS;

                     }, null);

                     group.datasets = datasets;

                     return; // all but intial call only accessible via
linked-list

              }

              public static long[] GetDims(H5Group group, HDF5.H5Dataset
set)

              {

                     H5DataSetId idDataSet =

                           H5D.open(group.id, group.Name + "/" + set.Name);

                     // Open Data Set in group

                     H5DataSpaceId idFileSpace = H5D.getSpace(idDataSet);

                     // Get Data Space Id for Data Set in file.

                     int nRank = H5S.getSimpleExtentNDims(idFileSpace);

                     // Get rank of data from data space

                     long[] dims = H5S.getSimpleExtentDims(idFileSpace);

                     return dims;

              }

              // don't use this one before the group is all set up

              public static long[] GetDims(H5Group group, int idxDataset)

              {

                     H5Dataset set = group.datasets[idxDataset];

                     return GetDims(group, group.datasets[idxDataset]);

              }

              public static Array ReadFrame(H5Group group, int dataset = 0,
int nFrame = 0)

              {

                     H5DataSetId idDataSet =

                           H5D.open(group.id, group.Name + "/" +
group.datasets[dataset].Name);

                     // Open Data Set in group

                     H5DataSpaceId idFileSpace = H5D.getSpace(idDataSet);

                     // Get Data Space Id for Data Set in file.

                     int nRank = H5S.getSimpleExtentNDims(idFileSpace);

                     // Get rank of data from data space

                     long[] dims = H5S.getSimpleExtentDims(idFileSpace);

                     long[] start, stride, count;

                     long nFrames, nx, ny;

                     if (nRank > 2) // dims are z y x here

                     {

                           nFrames = dims[0]; nx = dims[1]; ny = dims[2];

                           start = new long[] { nFrame, 0, 0 };

                           stride = new long[] { 1, 1, 1 };

                           count = new long[] { 1, nx, ny };

                     }

                     else // assume nRank == 2 !!

                     {

                           nFrames = 1; nx = dims[0]; ny = dims[1];

                           start = new long[] { nFrame };

                           stride = new long[] { 1 };

                           count = new long[] { 1 };

                     }

                     // setup counts and offsets to just read 1st "slice" of
data

                     H5DataSpaceId idMemSpace = H5S.create_simple(nRank,
count);

                     H5S.selectHyperslab(idMemSpace, H5S.SelectOperator.SET,
new long[] { 0, 0, 0}, count);

                     idFileSpace = H5D.getSpace(idDataSet);

                     // select hyperslab in filespace

                     H5S.selectHyperslab(idFileSpace,
H5S.SelectOperator.SET, start, count);

                     // ************* Assuming data is 16-bit integer
here!!! **********

                     // Set up memory to read into

                     Int16[,] rdata = new Int16[nx, ny];

                     H5Array<Int16> udata = new H5Array<Int16>(rdata);

                     H5D.read(idDataSet, new
H5DataTypeId(H5T.H5Type.NATIVE_SHORT), idMemSpace, idFileSpace,

                           new H5PropertyListId(new H5P.Template()), new
H5Array<short>(rdata));

                     return rdata;

              }

       }

}

In a somewhat broader sense, using the .NET Type class, it's relatively easy to convert primitive types to HDF H5T.H5Types at runtime.
switch (primitiveType.Name)
{
case "Char":
     hdfType = H5T.H5Type.NATIVE_UCHAR;
     break;
...
}

The reverse can be done with a bit more work and some caveats. In the current .NET wrappers, You can get H5T.H5TClass, size, and H5T.Sign. From there you can derive most types... H5T.H5TClass.FLOAT is typeof(float) when size == 4, typeof(double) when size == 8, and so on. The problem children are chars vs. 1 byte ints, and bools. There is no way in HDF5.NET to differentiate a 1 byte integer from a char (signed or unsigned).

Booleans are an additional complication because the H5T_NATIVE_HBOOL results in a 4 byte integer in storage. And aside from spacewise overkill, you end up with the same problem as with chars & 1 byte ints. For my own code, I decided Booleans were stored as 1 byte signed, 1 byte unsigned as bytes, and ignored chars. I also have a special case for time values.

This is a start of code that can dynamically generate H5TClass.Compound structures to/from .NET structs using reflection. I've found it very useful. If anyone would like to pursue this let me know. I cannot provide code, but I can assist.

Gerd, can any of these other values help differentiate between ints and the others?

Scott

···

From: hdf-forum-bounces@hdfgroup.org [mailto:hdf-forum-bounces@hdfgroup.org] On Behalf Of Gerd Heber
Sent: Monday, August 22, 2011 8:32 AM
To: 'HDF Users Discussion List'
Subject: Re: [Hdf-forum] More general reader

Morris, how are you? I'm not sure what you mean by the 'exact numeric data type'.
In your example, you are using H5T_NATIVE_SHORT (in memory) which, using
the H5T API (H5Tget_size, H5Tget_sign etc.), can be parsed out as:

Name Value
---- -----
SignType 2
BitOffset 0
Class INTEGER
MSBitPadding Zero
Precision 16
LSBitPadding Zero
ByteOrder LE
Size 2

Yes, not all of these functions have been wrapped as part of HDF5DotNet, if that's
your question. (Typically, only functions asked and paid for by our customers have been
wrapped.)

Best, G.

From: hdf-forum-bounces@hdfgroup.org [mailto:hdf-forum-bounces@hdfgroup.org] On Behalf Of Morris Maynard
Sent: Friday, August 19, 2011 3:59 PM
To: hdf-forum@hdfgroup.org
Subject: [Hdf-forum] More general reader

The code below illustrates how to use the HDF5DotNet library to traverse the group/dataset structure of a file, and then how to read data from a dataset. The code presented assumes only a two-level scheme of groups and associated datasets. It also assumes 16-bit integer data. I am studying how to get the exact numeric data type from a file but like most of the things commonly needed, this does not seem to be easy to do with the HDF5 libraries at this time.

// HDF5 Generic File Access in C#
// Copyright (C) 2011 Morris Maynard
// All rights reserved
// Please give credit to the author when using this code.
using HDF5DotNet;

namespace HDF5
{
       public class H5Dataset
       {
              public H5GroupId id;
              public String Name;
              public ulong lIndex;
              public long[] dims;
              public H5Dataset(H5GroupId _idg, String strN, ulong _idx)
              { id = _idg; Name = strN; lIndex = _idx; dims = null; }
       }

       public class H5Group
       {
              public H5GroupId id;
              public H5FileId idFile;
              public String Name;
              public ulong lIndex;
              public List<H5Dataset> datasets;
              public H5Group(H5GroupId _id, String strN, ulong _idx) { id = _id; Name = strN; lIndex = _idx; datasets = null; }
       }

       // HDF File Reader
       // Usage example:
                     //String fileName = textBox1.Text;
                     //H5Group[] groups = HDF5.H5Reader.GetGroups(fileName);
                     //long[] dims = HDF5.H5Reader.GetDims(H5Group group, HDF5.H5Dataset set);
                     //Array arr = HDF5.H5Reader.ReadFrame(groups[0], nFrame: 0);
                     //Console.Write((Int16[,])arr)[0,1]);
       public class H5Reader
       {
              public static H5Group[] GetGroups(String fileName)
              {
                     H5FileId idFile = H5F.open(fileName, H5F.OpenMode.ACC_RDONLY);
                     H5GroupId idGroup = H5G.open(idFile, "/");
                     List<H5Group> groups = new List<H5Group>();
                     ulong lIndex = 0;
                     H5L.iterate(idGroup, H5IndexType.NAME, H5IterationOrder.INCREASING, ref lIndex, (H5GroupId iGroup, string gName, H5LinkInfo info, object objData) =>
                     {
                           H5Group newGroup = new H5Group(iGroup, gName, lIndex);
                           groups.Add(newGroup);
                           return H5IterationResult.SUCCESS;
                     }, null);
                     foreach (H5Group g in groups)
                           GetDatasets(g, idFile);

                     return groups.ToArray();
              }

              public static void GetDatasets(H5Group group, H5LocId idFileOrGroup)
              {
                     H5GroupId idGroup = null;
                     idGroup = H5G.open(idFileOrGroup, group.Name);

                     ulong lIndex = 0;
                     // Going to get all of the subgroups of this group (no recursion - 1 level)
                     // Assuming here that subgroups are the same as datasets
                     List<H5Dataset> datasets = new List<H5Dataset>();
                     H5L.iterate(idGroup, H5IndexType.NAME, H5IterationOrder.INCREASING, ref lIndex, (H5GroupId iGroup, string gName, H5LinkInfo info, object objData) =>
                     {
                           H5Dataset newDataset = new H5Dataset(iGroup, gName, lIndex);
                           newDataset.dims = GetDims(group, newDataset);
                           datasets.Add(newDataset);
                           return H5IterationResult.SUCCESS;
                     }, null);

                     group.datasets = datasets;
                     return; // all but intial call only accessible via linked-list
              }

              public static long[] GetDims(H5Group group, HDF5.H5Dataset set)
              {
                     H5DataSetId idDataSet =
                           H5D.open(group.id, group.Name + "/" + set.Name);
                     // Open Data Set in group
                     H5DataSpaceId idFileSpace = H5D.getSpace(idDataSet);
                     // Get Data Space Id for Data Set in file.
                     int nRank = H5S.getSimpleExtentNDims(idFileSpace);
                     // Get rank of data from data space
                     long[] dims = H5S.getSimpleExtentDims(idFileSpace);
                     return dims;
              }

              // don't use this one before the group is all set up
              public static long[] GetDims(H5Group group, int idxDataset)
              {
                     H5Dataset set = group.datasets[idxDataset];
                     return GetDims(group, group.datasets[idxDataset]);
              }

              public static Array ReadFrame(H5Group group, int dataset = 0, int nFrame = 0)
              {
                     H5DataSetId idDataSet =
                           H5D.open(group.id, group.Name + "/" + group.datasets[dataset].Name);
                     // Open Data Set in group
                     H5DataSpaceId idFileSpace = H5D.getSpace(idDataSet);
                     // Get Data Space Id for Data Set in file.
                     int nRank = H5S.getSimpleExtentNDims(idFileSpace);
                     // Get rank of data from data space
                     long[] dims = H5S.getSimpleExtentDims(idFileSpace);
                     long[] start, stride, count;
                     long nFrames, nx, ny;
                     if (nRank > 2) // dims are z y x here
                     {
                           nFrames = dims[0]; nx = dims[1]; ny = dims[2];
                           start = new long[] { nFrame, 0, 0 };
                           stride = new long[] { 1, 1, 1 };
                           count = new long[] { 1, nx, ny };
                     }
                     else // assume nRank == 2 !!
                     {
                           nFrames = 1; nx = dims[0]; ny = dims[1];
                           start = new long[] { nFrame };
                           stride = new long[] { 1 };
                           count = new long[] { 1 };
                     }

                     // setup counts and offsets to just read 1st "slice" of data
                     H5DataSpaceId idMemSpace = H5S.create_simple(nRank, count);
                     H5S.selectHyperslab(idMemSpace, H5S.SelectOperator.SET, new long[] { 0, 0, 0}, count);
                     idFileSpace = H5D.getSpace(idDataSet);
                     // select hyperslab in filespace
                     H5S.selectHyperslab(idFileSpace, H5S.SelectOperator.SET, start, count);

                     // ************* Assuming data is 16-bit integer here!!! **********
                     // Set up memory to read into
                     Int16[,] rdata = new Int16[nx, ny];
                     H5Array<Int16> udata = new H5Array<Int16>(rdata);

                     H5D.read(idDataSet, new H5DataTypeId(H5T.H5Type.NATIVE_SHORT), idMemSpace, idFileSpace,
                           new H5PropertyListId(new H5P.Template()), new H5Array<short>(rdata));
                     return rdata;
              }
       }
}

________________________________
This e-mail and any files transmitted with it may be proprietary and are intended solely for the use of the individual or entity to whom they are addressed. If you have received this e-mail in error please notify the sender.
Please note that any views or opinions presented in this e-mail are solely those of the author and do not necessarily represent those of ITT Corporation. The recipient should check this e-mail and any attachments for the presence of viruses. ITT accepts no liability for any damage caused by any virus transmitted by this e-mail.