parallel hdf5 with independent I/O to different data sets

Hi all,

I am having trouble with the following piece of code (I hacked apart one of
the parallel HDF5 tutorial examples and changed the API to 1.8.4).

I am trying get each process to write to a different data set simultaneously
(but all data sets are in the same file) using independent I/O. I suspect
this may not be possible, but the documentation seems to say otherwise (see
http://www.hdfgroup.org/HDF5/Tutor/pprog.html, at the bottom it says "Each
process writes to a individual dataset.").

I have enforced that the data sets are created collectively, but the
simultaneous write fails.

Do you all know why this does not work?

The code is listed below.

Cheers,
Matt

/*
* This example writes data to the HDF5 file.
* Number of processes is assumed to be 1 or multiples of 2 (up to 8)
*/
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <assert.h>
#include <mpi.h>
#include <hdf5.h>

#define H5FILE_NAME "SDS.h5"
#define DATASETNAME "IntArray"
#define NX 8 /* dataset dimensions */
#define NY 5
#define RANK 2

int main (int argc, char **argv)
{
    /*
     * HDF5 APIs definitions
     */
    hid_t file_id, dset_id, group_id; /* file and dataset
identifiers */
    hid_t filespace; /* file and memory dataspace identifiers */
    hsize_t dimsf[] = {NX, NY}; /* dataset dimensions */
    int *data; /* pointer to data buffer to write
*/
    hid_t plist_id; /* property list identifier */
    int i;
    herr_t status;
    char name[500];

    /*
     * MPI variables
     */
    int mpi_size, mpi_rank;
    MPI_Comm comm = MPI_COMM_WORLD;
    MPI_Info info = MPI_INFO_NULL;

    /*
     * Initialize MPI
     */
    MPI_Init(&argc, &argv);
    MPI_Comm_size(comm, &mpi_size);
    MPI_Comm_rank(comm, &mpi_rank);

    /*
     * Initialize data buffer
     */
    data = (int *) malloc(sizeof(int)*dimsf[0]*
dimsf[1]);
    for (i=0; i < dimsf[0]*dimsf[1]; i++) {
        data[i] = mpi_rank;
    }
    /*
     * Set up file access property list with parallel I/O access
     */
    plist_id = H5Pcreate(H5P_FILE_ACCESS);
    H5Pset_fapl_mpio(plist_id, comm, info);

    /*
     * Create a new file collectively and release property list identifier.
     */
    file_id = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
    H5Pclose(plist_id);

    /*
     * Create property list for collective dataset write.
     */
    plist_id = H5Pcreate(H5P_DATASET_XFER);
    //H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);
    H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT);

    /*
     * Create the dataspace for the dataset.
     */
    filespace = H5Screate_simple(RANK, dimsf, NULL);

    for(i=0;i<mpi_size;++i)
      {
    MPI_Barrier(MPI_COMM_WORLD);
    sprintf(name,"node%d",i);
    dset_id = H5Dcreate(file_id, name, H5T_NATIVE_INT, filespace,
H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
    H5Dclose(dset_id);
    MPI_Barrier(MPI_COMM_WORLD);
      }
    MPI_Barrier(MPI_COMM_WORLD);

    sprintf(name,"node%d",i);
    dset_id = H5Dopen(file_id, name, H5P_DEFAULT);
    status = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, plist_id,
data);
    H5Dclose(dset_id);

    free(data);

    /*
     * Close/release resources.
     */
    H5Sclose(filespace);
    H5Pclose(plist_id);
    H5Fclose(file_id);

    MPI_Finalize();

    return 0;
}

Hi Matthew, have you tried using the MPI-POSIX VFD instead? Mark

···

On Tue, Mar 2, 2010 at 3:56 PM, Matthew Becker <beckermr@uchicago.edu> wrote:

Hi all,

I am having trouble with the following piece of code (I hacked apart one of
the parallel HDF5 tutorial examples and changed the API to 1.8.4).

I am trying get each process to write to a different data set simultaneously
(but all data sets are in the same file) using independent I/O. I suspect
this may not be possible, but the documentation seems to say otherwise (see
http://www.hdfgroup.org/HDF5/Tutor/pprog.html, at the bottom it says "Each
process writes to a individual dataset.").

I have enforced that the data sets are created collectively, but the
simultaneous write fails.

Do you all know why this does not work?

The code is listed below.

Cheers,
Matt

/*
* This example writes data to the HDF5 file.
* Number of processes is assumed to be 1 or multiples of 2 (up to 8)
*/
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <assert.h>
#include <mpi.h>
#include <hdf5.h>

#define H5FILE_NAME "SDS.h5"
#define DATASETNAME "IntArray"
#define NX 8 /* dataset dimensions */
#define NY 5
#define RANK 2

int main (int argc, char **argv)
{
/*
* HDF5 APIs definitions
*/
hid_t file_id, dset_id, group_id; /* file and dataset
identifiers */
hid_t filespace; /* file and memory dataspace identifiers */
hsize_t dimsf[] = {NX, NY}; /* dataset dimensions */
int *data; /* pointer to data buffer to write
*/
hid_t plist_id; /* property list identifier */
int i;
herr_t status;
char name[500];

/\*
 \* MPI variables
 \*/
int mpi\_size, mpi\_rank;
MPI\_Comm comm  = MPI\_COMM\_WORLD;
MPI\_Info info  = MPI\_INFO\_NULL;

/\*
 \* Initialize MPI
 \*/
MPI\_Init\(&amp;argc, &amp;argv\);
MPI\_Comm\_size\(comm, &amp;mpi\_size\);
MPI\_Comm\_rank\(comm, &amp;mpi\_rank\);

/\*
 \* Initialize data buffer
 \*/
data = \(int \*\) malloc\(sizeof\(int\)\*dimsf\[0\]\*

dimsf[1]);
for (i=0; i < dimsf[0]*dimsf[1]; i++) {
data[i] = mpi_rank;
}
/*
* Set up file access property list with parallel I/O access
*/
plist_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist_id, comm, info);

/\*
 \* Create a new file collectively and release property list identifier\.
 \*/
file\_id = H5Fcreate\(H5FILE\_NAME, H5F\_ACC\_TRUNC, H5P\_DEFAULT, plist\_id\);
H5Pclose\(plist\_id\);

/\*
 \* Create property list for collective dataset write\.
 \*/
plist\_id = H5Pcreate\(H5P\_DATASET\_XFER\);
//H5Pset\_dxpl\_mpio\(plist\_id, H5FD\_MPIO\_COLLECTIVE\);
H5Pset\_dxpl\_mpio\(plist\_id, H5FD\_MPIO\_INDEPENDENT\);

/\*
 \* Create the dataspace for the dataset\.
 \*/
filespace = H5Screate\_simple\(RANK, dimsf, NULL\);

for\(i=0;i&lt;mpi\_size;\+\+i\)
  \{
MPI\_Barrier\(MPI\_COMM\_WORLD\);
sprintf\(name,&quot;node%d&quot;,i\);
dset\_id = H5Dcreate\(file\_id, name, H5T\_NATIVE\_INT, filespace,

H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Dclose(dset_id);
MPI_Barrier(MPI_COMM_WORLD);
}
MPI_Barrier(MPI_COMM_WORLD);

sprintf\(name,&quot;node%d&quot;,i\);
dset\_id = H5Dopen\(file\_id, name, H5P\_DEFAULT\);
status = H5Dwrite\(dset\_id, H5T\_NATIVE\_INT, H5S\_ALL, H5S\_ALL, plist\_id,

data);
H5Dclose(dset_id);

free\(data\);

/\*
 \* Close/release resources\.
 \*/
H5Sclose\(filespace\);
H5Pclose\(plist\_id\);
H5Fclose\(file\_id\);

MPI\_Finalize\(\);

return 0;

}
_______________________________________________
Hdf-forum is for HDF software users discussion.
Hdf-forum@hdfgroup.org
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org

Hi Mark and all,

Sorry for the spam. This was a simple bug. It works with both mpio and
mpiposix io. I still have a question.

I have been through the tutorials, but it still is not clear to me.

Does the function H5Dopen need to be called collectively? The code below
(or above) seems to indicate that H5Dcreate must be called collectively (it
doesn't work for me if it is not...), but that H5Dopen does *not* need to be
called collectively?

What is the logic here?

Cheers,
Matt

···

On Tue, Mar 2, 2010 at 6:01 PM, Mark Howison <mark.howison@gmail.com> wrote:

Hi Matthew, have you tried using the MPI-POSIX VFD instead? Mark

On Tue, Mar 2, 2010 at 3:56 PM, Matthew Becker <beckermr@uchicago.edu> > wrote:
> Hi all,
>
> I am having trouble with the following piece of code (I hacked apart one
of
> the parallel HDF5 tutorial examples and changed the API to 1.8.4).
>
> I am trying get each process to write to a different data set
simultaneously
> (but all data sets are in the same file) using independent I/O. I
suspect
> this may not be possible, but the documentation seems to say otherwise
(see
> http://www.hdfgroup.org/HDF5/Tutor/pprog.html, at the bottom it says
"Each
> process writes to a individual dataset.").
>
> I have enforced that the data sets are created collectively, but the
> simultaneous write fails.
>
> Do you all know why this does not work?
>
> The code is listed below.
>
> Cheers,
> Matt
>
> /*
> * This example writes data to the HDF5 file.
> * Number of processes is assumed to be 1 or multiples of 2 (up to 8)
> */
> #include <stdlib.h>
> #include <stdio.h>
> #include <math.h>
> #include <assert.h>
> #include <mpi.h>
> #include <hdf5.h>
>
>
> #define H5FILE_NAME "SDS.h5"
> #define DATASETNAME "IntArray"
> #define NX 8 /* dataset dimensions */
> #define NY 5
> #define RANK 2
>
> int main (int argc, char **argv)
> {
> /*
> * HDF5 APIs definitions
> */
> hid_t file_id, dset_id, group_id; /* file and dataset
> identifiers */
> hid_t filespace; /* file and memory dataspace identifiers
*/
> hsize_t dimsf[] = {NX, NY}; /* dataset dimensions
*/
> int *data; /* pointer to data buffer to
write
> */
> hid_t plist_id; /* property list identifier */
> int i;
> herr_t status;
> char name[500];
>
> /*
> * MPI variables
> */
> int mpi_size, mpi_rank;
> MPI_Comm comm = MPI_COMM_WORLD;
> MPI_Info info = MPI_INFO_NULL;
>
> /*
> * Initialize MPI
> */
> MPI_Init(&argc, &argv);
> MPI_Comm_size(comm, &mpi_size);
> MPI_Comm_rank(comm, &mpi_rank);
>
> /*
> * Initialize data buffer
> */
> data = (int *) malloc(sizeof(int)*dimsf[0]*
> dimsf[1]);
> for (i=0; i < dimsf[0]*dimsf[1]; i++) {
> data[i] = mpi_rank;
> }
> /*
> * Set up file access property list with parallel I/O access
> */
> plist_id = H5Pcreate(H5P_FILE_ACCESS);
> H5Pset_fapl_mpio(plist_id, comm, info);
>
> /*
> * Create a new file collectively and release property list
identifier.
> */
> file_id = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT,
plist_id);
> H5Pclose(plist_id);
>
> /*
> * Create property list for collective dataset write.
> */
> plist_id = H5Pcreate(H5P_DATASET_XFER);
> //H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);
> H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT);
>
> /*
> * Create the dataspace for the dataset.
> */
> filespace = H5Screate_simple(RANK, dimsf, NULL);
>
> for(i=0;i<mpi_size;++i)
> {
> MPI_Barrier(MPI_COMM_WORLD);
> sprintf(name,"node%d",i);
> dset_id = H5Dcreate(file_id, name, H5T_NATIVE_INT, filespace,
> H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
> H5Dclose(dset_id);
> MPI_Barrier(MPI_COMM_WORLD);
> }
> MPI_Barrier(MPI_COMM_WORLD);
>
> sprintf(name,"node%d",i);
> dset_id = H5Dopen(file_id, name, H5P_DEFAULT);
> status = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
plist_id,
> data);
> H5Dclose(dset_id);
>
> free(data);
>
> /*
> * Close/release resources.
> */
> H5Sclose(filespace);
> H5Pclose(plist_id);
> H5Fclose(file_id);
>
> MPI_Finalize();
>
> return 0;
> }
> _______________________________________________
> Hdf-forum is for HDF software users discussion.
> Hdf-forum@hdfgroup.org
> http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
>
>

_______________________________________________
Hdf-forum is for HDF software users discussion.
Hdf-forum@hdfgroup.org
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org

Hi Matt,

Hi Mark and all,

Sorry for the spam. This was a simple bug. It works with both mpio and mpiposix io. I still have a question.

I have been through the tutorials, but it still is not clear to me.

Does the function H5Dopen need to be called collectively? The code below (or above) seems to indicate that H5Dcreate must be called collectively (it doesn't work for me if it is not...), but that H5Dopen does *not* need to be called collectively?

What is the logic here?

  Here's a link with some information about which routines need to be called collectively.

http://www.hdfgroup.org/HDF5/faq/parallel-apis.html#coll

  Quincey

···

On Mar 2, 2010, at 8:38 PM, Matthew Becker wrote:

Cheers,
Matt

On Tue, Mar 2, 2010 at 6:01 PM, Mark Howison <mark.howison@gmail.com> wrote:
Hi Matthew, have you tried using the MPI-POSIX VFD instead? Mark

On Tue, Mar 2, 2010 at 3:56 PM, Matthew Becker <beckermr@uchicago.edu> wrote:
> Hi all,
>
> I am having trouble with the following piece of code (I hacked apart one of
> the parallel HDF5 tutorial examples and changed the API to 1.8.4).
>
> I am trying get each process to write to a different data set simultaneously
> (but all data sets are in the same file) using independent I/O. I suspect
> this may not be possible, but the documentation seems to say otherwise (see
> http://www.hdfgroup.org/HDF5/Tutor/pprog.html, at the bottom it says "Each
> process writes to a individual dataset.").
>
> I have enforced that the data sets are created collectively, but the
> simultaneous write fails.
>
> Do you all know why this does not work?
>
> The code is listed below.
>
> Cheers,
> Matt
>
> /*
> * This example writes data to the HDF5 file.
> * Number of processes is assumed to be 1 or multiples of 2 (up to 8)
> */
> #include <stdlib.h>
> #include <stdio.h>
> #include <math.h>
> #include <assert.h>
> #include <mpi.h>
> #include <hdf5.h>
>
>
> #define H5FILE_NAME "SDS.h5"
> #define DATASETNAME "IntArray"
> #define NX 8 /* dataset dimensions */
> #define NY 5
> #define RANK 2
>
> int main (int argc, char **argv)
> {
> /*
> * HDF5 APIs definitions
> */
> hid_t file_id, dset_id, group_id; /* file and dataset
> identifiers */
> hid_t filespace; /* file and memory dataspace identifiers */
> hsize_t dimsf[] = {NX, NY}; /* dataset dimensions */
> int *data; /* pointer to data buffer to write
> */
> hid_t plist_id; /* property list identifier */
> int i;
> herr_t status;
> char name[500];
>
> /*
> * MPI variables
> */
> int mpi_size, mpi_rank;
> MPI_Comm comm = MPI_COMM_WORLD;
> MPI_Info info = MPI_INFO_NULL;
>
> /*
> * Initialize MPI
> */
> MPI_Init(&argc, &argv);
> MPI_Comm_size(comm, &mpi_size);
> MPI_Comm_rank(comm, &mpi_rank);
>
> /*
> * Initialize data buffer
> */
> data = (int *) malloc(sizeof(int)*dimsf[0]*
> dimsf[1]);
> for (i=0; i < dimsf[0]*dimsf[1]; i++) {
> data[i] = mpi_rank;
> }
> /*
> * Set up file access property list with parallel I/O access
> */
> plist_id = H5Pcreate(H5P_FILE_ACCESS);
> H5Pset_fapl_mpio(plist_id, comm, info);
>
> /*
> * Create a new file collectively and release property list identifier.
> */
> file_id = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
> H5Pclose(plist_id);
>
> /*
> * Create property list for collective dataset write.
> */
> plist_id = H5Pcreate(H5P_DATASET_XFER);
> //H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);
> H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT);
>
> /*
> * Create the dataspace for the dataset.
> */
> filespace = H5Screate_simple(RANK, dimsf, NULL);
>
> for(i=0;i<mpi_size;++i)
> {
> MPI_Barrier(MPI_COMM_WORLD);
> sprintf(name,"node%d",i);
> dset_id = H5Dcreate(file_id, name, H5T_NATIVE_INT, filespace,
> H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
> H5Dclose(dset_id);
> MPI_Barrier(MPI_COMM_WORLD);
> }
> MPI_Barrier(MPI_COMM_WORLD);
>
> sprintf(name,"node%d",i);
> dset_id = H5Dopen(file_id, name, H5P_DEFAULT);
> status = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, plist_id,
> data);
> H5Dclose(dset_id);
>
> free(data);
>
> /*
> * Close/release resources.
> */
> H5Sclose(filespace);
> H5Pclose(plist_id);
> H5Fclose(file_id);
>
> MPI_Finalize();
>
> return 0;
> }
> _______________________________________________
> Hdf-forum is for HDF software users discussion.
> Hdf-forum@hdfgroup.org
> http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org
>
>

_______________________________________________
Hdf-forum is for HDF software users discussion.
Hdf-forum@hdfgroup.org
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org

_______________________________________________
Hdf-forum is for HDF software users discussion.
Hdf-forum@hdfgroup.org
http://mail.hdfgroup.org/mailman/listinfo/hdf-forum_hdfgroup.org