Hello,
I have a problem to read subgroup names parallel. I have to work with HDF5 files with many subgroups (>100k). Due to time requirements it would be nice to parallelize the task to read all subgroup names of a specific group. I have created a minimal example but it hangs in some internal MPI communication:
#include <string>
#include <vector>
#include <iostream>
#include "hdf5.h"
#include "mpi.h"
#define FILENAME "myFile.h5"
#define GROUPNAME "myGroup"
#define NUMSUBGROUPS 100
void createTestFile(){
hid_t file = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
hid_t group = H5Gcreate (file, GROUPNAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
/* Create multiple subgroups */
for (uint32_t i = 0; i < NUMSUBGROUPS; i++){
const std::string cSubgroupName = std::to_string(i);
hid_t subgroup = H5Gcreate (group, cSubgroupName.c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
H5Gclose (subgroup);
}
H5Gclose (group);
H5Fclose (file);
}
void getChunkOfSubgroupNames(const hid_t cFile, const std::string& cGroupName, const hsize_t cStartIdx, const hsize_t cNumSubgroupNamesToRead,
std::vector<std::string>* pReadSubgrouNames){
const size_t cNameArraySize = 64;
char* pNameArray = (char*)std::malloc(sizeof(char) * cNameArraySize);
for (hsize_t i = cStartIdx; i < cStartIdx + cNumSubgroupNamesToRead; i++){
H5Lget_name_by_idx(cFile, cGroupName.c_str(), H5_INDEX_NAME, H5_ITER_NATIVE, i, pNameArray, cNameArraySize, H5P_DEFAULT);
const std::string cFoundSubgroupName(pNameArray);
pReadSubgrouNames->push_back(cFoundSubgroupName);
}
std::free(pNameArray);
}
int main(int argc, char** argv){
MPI_Init(&argc, &argv);
MPI_Comm cComm = MPI_COMM_WORLD;
int procId = 0;
int nProcs = 0;
MPI_Comm_rank(cComm, &procId);
MPI_Comm_size(cComm, &nProcs);
/* Create file by process 0 */
if (procId == 0){
createTestFile();
}
MPI_Barrier(cComm);
/* Open created HDF5 file parallel for all processes */
hid_t parallel_plist = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(parallel_plist, cComm, MPI_INFO_NULL);
hid_t file = H5Fopen(FILENAME, H5F_ACC_RDONLY, parallel_plist);
// hid_t file = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT);
/* Read subgroup names parallel */
const hsize_t cLocNumSubgroupNamesToRead = NUMSUBGROUPS / nProcs;
const hsize_t cLocOffsetSubgroupNamesToRead = procId * cLocNumSubgroupNamesToRead;
std::cout << "Proc " << procId << " tries to read " << cLocNumSubgroupNamesToRead << " subgroup names at offest " << cLocOffsetSubgroupNamesToRead << "." << std::endl;
std::vector<std::string> localReadSubgrouNames = {};
getChunkOfSubgroupNames(file, GROUPNAME, cLocOffsetSubgroupNamesToRead, cLocNumSubgroupNamesToRead, &localReadSubgrouNames);
std::cout << "Proc " << procId << " has found " << localReadSubgrouNames.size() << " subgroups." << std::endl;
H5Fclose (file);
MPI_Finalize();
return 0;
}
This can be compiled with:
g++ -c -g -std=c++11 -MMD -MP -MF main.o.d -o main.o main.cpp
g++ -o parallelloopsubgroups main.o -lhdf5 -lmpi
Running the program with
mpirun -np 2 parallelloopsubgroups
leads to a dead lock.
If I simply open the file without the MPI communicator with
H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT);
then everthing works fine but this is not indented.
My environment:
OpenMpi 4.1.2
Parallel HDF5 1.12.1
Can anybody help?
Thanks