phdf5 fails to create file on aws hpc cluster


#1

Hi All,
I have a problem to run our application on AWS compute cluster, Ubuntu 16.04, using intel compilers and mpi (2019.4). On my local machine everithing looks fine, any idea what could goes wrong?

HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 0:
#000: H5F.c line 444 in H5Fcreate(): unable to create file
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 1:
#000: H5F.c line 444 in H5Fcreate(): unable to create file
major: File accessibilty
minor: Unable to open file
#001: H5Fint.c line 1498 in H5F_open(): unable to open file: time = Mon Jul 1 06:47:29 2019
, name = ‘c_out.h5’, tent_flags = 13
major: File accessibilty
minor: Unable to open file
#002: H5FD.c line 734 in H5FD_open(): open failed
major: Virtual File Layer
minor: Unable to initialize object
#003: H5FDmpio.c line 998 in H5FD_mpio_open(): MPI_File_open failed
major: Internal error (too specific to document in detail)
minor: Some MPI function failed
#004: H5FDmpio.c line 998 in H5FD_mpio_open(): File does not exist, error stack:
ADIOI_UFS_OPEN(39): File c_out.h5 does not exist
major: Internal error (too specific to document in detail)
major: File accessibilty
minor: Unable to open file
#001: H5Fint.c line 1498 in H5F_open(): unable to open file: time = Mon Jul 1 06:47:29 2019
, name = ‘c_out.h5’, tent_flags = 13
major: File accessibilty
minor: Unable to open file
#002: H5FD.c line 734 in H5FD_open(): open failed
major: Virtual File Layer
minor: Unable to initialize object
#003: H5FDmpio.c line 998 in H5FD_mpio_open(): MPI_File_open failed
major: Internal error (too specific to document in detail)
minor: Some MPI function failed
#004: H5FDmpio.c line 998 in H5FD_mpio_open(): Other I/O error , error stack:
ADIO_OPEN(219): open failed on a remote node

Regards,
SK


#2

From the error printout: c_out.h5 can’t be opened from some reason. What this printout doesn’t have: a Short Self Contained Compilable Example, therefore I can’t rerun the program snippet, and post the result for you.

For MPIIO there are details: The parallel filesystem, the OMPIO/ROMIO/ADIO layer of MPI system, and on top of all sits HDF5, then your application. They all affect the outcome.

best: steve


#3

Hi steven,
thanks for helping me!
here is the required SSCCE:

#include
#include <mkl_scalapack.h>
#include
#include
#include
#include <mkl_blacs.h>
#include “hdf5.h”

extern “C”
{
/* BLACS C interface /
//void Cblacs_pinfo(int
mypnum, int* nprocs);
void Cblacs_get( MKL_INT context, MKL_INT request, MKL_INT* value);
int Cblacs_gridinit( MKL_INT* context, char * order, MKL_INT np_row, MKL_INT np_col);
void Cblacs_gridinfo( MKL_INT context, MKL_INT* np_row, MKL_INT* np_col, MKL_INT* my_row,
MKL_INT* my_col);
void Cblacs_gridexit(MKL_INT ictxt);
void Cblacs_barrier(MKL_INT ictxt, char * order);
void Cblacs_exit(int);
void Czgerv2d(int, int, int, std::complex, int, int, int);
void Czgesd2d(int, int, int, std::complex
, int, int, int);
}

int main(int argc, char* argv[])
{
MKL_INT m_context;
int m_myid;
int m_nprocs;
MKL_INT m_nprows;
MKL_INT m_npcols;
MKL_INT m_myrow; //process coordinates
MKL_INT m_mycol;
MPI_Info m_info;
MPI_Comm m_comm;

m_info  = MPI_INFO_NULL;
m_comm  = MPI_COMM_WORLD;
// Initialize the MPI environment

// MPI_Init(NULL, NULL);

// Get the number of processes

// int world_size;
// MPI_Comm_size(MPI_COMM_WORLD, &world_size);

// Get the rank of the process

// int world_rank;
// MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);

int dims[] = {0, 0};
blacs_pinfo_(&m_myid, &m_nprocs);
MPI_Dims_create(m_nprocs, 2, dims);
m_nprows = (MKL_INT)dims[0];
m_npcols = (MKL_INT)dims[1];

//
int negone = -1, zero = 0;
Cblacs_get(negone, zero, &m_context); //default system context.
// m_comm = (MPI_Comm)m_context;
char row_major[] = {“Row”};
Cblacs_gridinit(&m_context, row_major, m_nprows, m_npcols);
Cblacs_gridinfo(m_context, &m_nprows, &m_npcols, &m_myrow, &m_mycol);

std::string fname("test.h5");
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); //file access pl id
H5Pset_fapl_mpio(plist_id, m_comm, m_info);
hid_t fid = H5Fcreate(fname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); //using the default creation pl
H5Pclose(plist_id);

H5Fclose(fid);

// MPI_Finalize();
char bs[] = {“All”};
Cblacs_barrier(m_context, bs);
Cblacs_gridexit(m_context);
Cblacs_exit(0);
}

CMakeLists.txt
cmake_minimum_required(VERSION 3.10)
project(mkl_test)

list(APPEND CMAKE_MODULE_PATH “${CMAKE_CURRENT_LIST_DIR}/cmake”)

set(CMAKE_CXX_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/bin/intel64/icpc”)
set(CMAKE_C_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/bin/intel64/icc”)

if (CMAKE_CXX_COMPILER_ID STREQUAL “GNU”)
set(MPI_C_COMPILER “${INTELROOT}/linux/mpi/intel64/bin/mpicc”)
set(MPI_CXX_COMPILER “${INTELROOT}/linux/mpi/intel64/bin/mpicxx”)
endif ()

if (CMAKE_CXX_COMPILER_ID STREQUAL “Intel”)
set(MPI_C_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/mpi/intel64/bin/mpiicc”)
set(MPI_CXX_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/mpi/intel64/bin/mpiicpc”)

endif()
message(“The value of MPI_CXX_COMPILER is ${MPI_CXX_COMPILER}”)

#set(CMAKE_CXX_FLAGS_DEBUG “${CMAKE_CXX_FLAGS_DEBUG} -Wall”)

add_executable(test main.cpp )

include_directories($ENV{MKLROOT}/include)

set(MPI_ROOT $ENV{MPI_ROOT})
find_package(MPI REQUIRED)
if (MPI_FOUND)
include_directories(SYSTEM ${MPI_INCLUDE_PATH})
else (MPI_FOUND)
message(SEND_ERROR “This application cannot compile without MPI”)
endif (MPI_FOUND)
include_directories(${MPI_INCLUDE_PATH})

set(HDF5_ROOT $ENV{HDF5_ROOT})
message(“The value of HDF5_ROOT is ${HDF5_ROOT}”)
set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE “true”)
set(HDF5_PREFER_PARALLEL “true”)
find_package(HDF5 MODULE REQUIRED)
include_directories(${HDF5_INCLUDE_DIRS})

if (CMAKE_CXX_COMPILER_ID STREQUAL “GNU”)
target_link_libraries(test
-Wl,–start-group
$ENV{MKLROOT}/lib/intel64/libmkl_scalapack_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_blacs_openmpi_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_intel_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_gnu_thread.a
$ENV{MKLROOT}/lib/intel64/libmkl_core.a
-Wl,–end-group
-lgomp -lpthread -lm -ldl -lz
${MPI_CXX_LIBRARIES}
${HDF5_C_LIBRARIES}

        )

endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL “Intel”)
target_link_libraries(test
-Wl,–start-group
$ENV{MKLROOT}/lib/intel64/libmkl_scalapack_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_cdft_core.a
$ENV{MKLROOT}/lib/intel64/libmkl_intel_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_intel_thread.a
$ENV{MKLROOT}/lib/intel64/libmkl_core.a
$ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.a
-Wl,–end-group
-liomp5 -lpthread -lm -ldl
${HDF5_LIBRARIES}
${MPI_LIBRARIES}

        )

endif ()

and the log:
srun -N 2 -n 2 ./test
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 1:
#000: H5F.c line 444 in H5Fcreate(): unable to create file
major: File accessibilty
minor: Unable to open file
#001: H5Fint.c line 1498 in H5F_open(): unable to open file: time = Tue Jul 2 11:17:52 2019
, name = ‘test.h5’, tent_flags = 13
major: File accessibilty
minor: Unable to open file
#002: H5FD.c line 734 in H5FD_open(): open failed
major: Virtual File Layer
minor: Unable to initialize object
#003: H5FDmpio.c line 998 in H5FD_mpio_open(): MPI_File_open failed
major: Internal error (too specific to document in detail)
minor: Some MPI function failed
#004: H5FDmpio.c line 998 in H5FD_mpio_open(): File does not exist, error stack:
ADIOI_UFS_OPEN(39): File test.h5 does not exist
major: Internal error (too specific to document in detail)
minor: MPI Error String
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 0:
#000: H5F.c line 444 in H5Fcreate(): unable to create file
major: File accessibilty
minor: Unable to open file
#001: H5Fint.c line 1498 in H5F_open(): unable to open file: time = Tue Jul 2 11:17:52 2019
, name = ‘test.h5’, tent_flags = 13
major: File accessibilty
minor: Unable to open file
#002: H5FD.c line 734 in H5FD_open(): open failed
major: Virtual File Layer
minor: Unable to initialize object
#003: H5FDmpio.c line 998 in H5FD_mpio_open(): MPI_File_open failed
major: Internal error (too specific to document in detail)
minor: Some MPI function failed
#004: H5FDmpio.c line 998 in H5FD_mpio_open(): Other I/O error , error stack:
ADIO_OPEN(219): open failed on a remote node
major: Internal error (too specific to document in detail)
minor: MPI Error String
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 1:
#000: H5F.c line 671 in H5Fclose(): not a file ID
major: File accessibilty
minor: Inappropriate type
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 0:
#000: H5F.c line 671 in H5Fclose(): not a file ID
major: File accessibilty
minor: Inappropriate type
ubuntu@ip-172-31-39-201:/shared/hdf5_test/build$ ls
CMakeCache.txt CMakeFiles cmake_install.cmake Makefile test test.h5

h5dump -H test.h5
h5dump error: unable to open file “test.h5”

running the application for second time doesn’t create any error messages however the hdf5 file cannot be read:
srun -N 2 -n 2 ./test
h5dump -H test.h5
h5dump error: unable to open file “test.h5”

the error messages shows only if there is no test.h5 file in the working directory

the sequential code runs properly:
./test
h5dump -H test.h5
HDF5 “test.h5” {
GROUP “/” {
}
}

Regards,
SK


#4

Do you think removing the MKL scalapack is possible? I don’t have it yet installed on my cluster as I am working on pure HDF5 / MPI-IO for now. The rest looks OK. Also there is a link on this confluence page to upload file, which allows to upload/download provided example and error output. The icon looks like a little bold arrow pointing upward on top of a bold flat line.

My general experience: this should work ( without scalapack involved ) as I just added MPI-IO to H5CPP for ISC’19 conference: full presentation is here , MPIIO secttion is here and had no problem creating a file, and do partial IO. So it should work for you as well. The underlying calls are same as yours.
If C++ is an option for you, you might be interested in trying this example. FYI: H5CPP supports many linear algebra libraries out of box, my preference is armadillo or eigen 3.

best wishes: steven


#5

C++ is not a problem but I must use MKL ScaLAPACK since it is the fastest implementation and we have Intel compiler licenses. Furthermore, MKL requires working with 2d block-cyclic distributed matrices so we use hypeslabs selections for IO. Is it possible to dump that kind of data with h5cc in the right order? is there h5cc + Intel mkl/mpi examples?
Regards,
sk


#6

Yes, it is possible. If interested then we can make it happen; please let me know.
For that I need the shape of the matrix and links to MKL_xxx calls. Possibly we could work with your example + mashed with H5CPP calls?
Also, do you use any wrappers to MKL SCALAPACK or you go direct?

For now all BLAS/LAPACK calls are handled through eigen | armadillo | … which may be linked against MKL. I am develping a matching H5CLUSTER in the past few months, which doesn’t have MKL installed yet; but if you can run it on yours we are good. – I am also consumer to BLAS/LAPACK, and that motivated me to write H5CPP.

H5CPP does support partial access, although I may have to refine it a little – but the posted example clearly shows how it works:

double * ptr = ... ; 
h5::read<double>(ds, ptr,  
   h5::offset{0,rank}, h5::count{nrows,1},
    h5::collective);

Provides you with raw memory – zero copy – access to h5::read, similarly there is one for h5::write. From then on you have to organise the processes | ranks such that each of them get the correct data chunk.
Manipulating data chunk size such that it reflects the shape of cyclic blocks will maximise IO throughput.

Can you please check if this basic example compiles for you on your system with scalapack?

best wishes: steve


#7

I found this intel article on C and attached the C file here: scalapack.c (2.0 KB)

The way I understand this there are two options: there is enough global memory space for the entire matrix, then you can load it from HDF5 hypercube directly, then redistribute it with BLASC calls. This is less efficient with huge datasets.

The alternative is to skip the global distribution part – not sure if that is possible --, then load the local matrices directly from HDF5 hypercube chunks, into the allocated memory space.
This is easy to do with H5CPP as IO templates such as h5::read and h5::write accept raw pointers, as well as h5::offset{..}, h5::count{..}, h5::stride{..}, h5::block{..} parameters to do partial IO from a hypercube.

Does this make sense?

ps.: I downloaded and installed MKL’19, some other time will look at the compile issue – if any.

best: steve


#8

It looks like using h5cpp will require major refactoring of our code and i do not have time to work on this right now. something more, i did not manage even to install the lib on the cluster:
– Conan: Automatic detection of conan settings from cmake
CMake Error at cmake/conan.cmake:200 (message):
Conan: compiler setup not recognized
Call Stack (most recent call first):
cmake/conan.cmake:447 (conan_cmake_settings)
CMakeLists.txt:21 (conan_cmake_run)

– Configuring incomplete, errors occurred!
See also “/shared/h5cpp/build/CMakeFiles/CMakeOutput.log”.

I should find a faster solution
Regard,
sk


#9

H5CPP has no libraries. It is HEADER only. ALL H5CPP calls take CAPI descriptors without any modifications. Full seamless interop of the C++ interface was one of the design guidelines. No modifications is needed to any existing project that has compiled against recent HDF5 CAPI.

From the printout I only see conan errors, a package manager that I do not support. Did you VERIFY to download the HEADER ONLY project from H5CPP or from my GITHUB page as opposed to some third party package named h5cpp fetched with [conan | spack | etc] ?

in any event, let us know how the fast way worked out for you. I am curious in the efficient data distribution of massive matrix across the computational nodes, and how you did it. I looked into the topic for you with my limited time. Will post the solution for SC’19 at the end of year.

best wishes:
steven