Hi steven,
thanks for helping me!
here is the required SSCCE:
#include
#include <mkl_scalapack.h>
#include
#include
#include
#include <mkl_blacs.h>
#include “hdf5.h”
extern “C”
{
/* BLACS C interface /
//void Cblacs_pinfo(int mypnum, int* nprocs);
void Cblacs_get( MKL_INT context, MKL_INT request, MKL_INT* value);
int Cblacs_gridinit( MKL_INT* context, char * order, MKL_INT np_row, MKL_INT np_col);
void Cblacs_gridinfo( MKL_INT context, MKL_INT* np_row, MKL_INT* np_col, MKL_INT* my_row,
MKL_INT* my_col);
void Cblacs_gridexit(MKL_INT ictxt);
void Cblacs_barrier(MKL_INT ictxt, char * order);
void Cblacs_exit(int);
void Czgerv2d(int, int, int, std::complex, int, int, int);
void Czgesd2d(int, int, int, std::complex, int, int, int);
}
int main(int argc, char* argv[])
{
MKL_INT m_context;
int m_myid;
int m_nprocs;
MKL_INT m_nprows;
MKL_INT m_npcols;
MKL_INT m_myrow; //process coordinates
MKL_INT m_mycol;
MPI_Info m_info;
MPI_Comm m_comm;
m_info = MPI_INFO_NULL;
m_comm = MPI_COMM_WORLD;
// Initialize the MPI environment
// MPI_Init(NULL, NULL);
// Get the number of processes
// int world_size;
// MPI_Comm_size(MPI_COMM_WORLD, &world_size);
// Get the rank of the process
// int world_rank;
// MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
int dims[] = {0, 0};
blacs_pinfo_(&m_myid, &m_nprocs);
MPI_Dims_create(m_nprocs, 2, dims);
m_nprows = (MKL_INT)dims[0];
m_npcols = (MKL_INT)dims[1];
//
int negone = -1, zero = 0;
Cblacs_get(negone, zero, &m_context); //default system context.
// m_comm = (MPI_Comm)m_context;
char row_major[] = {“Row”};
Cblacs_gridinit(&m_context, row_major, m_nprows, m_npcols);
Cblacs_gridinfo(m_context, &m_nprows, &m_npcols, &m_myrow, &m_mycol);
std::string fname("test.h5");
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); //file access pl id
H5Pset_fapl_mpio(plist_id, m_comm, m_info);
hid_t fid = H5Fcreate(fname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); //using the default creation pl
H5Pclose(plist_id);
H5Fclose(fid);
// MPI_Finalize();
char bs[] = {“All”};
Cblacs_barrier(m_context, bs);
Cblacs_gridexit(m_context);
Cblacs_exit(0);
}
CMakeLists.txt
cmake_minimum_required(VERSION 3.10)
project(mkl_test)
list(APPEND CMAKE_MODULE_PATH “${CMAKE_CURRENT_LIST_DIR}/cmake”)
set(CMAKE_CXX_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/bin/intel64/icpc”)
set(CMAKE_C_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/bin/intel64/icc”)
if (CMAKE_CXX_COMPILER_ID STREQUAL “GNU”)
set(MPI_C_COMPILER “${INTELROOT}/linux/mpi/intel64/bin/mpicc”)
set(MPI_CXX_COMPILER “${INTELROOT}/linux/mpi/intel64/bin/mpicxx”)
endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL “Intel”)
set(MPI_C_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/mpi/intel64/bin/mpiicc”)
set(MPI_CXX_COMPILER “/opt/intel/compilers_and_libraries_2019.4.243/linux/mpi/intel64/bin/mpiicpc”)
endif()
message(“The value of MPI_CXX_COMPILER is ${MPI_CXX_COMPILER}”)
#set(CMAKE_CXX_FLAGS_DEBUG “${CMAKE_CXX_FLAGS_DEBUG} -Wall”)
add_executable(test main.cpp )
include_directories($ENV{MKLROOT}/include)
set(MPI_ROOT $ENV{MPI_ROOT})
find_package(MPI REQUIRED)
if (MPI_FOUND)
include_directories(SYSTEM ${MPI_INCLUDE_PATH})
else (MPI_FOUND)
message(SEND_ERROR “This application cannot compile without MPI”)
endif (MPI_FOUND)
include_directories(${MPI_INCLUDE_PATH})
set(HDF5_ROOT $ENV{HDF5_ROOT})
message(“The value of HDF5_ROOT is ${HDF5_ROOT}”)
set(HDF5_NO_FIND_PACKAGE_CONFIG_FILE “true”)
set(HDF5_PREFER_PARALLEL “true”)
find_package(HDF5 MODULE REQUIRED)
include_directories(${HDF5_INCLUDE_DIRS})
if (CMAKE_CXX_COMPILER_ID STREQUAL “GNU”)
target_link_libraries(test
-Wl,–start-group
$ENV{MKLROOT}/lib/intel64/libmkl_scalapack_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_blacs_openmpi_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_intel_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_gnu_thread.a
$ENV{MKLROOT}/lib/intel64/libmkl_core.a
-Wl,–end-group
-lgomp -lpthread -lm -ldl -lz
${MPI_CXX_LIBRARIES}
${HDF5_C_LIBRARIES}
)
endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL “Intel”)
target_link_libraries(test
-Wl,–start-group
$ENV{MKLROOT}/lib/intel64/libmkl_scalapack_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_cdft_core.a
$ENV{MKLROOT}/lib/intel64/libmkl_intel_lp64.a
$ENV{MKLROOT}/lib/intel64/libmkl_intel_thread.a
$ENV{MKLROOT}/lib/intel64/libmkl_core.a
$ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.a
-Wl,–end-group
-liomp5 -lpthread -lm -ldl
${HDF5_LIBRARIES}
${MPI_LIBRARIES}
)
endif ()
and the log:
srun -N 2 -n 2 ./test
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 1:
#000: H5F.c line 444 in H5Fcreate(): unable to create file
major: File accessibilty
minor: Unable to open file
#001: H5Fint.c line 1498 in H5F_open(): unable to open file: time = Tue Jul 2 11:17:52 2019
, name = ‘test.h5’, tent_flags = 13
major: File accessibilty
minor: Unable to open file
#002: H5FD.c line 734 in H5FD_open(): open failed
major: Virtual File Layer
minor: Unable to initialize object
#003: H5FDmpio.c line 998 in H5FD_mpio_open(): MPI_File_open failed
major: Internal error (too specific to document in detail)
minor: Some MPI function failed
#004: H5FDmpio.c line 998 in H5FD_mpio_open(): File does not exist, error stack:
ADIOI_UFS_OPEN(39): File test.h5 does not exist
major: Internal error (too specific to document in detail)
minor: MPI Error String
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 0:
#000: H5F.c line 444 in H5Fcreate(): unable to create file
major: File accessibilty
minor: Unable to open file
#001: H5Fint.c line 1498 in H5F_open(): unable to open file: time = Tue Jul 2 11:17:52 2019
, name = ‘test.h5’, tent_flags = 13
major: File accessibilty
minor: Unable to open file
#002: H5FD.c line 734 in H5FD_open(): open failed
major: Virtual File Layer
minor: Unable to initialize object
#003: H5FDmpio.c line 998 in H5FD_mpio_open(): MPI_File_open failed
major: Internal error (too specific to document in detail)
minor: Some MPI function failed
#004: H5FDmpio.c line 998 in H5FD_mpio_open(): Other I/O error , error stack:
ADIO_OPEN(219): open failed on a remote node
major: Internal error (too specific to document in detail)
minor: MPI Error String
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 1:
#000: H5F.c line 671 in H5Fclose(): not a file ID
major: File accessibilty
minor: Inappropriate type
HDF5-DIAG: Error detected in HDF5 (1.10.5) MPI-process 0:
#000: H5F.c line 671 in H5Fclose(): not a file ID
major: File accessibilty
minor: Inappropriate type
ubuntu@ip-172-31-39-201:/shared/hdf5_test/build$ ls
CMakeCache.txt CMakeFiles cmake_install.cmake Makefile test test.h5
h5dump -H test.h5
h5dump error: unable to open file “test.h5”
running the application for second time doesn’t create any error messages however the hdf5 file cannot be read:
srun -N 2 -n 2 ./test
h5dump -H test.h5
h5dump error: unable to open file “test.h5”
the error messages shows only if there is no test.h5 file in the working directory
the sequential code runs properly:
./test
h5dump -H test.h5
HDF5 “test.h5” {
GROUP “/” {
}
}
Regards,
SK
