Greetings,
I've recently noticed a problem with the group heap size in the current
distribution. I created a sample program that demonstrates the condition
that I am seeing. Here is the write up of the situation.
The problem doesn't seem to be happening in older versions of the HDF-JAVA.
Please let me know if I am missing something in the new API.
···
-------------------------------
The problem occurs in
- hdf-java-2.7-bin for Windows 64bit and Java 1.6.0_26-b03 for Windows 64bit
- hdf-java-2.7-bin for Windows 32bit and Java jdk1.6.0_25 for Windows 32 bit
Fortunately, it seems that the problem does not occur in
- hdf-java-2.6.1-bin for Windows 64bit and Java 1.6.0_26-b03 for Windows
64bit
-------------------------------
I wrote a Java program that demonstrates this problem called
H5GroupHeapMemoryLeak. The demo basically runs two tests.
Test 1 - working test with _no_ leak
step1. open file
step2. create a unique group
step3. write a 20 datasets with 10k values in them under the group
step4. close file
step5. repeat steps1-4 25 times
Test 2 - demonstrates the Group Heap Leak problem
step1. open file
step2. create a group called "levelOneGroup" or get it from the file
step3. create a unique group
step4. write a 20 datasets with 10k values in them under the group
step5. close file
step6. repeat steps1-4 25 times
The difference between Test 1 and Test 2 seems to be the reuse of the
"levelOneGroup" group in Test 2. In this case, we must call
fileFormat.get(String) to get the group. It seems that this call is what
creates the heap problem.
-------------------------------
The output of the program is the following:
good file size: 202392
leak file size: 23187400
-------------------------------
The output of h5stat shows the differences in the Group Heap
good file size: 202392
File space information for file metadata (in bytes):
Groups:
B-tree/List: 48584
Heap: 9984
//leak file size: 23187400
File space information for file metadata (in bytes):
Groups:
B-tree/List: 49456
Heap: 11544088
-------------------------------
Here is the code for this demonstration. I based the demo on some tests that
were already in the distribution (excuse the lack of documentation)
import java.util.List;
import ncsa.hdf.object.Dataset;
import ncsa.hdf.object.Datatype;
import ncsa.hdf.object.FileFormat;
import ncsa.hdf.object.Group;
import ncsa.hdf.object.h5.H5File;
import
com.referentia.liveaction.server.core.nfstore.impl.hdf5.util.HDF5SdfConstants;
/**
* Implements an example of a memory leak in the groups.
*/
public class H5GroupHeapMemoryLeak {
private static final boolean USEGET_TO_FIND_GROUP = true;
public static void main(String args[]) throws Exception {
// create the file and add groups ans dataset into the file
int repeats = 25;
int datasets = 20;
int datasetSize = 10000;
H5GroupHeapMemoryLeak test = new H5GroupHeapMemoryLeak();
long goodFileSize = test.runGoodDemo(repeats, datasets, datasetSize);
long leakFileSize = test.runMemoryLeakDemo(repeats, datasets,
datasetSize);
System.out.println("good file size: " + goodFileSize);
System.out.println("leak file size: " + leakFileSize);
}
private long runMemoryLeakDemo(int repeats, int datasets, int datasetSize)
throws Exception {
String fileName = H5GroupHeapMemoryLeak.class.getSimpleName()
+ "-LEAK" + "-" + System.currentTimeMillis() + "-repeats" + repeats
+ "-datasets" + datasets + "-datasetSize" + datasetSize + ".h5";
createFile(fileName);
long finalFileSize = 0;
for (int i = 0; i < repeats; i++) {
FileFormat testFile = getFile(fileName);
testFile.open();
Group datasetGroup = getGroupWithMemoryLeak(testFile, "group" + i);
writeToDatasetGroup(testFile, datasetGroup, datasets, datasetSize);
testFile.close();
finalFileSize = testFile.length();
//ystem.out.println("\tfile length " + i + ": " + finalFileSize);
}
return finalFileSize;
}
private long runGoodDemo(int repeats, int datasets, int datasetSize)
throws Exception {
String fileName = H5GroupHeapMemoryLeak.class.getSimpleName()
+ "-GOOD" + "-" + System.currentTimeMillis() + "-repeats" + repeats
+ "-datasets" + datasets + "-datasetSize" + datasetSize + ".h5";
createFile(fileName);
long finalFileSize = 0;
for (int i = 0; i < repeats; i++) {
FileFormat testFile = getFile(fileName);
testFile.open();
Group datasetGroup = getGoodGroup(testFile, "group" + i);
writeToDatasetGroup(testFile, datasetGroup, datasets, datasetSize);
testFile.close();
finalFileSize = testFile.length();
//System.out.println("\tfile length " + i + ": " + finalFileSize);
}
return finalFileSize;
}
private FileFormat getFile(String fileName) throws Exception {
// retrieve an instance of H5File
FileFormat fileFormat =
FileFormat.getFileFormat(FileFormat.FILE_TYPE_HDF5);
if (fileFormat == null) {
System.err.println("Cannot find HDF5 FileFormat.");
return null;
}
// open the file with read and write access
FileFormat testFile = (H5File) fileFormat.createFile(fileName,
FileFormat.FILE_CREATE_OPEN);
if (testFile == null) {
System.err.println("Failed to open file: " + fileName);
return null;
}
return testFile;
}
private Group getGoodGroup(FileFormat testFile, String groupName) throws
Exception {
Group root = (Group) ((javax.swing.tree.DefaultMutableTreeNode)
testFile.getRootNode()).getUserObject();
Group datasetGroup = testFile.createGroup(groupName, root);
return datasetGroup;
}
private Group getGroupWithMemoryLeak(FileFormat testFile, String
groupName) throws Exception {
Group root = (Group) ((javax.swing.tree.DefaultMutableTreeNode)
testFile.getRootNode()).getUserObject();
Group levelOneGroup = null;
if (USEGET_TO_FIND_GROUP) {
levelOneGroup = (Group) testFile.get("levelOneGroup");
}
else {
List<?> members = root.getMemberList();
for (Object tryingToFindGroup : members) {
if (tryingToFindGroup instanceof Group &&
"levelOneGroup".equals(((Group) tryingToFindGroup).getName())) {
levelOneGroup = (Group) tryingToFindGroup;
break;
}
}
}
if (levelOneGroup == null) {
levelOneGroup = testFile.createGroup("levelOneGroup", root);
}
Group datasetGroup = testFile.createGroup(groupName, levelOneGroup);
return datasetGroup;
}
private void writeToDatasetGroup(FileFormat testFile, Group datasetGroup,
int datasets, int datasetSize) throws Exception {
for (int i = 0; i < datasets; i++) {
int[] args = new int[] { Datatype.CLASS_INTEGER, 8, Datatype.NATIVE,
Datatype.SIGN_NONE };
Datatype datatype = testFile.createDatatype(args[0], args[1], args[2],
args[3]);
writeDataset("data" + i, datasetSize, testFile, datasetGroup,
datatype);
}
}
private void writeDataset(String datasetName, int datasetSize, FileFormat
testFile, Group group, Datatype datatype) throws Exception {
int size = datasetSize;
long[] initialSize = new long[] { size };
long[] maxSize = new long[] { Long.MAX_VALUE };
long[] chunkSize = new long[] { 60000 };
int gzipCompressionLevel = HDF5SdfConstants.COMPRESSION_LEVEL;
Dataset dataset = testFile.createScalarDS(datasetName, group, datatype,
initialSize, maxSize, chunkSize, gzipCompressionLevel, null);
dataset.init();
long[] data = new long[size];
for (int i = 0; i < size; i++) {
data[i] = i;
}
//we don't have to write data to show the group memory leak
//dataset.write(data);
dataset.close(dataset.getFID());
}
/**
* create the file and add groups ans dataset into the file, which is the
same
* as javaExample.H5DatasetCreate
* @see javaExample.H5DatasetCreate
* @throws Exception
*/
private void createFile(String fileName) throws Exception {
// retrieve an instance of H5File
FileFormat fileFormat =
FileFormat.getFileFormat(FileFormat.FILE_TYPE_HDF5);
if (fileFormat == null) {
System.err.println("Cannot find HDF5 FileFormat.");
return;
}
// create a new file with a given file name.
H5File testFile = (H5File) fileFormat.createFile(fileName,
FileFormat.FILE_CREATE_OPEN);
if (testFile == null) {
System.err.println("Failed to create file:" + fileName);
return;
}
// open the file and retrieve the root group
testFile.open();
// close file resource
testFile.close();
}
}
thanks, Aaron Kagawa