mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

JNI: Improve varlen datatype handling in H5A/H5D read/write functions (#2156)

Browse Source 
* Improve JNI VL datatype read/write to handle complex combinations

* Implement VL of VL JNI writes and reads

* Add Java VLofVL test for attributtes

* Changes to address review issues

* Fix H5Aread vl blocks
This commit is contained in:
Allen Byrne
2022-10-25 23:33:30 -05:00
committed by GitHub
parent ad2d77397a
commit 9c61f7b1e1
12 changed files with 1089 additions and 971 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/test/TestH5A.java
+157 -4
View File
@@ -1415,7 +1415,7 @@ public class TestH5A {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5AVLwr: ", attr_int_id >= 0);
H5.H5AwriteVL(attr_int_id, atype_int_id, vl_int_data);
H5.H5Awrite(attr_int_id, atype_int_id, vl_int_data);
}
catch (Exception err) {
if (attr_int_id > 0)
@@ -1472,7 +1472,7 @@ public class TestH5A {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5AVLwr: ", attr_dbl_id >= 0);
H5.H5AwriteVL(attr_dbl_id, atype_dbl_id, vl_dbl_data);
H5.H5Awrite(attr_dbl_id, atype_dbl_id, vl_dbl_data);
}
catch (Exception err) {
if (attr_dbl_id > 0)
@@ -1511,7 +1511,7 @@ public class TestH5A {
vl_readbuf[j] = new ArrayList<Integer>();
try {
H5.H5AreadVL(attr_int_id, atype_int_id, vl_readbuf);
H5.H5Aread(attr_int_id, atype_int_id, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -1531,7 +1531,7 @@ public class TestH5A {
vl_readbuf[j] = new ArrayList<Double>();
try {
H5.H5AreadVL(attr_dbl_id, atype_dbl_id, vl_readbuf);
H5.H5Aread(attr_dbl_id, atype_dbl_id, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -1576,4 +1576,157 @@ public class TestH5A {
}
}
}
@Test
public void testH5AVLwrVL()
{
String attr_int_name = "VLIntdata";
long attr_int_id = HDF5Constants.H5I_INVALID_HID;
long atype_int_id = HDF5Constants.H5I_INVALID_HID;
long base_atype_int_id = HDF5Constants.H5I_INVALID_HID;
long aspace_id = HDF5Constants.H5I_INVALID_HID;
long[] dims = {4};
long lsize = 1;
ArrayList[] base_vl_int_data = new ArrayList[4];
ArrayList[] vl_int_data = new ArrayList[4];
try {
// Write Integer data
vl_int_data[0] = new ArrayList<Integer>(Arrays.asList(1));
vl_int_data[1] = new ArrayList<Integer>(Arrays.asList(2, 3));
vl_int_data[2] = new ArrayList<Integer>(Arrays.asList(4, 5, 6));
vl_int_data[3] = new ArrayList<Integer>(Arrays.asList(7, 8, 9, 10));
Class dataClass = vl_int_data.getClass();
assertTrue("testH5AVLwrVL.getClass: " + dataClass, dataClass.isArray());
// Write VL data
base_vl_int_data[0] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[0].add(vl_int_data[0]);
base_vl_int_data[1] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[1].add(vl_int_data[0]);
base_vl_int_data[1].add(vl_int_data[1]);
base_vl_int_data[2] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[2].add(vl_int_data[0]);
base_vl_int_data[2].add(vl_int_data[1]);
base_vl_int_data[2].add(vl_int_data[2]);
base_vl_int_data[3] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[3].add(vl_int_data[0]);
base_vl_int_data[3].add(vl_int_data[1]);
base_vl_int_data[3].add(vl_int_data[2]);
base_vl_int_data[3].add(vl_int_data[3]);
try {
atype_int_id = H5.H5Tvlen_create(HDF5Constants.H5T_STD_U32LE);
assertTrue("testH5AVLwr.H5Tvlen_create: ", atype_int_id >= 0);
base_atype_int_id = H5.H5Tvlen_create(atype_int_id);
assertTrue("testH5AVLwrVL.H5Tvlen_create: ", base_atype_int_id >= 0);
}
catch (Exception err) {
if (base_atype_int_id > 0)
try {
H5.H5Tclose(base_atype_int_id);
}
catch (Exception ex) {
}
err.printStackTrace();
fail("H5.testH5AVLwrVL: " + err);
}
try {
aspace_id = H5.H5Screate_simple(1, dims, null);
assertTrue(aspace_id > 0);
attr_int_id = H5.H5Acreate(H5did, attr_int_name, base_atype_int_id, aspace_id,
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5AVLwrVL: ", attr_int_id >= 0);
H5.H5Awrite(attr_int_id, base_atype_int_id, base_vl_int_data);
}
catch (Exception err) {
if (attr_int_id > 0)
try {
H5.H5Aclose(attr_int_id);
}
catch (Exception ex) {
}
if (atype_int_id > 0)
try {
H5.H5Tclose(atype_int_id);
}
catch (Exception ex) {
}
err.printStackTrace();
fail("H5.testH5AVLwrVL: " + err);
}
finally {
if (aspace_id > 0)
try {
H5.H5Sclose(aspace_id);
}
catch (Exception ex) {
}
}
H5.H5Fflush(H5fid, HDF5Constants.H5F_SCOPE_LOCAL);
for (int j = 0; j < dims.length; j++)
lsize *= dims[j];
// Read Integer data
ArrayList[] base_vl_readbuf = new ArrayList[4];
for (int j = 0; j < lsize; j++)
base_vl_readbuf[j] = new ArrayList<ArrayList<Integer>>();
try {
H5.H5Aread(attr_int_id, base_atype_int_id, base_vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
}
ArrayList vl_readbuf = (ArrayList)base_vl_readbuf[0];
assertTrue("vl_readbuf exists", vl_readbuf != null);
assertTrue("vl_readbuf has size " + vl_readbuf.size(), vl_readbuf.size() > 0);
ArrayList vl_readbuf_int = (ArrayList)vl_readbuf.get(0);
assertTrue("testH5AVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[0].get(0).equals(vl_readbuf_int.get(0)));
vl_readbuf = (ArrayList)base_vl_readbuf[1];
vl_readbuf_int = (ArrayList)vl_readbuf.get(1);
assertTrue("testH5AVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[1].get(0).equals(vl_readbuf_int.get(0)));
vl_readbuf = (ArrayList)base_vl_readbuf[2];
vl_readbuf_int = (ArrayList)vl_readbuf.get(2);
assertTrue("testH5AVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[2].get(0).equals(vl_readbuf_int.get(0)));
vl_readbuf = (ArrayList)base_vl_readbuf[3];
vl_readbuf_int = (ArrayList)vl_readbuf.get(3);
assertTrue("testH5AVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[3].get(0).equals(vl_readbuf_int.get(0)));
}
catch (Throwable err) {
err.printStackTrace();
fail("H5.testH5AVLwrVL: " + err);
}
finally {
if (attr_int_id > 0)
try {
H5.H5Aclose(attr_int_id);
}
catch (Exception ex) {
}
if (atype_int_id > 0)
try {
H5.H5Tclose(atype_int_id);
}
catch (Exception ex) {
}
if (base_atype_int_id > 0)
try {
H5.H5Tclose(base_atype_int_id);
}
catch (Exception ex) {
}
}
}
}
java/test/TestH5D.java
+171 -10
View File
@@ -1064,8 +1064,8 @@ public class TestH5D {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5DVLwr: ", dset_int_id >= 0);
H5.H5DwriteVL(dset_int_id, dtype_int_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_int_data);
H5.H5Dwrite(dset_int_id, dtype_int_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_int_data);
}
catch (Exception err) {
if (dset_int_id > 0)
@@ -1123,8 +1123,8 @@ public class TestH5D {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5DVLwr: ", dset_dbl_id >= 0);
H5.H5DwriteVL(dset_dbl_id, dtype_dbl_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_dbl_data);
H5.H5Dwrite(dset_dbl_id, dtype_dbl_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_dbl_data);
}
catch (Exception err) {
if (dset_dbl_id > 0)
@@ -1153,9 +1153,8 @@ public class TestH5D {
H5.H5Fflush(H5fid, HDF5Constants.H5F_SCOPE_LOCAL);
for (int j = 0; j < dims.length; j++) {
for (int j = 0; j < dims.length; j++)
lsize *= dims[j];
}
// Read Integer data
ArrayList[] vl_readbuf = new ArrayList[4];
@@ -1163,8 +1162,8 @@ public class TestH5D {
vl_readbuf[j] = new ArrayList<Integer>();
try {
H5.H5DreadVL(dset_int_id, dtype_int_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
H5.H5Dread(dset_int_id, dtype_int_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -1184,8 +1183,8 @@ public class TestH5D {
vl_readbuf[j] = new ArrayList<Double>();
try {
H5.H5DreadVL(dset_dbl_id, dtype_dbl_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
H5.H5Dread(dset_dbl_id, dtype_dbl_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -1230,4 +1229,166 @@ public class TestH5D {
}
}
}
@Test
public void testH5DVLwrVL()
{
String dset_int_name = "VLIntdata";
long dset_int_id = HDF5Constants.H5I_INVALID_HID;
long dtype_int_id = HDF5Constants.H5I_INVALID_HID;
long base_dtype_int_id = HDF5Constants.H5I_INVALID_HID;
long dspace_id = HDF5Constants.H5I_INVALID_HID;
long[] dims = {4};
long lsize = 1;
ArrayList[] base_vl_int_data = new ArrayList[4];
ArrayList[] vl_int_data = new ArrayList[4];
try {
// Write Integer data
vl_int_data[0] = new ArrayList<Integer>(Arrays.asList(1));
vl_int_data[1] = new ArrayList<Integer>(Arrays.asList(2, 3));
vl_int_data[2] = new ArrayList<Integer>(Arrays.asList(4, 5, 6));
vl_int_data[3] = new ArrayList<Integer>(Arrays.asList(7, 8, 9, 10));
Class dataClass = vl_int_data.getClass();
assertTrue("testH5DVLwrVL.getClass: " + dataClass, dataClass.isArray());
// Write VL data
base_vl_int_data[0] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[0].add(vl_int_data[0]);
base_vl_int_data[1] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[1].add(vl_int_data[0]);
base_vl_int_data[1].add(vl_int_data[1]);
base_vl_int_data[2] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[2].add(vl_int_data[0]);
base_vl_int_data[2].add(vl_int_data[1]);
base_vl_int_data[2].add(vl_int_data[2]);
base_vl_int_data[3] = new ArrayList<ArrayList<Integer>>();
base_vl_int_data[3].add(vl_int_data[0]);
base_vl_int_data[3].add(vl_int_data[1]);
base_vl_int_data[3].add(vl_int_data[2]);
base_vl_int_data[3].add(vl_int_data[3]);
try {
dtype_int_id = H5.H5Tvlen_create(HDF5Constants.H5T_STD_U32LE);
assertTrue("testH5DVLwrVL.H5Tvlen_create: ", dtype_int_id >= 0);
base_dtype_int_id = H5.H5Tvlen_create(dtype_int_id);
assertTrue("testH5DVLwrVL.H5Tvlen_create: ", base_dtype_int_id >= 0);
}
catch (Exception err) {
if (base_dtype_int_id > 0)
try {
H5.H5Tclose(base_dtype_int_id);
}
catch (Exception ex) {
}
if (dtype_int_id > 0)
try {
H5.H5Tclose(dtype_int_id);
}
catch (Exception ex) {
}
err.printStackTrace();
fail("H5.testH5DVLwrVL: " + err);
}
try {
dspace_id = H5.H5Screate_simple(1, dims, null);
assertTrue(dspace_id > 0);
dset_int_id = H5.H5Dcreate(H5fid, dset_int_name, base_dtype_int_id, dspace_id,
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT,
HDF5Constants.H5P_DEFAULT);
assertTrue("testH5DVLwrVL: ", dset_int_id >= 0);
H5.H5Dwrite(dset_int_id, base_dtype_int_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, base_vl_int_data);
}
catch (Exception err) {
if (dset_int_id > 0)
try {
H5.H5Dclose(dset_int_id);
}
catch (Exception ex) {
}
if (dtype_int_id > 0)
try {
H5.H5Tclose(dtype_int_id);
}
catch (Exception ex) {
}
err.printStackTrace();
fail("H5.testH5DVLwrVL: " + err);
}
finally {
if (dspace_id > 0)
try {
H5.H5Sclose(dspace_id);
}
catch (Exception ex) {
}
}
H5.H5Fflush(H5fid, HDF5Constants.H5F_SCOPE_LOCAL);
for (int j = 0; j < dims.length; j++)
lsize *= dims[j];
// Read Integer data
ArrayList[] base_vl_readbuf = new ArrayList[4];
for (int j = 0; j < lsize; j++)
base_vl_readbuf[j] = new ArrayList<ArrayList<Integer>>();
try {
H5.H5Dread(dset_int_id, base_dtype_int_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, base_vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
}
ArrayList vl_readbuf = (ArrayList)base_vl_readbuf[0];
assertTrue("vl_readbuf exists", vl_readbuf != null);
assertTrue("vl_readbuf has size " + vl_readbuf.size(), vl_readbuf.size() > 0);
ArrayList vl_readbuf_int = (ArrayList)vl_readbuf.get(0);
assertTrue("testH5DVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[0].get(0).equals(vl_readbuf_int.get(0)));
vl_readbuf = (ArrayList)base_vl_readbuf[1];
vl_readbuf_int = (ArrayList)vl_readbuf.get(1);
assertTrue("testH5DVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[1].get(0).equals(vl_readbuf_int.get(0)));
vl_readbuf = (ArrayList)base_vl_readbuf[2];
vl_readbuf_int = (ArrayList)vl_readbuf.get(2);
assertTrue("testH5DVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[2].get(0).equals(vl_readbuf_int.get(0)));
vl_readbuf = (ArrayList)base_vl_readbuf[3];
vl_readbuf_int = (ArrayList)vl_readbuf.get(3);
assertTrue("testH5DVLwrVL:" + vl_readbuf_int.get(0),
vl_int_data[3].get(0).equals(vl_readbuf_int.get(0)));
}
catch (Throwable err) {
err.printStackTrace();
fail("H5.testH5DVLwrVL: " + err);
}
finally {
if (dset_int_id > 0)
try {
H5.H5Dclose(dset_int_id);
}
catch (Exception ex) {
}
if (dtype_int_id > 0)
try {
H5.H5Tclose(dtype_int_id);
}
catch (Exception ex) {
}
if (base_dtype_int_id > 0)
try {
H5.H5Tclose(base_dtype_int_id);
}
catch (Exception ex) {
}
}
}
}
java/test/TestH5R.java
+12 -12
View File
@@ -600,7 +600,7 @@ public class TestH5R {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5RVLattr_ref: ", attr_obj_id >= 0);
H5.H5AwriteVL(attr_obj_id, atype_obj_id, vl_obj_data);
H5.H5Awrite(attr_obj_id, atype_obj_id, vl_obj_data);
}
catch (Exception err) {
if (attr_obj_id > 0)
@@ -657,7 +657,7 @@ public class TestH5R {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5RVLattr_ref: ", attr_reg_id >= 0);
H5.H5AwriteVL(attr_reg_id, atype_reg_id, vl_reg_data);
H5.H5Awrite(attr_reg_id, atype_reg_id, vl_reg_data);
}
catch (Exception err) {
if (attr_reg_id > 0)
@@ -696,7 +696,7 @@ public class TestH5R {
vl_readbuf[j] = new ArrayList<byte[]>();
try {
H5.H5AreadVL(attr_obj_id, atype_obj_id, vl_readbuf);
H5.H5Aread(attr_obj_id, atype_obj_id, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -716,7 +716,7 @@ public class TestH5R {
vl_readbuf[j] = new ArrayList<byte[]>();
try {
H5.H5AreadVL(attr_reg_id, atype_reg_id, vl_readbuf);
H5.H5Aread(attr_reg_id, atype_reg_id, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -831,8 +831,8 @@ public class TestH5R {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5RVLdset_ref: ", dset_obj_id >= 0);
H5.H5DwriteVL(dset_obj_id, dtype_obj_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_obj_data);
H5.H5Dwrite(dset_obj_id, dtype_obj_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_obj_data);
}
catch (Exception err) {
if (dset_obj_id > 0)
@@ -890,8 +890,8 @@ public class TestH5R {
HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
assertTrue("testH5RVLdset_ref: ", dset_reg_id >= 0);
H5.H5DwriteVL(dset_reg_id, dtype_reg_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_reg_data);
H5.H5Dwrite(dset_reg_id, dtype_reg_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_reg_data);
}
catch (Exception err) {
if (dset_reg_id > 0)
@@ -930,8 +930,8 @@ public class TestH5R {
vl_readbuf[j] = new ArrayList<byte[]>();
try {
H5.H5DreadVL(dset_obj_id, dtype_obj_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
H5.H5Dread(dset_obj_id, dtype_obj_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
@@ -951,8 +951,8 @@ public class TestH5R {
vl_readbuf[j] = new ArrayList<byte[]>();
try {
H5.H5DreadVL(dset_reg_id, dtype_reg_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
H5.H5Dread(dset_reg_id, dtype_reg_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
HDF5Constants.H5P_DEFAULT, vl_readbuf);
}
catch (Exception ex) {
ex.printStackTrace();
java/test/testfiles/JUnit-TestH5A.txt
+2 -1
View File
@@ -11,6 +11,7 @@ JUnit version 4.11
.testH5Aget_info_by_name
.testH5Aget_create_plist
.testH5Adelete_by_name
.testH5AVLwrVL
.testH5Aopen_by_name
.testH5Aget_info
.testH5Aget_name
@@ -31,5 +32,5 @@ JUnit version 4.11
Time: XXXX
OK (29 tests)
OK (30 tests)
java/test/testfiles/JUnit-TestH5D.txt
+2 -1
View File
@@ -1,4 +1,5 @@
JUnit version 4.11
.testH5DVLwrVL
.testH5Dget_storage_size
.testH5Diterate_write
.testH5Dcreate
@@ -22,5 +23,5 @@ JUnit version 4.11
Time: XXXX
OK (20 tests)
OK (21 tests)