Fix JNI datatype ID leak in h5str_detect_vlen_str() (#6522)

* Fix JNI datatype ID leak in h5str_detect_vlen_str()

The JNI H5Dread/H5Dwrite/H5Aread/H5Awrite wrappers call h5str_detect_vlen()
on the memory type. For an H5T_ARRAY/H5T_VLEN of a fixed (non-vlen-string)
base type, h5str_detect_vlen_str() acquired the base type via H5Tget_super()
but only closed it when the recursive check returned 1 or a negative error.
When the recursive call returned 0 because no vlen string was found, the base type ID
was leaked.

This PR changes h5str_detect_vlen_str() to close the id unconditionally after the recursive check,
in the same style as the compound-member case in the same function.

A JNI regression test exists at TestH5D.testH5DArray_super_no_id_leak, which reads
an H5T_ARRAY-of-int dataset in a loop and asserts via H5Fget_obj_count() that
no datatype IDs leak.

* Assert non-negative H5Fget_obj_count in array datatype ID leak test

Guard the before/after open-datatype counts against a negative
(failed) H5Fget_obj_count return, which would otherwise let the
equality check pass spuriously. Keep the count scoped to
H5F_OBJ_ALL: the leaked IDs are transient datatypes not attached to
any file, so a per-file count would not see them.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Modify CHANGELOG entry

* Modify CHANGELOG again

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: H. Joe Lee <hyoklee@hdfgroup.org>

java/src-jni/jni/h5util.c

@@ -2075,8 +2075,8 @@ h5str_detect_vlen_str(hid_t tid)
             goto done;
         } /* end if */
         ret = h5str_detect_vlen_str(btid);
+        H5Tclose(btid);
         if ((ret == 1) || (ret < 0)) {
-            H5Tclose(btid);
             goto done;
         } /* end if */
     }     /* end if */

java/src-jni/test/TestH5D.java

@@ -1624,6 +1624,77 @@ public class TestH5D {
         }
     }
 
+    /*
+     * Reading or writing a dataset whose memory type is an array of a fixed base type
+     * must not accumulate open datatype IDs. Write and read such a dataset repeatedly
+     * and verify the library's open-datatype count is unchanged.
+     */
+    @Test
+    public void testH5DArray_datatype_ids_stable() throws Throwable
+    {
+        String dset_name = "ArrayIntData";
+        long dtype_id    = HDF5Constants.H5I_INVALID_HID;
+        long dspace_id   = HDF5Constants.H5I_INVALID_HID;
+        long dset_id     = HDF5Constants.H5I_INVALID_HID;
+        long[] arr_dims  = {3};
+        final int NITER  = 50;
+        int[] wbuf       = {10, 20, 30};
+        int[] rbuf       = new int[3];
+
+        try {
+            dtype_id = H5.H5Tarray_create(HDF5Constants.H5T_NATIVE_INT, 1, arr_dims);
+            assertTrue("testH5DArray_datatype_ids_stable.H5Tarray_create: ", dtype_id >= 0);
+            dspace_id = H5.H5Screate(HDF5Constants.H5S_SCALAR);
+            assertTrue("testH5DArray_datatype_ids_stable.H5Screate: ", dspace_id >= 0);
+            dset_id = H5.H5Dcreate(H5fid, dset_name, dtype_id, dspace_id, HDF5Constants.H5P_DEFAULT,
+                                   HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);
+            assertTrue("testH5DArray_datatype_ids_stable.H5Dcreate: ", dset_id >= 0);
+
+            /*
+             * The leaked IDs are transient datatypes (from H5Tget_super), which are not
+             * attached to any file, so they are only visible through the H5F_OBJ_ALL path
+             * that walks the global datatype ID list. Scoping the count to a single file id
+             * would miss them entirely.
+             */
+            long before = H5.H5Fget_obj_count(HDF5Constants.H5F_OBJ_ALL, HDF5Constants.H5F_OBJ_DATATYPE);
+            assertTrue("testH5DArray_datatype_ids_stable.H5Fget_obj_count(before): ", before >= 0);
+
+            for (int i = 0; i < NITER; i++) {
+                H5.H5Dwrite(dset_id, dtype_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
+                            HDF5Constants.H5P_DEFAULT, wbuf);
+                H5.H5Dread(dset_id, dtype_id, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL,
+                           HDF5Constants.H5P_DEFAULT, rbuf);
+            }
+
+            long after = H5.H5Fget_obj_count(HDF5Constants.H5F_OBJ_ALL, HDF5Constants.H5F_OBJ_DATATYPE);
+            assertTrue("testH5DArray_datatype_ids_stable.H5Fget_obj_count(after): ", after >= 0);
+
+            assertEquals("testH5DArray_datatype_ids_stable: open datatype count changed after " + NITER +
+                             " write/read cycles on an array datatype",
+                         before, after);
+        }
+        finally {
+            if (dset_id >= 0)
+                try {
+                    H5.H5Dclose(dset_id);
+                }
+                catch (Exception ex) {
+                }
+            if (dtype_id >= 0)
+                try {
+                    H5.H5Tclose(dtype_id);
+                }
+                catch (Exception ex) {
+                }
+            if (dspace_id >= 0)
+                try {
+                    H5.H5Sclose(dspace_id);
+                }
+                catch (Exception ex) {
+                }
+        }
+    }
+
     @Test
     public void testH5DArray_string_buffer() throws Throwable
     {

java/src-jni/test/testfiles/JUnit-TestH5D.txt

@@ -17,6 +17,7 @@ JUnit version 4.13.2
 .testH5DwriteVL_compound_wrong_member_type
 .testH5Dget_access_plist
 .testH5Dread_vlen_of_nested_compound
+.testH5DArray_datatype_ids_stable
 .testH5DArray_string_buffer_flat_StringArray
 .testH5Dget_space_closed
 .testH5DArray_string_buffer_flat_StringArray_write
@@ -34,5 +35,5 @@ JUnit version 4.13.2
 
 Time:  XXXX
 
-OK (32 tests)
+OK (33 tests)