mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Optimizations for repeated VDS source file and dataset names (#5640)

Browse Source 
Add hash tables for detecting repeated source file and dataset names.
Share these repeated strings between mappings in memory. Add new
encoding format for VDS for shared names. Add tests and documentation
for these changes.

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Matthew Larson <matthewjlar@gmail.com>
This commit is contained in:
Neil Fortner
2025-07-16 09:23:59 -05:00
committed by GitHub
co-authored by github-actions Matthew Larson
parent 434b94c1b3
commit bc9914d2f0
12 changed files with 14188 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doxygen/aliases
+1 -1
View File
@@ -259,7 +259,7 @@ ALIASES += sa_metadata_ops="\sa \li H5Pget_all_coll_metadata_ops() \li H5Pget_co
# Specifications
################################################################################
ALIASES += ref_spec_fileformat="\ref FMT3"
ALIASES += ref_spec_fileformat="\ref FMT4"
ALIASES += ref_spec_fileformat_btrees_v1="\ref subsubsec_fmt3_infra_btrees_v1"
################################################################################
doxygen/dox/FileFormatDisc.dox
+1 -1
View File
@@ -10,7 +10,7 @@ Navigate back: \ref index "Main" / \ref TN
<dd>Current H5 library designers and knowledgeable external developers.</dd>
<dt>Background Reading:</dt>
<dd>\ref FMT3 <br />This describes the current HDF5 file format.</dd>
<dd>\ref FMT4 <br />This describes the current HDF5 file format.</dd>
</dl>
\section sec_fmtdisc_intro Introduction
doxygen/dox/FileFormatSpec.dox
+2
View File
@@ -1,3 +1,5 @@
\ref FMT4
\ref FMT3
\ref FMT2
doxygen/dox/H5.format.4.0.dox
+12706
View File
File diff suppressed because it is too large Load Diff
doxygen/dox/Specifications.dox
+1
View File
@@ -16,6 +16,7 @@ Navigate back: \ref index "Main"
\li \ref FMT11
\li \ref FMT2
\li \ref FMT3
\li \ref FMT4
\section sec_spec_other Other
release_docs/RELEASE.txt
+8
View File
@@ -200,6 +200,14 @@ New Features
Library:
--------
- The file format has been updated to 4.0
The Virtual Dataset Global Heap Block format has been updated to version 1
to support shared string storage for source filenames and dataset names,
reducing file size when multiple mappings reference the same sources. This
new format is only used when the HDF5 library version bounds lower bound
is set to 2.0 or later.
- The H5Dread_chunk() signature has changed
A new parameter, nalloc, has been added to H5Dread_chunk(). This parameter
src/H5Dvirtual.c
+149 -26
View File
@@ -399,14 +399,16 @@ herr_t
H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout)
{
H5O_storage_virtual_t *virt = &layout->storage.u.virt;
uint8_t *heap_block = NULL; /* Block to add to heap */
size_t *str_size = NULL; /* Array for VDS entry string lengths */
uint8_t *heap_block_p; /* Pointer into the heap block, while encoding */
size_t block_size; /* Total size of block needed */
hsize_t tmp_nentries; /* Temp. variable for # of VDS entries */
uint32_t chksum; /* Checksum for heap data */
size_t i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
uint8_t *heap_block = NULL; /* Block to add to heap */
size_t *str_size = NULL; /* Array for VDS entry string lengths */
uint8_t *heap_block_p; /* Pointer into the heap block, while encoding */
size_t block_size; /* Total size of block needed */
hsize_t tmp_hsize; /* Temp. variable for encoding hsize_t */
uint32_t chksum; /* Checksum for heap data */
uint8_t max_version; /* Maximum encoding version allowed by version bounds */
uint8_t version = H5O_LAYOUT_VDS_GH_ENC_VERS_0; /* Encoding version */
size_t i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@@ -421,6 +423,12 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout)
/* Set the low/high bounds according to 'f' for the API context */
H5CX_set_libver_bounds(f);
/* Calculate maximum encoding version. Currently there are no features that require a later version,
* so we only upgrade if the lower bound is high enough that we don't worry about backward
* compatibility, and if there is a benefit (will calculate the benefit later). */
max_version =
H5F_LOW_BOUND(f) >= H5F_LIBVER_V200 ? H5O_LAYOUT_VDS_GH_ENC_VERS_1 : H5O_LAYOUT_VDS_GH_ENC_VERS_0;
/* Allocate array for caching results of strlen */
if (NULL == (str_size = (size_t *)H5MM_malloc(2 * virt->list_nused * sizeof(size_t))))
HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, FAIL, "unable to allocate string length array");
@@ -459,11 +467,64 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout)
if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_dset.virtual_select)) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size");
block_size += (size_t)select_serial_size;
} /* end for */
}
/* Checksum */
block_size += 4;
/*
* Calculate_heap_block_size for version 1, if available
*/
if (max_version >= H5O_LAYOUT_VDS_GH_ENC_VERS_1) {
size_t block_size_1; /* Block size if we use version 1 */
/* Version and number of entries */
block_size_1 = (size_t)1 + H5F_SIZEOF_SIZE(f);
/* Calculate size of each entry */
for (i = 0; i < virt->list_nused; i++) {
H5O_storage_virtual_ent_t *ent = &virt->list[i];
hssize_t select_serial_size; /* Size of serialized selection */
/* Flags */
block_size_1 += (size_t)1;
/* Source file name (no encoding necessary for ".") */
if (strcmp(ent->source_file_name, ".")) {
if (ent->source_file_orig == SIZE_MAX)
block_size_1 += str_size[2 * i];
else
block_size_1 += MIN(str_size[2 * i], H5F_SIZEOF_SIZE(f));
}
/* Source dset name */
if (ent->source_dset_orig == SIZE_MAX)
block_size_1 += str_size[(2 * i) + 1];
else
block_size_1 += MIN(str_size[(2 * i) + 1], H5F_SIZEOF_SIZE(f));
/* Source selection */
if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_select)) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size");
block_size_1 += (size_t)select_serial_size;
/* Virtual dataset selection */
if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_dset.virtual_select)) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size");
block_size_1 += (size_t)select_serial_size;
}
/* Checksum */
block_size_1 += 4;
/* Determine which version to use. Only use version 1 if we save space. In the case of a tie, use
* version 1 since it will allow faster decoding since we know (some of) which strings are shared
* and won't need to do hash table lookups for those. */
if (block_size_1 <= block_size) {
version = H5O_LAYOUT_VDS_GH_ENC_VERS_1;
block_size = block_size_1;
}
}
/* Allocate heap block */
if (NULL == (heap_block = (uint8_t *)H5MM_malloc(block_size)))
HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, FAIL, "unable to allocate heap block");
@@ -474,22 +535,57 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout)
heap_block_p = heap_block;
/* Encode heap block encoding version */
*heap_block_p++ = (uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS;
*heap_block_p++ = version;
/* Number of entries */
tmp_nentries = (hsize_t)virt->list_nused;
H5F_ENCODE_LENGTH(f, heap_block_p, tmp_nentries);
H5_CHECK_OVERFLOW(virt->list_nused, size_t, hsize_t);
tmp_hsize = (hsize_t)virt->list_nused;
H5F_ENCODE_LENGTH(f, heap_block_p, tmp_hsize);
/* Encode each entry */
for (i = 0; i < virt->list_nused; i++) {
H5O_storage_virtual_ent_t *ent = &virt->list[i];
H5O_storage_virtual_ent_t *ent = &virt->list[i];
uint8_t flags = 0;
/* Flags */
if (version >= H5O_LAYOUT_VDS_GH_ENC_VERS_1) {
if (!strcmp(ent->source_file_name, "."))
/* Source file in same file as VDS */
flags |= H5O_LAYOUT_VDS_SOURCE_SAME_FILE;
else if ((ent->source_file_orig != SIZE_MAX) && (str_size[2 * i] >= H5F_SIZEOF_SIZE(f)))
/* Source file name is shared (stored in another entry) */
flags |= H5O_LAYOUT_VDS_SOURCE_FILE_SHARED;
if ((ent->source_dset_orig != SIZE_MAX) && (str_size[(2 * i) + 1] >= H5F_SIZEOF_SIZE(f)))
/* Source dataset name is shared (stored in another entry) */
flags |= H5O_LAYOUT_VDS_SOURCE_DSET_SHARED;
*heap_block_p++ = flags;
}
/* Source file name */
H5MM_memcpy((char *)heap_block_p, ent->source_file_name, str_size[2 * i]);
heap_block_p += str_size[2 * i];
if (!(flags & H5O_LAYOUT_VDS_SOURCE_SAME_FILE)) {
if (flags & H5O_LAYOUT_VDS_SOURCE_FILE_SHARED) {
assert(ent->source_file_orig < i);
tmp_hsize = (hsize_t)ent->source_file_orig;
H5F_ENCODE_LENGTH(f, heap_block_p, tmp_hsize);
}
else {
H5MM_memcpy((char *)heap_block_p, ent->source_file_name, str_size[2 * i]);
heap_block_p += str_size[2 * i];
}
}
/* Source dataset name */
H5MM_memcpy((char *)heap_block_p, ent->source_dset_name, str_size[(2 * i) + 1]);
heap_block_p += str_size[(2 * i) + 1];
if (flags & H5O_LAYOUT_VDS_SOURCE_DSET_SHARED) {
assert(ent->source_dset_orig < i);
tmp_hsize = (hsize_t)ent->source_dset_orig;
H5F_ENCODE_LENGTH(f, heap_block_p, tmp_hsize);
}
else {
H5MM_memcpy((char *)heap_block_p, ent->source_dset_name, str_size[(2 * i) + 1]);
heap_block_p += str_size[(2 * i) + 1];
}
/* Source selection */
if (H5S_SELECT_SERIALIZE(ent->source_select, &heap_block_p) < 0)
@@ -498,7 +594,7 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout)
/* Virtual selection */
if (H5S_SELECT_SERIALIZE(ent->source_dset.virtual_select, &heap_block_p) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize virtual selection");
} /* end for */
}
/* Checksum */
chksum = H5_checksum_metadata(heap_block, block_size - (size_t)4, 0);
@@ -507,7 +603,7 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout)
/* Insert block into global heap */
if (H5HG_insert(f, block_size, heap_block, &(virt->serial_list_hobjid)) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to insert virtual dataset heap block");
} /* end if */
}
done:
heap_block = (uint8_t *)H5MM_xfree(heap_block);
@@ -544,6 +640,12 @@ H5D__virtual_copy_layout(H5O_layout_t *layout)
assert(layout);
assert(layout->type == H5D_VIRTUAL);
/* Reset hash tables (they are owned by the original list). No need to recreate here - they are only
* needed when adding mappings, and if we add a new mapping the code in H5Pset_virtual() will rebuild
* them). */
virt->source_file_hash_table = NULL;
virt->source_dset_hash_table = NULL;
/* Save original entry list and top-level property lists and reset in layout
* so the originals aren't closed on error */
orig_source_fapl = virt->source_fapl;
@@ -573,11 +675,26 @@ H5D__virtual_copy_layout(H5O_layout_t *layout)
H5S_copy(orig_list[i].source_dset.virtual_select, false, true)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection");
/* Copy original source names */
if (NULL == (ent->source_file_name = H5MM_strdup(orig_list[i].source_file_name)))
HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name");
if (NULL == (ent->source_dset_name = H5MM_strdup(orig_list[i].source_dset_name)))
HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name");
/* Copy source file name. If the original is shared, share it in the copy too. */
ent->source_file_orig = orig_list[i].source_file_orig;
if (ent->source_file_orig == SIZE_MAX) {
/* Source file name is not shared, simply strdup to new ent */
if (NULL == (ent->source_file_name = H5MM_strdup(orig_list[i].source_file_name)))
HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name");
}
else
/* Source file name is shared, link to correct index in new list */
ent->source_file_name = virt->list[ent->source_file_orig].source_file_name;
/* Copy source dataset name. If the original is shared, share it in the copy too. */
ent->source_dset_orig = orig_list[i].source_dset_orig;
if (ent->source_dset_orig == SIZE_MAX) {
if (NULL == (ent->source_dset_name = H5MM_strdup(orig_list[i].source_dset_name)))
HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name");
}
else
/* Source dataset name is shared, link to correct index in new list */
ent->source_dset_name = virt->list[ent->source_dset_orig].source_dset_name;
/* Copy source selection */
if (NULL == (ent->source_select = H5S_copy(orig_list[i].source_select, false, true)))
@@ -700,6 +817,10 @@ H5D__virtual_reset_layout(H5O_layout_t *layout)
assert(layout);
assert(layout->type == H5D_VIRTUAL);
/* Clear hash tables */
HASH_CLEAR(hh_source_file, virt->source_file_hash_table);
HASH_CLEAR(hh_source_dset, virt->source_dset_hash_table);
/* Free the list entries. Note we always attempt to free everything even in
* the case of a failure. Because of this, and because we free the list
* afterwards, we do not need to zero out the memory in the list. */
@@ -710,8 +831,10 @@ H5D__virtual_reset_layout(H5O_layout_t *layout)
HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset");
/* Free original source names */
(void)H5MM_xfree(ent->source_file_name);
(void)H5MM_xfree(ent->source_dset_name);
if (ent->source_file_orig == SIZE_MAX)
(void)H5MM_xfree(ent->source_file_name);
if (ent->source_dset_orig == SIZE_MAX)
(void)H5MM_xfree(ent->source_dset_name);
/* Free sub_dset */
for (j = 0; j < ent->sub_dset_nalloc; j++)
src/H5Olayout.c
+192 -31
View File
@@ -560,6 +560,9 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
hsize_t tmp_hsize = 0;
uint32_t stored_chksum;
uint32_t computed_chksum;
size_t first_same_file = SIZE_MAX;
bool clear_file_hash_table = false;
bool clear_dset_hash_table = false;
/* Read heap */
if (NULL == (heap_block = (uint8_t *)H5HG_read(
@@ -575,10 +578,12 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
"ran off end of input buffer while decoding");
heap_vers = (uint8_t)*heap_block_p++;
if ((uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS != heap_vers)
HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL,
"bad version # of encoded VDS heap information, expected %u, got %u",
(unsigned)H5O_LAYOUT_VDS_GH_ENC_VERS, (unsigned)heap_vers);
assert(H5O_LAYOUT_VDS_GH_ENC_VERS_0 == 0);
if (heap_vers > (uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS_1)
HGOTO_ERROR(
H5E_OHDR, H5E_VERSION, NULL,
"bad version # of encoded VDS heap information, expected %u or lower, got %u",
(unsigned)H5O_LAYOUT_VDS_GH_ENC_VERS_1, (unsigned)heap_vers);
/* Number of entries */
if (H5_IS_BUFFER_OVERFLOW(heap_block_p, H5F_sizeof_size(f), heap_block_p_end))
@@ -602,49 +607,199 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
/* Decode each entry */
for (size_t i = 0; i < mesg->storage.u.virt.list_nused; i++) {
H5O_storage_virtual_ent_t
*tmp_ent; /* Temporary VDS entry pointer, for hash table lookups */
ptrdiff_t avail_buffer_space;
uint8_t flags = 0;
avail_buffer_space = heap_block_p_end - heap_block_p + 1;
if (avail_buffer_space <= 0)
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
/* Flags */
if (heap_vers >= H5O_LAYOUT_VDS_GH_ENC_VERS_1) {
flags = *heap_block_p++;
if (flags & ~H5O_LAYOUT_ALL_VDS_FLAGS)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad flag value for VDS mapping");
}
avail_buffer_space = heap_block_p_end - heap_block_p + 1;
/* Source file name */
tmp_size = strnlen((const char *)heap_block_p, (size_t)avail_buffer_space);
if (tmp_size == (size_t)avail_buffer_space)
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
if (flags & H5O_LAYOUT_VDS_SOURCE_SAME_FILE) {
/* Source file in same file as VDS, use "." */
if (first_same_file == SIZE_MAX) {
/* No previous instance of ".", copy "." to entry and record this instance */
if (NULL ==
(mesg->storage.u.virt.list[i].source_file_name = (char *)H5MM_malloc(2)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL,
"memory allocation failed for source file string");
mesg->storage.u.virt.list[i].source_file_name[0] = '.';
mesg->storage.u.virt.list[i].source_file_name[1] = '\0';
mesg->storage.u.virt.list[i].source_file_orig = SIZE_MAX;
first_same_file = i;
/* Invalidate hash table for use after decoding since it is missing this "."
*/
clear_file_hash_table = true;
}
else {
/* Reference previous instance of "." */
assert(first_same_file < i);
mesg->storage.u.virt.list[i].source_file_name =
mesg->storage.u.virt.list[first_same_file].source_file_name;
mesg->storage.u.virt.list[i].source_file_orig = first_same_file;
}
}
else {
if (flags & H5O_LAYOUT_VDS_SOURCE_FILE_SHARED) {
if (avail_buffer_space < H5F_SIZEOF_SIZE(f))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
/* Source file is shared (stored in another entry), decode origin entry number
*/
H5F_DECODE_LENGTH(f, heap_block_p, tmp_hsize);
H5_CHECK_OVERFLOW(tmp_hsize, hsize_t, size_t);
if ((size_t)tmp_hsize >= i)
HGOTO_ERROR(
H5E_OHDR, H5E_BADVALUE, NULL,
"origin source file entry has higher index than current entry");
mesg->storage.u.virt.list[i].source_file_orig = (size_t)tmp_hsize;
/* Use source file name from origin entry */
mesg->storage.u.virt.list[i].source_file_name =
mesg->storage.u.virt.list[tmp_hsize].source_file_name;
}
else {
tmp_size = strnlen((const char *)heap_block_p, (size_t)avail_buffer_space);
if (tmp_size == (size_t)avail_buffer_space)
HGOTO_ERROR(
H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding - unterminated source "
"file name string");
else
tmp_size += 1; /* Add space for NUL terminator */
else
tmp_size += 1; /* Add space for NUL terminator */
if (NULL ==
(mesg->storage.u.virt.list[i].source_file_name = (char *)H5MM_malloc(tmp_size)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL,
"unable to allocate memory for source file name");
H5MM_memcpy(mesg->storage.u.virt.list[i].source_file_name, heap_block_p, tmp_size);
heap_block_p += tmp_size;
/* Check for source file name in hash table. While this normally shouldn't be
* necessary if it is version 1 or greater and it is at least as long as "size
* of lengths", we should still check since if we don't and it's not shared in
* the file for whatever reason it could cause the library to insert a
* duplicate key if it rebuilds the hash table. */
tmp_ent = NULL;
if (i > 0)
HASH_FIND(hh_source_file, mesg->storage.u.virt.source_file_hash_table,
heap_block_p, tmp_size - 1, tmp_ent);
if (tmp_ent) {
/* Found source file name in previous mapping, use link to that mapping's
* source file name */
assert(tmp_ent >= mesg->storage.u.virt.list &&
tmp_ent < &mesg->storage.u.virt.list[i]);
mesg->storage.u.virt.list[i].source_file_orig =
(size_t)(tmp_ent - mesg->storage.u.virt.list);
mesg->storage.u.virt.list[i].source_file_name = tmp_ent->source_file_name;
}
else {
/* Did not find source file name, copy it to the entry and add it to the
* hash table */
if (NULL == (mesg->storage.u.virt.list[i].source_file_name =
(char *)H5MM_malloc(tmp_size)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL,
"unable to allocate memory for source file name");
mesg->storage.u.virt.list[i].source_file_orig = SIZE_MAX;
H5MM_memcpy(mesg->storage.u.virt.list[i].source_file_name, heap_block_p,
tmp_size);
/* Add to source file name hash table. If we eventually make the library
* resilient to repeated strings not stored shared in memory, possibly by
* permanently disabling the hash table, or marking it as needing a
* careful rebuild, we can avoid this step if the version is 1 or greater
* and the name is at least as long as "size of lengths". See comment
* above about HASH_FIND line. */
HASH_ADD_KEYPTR(hh_source_file,
mesg->storage.u.virt.source_file_hash_table,
mesg->storage.u.virt.list[i].source_file_name,
tmp_size - 1, &(mesg->storage.u.virt.list[i]));
}
heap_block_p += tmp_size;
}
}
avail_buffer_space = heap_block_p_end - heap_block_p + 1;
if (avail_buffer_space <= 0)
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
/* Source dataset name */
tmp_size = strnlen((const char *)heap_block_p, (size_t)avail_buffer_space);
if (tmp_size == (size_t)avail_buffer_space)
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding - unterminated source "
"dataset name string");
else
tmp_size += 1; /* Add space for NUL terminator */
if (flags & H5O_LAYOUT_VDS_SOURCE_DSET_SHARED) {
if (avail_buffer_space < H5F_SIZEOF_SIZE(f))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
if (NULL ==
(mesg->storage.u.virt.list[i].source_dset_name = (char *)H5MM_malloc(tmp_size)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL,
"unable to allocate memory for source dataset name");
H5MM_memcpy(mesg->storage.u.virt.list[i].source_dset_name, heap_block_p, tmp_size);
heap_block_p += tmp_size;
/* Source dataset is shared (stored in another entry), decode origin entry number
*/
H5F_DECODE_LENGTH(f, heap_block_p, tmp_hsize);
H5_CHECK_OVERFLOW(tmp_hsize, hsize_t, size_t);
if ((size_t)tmp_hsize >= i)
HGOTO_ERROR(
H5E_OHDR, H5E_BADVALUE, NULL,
"origin source dataset entry has higher index than current entry");
mesg->storage.u.virt.list[i].source_dset_orig = (size_t)tmp_hsize;
/* Use source dataset name from origin entry */
mesg->storage.u.virt.list[i].source_dset_name =
mesg->storage.u.virt.list[tmp_hsize].source_dset_name;
}
else {
tmp_size = strnlen((const char *)heap_block_p, (size_t)avail_buffer_space);
if (tmp_size == (size_t)avail_buffer_space)
HGOTO_ERROR(
H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding - unterminated source "
"dataset name string");
else
tmp_size += 1; /* Add space for NUL terminator */
/* Check for source dataset name in hash table. While this normally shouldn't be
* necessary if it is version 1 or greater and it is at least as long as "size of
* lengths", we should still check since if we don't and it's not shared in the
* file for whatever reason it could cause the library to insert a duplicate key
* if it rebuilds the hash table. */
tmp_ent = NULL;
if (i > 0)
HASH_FIND(hh_source_dset, mesg->storage.u.virt.source_dset_hash_table,
heap_block_p, tmp_size - 1, tmp_ent);
if (tmp_ent) {
/* Found source dataset name in previous mapping, use link to that mapping's
* source dataset name */
assert(tmp_ent >= mesg->storage.u.virt.list &&
tmp_ent < &mesg->storage.u.virt.list[i]);
mesg->storage.u.virt.list[i].source_dset_orig =
(size_t)(tmp_ent - mesg->storage.u.virt.list);
mesg->storage.u.virt.list[i].source_dset_name = tmp_ent->source_dset_name;
}
else {
/* Did not find source dataset name, copy it to the entry and add it to the
* hash table */
if (NULL == (mesg->storage.u.virt.list[i].source_dset_name =
(char *)H5MM_malloc(tmp_size)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL,
"unable to allocate memory for source dataset name");
mesg->storage.u.virt.list[i].source_dset_orig = SIZE_MAX;
H5MM_memcpy(mesg->storage.u.virt.list[i].source_dset_name, heap_block_p,
tmp_size);
/* Add to source dataset name hash table. If we eventually make the library
* resilient to repeated strings not stored shared in memory, possibly by
* permanently disabling the hash table, or marking it as needing a careful
* rebuild, we can avoid this step if the version is 1 or greater and the name
* is at least as long as "size of lengths". See comment above about HASH_FIND
* line. */
HASH_ADD_KEYPTR(hh_source_dset, mesg->storage.u.virt.source_dset_hash_table,
mesg->storage.u.virt.list[i].source_dset_name, tmp_size - 1,
&(mesg->storage.u.virt.list[i]));
}
heap_block_p += tmp_size;
}
/* Source selection */
avail_buffer_space = heap_block_p_end - heap_block_p + 1;
@@ -755,6 +910,12 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
/* Verify that the heap block size is correct */
if ((size_t)(heap_block_p - heap_block) != block_size)
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect heap block size");
/* Clear hash tables if requested */
if (clear_file_hash_table)
HASH_CLEAR(hh_source_file, mesg->storage.u.virt.source_file_hash_table);
if (clear_dset_hash_table)
HASH_CLEAR(hh_source_dset, mesg->storage.u.virt.source_dset_hash_table);
} /* end if */
/* Set the layout operations */
src/H5Oprivate.h
+23 -2
View File
@@ -402,8 +402,19 @@ typedef struct H5O_efl_t {
#define H5O_LAYOUT_ALL_CHUNK_FLAGS \
(H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS | H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER)
/* Version number of encoded virtual dataset global heap blocks */
#define H5O_LAYOUT_VDS_GH_ENC_VERS 0
/* Initial version of encoded virtual dataset global heap blocks */
#define H5O_LAYOUT_VDS_GH_ENC_VERS_0 0
/* This version added support for shared source file and dataset names, as well as not storing the source file
* name when it is "." */
#define H5O_LAYOUT_VDS_GH_ENC_VERS_1 1
/* Flags for virtual dataset mappings */
#define H5O_LAYOUT_VDS_SOURCE_FILE_SHARED 0x01
#define H5O_LAYOUT_VDS_SOURCE_DSET_SHARED 0x02
#define H5O_LAYOUT_VDS_SOURCE_SAME_FILE 0x04
#define H5O_LAYOUT_ALL_VDS_FLAGS \
(H5O_LAYOUT_VDS_SOURCE_FILE_SHARED | H5O_LAYOUT_VDS_SOURCE_DSET_SHARED | H5O_LAYOUT_VDS_SOURCE_SAME_FILE)
/* Initial version of the layout information. Used when space is allocated */
#define H5O_LAYOUT_VERSION_1 1
@@ -529,7 +540,9 @@ typedef struct H5O_storage_virtual_ent_t {
/* Stored */
H5O_storage_virtual_srcdset_t source_dset; /* Information about the source dataset */
char *source_file_name; /* Original (unparsed) source file name */
size_t source_file_orig; /* Index of first entry containing source_file_name */
char *source_dset_name; /* Original (unparsed) source dataset name */
size_t source_dset_orig; /* Index of first entry containing source_dset_name */
struct H5S_t *source_select; /* Selection in the source dataset for mapping */
/* Not stored */
@@ -559,6 +572,8 @@ typedef struct H5O_storage_virtual_ent_t {
unlim_extent_virtual */
H5O_virtual_space_status_t source_space_status; /* Extent patching status of source_select */
H5O_virtual_space_status_t virtual_space_status; /* Extent patching status of virtual_select */
UT_hash_handle hh_source_file; /* Hash handle for this entry in the source file name hash table */
UT_hash_handle hh_source_dset; /* Hash handle for this entry in the source dataset name hash table */
} H5O_storage_virtual_ent_t;
typedef struct H5O_storage_virtual_t {
@@ -580,6 +595,12 @@ typedef struct H5O_storage_virtual_t {
hid_t source_fapl; /* FAPL to use to open source files */
hid_t source_dapl; /* DAPL to use to open source datasets */
bool init; /* Whether all information has been completely initialized */
H5O_storage_virtual_ent_t
*source_file_hash_table; /* Hash table of virtual entries sorted by source file name. Only the first
occurrence of each source file name is stored. */
H5O_storage_virtual_ent_t
*source_dset_hash_table; /* Hash table of virtual entries sorted by source dataset name. Only the
first occurrence of each source dataset name is stored. */
} H5O_storage_virtual_t;
typedef struct H5O_storage_t {
src/H5Pdcpl.c
+93 -15
View File
@@ -89,7 +89,7 @@
{ \
{HADDR_UNDEF, 0}, 0, NULL, 0, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, \
H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false \
H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, false, NULL, NULL \
}
#define H5D_DEF_STORAGE_COMPACT \
{ \
@@ -2023,12 +2023,14 @@ herr_t
H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const char *src_dset_name,
hid_t src_space_id)
{
H5P_genplist_t *plist = NULL; /* Property list pointer */
H5O_layout_t virtual_layout; /* Layout information for setting virtual info */
H5S_t *vspace; /* Virtual dataset space selection */
H5S_t *src_space; /* Source dataset space selection */
H5O_storage_virtual_ent_t *old_list = NULL; /* List pointer previously on property list */
H5O_storage_virtual_ent_t *ent = NULL; /* Convenience pointer to new VDS entry */
H5P_genplist_t *plist = NULL; /* Property list pointer */
H5O_layout_t virtual_layout; /* Layout information for setting virtual info */
H5S_t *vspace; /* Virtual dataset space selection */
H5S_t *src_space; /* Source dataset space selection */
H5O_storage_virtual_ent_t *old_list = NULL; /* List pointer previously on property list */
H5O_storage_virtual_ent_t *ent = NULL; /* Convenience pointer to new VDS entry */
H5O_storage_virtual_ent_t *tmp_ent; /* Temporary VDS entry pointer, for hash table lookups */
size_t tmp_len; /* Temporary variable holding a string length */
bool retrieved_layout = false; /* Whether the layout has been retrieved */
bool free_list = false; /* Whether to free the list of virtual entries */
herr_t ret_value = SUCCEED; /* Return value */
@@ -2078,25 +2080,95 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const
/* Expand list if necessary */
if (virtual_layout.storage.u.virt.list_nused == virtual_layout.storage.u.virt.list_nalloc) {
H5O_storage_virtual_ent_t *x; /* Pointer to the new list */
size_t new_alloc = MAX(H5D_VIRTUAL_DEF_LIST_SIZE, virtual_layout.storage.u.virt.list_nalloc * 2);
size_t new_alloc = MAX(H5D_VIRTUAL_DEF_LIST_SIZE, virtual_layout.storage.u.virt.list_nalloc * 2);
ptrdiff_t buf_diff;
/* Expand size of entry list */
if (NULL == (x = (H5O_storage_virtual_ent_t *)H5MM_realloc(
virtual_layout.storage.u.virt.list, new_alloc * sizeof(H5O_storage_virtual_ent_t))))
HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't reallocate virtual dataset mapping list");
buf_diff = (char *)x - (char *)virtual_layout.storage.u.virt.list;
virtual_layout.storage.u.virt.list = x;
virtual_layout.storage.u.virt.list_nalloc = new_alloc;
/* Adjust pointers in the hash tables in case realloc moved the buffers, and hence all the elements
* and hash handles in the hash tables */
HASH_ADJUST_PTRS(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table, buf_diff);
HASH_ADJUST_PTRS(hh_source_dset, virtual_layout.storage.u.virt.source_dset_hash_table, buf_diff);
} /* end if */
/* Add virtual dataset mapping entry */
/* Check if we need to (re)build the hash tables */
assert((virtual_layout.storage.u.virt.list_nused &&
virtual_layout.storage.u.virt.source_file_hash_table &&
virtual_layout.storage.u.virt.source_dset_hash_table) ||
(!virtual_layout.storage.u.virt.source_file_hash_table &&
!virtual_layout.storage.u.virt.source_dset_hash_table));
if (virtual_layout.storage.u.virt.list_nused && !virtual_layout.storage.u.virt.source_file_hash_table) {
for (size_t i = 0; i < virtual_layout.storage.u.virt.list_nused; i++) {
if (virtual_layout.storage.u.virt.list[i].source_file_orig == SIZE_MAX)
HASH_ADD_KEYPTR(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table,
virtual_layout.storage.u.virt.list[i].source_file_name,
strlen(virtual_layout.storage.u.virt.list[i].source_file_name),
&(virtual_layout.storage.u.virt.list[i]));
if (virtual_layout.storage.u.virt.list[i].source_dset_orig == SIZE_MAX)
HASH_ADD_KEYPTR(hh_source_dset, virtual_layout.storage.u.virt.source_dset_hash_table,
virtual_layout.storage.u.virt.list[i].source_dset_name,
strlen(virtual_layout.storage.u.virt.list[i].source_dset_name),
&(virtual_layout.storage.u.virt.list[i]));
}
}
/*
* Add virtual dataset mapping entry
*/
ent = &virtual_layout.storage.u.virt.list[virtual_layout.storage.u.virt.list_nused];
memset(ent, 0, sizeof(H5O_storage_virtual_ent_t)); /* Clear before starting to set up */
if (NULL == (ent->source_dset.virtual_select = H5S_copy(vspace, false, true)))
HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy virtual selection");
if (NULL == (ent->source_file_name = H5MM_xstrdup(src_file_name)))
HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name");
if (NULL == (ent->source_dset_name = H5MM_xstrdup(src_dset_name)))
HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name");
/* Check for source file name in hash table */
tmp_ent = NULL;
tmp_len = strlen(src_file_name);
if (virtual_layout.storage.u.virt.list_nused > 0)
HASH_FIND(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table, src_file_name,
tmp_len, tmp_ent);
if (tmp_ent) {
/* Found source file name in previous mapping, use link to that mapping's source file name */
assert(tmp_ent >= virtual_layout.storage.u.virt.list && tmp_ent < ent);
ent->source_file_orig = (size_t)(tmp_ent - virtual_layout.storage.u.virt.list);
ent->source_file_name = tmp_ent->source_file_name;
}
else {
/* Did not find source file name, copy it to the entry and add it to the hash table */
if (NULL == (ent->source_file_name = H5MM_xstrdup(src_file_name)))
HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name");
ent->source_file_orig = SIZE_MAX;
HASH_ADD_KEYPTR(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table,
ent->source_file_name, tmp_len, ent);
}
/* Check for source dataset name in hash table */
tmp_ent = NULL;
tmp_len = strlen(src_dset_name);
if (virtual_layout.storage.u.virt.list_nused > 0)
HASH_FIND(hh_source_dset, virtual_layout.storage.u.virt.source_dset_hash_table, src_dset_name,
tmp_len, tmp_ent);
if (tmp_ent) {
/* Found source dataset name in previous mapping, use link to that mapping's source dataset name */
assert(tmp_ent >= virtual_layout.storage.u.virt.list && tmp_ent < ent);
ent->source_dset_orig = (size_t)(tmp_ent - virtual_layout.storage.u.virt.list);
ent->source_dset_name = tmp_ent->source_dset_name;
}
else {
/* Did not find source dataset name, copy it to the entry and add it to the hash table */
if (NULL == (ent->source_dset_name = H5MM_xstrdup(src_dset_name)))
HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source dataset name");
ent->source_dset_orig = SIZE_MAX;
HASH_ADD_KEYPTR(hh_source_dset, virtual_layout.storage.u.virt.source_dset_hash_table,
ent->source_dset_name, tmp_len, ent);
}
if (NULL == (ent->source_select = H5S_copy(src_space, false, true)))
HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy source selection");
if (H5D_virtual_parse_source_name(ent->source_file_name, &ent->parsed_source_file_name,
@@ -2155,8 +2227,14 @@ done:
if (ret_value < 0) {
/* Free incomplete entry if present */
if (ent) {
ent->source_file_name = (char *)H5MM_xfree(ent->source_file_name);
ent->source_dset_name = (char *)H5MM_xfree(ent->source_dset_name);
if (ent->source_file_orig == SIZE_MAX)
ent->source_file_name = (char *)H5MM_xfree(ent->source_file_name);
else
HASH_DELETE(hh_source_file, virtual_layout.storage.u.virt.source_file_hash_table, ent);
if (ent->source_dset_orig == SIZE_MAX)
ent->source_dset_name = (char *)H5MM_xfree(ent->source_dset_name);
else
HASH_DELETE(hh_source_dset, virtual_layout.storage.u.virt.source_dset_hash_table, ent);
if (ent->source_dset.virtual_select && H5S_close(ent->source_dset.virtual_select) < 0)
HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release virtual selection");
ent->source_dset.virtual_select = NULL;
src/uthash.h
+31
View File
@@ -1112,6 +1112,37 @@ typedef unsigned char uint8_t;
#define HASH_COUNT(head) HASH_CNT(hh, head)
#define HASH_CNT(hh, head) ((head != NULL) ? ((head)->hh.tbl->num_items) : 0U)
/* Adjust all element and hash handle pointers by ptr_adj bytes. Does not adjust key pointers. Intended for
* the case where all elements are stored in a flat array and that array is realloced. */
#define HASH_ADJUST_PTRS(hh, head, ptr_adj) \
do { \
ptrdiff_t _ptr_adj = (ptrdiff_t)(ptr_adj); \
if (head && (_ptr_adj != 0)) { \
unsigned _tmp_bkt; \
(head) = (void *)((char *)head + _ptr_adj); \
(head)->hh.tbl->tail = (UT_hash_handle *)(void *)((char *)(head)->hh.tbl->tail + _ptr_adj); \
for (_tmp_bkt = 0; _tmp_bkt < (head)->hh.tbl->num_buckets; _tmp_bkt++) \
if ((head)->hh.tbl->buckets[_tmp_bkt].hh_head) { \
(head)->hh.tbl->buckets[_tmp_bkt].hh_head = \
(UT_hash_handle *)(void *)((char *)(head)->hh.tbl->buckets[_tmp_bkt].hh_head + \
_ptr_adj); \
for (UT_hash_handle *_tmp_hh = (head)->hh.tbl->buckets[_tmp_bkt].hh_head; \
_tmp_hh != NULL; _tmp_hh = _tmp_hh->hh_next) { \
if (_tmp_hh->prev) \
_tmp_hh->prev = (UT_hash_handle *)(void *)((char *)_tmp_hh->prev + _ptr_adj); \
if (_tmp_hh->next) \
_tmp_hh->next = (UT_hash_handle *)(void *)((char *)_tmp_hh->next + _ptr_adj); \
if (_tmp_hh->hh_prev) \
_tmp_hh->hh_prev = \
(UT_hash_handle *)(void *)((char *)_tmp_hh->hh_prev + _ptr_adj); \
if (_tmp_hh->hh_next) \
_tmp_hh->hh_next = \
(UT_hash_handle *)(void *)((char *)_tmp_hh->hh_next + _ptr_adj); \
} \
} \
} \
} while (0)
typedef struct UT_hash_bucket {
struct UT_hash_handle *hh_head;
unsigned count;
test/dsets.c
+981
View File
@@ -78,6 +78,7 @@ static const char *FILENAME[] = {"dataset", /* 0 */
"alloc_0sized", /* 26 */
"h5s_block", /* 27 */
"h5s_plist", /* 28 */
"vds_strings", /* 29 */
NULL};
#define OHMIN_FILENAME_A "ohdr_min_a"
@@ -308,6 +309,9 @@ const char *OLD_FILENAME[] = {
#define DCPL_LAYOUT_DIM2 10
#define DCPL_LAYOUT_NUM_SRC_DSETS 2
/* Declarations for test test_vds_shared_strings */
#define NUM_MAPPINGS_MANY 1000
/* Local prototypes for filter functions */
static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes,
size_t *buf_size, void **buf);
@@ -16147,6 +16151,980 @@ error:
return -1;
}
/*-------------------------------------------------------------------------
* Function: test_vds_shared_strings
*
* Purpose: Tests VDS (Virtual Dataset) shared strings functionality.
* Verifies that string sharing works as expected and that
* the correct encoding format is used.
*
* Return: Success: 0
* Failure: -1
*-------------------------------------------------------------------------
*/
static int
test_vds_shared_strings(hid_t fapl)
{
char filename[FILENAME_BUF_SIZE];
hid_t file_id = H5I_INVALID_HID; /* File */
hid_t dcpl_id = H5I_INVALID_HID; /* Dataset creation property list */
hid_t src_space_id = H5I_INVALID_HID; /* Source dataspace */
hid_t virt_space_id = H5I_INVALID_HID; /* Virtual dataspace */
hid_t dset_id = H5I_INVALID_HID; /* Virtual dataset */
hsize_t dims[1] = {10}; /* Dataset dimensions */
H5O_storage_virtual_t *virt_layout = NULL; /* Virtual storage layout */
H5D_t *dset_int = NULL; /* Internal dataset structure */
TESTING("VDS sharing of file/dataset names");
/* Set up file name */
h5_fixname(FILENAME[29], fapl, filename, sizeof(filename));
/* Create source and virtual dataspaces */
if ((src_space_id = H5Screate_simple(1, dims, NULL)) < 0)
TEST_ERROR;
if ((virt_space_id = H5Screate_simple(1, dims, NULL)) < 0)
TEST_ERROR;
/*
* Test 1: VDS with no sharing
*/
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Add virtual mappings with completely different strings */
if (H5Pset_virtual(dcpl_id, virt_space_id, "file1.h5", "/dataset1", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "file2.h5", "/dataset2", src_space_id) < 0)
TEST_ERROR;
if ((dset_id = H5Dcreate2(file_id, "vds_no_share", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name == virt_layout->list[1].source_file_name) {
H5_FAILED();
puts(" Source file names are erroneously shared");
goto error;
}
if (virt_layout->list[0].source_dset_name == virt_layout->list[1].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are erroneously shared");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[1].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source file names are erroneously marked as shared");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX ||
virt_layout->list[1].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source dataset names are erroneously marked as shared");
goto error;
}
/* Re-open verification for test 1 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_no_share", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name == virt_layout->list[1].source_file_name) {
H5_FAILED();
puts(" Source file names are erroneously shared after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_name == virt_layout->list[1].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are erroneously shared after re-open");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[1].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source file names are erroneously marked as shared after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX ||
virt_layout->list[1].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source dataset names are erroneously marked as shared after re-open");
goto error;
}
/* Close resources for test 1 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
/*
* Test 2: VDS with shared source filenames
*/
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Add virtual mappings with repeated source file, different datasets */
if (H5Pset_virtual(dcpl_id, virt_space_id, "shared_source.h5", "/dataset1", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "shared_source.h5", "/dataset2", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "shared_source.h5", "/dataset3", src_space_id) < 0)
TEST_ERROR;
if ((dset_id = H5Dcreate2(file_id, "vds_shared_file", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name != virt_layout->list[1].source_file_name ||
virt_layout->list[0].source_file_name != virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" Source file names are not shared");
goto error;
}
if (virt_layout->list[0].source_dset_name == virt_layout->list[1].source_dset_name ||
virt_layout->list[0].source_dset_name == virt_layout->list[2].source_dset_name ||
virt_layout->list[1].source_dset_name == virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are erroneously shared");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" First source file name incorrectly marked as shared");
goto error;
}
if (virt_layout->list[1].source_file_orig != 0 || virt_layout->list[2].source_file_orig != 0) {
H5_FAILED();
puts(" Source file name sharing indices are incorrect");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX ||
virt_layout->list[1].source_dset_orig != SIZE_MAX ||
virt_layout->list[2].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source dataset names are erroneously marked as shared");
goto error;
}
/* Re-open verification for test 2 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_shared_file", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name != virt_layout->list[1].source_file_name ||
virt_layout->list[0].source_file_name != virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" Source file names are not shared after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_name == virt_layout->list[1].source_dset_name ||
virt_layout->list[0].source_dset_name == virt_layout->list[2].source_dset_name ||
virt_layout->list[1].source_dset_name == virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are erroneously shared after re-open");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" First source file name incorrectly marked as shared after re-open");
goto error;
}
if (virt_layout->list[1].source_file_orig != 0 || virt_layout->list[2].source_file_orig != 0) {
H5_FAILED();
puts(" Source file name sharing indices are incorrect after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX ||
virt_layout->list[1].source_dset_orig != SIZE_MAX ||
virt_layout->list[2].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source dataset names are erroneously marked as shared after re-open");
goto error;
}
/* Close resources for test 2 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
/*
* Test 3: VDS with shared dataset names
*/
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Add virtual mappings with different source files, same dataset */
if (H5Pset_virtual(dcpl_id, virt_space_id, "source1.h5", "/shared_dataset", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "source2.h5", "/shared_dataset", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "source3.h5", "/shared_dataset", src_space_id) < 0)
TEST_ERROR;
if ((dset_id = H5Dcreate2(file_id, "vds_shared_dset", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_dset_name != virt_layout->list[1].source_dset_name ||
virt_layout->list[0].source_dset_name != virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are not shared");
goto error;
}
if (virt_layout->list[0].source_file_name == virt_layout->list[1].source_file_name ||
virt_layout->list[0].source_file_name == virt_layout->list[2].source_file_name ||
virt_layout->list[1].source_file_name == virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" Source file names are erroneously shared");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" First source dataset name incorrectly marked as shared");
goto error;
}
if (virt_layout->list[1].source_dset_orig != 0 || virt_layout->list[2].source_dset_orig != 0) {
H5_FAILED();
puts(" Source dataset name sharing indices are incorrect");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[1].source_file_orig != SIZE_MAX ||
virt_layout->list[2].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source file names are erroneously marked as shared");
goto error;
}
/* Re-open verification for test 3 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_shared_dset", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_dset_name != virt_layout->list[1].source_dset_name ||
virt_layout->list[0].source_dset_name != virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are not shared after re-open");
goto error;
}
if (virt_layout->list[0].source_file_name == virt_layout->list[1].source_file_name ||
virt_layout->list[0].source_file_name == virt_layout->list[2].source_file_name ||
virt_layout->list[1].source_file_name == virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" Source file names are erroneously shared after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" First source dataset name incorrectly marked as shared after re-open");
goto error;
}
if (virt_layout->list[1].source_dset_orig != 0 || virt_layout->list[2].source_dset_orig != 0) {
H5_FAILED();
puts(" Source dataset name sharing indices are incorrect after re-open");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[1].source_file_orig != SIZE_MAX ||
virt_layout->list[2].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" Source file names are erroneously marked as shared after re-open");
goto error;
}
/* Close resources for test 3 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
/*
* Test 4: VDS with both filenames and dataset names shared
*/
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Add identical virtual mappings */
if (H5Pset_virtual(dcpl_id, virt_space_id, "shared_source.h5", "/shared_dataset", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "shared_source.h5", "/shared_dataset", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "shared_source.h5", "/shared_dataset", src_space_id) < 0)
TEST_ERROR;
if ((dset_id = H5Dcreate2(file_id, "vds_shared_both", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name != virt_layout->list[1].source_file_name ||
virt_layout->list[0].source_file_name != virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" Source file names are not shared");
goto error;
}
if (virt_layout->list[0].source_dset_name != virt_layout->list[1].source_dset_name ||
virt_layout->list[0].source_dset_name != virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are not shared");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[0].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" First entry incorrectly marked as shared");
goto error;
}
if (virt_layout->list[1].source_file_orig != 0 || virt_layout->list[1].source_dset_orig != 0 ||
virt_layout->list[2].source_file_orig != 0 || virt_layout->list[2].source_dset_orig != 0) {
H5_FAILED();
puts(" String sharing indices are incorrect");
goto error;
}
/* Re-open verification for test 4 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_shared_both", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name != virt_layout->list[1].source_file_name ||
virt_layout->list[0].source_file_name != virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" Source file names are not shared after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_name != virt_layout->list[1].source_dset_name ||
virt_layout->list[0].source_dset_name != virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are not shared after re-open");
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[0].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" First entry incorrectly marked as shared after re-open");
goto error;
}
if (virt_layout->list[1].source_file_orig != 0 || virt_layout->list[1].source_dset_orig != 0 ||
virt_layout->list[2].source_file_orig != 0 || virt_layout->list[2].source_dset_orig != 0) {
H5_FAILED();
puts(" String sharing indices are incorrect after re-open");
goto error;
}
/* Close resources for test 4 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
/*
* Test 5: VDS with same-file reference (".")
*/
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Add virtual mappings using "." for same file */
if (H5Pset_virtual(dcpl_id, virt_space_id, ".", "/dataset1", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, ".", "/dataset2", src_space_id) < 0)
TEST_ERROR;
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dcreate2(file_id, "vds_same_file", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name != virt_layout->list[1].source_file_name) {
H5_FAILED();
puts(" Same-file strings are not shared");
goto error;
}
if (strcmp(virt_layout->list[0].source_file_name, ".") != 0) {
H5_FAILED();
printf(" Expected same-file reference '.', got '%s'\n", virt_layout->list[0].source_file_name);
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" First same-file entry incorrectly marked as shared");
goto error;
}
if (virt_layout->list[1].source_file_orig != 0) {
H5_FAILED();
puts(" Same-file sharing index is incorrect");
goto error;
}
if (virt_layout->list[0].source_dset_name == virt_layout->list[1].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are erroneously shared");
goto error;
}
/* Re-open verification for test 5 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_same_file", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
if (virt_layout->list[0].source_file_name != virt_layout->list[1].source_file_name) {
H5_FAILED();
puts(" Same-file strings are not shared after re-open");
goto error;
}
if (strcmp(virt_layout->list[0].source_file_name, ".") != 0) {
H5_FAILED();
printf(" Expected same-file reference '.', got '%s' after re-open\n",
virt_layout->list[0].source_file_name);
goto error;
}
if (virt_layout->list[0].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" First same-file entry incorrectly marked as shared after re-open");
goto error;
}
if (virt_layout->list[1].source_file_orig != 0) {
H5_FAILED();
puts(" Same-file sharing index is incorrect after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_name == virt_layout->list[1].source_dset_name) {
H5_FAILED();
puts(" Source dataset names are erroneously shared after re-open");
goto error;
}
/* Clean-up for test 5 */
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
/*
* Test 6: VDS with unusual pattern testing robust sharing detection
* Pattern: dset1, dset2, dset1, dset3, dset2
*/
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Add virtual mappings in unusual pattern: dset1, dset2, dset1, dset3, dset2 */
if (H5Pset_virtual(dcpl_id, virt_space_id, "file1.h5", "/dset1", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "file2.h5", "/dset2", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "file1.h5", "/dset1", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "file3.h5", "/dset3", src_space_id) < 0)
TEST_ERROR;
if (H5Pset_virtual(dcpl_id, virt_space_id, "file2.h5", "/dset2", src_space_id) < 0)
TEST_ERROR;
if ((dset_id = H5Dcreate2(file_id, "vds_unusual_pattern", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT,
dcpl_id, H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
/* Verify sharing pattern for files:
* [0]: file1.h5 (original)
* [1]: file2.h5 (original)
* [2]: file1.h5 (shared from [0])
* [3]: file3.h5 (original)
* [4]: file2.h5 (shared from [1])
*/
if (virt_layout->list[0].source_file_name != virt_layout->list[2].source_file_name) {
H5_FAILED();
puts(" File sharing failed: entries [0] and [2] should share file1.h5");
goto error;
}
if (virt_layout->list[1].source_file_name != virt_layout->list[4].source_file_name) {
H5_FAILED();
puts(" File sharing failed: entries [1] and [4] should share file2.h5");
goto error;
}
/* Verify dataset sharing pattern:
* [0]: /dset1 (original)
* [1]: /dset2 (original)
* [2]: /dset1 (shared from [0])
* [3]: /dset3 (original)
* [4]: /dset2 (shared from [1])
*/
if (virt_layout->list[0].source_dset_name != virt_layout->list[2].source_dset_name) {
H5_FAILED();
puts(" Dataset sharing failed: entries [0] and [2] should share /dset1");
goto error;
}
if (virt_layout->list[1].source_dset_name != virt_layout->list[4].source_dset_name) {
H5_FAILED();
puts(" Dataset sharing failed: entries [1] and [4] should share /dset2");
goto error;
}
/* Verify sharing indices are correct */
if (virt_layout->list[0].source_file_orig != SIZE_MAX ||
virt_layout->list[1].source_file_orig != SIZE_MAX ||
virt_layout->list[3].source_file_orig != SIZE_MAX) {
H5_FAILED();
puts(" File sharing indices: original entries incorrectly marked as shared");
goto error;
}
if (virt_layout->list[2].source_file_orig != 0 || virt_layout->list[4].source_file_orig != 1) {
H5_FAILED();
puts(" File sharing indices: shared entries have incorrect indices");
goto error;
}
if (virt_layout->list[0].source_dset_orig != SIZE_MAX ||
virt_layout->list[1].source_dset_orig != SIZE_MAX ||
virt_layout->list[3].source_dset_orig != SIZE_MAX) {
H5_FAILED();
puts(" Dataset sharing indices: original entries incorrectly marked as shared");
goto error;
}
if (virt_layout->list[2].source_dset_orig != 0 || virt_layout->list[4].source_dset_orig != 1) {
H5_FAILED();
puts(" Dataset sharing indices: shared entries have incorrect indices");
goto error;
}
/* Re-open verification for test 6 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_unusual_pattern", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
/* Re-verify sharing after re-open */
if (virt_layout->list[0].source_file_name != virt_layout->list[2].source_file_name ||
virt_layout->list[1].source_file_name != virt_layout->list[4].source_file_name) {
H5_FAILED();
puts(" File sharing failed after re-open");
goto error;
}
if (virt_layout->list[0].source_dset_name != virt_layout->list[2].source_dset_name ||
virt_layout->list[1].source_dset_name != virt_layout->list[4].source_dset_name) {
H5_FAILED();
puts(" Dataset sharing failed after re-open");
goto error;
}
/* Re-verify sharing indices after re-open */
if (virt_layout->list[2].source_file_orig != 0 || virt_layout->list[4].source_file_orig != 1 ||
virt_layout->list[2].source_dset_orig != 0 || virt_layout->list[4].source_dset_orig != 1) {
H5_FAILED();
puts(" Sharing indices are incorrect after re-open");
goto error;
}
/* Clean up test 6 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
/*
* Test 7: VDS with many mappings to test hash table resizing
* Creates many mappings with a mix of shared and unique strings
*/
if ((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_layout(dcpl_id, H5D_VIRTUAL) < 0)
TEST_ERROR;
/* Create NUM_MAPPINGS_MANY mappings with a pattern that includes sharing:
* - Every 10th mapping uses "shared_file.h5"
* - Every 5th mapping uses "/shared_dataset"
* - Others use unique file/dataset names
*/
char file_name[64];
char dset_name[64];
int shared_file_count = 0;
int shared_dset_count = 0;
for (int i = 0; i < NUM_MAPPINGS_MANY; i++) {
if (i % 10 == 0) {
strcpy(file_name, "shared_file.h5");
shared_file_count++;
}
else {
snprintf(file_name, sizeof(file_name), "file_%d.h5", i);
}
if (i % 5 == 0) {
strcpy(dset_name, "/shared_dataset");
shared_dset_count++;
}
else {
snprintf(dset_name, sizeof(dset_name), "/dataset_%d", i);
}
if (H5Pset_virtual(dcpl_id, virt_space_id, file_name, dset_name, src_space_id) < 0)
TEST_ERROR;
}
if ((dset_id = H5Dcreate2(file_id, "vds_many_mappings", H5T_NATIVE_INT, virt_space_id, H5P_DEFAULT,
dcpl_id, H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
/* Verify that we have the expected number of mappings */
if (virt_layout->list_nused != NUM_MAPPINGS_MANY) {
H5_FAILED();
printf(" Expected %d mappings, got %zu\n", NUM_MAPPINGS_MANY, virt_layout->list_nused);
goto error;
}
for (int i = 0; i < NUM_MAPPINGS_MANY; i++) {
/* Check file sharing */
if (i % 10 == 0) { /* Should share "shared_file.h5" */
if (i > 0 && virt_layout->list[0].source_file_name != virt_layout->list[i].source_file_name) {
H5_FAILED();
printf(" File sharing failed: entry [%d] should share shared_file.h5 with entry [0]\n", i);
goto error;
}
if (i == 0) {
if (virt_layout->list[i].source_file_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [0] incorrectly marked as shared file (expected SIZE_MAX, got %zu)\n",
virt_layout->list[i].source_file_orig);
goto error;
}
}
else {
if (virt_layout->list[i].source_file_orig != 0) {
H5_FAILED();
printf(" File sharing index incorrect for entry [%d]: expected 0, got %zu\n", i,
virt_layout->list[i].source_file_orig);
goto error;
}
}
}
else { /* Should not share file */
if (virt_layout->list[i].source_file_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [%d] incorrectly marked as sharing file (expected SIZE_MAX, got %zu)\n", i,
virt_layout->list[i].source_file_orig);
goto error;
}
}
/* Check dataset sharing */
if (i % 5 == 0) { /* Should share "/shared_dataset" */
if (i > 0 && virt_layout->list[0].source_dset_name != virt_layout->list[i].source_dset_name) {
H5_FAILED();
printf(" Dataset sharing failed: entry [%d] should share /shared_dataset with entry [0]\n",
i);
goto error;
}
if (i == 0) {
if (virt_layout->list[i].source_dset_orig != SIZE_MAX) {
H5_FAILED();
printf(
" Entry [0] incorrectly marked as shared dataset (expected SIZE_MAX, got %zu)\n",
virt_layout->list[i].source_dset_orig);
goto error;
}
}
else {
if (virt_layout->list[i].source_dset_orig != 0) {
H5_FAILED();
printf(" Dataset sharing index incorrect for entry [%d]: expected 0, got %zu\n", i,
virt_layout->list[i].source_dset_orig);
goto error;
}
}
}
else { /* Should not share dataset */
if (virt_layout->list[i].source_dset_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [%d] incorrectly marked as sharing dataset (expected SIZE_MAX, got %zu)\n",
i, virt_layout->list[i].source_dset_orig);
goto error;
}
}
}
/* Re-open verification for test 7 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset_id = H5Dopen2(file_id, "vds_many_mappings", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_int = (H5D_t *)H5VL_object(dset_id)) == NULL)
TEST_ERROR;
virt_layout = &(dset_int->shared->layout.storage.u.virt);
/* Full verification after re-open */
for (int i = 0; i < NUM_MAPPINGS_MANY; i++) {
/* Check file sharing */
if (i % 10 == 0) { /* Should share "shared_file.h5" */
if (i > 0 && virt_layout->list[0].source_file_name != virt_layout->list[i].source_file_name) {
H5_FAILED();
printf(" File sharing failed after re-open: entry [%d] should share shared_file.h5 with "
"entry [0]\n",
i);
goto error;
}
if (i == 0) {
if (virt_layout->list[i].source_file_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [0] incorrectly marked as shared file after re-open (expected "
"SIZE_MAX, got %zu)\n",
virt_layout->list[i].source_file_orig);
goto error;
}
}
else {
if (virt_layout->list[i].source_file_orig != 0) {
H5_FAILED();
printf(" File sharing index incorrect after re-open for entry [%d]: expected 0, got "
"%zu\n",
i, virt_layout->list[i].source_file_orig);
goto error;
}
}
}
else { /* Should not share file */
if (virt_layout->list[i].source_file_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [%d] incorrectly marked as sharing file after re-open (expected SIZE_MAX, "
"got %zu)\n",
i, virt_layout->list[i].source_file_orig);
goto error;
}
}
/* Check dataset sharing */
if (i % 5 == 0) { /* Should share "/shared_dataset" */
if (i > 0 && virt_layout->list[0].source_dset_name != virt_layout->list[i].source_dset_name) {
H5_FAILED();
printf(" Dataset sharing failed after re-open: entry [%d] should share /shared_dataset "
"with entry [0]\n",
i);
goto error;
}
if (i == 0) {
if (virt_layout->list[i].source_dset_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [0] incorrectly marked as shared dataset after re-open (expected "
"SIZE_MAX, got %zu)\n",
virt_layout->list[i].source_dset_orig);
goto error;
}
}
else {
if (virt_layout->list[i].source_dset_orig != 0) {
H5_FAILED();
printf(" Dataset sharing index incorrect after re-open for entry [%d]: expected 0, "
"got %zu\n",
i, virt_layout->list[i].source_dset_orig);
goto error;
}
}
}
else { /* Should not share dataset */
if (virt_layout->list[i].source_dset_orig != SIZE_MAX) {
H5_FAILED();
printf(" Entry [%d] incorrectly marked as sharing dataset after re-open (expected "
"SIZE_MAX, got %zu)\n",
i, virt_layout->list[i].source_dset_orig);
goto error;
}
}
}
/* Clean up test 7 */
if (H5Dclose(dset_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl_id) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
/* Clean up */
if (H5Sclose(src_space_id) < 0)
TEST_ERROR;
if (H5Sclose(virt_space_id) < 0)
TEST_ERROR;
PASSED();
return SUCCEED;
error:
H5E_BEGIN_TRY
{
H5Dclose(dset_id);
H5Pclose(dcpl_id);
H5Fclose(file_id);
H5Sclose(src_space_id);
H5Sclose(virt_space_id);
}
H5E_END_TRY;
return FAIL;
} /* end test_vds_shared_strings() */
/*-------------------------------------------------------------------------
* Function: main
*
@@ -16432,6 +17410,9 @@ main(void)
nerrors += (test_dcpl_layout_caching(H5D_CHUNKED) < 0 ? 1 : 0);
nerrors += (test_dcpl_layout_caching(H5D_VIRTUAL) < 0 ? 1 : 0);
/* Verify that source file/dataset names are shared properly */
nerrors += (test_vds_shared_strings(fapl) < 0 ? 1 : 0);
if (nerrors)
goto error;
printf("All dataset tests passed.\n");