mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '6153' into blob

Browse Source 
# Conflicts:
#	src/H5Opline.c
#	src/H5Oprivate.h
#	src/H5Pocpl.c
#	src/H5Z.c
#	src/H5Zprivate.h
#	test/tfilter2.c
This commit is contained in:
Scot Breitenfeld
2026-07-15 19:41:16 -05:00
parent 6273b96655 a9ada8bef2
commit 83f70cedbf
10 changed files with 751 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
release_docs/CHANGELOG.md
+22 -6
View File
@@ -170,15 +170,31 @@ We would like to thank the many HDF5 community members who contributed to this r
libhdf5. Hex-float literals (`0x1.8p+1`) in parameter strings are
transparently rewritten to decimal before parsing.
**On-disk format:** No new pipeline version is introduced. Parameter
strings are converted to `cd_values` by the filter's `set_config` callback
at `H5Pappend_filter` time and stored using the existing v2 pipeline
message. On read, `H5Pget_filter_params_by_idx` reconstructs the string
via the filter's `get_config` callback. This means the on-disk format is
unchanged and full backward read compatibility is preserved.
**On-disk format:** A new pipeline message version, `H5O_PLINE_VERSION_3`,
stores each filter's verbatim parameter string after the filter name, so
the exact string can be recovered without loading the filter plugin.
`H5Pget_filter_params_by_idx` returns that stored string when present and
otherwise falls back to the filter's `get_config` callback, then to a
`cd_values` listing. Version 3 is written only when a filter carries a
stored string and the file's high library-version bound admits it (see
`H5F_LIBVER_V300` below); otherwise the message is written at version 2
with the string omitted, so files without stored strings remain
byte-identical to previous releases. `H5Pmodify_filter` clears the stored
string for the modified filter; `H5Pcopy`, `H5Pencode`/`H5Pdecode`, and
`H5Ocopy` carry it with the entry.
Fixes GitHub issue [#6153](https://github.com/HDFGroup/hdf5/issues/6153)
### Added the H5F_LIBVER_V300 library version bound
The `H5F_libver_t` enumeration gains `H5F_LIBVER_V300`, and
`H5F_LIBVER_LATEST` now maps to it. A file access property list's high
bound must be at least `H5F_LIBVER_V300` to write the version-3 filter
pipeline message (used to persist filter parameter strings); all other
message versions are unchanged from `H5F_LIBVER_V200`. The constant is
mirrored in the Fortran (`H5F_LIBVER_V300_F`) and Java
(`HDF5Constants.H5F_LIBVER_V300`) bindings.
### Added optional digital signature verification for dynamically loaded plugins
When built with `-DHDF5_REQUIRE_SIGNED_PLUGINS=ON` and OpenSSL, HDF5 will cryptographically verify each plugin before loading it. Plugins are signed with the new `h5sign` tool, which appends an RSA signature and a compact footer to the plugin binary. Verification uses a keystore directory of trusted public keys, configurable at compile time (`-DHDF5_PLUGIN_KEYSTORE_DIR=<path>`) or at runtime via the `HDF5_PLUGIN_KEYSTORE` environment variable. Individual signatures can be revoked without removing the entire public key by listing their SHA-256 hashes in a `revoked_signatures.txt` file in the keystore directory. Supported algorithms include SHA-256, SHA-384, and SHA-512 with both PKCS#1 v1.5 and PSS padding. See `docs/PLUGIN_SIGNATURE_README.md` for details.
src/H5Opline.c
+71 -1
View File
@@ -244,6 +244,31 @@ H5O__pline_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS
}
}
/* Verbatim configuration string, for version 3+. A zero length means
* no string was stored for this filter. */
if (pline->version >= H5O_PLINE_VERSION_3) {
size_t config_length;
if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
UINT16DECODE(p, config_length);
if (config_length > H5Z_CONFIG_STRING_MAX)
HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "filter config string exceeds maximum length");
if (config_length) {
if (H5_IS_BUFFER_OVERFLOW(p, config_length, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
/* Stored without a NUL terminator; add one on the way in */
if (NULL == (filter->config = (char *)H5MM_malloc(config_length + 1)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
"memory allocation failed for filter config string");
H5MM_memcpy(filter->config, p, config_length);
filter->config[config_length] = '\0';
p += config_length;
}
}
/* Blob locator, for version 3+. Only the locator is stored in the
* message; the blob bytes live in the global heap and are recovered
* via the filter's read_blob callback at dataset-open time. */
@@ -374,6 +399,18 @@ H5O__pline_encode(H5F_t *f, uint8_t *p /*out*/, const void *mesg)
if (filter->cd_nelmts % 2)
UINT32ENCODE(p, 0);
/* Verbatim configuration string, for version 3+. Written without a
* NUL terminator; a zero length means no string for this filter. */
if (pline->version >= H5O_PLINE_VERSION_3) {
size_t config_length = filter->config ? strlen(filter->config) : 0;
UINT16ENCODE(p, config_length);
if (config_length > 0) {
H5MM_memcpy(p, filter->config, config_length);
p += config_length;
}
}
/* Blob locator, for version 3+. The locator is defined only after
* the blob has been written at dataset-creation time. */
if (pline->version >= H5O_PLINE_VERSION_3) {
@@ -464,6 +501,13 @@ H5O__pline_copy(const void *_src, void *_dst /*out*/)
dst->filter[i].cd_values = dst->filter[i]._cd_values;
} /* end if */
/* Verbatim configuration string */
if (src->filter[i].config) {
if (NULL == (dst->filter[i].config = (char *)H5MM_strdup(src->filter[i].config)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
"memory allocation failed for filter config string");
} /* end if */
/* Blob bytes: deep copy. The locator is copied verbatim but is
* only meaningful within the originating file; dataset creation
* always writes the blob afresh and assigns a new locator. The
@@ -552,6 +596,11 @@ H5O__pline_size(const H5F_t *f, const void *mesg)
if (pline->filter[i].cd_nelmts % 2)
ret_value += 4;
/* Verbatim configuration string, for version 3+: 2-byte length prefix
* plus the string bytes (no NUL terminator on disk) */
if (pline->version >= H5O_PLINE_VERSION_3)
ret_value += 2 + (pline->filter[i].config ? strlen(pline->filter[i].config) : 0);
/* Blob locator, for version 3+ */
if (pline->version >= H5O_PLINE_VERSION_3) {
ret_value += 1; /* has_aux flag */
@@ -599,6 +648,7 @@ H5O__pline_reset(void *mesg)
assert(pline->filter[i].cd_nelmts > H5Z_COMMON_CD_VALUES);
if (pline->filter[i].cd_values != pline->filter[i]._cd_values)
pline->filter[i].cd_values = (unsigned *)H5MM_xfree(pline->filter[i].cd_values);
pline->filter[i].config = (char *)H5MM_xfree(pline->filter[i].config);
H5Z_blob_release(&pline->filter[i]);
} /* end for */
@@ -819,7 +869,27 @@ H5O_pline_set_version(H5F_t *f, H5O_pline_t *pline)
/* Upgrade to the version indicated by the file's low bound if higher */
version = MAX(pline->version, H5O_pline_ver_bounds[H5F_LOW_BOUND(f)]);
/* A blob-bearing filter requires the version-3 encoding for its locator */
/* A filter carrying a verbatim configuration string needs the version-3
* encoding to persist it. Request v3 only when the file's high bound
* admits it; otherwise fall back silently to the current version -- the
* strings are simply not written and introspection relies on get_config. */
if (version < H5O_PLINE_VERSION_3) {
bool have_config = false;
for (size_t u = 0; u < pline->nused; u++)
if (pline->filter[u].config) {
have_config = true;
break;
}
if (have_config && H5O_pline_ver_bounds[H5F_HIGH_BOUND(f)] >= H5O_PLINE_VERSION_3)
version = H5O_PLINE_VERSION_3;
}
/* A blob-bearing filter requires the version-3 encoding for its locator.
* Unlike the configuration string above, the blob bytes have nowhere
* else to go, so this is not silently downgraded -- the version bounds
* check below reports an error if the file's high bound doesn't admit it. */
for (size_t u = 0; u < pline->nused; u++)
if (pline->filter[u].aux_data != NULL) {
version = MAX(version, H5O_PLINE_VERSION_3);
src/H5Oprivate.h
+6 -3
View File
@@ -746,9 +746,12 @@ typedef struct H5O_ginfo_t {
*/
#define H5O_PLINE_VERSION_2 2
/* This version appends a has_aux flag byte after each filter's cd_values
* and, when set, a fixed-size locator (address + global-heap index)
* for the filter's blob stored as a global-heap object
/* This version stores each filter's verbatim key=value configuration string
* (as passed to H5Pappend_filter) after the filter name, so it can be
* recovered losslessly without loading the filter plugin. It also
* appends a has_aux flag byte after that string and, when set, a
* fixed-size locator (address + global-heap index) for the filter's
* blob stored as a global-heap object
*/
#define H5O_PLINE_VERSION_3 3
src/H5Pocpl.c
+92 -5
View File
@@ -1321,6 +1321,21 @@ H5P__ocrt_pipeline_enc(const void *value, void **_pp, size_t *size)
for (v = 0; v < pline->filter[u].cd_nelmts; v++)
H5_ENCODE_UNSIGNED(*pp, pline->filter[u].cd_values[v]);
/* encode the verbatim config string inline (a self-contained
* encoded DCPL carries the string, not a file locator) */
if (NULL != pline->filter[u].config) {
uint64_t config_len = (uint64_t)strlen(pline->filter[u].config);
*(*pp)++ = (uint8_t) true;
enc_size = H5VM_limit_enc_size(config_len);
*(*pp)++ = (uint8_t)enc_size;
UINT64ENCODE_VAR(*pp, config_len, enc_size);
H5MM_memcpy(*pp, pline->filter[u].config, (size_t)config_len);
*pp += config_len;
} /* end if */
else
*(*pp)++ = (uint8_t) false;
/* encode the blob bytes inline. A serialized property must be
* self-contained; the on-disk locator is file-specific and is
* regenerated when the decoded plist is used with H5Dcreate. */
@@ -1346,10 +1361,15 @@ H5P__ocrt_pipeline_enc(const void *value, void **_pp, size_t *size)
*size += H5Z_COMMON_NAME_LEN;
*size += (1 + H5VM_limit_enc_size((uint64_t)pline->filter[u].cd_nelmts));
*size += pline->filter[u].cd_nelmts * sizeof(unsigned);
*size += 1; /* has_config flag */
if (NULL != pline->filter[u].config) {
uint64_t config_len = (uint64_t)strlen(pline->filter[u].config);
*size += (1 + H5VM_limit_enc_size(config_len) + (size_t)config_len);
}
*size += 1; /* has_aux flag */
if (NULL != pline->filter[u].aux_data)
*size += 8 + pline->filter[u].aux_size; /* aux_size + blob bytes */
} /* end for */
} /* end for */
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5P__ocrt_pipeline_enc() */
@@ -1399,6 +1419,8 @@ H5P__ocrt_pipeline_dec(const void **_pp, void *_value)
for (u = 0; u < nused; u++) {
H5Z_filter_info_t filter; /* Filter info, for pipeline */
uint8_t has_name; /* Flag to indicate whether filter has a name */
uint8_t has_config; /* Flag to indicate whether filter has a config string */
char *config = NULL; /* Decoded verbatim config string */
uint8_t has_aux; /* Flag to indicate whether filter has a blob */
void *aux_data = NULL; /* Decoded blob bytes */
size_t aux_size = 0; /* Decoded blob length */
@@ -1437,6 +1459,23 @@ H5P__ocrt_pipeline_dec(const void **_pp, void *_value)
for (v = 0; v < filter.cd_nelmts; v++)
H5_DECODE_UNSIGNED(*pp, filter.cd_values[v]);
/* decode the verbatim config string, if present */
has_config = *(*pp)++;
if (has_config) {
uint64_t config_len;
enc_size = *(*pp)++;
assert(enc_size < 256);
UINT64DECODE_VAR(*pp, config_len, enc_size);
if (NULL == (config = (char *)H5MM_malloc((size_t)config_len + 1))) {
filter.cd_values = (unsigned *)H5MM_xfree(filter.cd_values);
HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "memory allocation failed for config string");
}
H5MM_memcpy(config, *pp, (size_t)config_len);
config[config_len] = '\0';
*pp += config_len;
}
/* decode the blob bytes, if present. The on-disk locator is not
* serialized; a later H5Dcreate writes the blob afresh. */
has_aux = *(*pp)++;
@@ -1445,21 +1484,30 @@ H5P__ocrt_pipeline_dec(const void **_pp, void *_value)
UINT64DECODE(*pp, aux_size64);
aux_size = (size_t)aux_size64;
if (aux_size == 0)
if (aux_size == 0) {
H5MM_xfree(config);
filter.cd_values = (unsigned *)H5MM_xfree(filter.cd_values);
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "encoded filter blob has zero length");
if (NULL == (aux_data = H5MM_malloc(aux_size)))
}
if (NULL == (aux_data = H5MM_malloc(aux_size))) {
H5MM_xfree(config);
filter.cd_values = (unsigned *)H5MM_xfree(filter.cd_values);
HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "memory allocation failed for filter blob");
}
H5MM_memcpy(aux_data, *pp, aux_size);
*pp += aux_size;
} /* end if */
/* Add the filter to the I/O pipeline */
if (H5Z_append(pline, filter.id, filter.flags, filter.cd_nelmts, filter.cd_values) < 0) {
H5MM_xfree(config);
H5MM_xfree(aux_data);
filter.cd_values = (unsigned *)H5MM_xfree(filter.cd_values);
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to add filter to pipeline");
}
/* Attach the blob to the just-appended entry */
/* Attach the decoded config string and blob to the just-appended entry */
pline->filter[pline->nused - 1].config = config;
pline->filter[pline->nused - 1].aux_data = aux_data;
pline->filter[pline->nused - 1].aux_size = aux_size;
@@ -1875,6 +1923,8 @@ H5Pappend_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, const
const unsigned *cd_values = NULL;
unsigned *allocated_cd_values = NULL; /* owns heap mem for string path */
char *fi_name_heap = NULL; /* non-NULL if fi->name was H5MM_strdup'd here */
const char *retain_config = NULL; /* verbatim string to persist (STRING path only) */
char *fi_config_heap = NULL; /* non-NULL if fi->config was H5MM_strdup'd here */
size_t cd_nelmts = 0;
herr_t ret_value = SUCCEED;
@@ -1943,6 +1993,12 @@ H5Pappend_filter(hid_t plist_id, H5Z_filter_t filter, unsigned int flags, const
"filter does not support string configuration (no set_config callback)");
}
/* Persist the caller's verbatim parameter string so it can be
* recovered losslessly (pipeline v3) without loading the plugin.
* An empty input stores nothing. */
if (!empty_input)
retain_config = param_str;
/* set_config is present: invoke it. Normalise an empty input to
* params = NULL so callbacks only need to handle one form. */
{
@@ -2002,13 +2058,26 @@ append_to_pipeline:
* performs a single speculative lookup. */
H5P__pline_persist_name(&pline, filter, entry, &fi_name_heap);
/* Persist the verbatim configuration string on the entry. Track the
* allocation so it can be freed if H5P_poke fails below (ownership
* transfers to the plist on success). */
if (retain_config) {
H5Z_filter_info_t *fi = &pline.filter[pline.nused - 1];
if (NULL == (fi->config = (char *)H5MM_strdup(retain_config)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for filter config string");
fi_config_heap = fi->config;
}
/* Store updated pipeline back in property list */
if (H5P_poke(plist, H5O_CRT_PIPELINE_NAME, &pline) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set pipeline");
done:
if (ret_value < 0)
if (ret_value < 0) {
H5MM_xfree(fi_name_heap);
H5MM_xfree(fi_config_heap);
}
H5MM_xfree(allocated_cd_values);
FUNC_LEAVE_API(ret_value)
} /* end H5Pappend_filter() */
@@ -2159,6 +2228,24 @@ H5Pget_filter_params_by_idx(hid_t plist_id, unsigned idx, char *params_buf, size
filter = &pline.filter[idx];
/* Source 1 (highest fidelity): a verbatim configuration string stored on
* the entry. This is byte-for-byte lossless and needs no plugin, so it
* short-circuits both the get_config and cd_values paths below. */
if (filter->config) {
size_t needed = strlen(filter->config);
if (params_len)
*params_len = needed;
if (params_buf && params_buf_size > 0) {
size_t copy_len = (needed < params_buf_size - 1) ? needed : params_buf_size - 1;
H5MM_memcpy(params_buf, filter->config, copy_len);
params_buf[copy_len] = '\0';
}
HGOTO_DONE(SUCCEED);
}
/* Trigger a plugin load if the filter isn't registered yet -- mirrors
* H5Zget_filter_class_info()'s use of H5Z_filter_avail() to get get_config
* available on the first query, not just subsequent ones. Availability
src/H5Z.c
+59 -41
View File
@@ -1313,6 +1313,10 @@ H5Z_modify(const H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags, size_t
pline->filter[idx].flags = flags;
pline->filter[idx].cd_nelmts = cd_nelmts;
/* Modifying the raw cd_values invalidates any stored configuration string;
* drop it so introspection falls back to the filter's get_config callback */
pline->filter[idx].config = (char *)H5MM_xfree(pline->filter[idx].config);
/* Free any existing parameters */
if (pline->filter[idx].cd_values != NULL && pline->filter[idx].cd_values != pline->filter[idx]._cd_values)
H5MM_xfree(pline->filter[idx].cd_values);
@@ -1416,6 +1420,7 @@ H5Z_append(H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags, size_t cd_ne
pline->filter[idx].flags = flags;
pline->filter[idx].name = NULL; /*we'll pick it up later*/
pline->filter[idx].cd_nelmts = cd_nelmts;
pline->filter[idx].config = NULL; /*set by H5Pappend_filter or pline decode*/
pline->filter[idx].aux_data = NULL; /*set by H5Pappend_filter_blob or pline decode*/
pline->filter[idx].aux_size = 0;
pline->filter[idx].aux_loc.addr = HADDR_UNDEF;
@@ -1959,6 +1964,7 @@ H5Z_delete(H5O_pline_t *pline, H5Z_filter_t filter)
assert(pline->filter[idx].cd_nelmts > H5Z_COMMON_CD_VALUES);
if (pline->filter[idx].cd_values != pline->filter[idx]._cd_values)
pline->filter[idx].cd_values = (unsigned *)H5MM_xfree(pline->filter[idx].cd_values);
pline->filter[idx].config = (char *)H5MM_xfree(pline->filter[idx].config);
H5Z_blob_release(&pline->filter[idx]);
/* Remove filter from pipeline array */
@@ -2032,11 +2038,22 @@ H5Z_blob_release(H5Z_filter_info_t *fi)
* callback control their own storage; otherwise the bytes are
* inserted into the file's global heap.
*
* For parallel access, only rank 0 writes; the resulting locator
* is broadcast so all ranks encode an identical pipeline message.
* H5HG_insert allocates file space outside collective-I/O
* coordination, so independent per-rank writes would produce
* divergent locators.
* For parallel access, EVERY rank performs the identical write
* (same bytes, same point in H5D__create's collective metadata
* sequence) rather than having rank 0 write and broadcast the
* locator: dataset creation is already collective and requires
* an identical DCPL -- and therefore identical blob bytes -- on
* every rank, so H5MF_alloc allocates the same file-space address
* deterministically on every rank, exactly as it already does for
* every other piece of metadata dataset creation writes (object
* header, layout message, etc). A rank-0-writes-then-broadcasts
* protocol was considered and rejected: H5HG_insert dirties a
* metadata-cache entry, and only rank 0 doing so would desync the
* parallel metadata cache's identical-operations invariant across
* ranks. A custom write_blob callback MUST follow the same rule:
* perform identical file-modifying operations on every rank (or
* none, by delegating to the default writer); per-rank divergent
* behavior is undefined.
*
* Return: Non-negative on success / Negative on failure
*-------------------------------------------------------------------------
@@ -2055,7 +2072,6 @@ H5Z_blob_write(H5F_t *f, H5O_pline_t *pline)
H5Z_filter_info_t *fi = &pline->filter[u];
H5Z_entry_t *entry = NULL;
H5Z_blob_loc_t loc;
bool do_write = true;
if (fi->aux_data == NULL)
continue;
@@ -2065,50 +2081,52 @@ H5Z_blob_write(H5F_t *f, H5O_pline_t *pline)
(void)H5Z_find_entry(true, fi->id, &entry);
#ifdef H5_HAVE_PARALLEL
if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI))
do_write = (H5F_mpi_get_rank(f) == 0);
#endif
if (entry && entry->write_blob) {
hid_t file_id;
if (do_write) {
if (entry && entry->write_blob) {
hid_t file_id;
if ((file_id = H5F_get_id(f)) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get file ID for blob callback");
if ((entry->write_blob)(file_id, fi->aux_data, fi->aux_size, &loc) < 0) {
(void)H5I_dec_ref(file_id);
HGOTO_ERROR(H5E_PLINE, H5E_CALLBACK, FAIL, "filter write_blob callback failed");
}
if (H5I_dec_ref(file_id) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTDEC, FAIL, "can't release file ID");
}
else {
H5HG_t hobj;
if (H5HG_insert(f, fi->aux_size, fi->aux_data, &hobj) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, FAIL,
"unable to insert filter blob into global heap");
loc.addr = hobj.addr;
loc.idx = hobj.idx;
if ((file_id = H5F_get_id(f)) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get file ID for blob callback");
if ((entry->write_blob)(file_id, fi->aux_data, fi->aux_size, &loc) < 0) {
(void)H5I_dec_ref(file_id);
HGOTO_ERROR(H5E_PLINE, H5E_CALLBACK, FAIL, "filter write_blob callback failed");
}
if (H5I_dec_ref(file_id) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTDEC, FAIL, "can't release file ID");
}
else {
H5HG_t hobj;
#ifdef H5_HAVE_PARALLEL
/* All ranks must agree on the locator before the pipeline message is
* encoded. An undefined address after the broadcast means rank 0
* failed, so every rank fails consistently instead of deadlocking. */
if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
int mpi_code;
if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&loc, sizeof(loc), MPI_BYTE, 0, H5F_mpi_get_comm(f))))
HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code)
if (H5HG_insert(f, fi->aux_size, fi->aux_data, &hobj) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, FAIL,
"unable to insert filter blob into global heap");
loc.addr = hobj.addr;
loc.idx = hobj.idx;
}
#endif
if (!H5_addr_defined(loc.addr))
HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, FAIL, "filter blob was not assigned a valid locator");
#if defined(H5_HAVE_PARALLEL) && !defined(NDEBUG)
/* Debug-build safety net: every rank just performed what should be
* an identical write, so every rank must have landed on the same
* locator. This is not a substitute for collective consistency
* (a hang here means ranks already diverged before this point) but
* catches a non-collective or non-deterministic write_blob callback
* before it silently corrupts the file. */
if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
H5Z_blob_loc_t rank0_loc = loc;
int mpi_code;
if (MPI_SUCCESS !=
(mpi_code = MPI_Bcast(&rank0_loc, sizeof(rank0_loc), MPI_BYTE, 0, H5F_mpi_get_comm(f))))
HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code)
assert(H5_addr_eq(rank0_loc.addr, loc.addr) && rank0_loc.idx == loc.idx &&
"write_blob produced a different locator on this rank than on rank 0 -- "
"custom write_blob callbacks must perform identical file-modifying "
"operations on every rank");
}
#endif
fi->aux_loc = loc;
}
src/H5Zdevelop.h
+10 -9
View File
@@ -232,15 +232,16 @@ typedef herr_t (*H5Z_set_config_func_t)(const char *params, unsigned *flags, siz
*
* \return Non-negative on success; negative on failure.
*
* \note Implementations that format \c float or \c double values \b must use
* the C99 \c \%a format specifier (e.g., \c snprintf(buf,*buf_size,"\%a",val))
* rather than \c \%g, \c \%f, or \c \%e. \c \%a encodes the exact
* IEEE 754 bit pattern as a hexadecimal float literal, guaranteeing that
* \c strtod parses the output back to the identical value with no
* rounding. This makes exact round-trips possible for filters that store
* \c float or \c double parameters via the cd_values packing convention.
* Decimal float input (e.g., \c rate=3.5) remains valid for user
* convenience; the asymmetry (decimal in, hex-float out) is intentional.
* \details This callback is only a fallback for introspection: when a filter
* was configured with a parameter string, H5Pget_filter_params_by_idx()
* returns that stored string verbatim and never calls get_config.
* get_config is used when no string was stored (for example, a filter
* added through the raw cd_values API). How a filter encodes values
* into cd_values is entirely private to that filter.
*
* \note When reconstructing \c float or \c double values, formatting with the
* C99 \c \%a specifier (a hexadecimal float literal) round-trips through
* \c strtod with no rounding, unlike \c \%g / \c \%f / \c \%e.
*
* \since 3.0.0
*/
src/H5Zprivate.h
+2
View File
@@ -61,6 +61,8 @@ struct H5Z_filter_info_t {
size_t cd_nelmts; /*number of elements in cd_values[] */
unsigned _cd_values[H5Z_COMMON_CD_VALUES]; /*internal client data values */
unsigned *cd_values; /*client data values */
char *config; /*verbatim key=value config string, or
*NULL; persisted in pipeline v3 */
void *aux_data; /*in-memory blob; NULL if none */
size_t aux_size; /*byte length of aux_data */
H5Z_blob_loc_t aux_loc; /*on-disk locator; undefined until the
test/tfilter2.c
+290 -3
View File
@@ -21,7 +21,8 @@
#include "h5test.h"
static const char *FILENAME[] = {"tfilter2", "tfilter2_blob", "tfilter2_blob_custom", NULL};
static const char *FILENAME[] = {"tfilter2", "tfilter2_blob", "tfilter2_blob_custom",
"tfilter2_cfg", "tfilter2_cfg_copy", NULL};
/* -----------------------------------------------------------------------
* Parser tests - typed TOML accessor functions
@@ -495,8 +496,9 @@ test_callback_contracts(void)
TEST_ERROR;
if (plen == 0)
TEST_ERROR;
/* Should contain "level = 9" (TOML output format) */
if (strstr(pbuf, "level = 9") == NULL)
/* The verbatim configuration string is retained and returned as-is
* ("level=9"), taking precedence over the get_config reconstruction. */
if (strcmp(pbuf, "level=9") != 0)
TEST_ERROR;
H5Pclose(dcpl);
dcpl = H5I_INVALID_HID;
@@ -2141,6 +2143,288 @@ error:
return -1;
}
/* -----------------------------------------------------------------------
/* -----------------------------------------------------------------------
* On-disk configuration-string storage (pipeline v3, RFC-HDFG-2026-001)
*
* This filter's stored parameter string ("level=N", no spaces) differs
* from its get_config reconstruction ("level = N", with spaces) so tests
* can tell whether a returned string came from the persisted verbatim
* string or from the get_config fallback.
* ---------------------------------------------------------------------- */
#define CFG_ONDISK_FILTER_ID 531
static herr_t
cfg_ondisk_set_config(const char *params, unsigned H5_ATTR_UNUSED *flags, size_t *cd_nelmts,
unsigned cd_values[], size_t cd_values_size)
{
int64_t level = 0;
*cd_nelmts = 1;
if (cd_values && cd_values_size >= 1) {
if (params && *params)
H5Zconfig_get_int(params, "level", &level);
cd_values[0] = (unsigned)level;
}
return SUCCEED;
}
static herr_t
cfg_ondisk_get_config(unsigned H5_ATTR_UNUSED flags, size_t cd_nelmts, const unsigned cd_values[], char *buf,
size_t *buf_size)
{
unsigned level = (cd_nelmts >= 1) ? cd_values[0] : 0;
size_t needed = (size_t)snprintf(NULL, 0, "level = %u", level) + 1;
if (buf_size)
*buf_size = needed;
if (buf)
snprintf(buf, needed, "level = %u", level);
return SUCCEED;
}
static size_t
cfg_ondisk_filter_func(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values,
hid_t H5_ATTR_UNUSED dxpl_id, const hsize_t H5_ATTR_UNUSED *scaled,
size_t H5_ATTR_UNUSED ndims, size_t nbytes, size_t *buf_size, void **buf)
{
(void)flags;
(void)cd_nelmts;
(void)cd_values;
(void)buf_size;
(void)buf;
return nbytes; /* pass-through */
}
static const H5Z_class3_t cfg_ondisk_cls = {
2, /* version */
CFG_ONDISK_FILTER_ID, /* id */
1, /* encoder_present */
1, /* decoder_present */
"cfg_ondisk_filter", /* name */
NULL, /* description */
NULL, /* can_apply */
NULL, /* set_local */
cfg_ondisk_filter_func, /* filter */
cfg_ondisk_set_config, /* set_config */
cfg_ondisk_get_config, /* get_config */
};
/* Build a chunked, filter-configured DCPL from a parameter string */
static hid_t
cfg_ondisk_make_dcpl(const char *params)
{
hid_t dcpl = H5I_INVALID_HID;
hsize_t chunk[2] = {4, 4};
H5Z_params_t p;
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
return H5I_INVALID_HID;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
p.type = H5Z_PARAMS_STRING;
p.u.str = params;
if (H5Pappend_filter(dcpl, CFG_ONDISK_FILTER_ID, 0, &p) < 0)
goto error;
return dcpl;
error:
H5E_BEGIN_TRY
{
H5Pclose(dcpl);
}
H5E_END_TRY
return H5I_INVALID_HID;
}
/* Fetch filter 0's parameter string from a DCPL into buf */
static herr_t
cfg_ondisk_get_params(hid_t dcpl, char *buf, size_t buf_size)
{
size_t len = 0;
if (H5Pget_filter_params_by_idx(dcpl, 0, buf, buf_size, &len) < 0)
return FAIL;
return SUCCEED;
}
static int
test_config_string_ondisk(hid_t fapl)
{
hid_t file = H5I_INVALID_HID, sid = H5I_INVALID_HID, dcpl = H5I_INVALID_HID, dset = H5I_INVALID_HID;
hid_t dcpl_out = H5I_INVALID_HID, fapl_dg = H5I_INVALID_HID, dcpl_dec = H5I_INVALID_HID;
hid_t file2 = H5I_INVALID_HID;
hsize_t dims[2] = {8, 8};
char filename[1024], filename2[1024];
char pbuf[H5Z_CONFIG_STRING_MAX + 1];
void *enc_buf = NULL;
size_t enc_size = 0;
if (H5Zregister(&cfg_ondisk_cls) < 0)
TEST_ERROR;
if ((sid = H5Screate_simple(2, dims, NULL)) < 0)
TEST_ERROR;
h5_fixname(FILENAME[3], fapl, filename, sizeof(filename));
h5_fixname(FILENAME[4], fapl, filename2, sizeof(filename2));
/* --- fmt-01/02: verbatim round-trip, recovered without the plugin --- */
TESTING("config string: verbatim on-disk round-trip without plugin");
if ((dcpl = cfg_ondisk_make_dcpl("level=7")) < 0)
TEST_ERROR;
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dset = H5Dcreate2(file, "dset", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0 || H5Pclose(dcpl) < 0 || H5Fclose(file) < 0)
TEST_ERROR;
dset = dcpl = file = H5I_INVALID_HID;
/* Drop the plugin: the only remaining source is the persisted string */
if (H5Zunregister(CFG_ONDISK_FILTER_ID) < 0)
TEST_ERROR;
if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file, "dset", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dcpl_out = H5Dget_create_plist(dset)) < 0)
TEST_ERROR;
if (cfg_ondisk_get_params(dcpl_out, pbuf, sizeof(pbuf)) < 0)
TEST_ERROR;
if (strcmp(pbuf, "level=7") != 0) /* verbatim, not the "level = 7" get_config form */
TEST_ERROR;
if (H5Pclose(dcpl_out) < 0 || H5Dclose(dset) < 0 || H5Fclose(file) < 0)
TEST_ERROR;
dcpl_out = dset = file = H5I_INVALID_HID;
if (H5Zregister(&cfg_ondisk_cls) < 0) /* restore for later cases */
TEST_ERROR;
PASSED();
/* --- fmt-05: libver high bound below V300 silently omits the string --- */
TESTING("config string: silent v2 downgrade when libver bound too low");
if ((fapl_dg = H5Pcopy(fapl)) < 0)
TEST_ERROR;
if (H5Pset_libver_bounds(fapl_dg, H5F_LIBVER_EARLIEST, H5F_LIBVER_V200) < 0)
TEST_ERROR;
if ((dcpl = cfg_ondisk_make_dcpl("level=5")) < 0)
TEST_ERROR;
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_dg)) < 0)
TEST_ERROR;
if ((dset = H5Dcreate2(file, "dset", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0 || H5Pclose(dcpl) < 0 || H5Fclose(file) < 0)
TEST_ERROR;
dset = dcpl = file = H5I_INVALID_HID;
/* Plugin still registered: getter falls back to get_config ("level = 5"),
* proving the verbatim string was not persisted at v2. */
if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file, "dset", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dcpl_out = H5Dget_create_plist(dset)) < 0)
TEST_ERROR;
if (cfg_ondisk_get_params(dcpl_out, pbuf, sizeof(pbuf)) < 0)
TEST_ERROR;
if (strcmp(pbuf, "level = 5") != 0) /* get_config form, not the stored "level=5" */
TEST_ERROR;
if (H5Pclose(dcpl_out) < 0 || H5Dclose(dset) < 0 || H5Fclose(file) < 0 || H5Pclose(fapl_dg) < 0)
TEST_ERROR;
dcpl_out = dset = file = fapl_dg = H5I_INVALID_HID;
PASSED();
/* --- fmt-07: H5Pmodify_filter clears the stored string --- */
TESTING("config string: H5Pmodify_filter clears the stored string");
if ((dcpl = cfg_ondisk_make_dcpl("level=3")) < 0)
TEST_ERROR;
{
unsigned cd[1] = {8};
if (H5Pmodify_filter(dcpl, CFG_ONDISK_FILTER_ID, 0, 1, cd) < 0)
TEST_ERROR;
}
if (cfg_ondisk_get_params(dcpl, pbuf, sizeof(pbuf)) < 0)
TEST_ERROR;
if (strcmp(pbuf, "level = 8") != 0) /* get_config of new cd_values, not "level=3" */
TEST_ERROR;
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
dcpl = H5I_INVALID_HID;
PASSED();
/* --- reg-07: stored string survives H5Pencode/H5Pdecode --- */
TESTING("config string: survives H5Pencode/H5Pdecode");
if ((dcpl = cfg_ondisk_make_dcpl("level=9")) < 0)
TEST_ERROR;
if (H5Pencode2(dcpl, NULL, &enc_size, H5P_DEFAULT) < 0)
TEST_ERROR;
if (NULL == (enc_buf = malloc(enc_size)))
TEST_ERROR;
if (H5Pencode2(dcpl, enc_buf, &enc_size, H5P_DEFAULT) < 0)
TEST_ERROR;
if ((dcpl_dec = H5Pdecode(enc_buf)) < 0)
TEST_ERROR;
if (cfg_ondisk_get_params(dcpl_dec, pbuf, sizeof(pbuf)) < 0)
TEST_ERROR;
if (strcmp(pbuf, "level=9") != 0)
TEST_ERROR;
free(enc_buf);
enc_buf = NULL;
if (H5Pclose(dcpl_dec) < 0 || H5Pclose(dcpl) < 0)
TEST_ERROR;
dcpl_dec = dcpl = H5I_INVALID_HID;
PASSED();
/* --- fmt-06: H5Ocopy deep-copies the stored string to a new file --- */
TESTING("config string: survives H5Ocopy to another file");
if ((dcpl = cfg_ondisk_make_dcpl("level=9")) < 0)
TEST_ERROR;
if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if ((dset = H5Dcreate2(file, "dset", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if ((file2 = H5Fcreate(filename2, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
if (H5Ocopy(file, "dset", file2, "dset_copy", H5P_DEFAULT, H5P_DEFAULT) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file2, "dset_copy", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dcpl_out = H5Dget_create_plist(dset)) < 0)
TEST_ERROR;
if (cfg_ondisk_get_params(dcpl_out, pbuf, sizeof(pbuf)) < 0)
TEST_ERROR;
if (strcmp(pbuf, "level=9") != 0)
TEST_ERROR;
if (H5Pclose(dcpl_out) < 0 || H5Dclose(dset) < 0 || H5Pclose(dcpl) < 0 || H5Fclose(file) < 0 ||
H5Fclose(file2) < 0)
TEST_ERROR;
dcpl_out = dset = dcpl = file = file2 = H5I_INVALID_HID;
PASSED();
if (H5Sclose(sid) < 0)
TEST_ERROR;
if (H5Zunregister(CFG_ONDISK_FILTER_ID) < 0)
TEST_ERROR;
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(dcpl);
H5Pclose(dcpl_out);
H5Pclose(dcpl_dec);
H5Pclose(fapl_dg);
H5Dclose(dset);
H5Sclose(sid);
H5Fclose(file);
H5Fclose(file2);
H5Zunregister(CFG_ONDISK_FILTER_ID);
}
H5E_END_TRY
free(enc_buf);
return -1;
}
/* -----------------------------------------------------------------------
* In-file blob configuration storage (H5Pappend_filter_blob)
* ---------------------------------------------------------------------- */
@@ -2706,6 +2990,9 @@ main(void)
/* filter2 context passthrough: dxpl_id, scaled, ndims */
nerrors += test_filter2_context_passthrough(file) < 0 ? 1 : 0;
/* On-disk configuration-string storage (pipeline v3) */
nerrors += test_config_string_ondisk(fapl) < 0 ? 1 : 0;
/* In-file blob configuration storage (H5Pappend_filter_blob) */
nerrors += test_blob_default_storage(fapl) < 0 ? 1 : 0;
nerrors += test_blob_custom_callbacks(fapl) < 0 ? 1 : 0;
testpar/t_filters_parallel.c
+192
View File
@@ -10311,6 +10311,197 @@ test_par_append_filter_builtin_string_pipeline(hid_t fapl_id)
VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded");
}
/* -----------------------------------------------------------------------
* RFC-HDFG-2026-003: In-file blob configuration storage, parallel write
* ---------------------------------------------------------------------- */
#define PAR_BLOB_FILTER_ID 540
#define PAR_BLOB_SIZE 4096
#define PAR_BLOB_MAGIC "TFILTERPARBLOBMAGIC"
#define PAR_BLOB_MAGIC_LEN (sizeof(PAR_BLOB_MAGIC) - 1)
static size_t
par_blob_passthrough_func(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values,
hid_t H5_ATTR_UNUSED dxpl_id, const hsize_t H5_ATTR_UNUSED *scaled,
size_t H5_ATTR_UNUSED ndims, size_t nbytes, size_t *buf_size, void **buf)
{
(void)flags;
(void)cd_nelmts;
(void)cd_values;
(void)buf_size;
(void)buf;
return nbytes; /* pass-through */
}
static const H5Z_class3_t par_blob_cls = {
2, /* version */
PAR_BLOB_FILTER_ID, /* id */
1, /* encoder_present */
1, /* decoder_present */
"par_blob_filter", /* name */
NULL, /* description */
NULL, /* can_apply */
NULL, /* set_local */
par_blob_passthrough_func, /* filter */
NULL, /* set_config */
NULL, /* get_config */
NULL, /* write_blob: use default global-heap storage */
NULL, /* read_blob */
NULL, /* close_blob */
};
/* Every rank must call H5Pappend_filter_blob with identical bytes: dataset
* creation is collective and requires an identical DCPL on every rank, and
* the blob-write protocol (H5Z_blob_write) relies on that to let every rank
* perform an identical H5HG_insert rather than broadcasting a locator from
* rank 0. This test exercises exactly that path with the default (NULL
* write_blob/read_blob) global-heap storage. */
static void
test_par_append_filter_blob(hid_t fapl_id)
{
hid_t file_id = H5I_INVALID_HID;
hid_t group_id = H5I_INVALID_HID;
hid_t dcpl_id = H5I_INVALID_HID;
hid_t dcpl_out = H5I_INVALID_HID;
hid_t dxpl_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t fspace_id = H5I_INVALID_HID;
hid_t mspace_id = H5I_INVALID_HID;
hsize_t dims[2] = {(hsize_t)(mpi_size * 4), 4};
hsize_t chunk[2] = {4, 4};
hsize_t start[2], count[2], block[2];
C_DATATYPE *wbuf = NULL;
C_DATATYPE *rbuf = NULL;
unsigned char *blob = NULL;
void *enc_buf = NULL;
size_t enc_size = 0;
size_t nbytes;
herr_t ret;
if (MAINPROCESS)
puts("Testing par-05: H5Pappend_filter_blob collective write across MPI ranks");
VRFY((H5Zregister(&par_blob_cls) >= 0), "H5Zregister(par_blob_cls) succeeded");
/* Identical, rank-independent content: every rank must supply the same
* blob bytes for the collective dataset create to be well-defined. */
blob = (unsigned char *)malloc(PAR_BLOB_SIZE);
VRFY((blob != NULL), "malloc blob succeeded");
memcpy(blob, PAR_BLOB_MAGIC, PAR_BLOB_MAGIC_LEN);
for (size_t i = PAR_BLOB_MAGIC_LEN; i < PAR_BLOB_SIZE; i++)
blob[i] = (unsigned char)(i * 7 + 3);
file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id);
VRFY((file_id >= 0), "H5Fopen succeeded");
group_id = H5Gcreate2(file_id, "par_append_filter_blob", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
VRFY((group_id >= 0), "H5Gcreate2 succeeded");
dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
VRFY((dcpl_id >= 0), "H5Pcreate DCPL succeeded");
VRFY((H5Pset_chunk(dcpl_id, 2, chunk) >= 0), "H5Pset_chunk succeeded");
ret = H5Pappend_filter_blob(dcpl_id, PAR_BLOB_FILTER_ID, 0, blob, PAR_BLOB_SIZE);
VRFY((ret >= 0), "H5Pappend_filter_blob succeeded");
fspace_id = H5Screate_simple(2, dims, NULL);
VRFY((fspace_id >= 0), "H5Screate_simple filespace succeeded");
/* Collective: every rank performs the identical H5HG_insert this
* exercises inside H5D__create. */
dset_id = H5Dcreate2(group_id, "dset", HDF5_DATATYPE_NAME, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
VRFY((dset_id >= 0), "H5Dcreate2 succeeded");
VRFY((H5Sclose(fspace_id) >= 0), "H5Sclose filespace succeeded");
fspace_id = H5I_INVALID_HID;
dxpl_id = H5Pcreate(H5P_DATASET_XFER);
VRFY((dxpl_id >= 0), "H5Pcreate DXPL succeeded");
VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) >= 0), "H5Pset_dxpl_mpio succeeded");
/* Each rank writes one 4x4 chunk, to confirm the blob-bearing pipeline
* doesn't interfere with ordinary collective chunked I/O. */
nbytes = 4 * 4 * sizeof(C_DATATYPE);
wbuf = (C_DATATYPE *)malloc(nbytes);
VRFY((wbuf != NULL), "malloc wbuf succeeded");
for (size_t i = 0; i < 16; i++)
wbuf[i] = (C_DATATYPE)(mpi_rank * 16 + (int)i);
start[0] = (hsize_t)(mpi_rank * 4);
start[1] = 0;
count[0] = 1;
count[1] = 1;
block[0] = 4;
block[1] = 4;
fspace_id = H5Dget_space(dset_id);
VRFY((fspace_id >= 0), "H5Dget_space succeeded");
VRFY((H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, NULL, count, block) >= 0),
"H5Sselect_hyperslab write succeeded");
mspace_id = H5Screate_simple(2, block, NULL);
VRFY((mspace_id >= 0), "H5Screate_simple mspace succeeded");
ret = H5Dwrite(dset_id, HDF5_DATATYPE_NAME, mspace_id, fspace_id, dxpl_id, wbuf);
VRFY((ret >= 0), "H5Dwrite succeeded");
VRFY((H5Sclose(mspace_id) >= 0), "H5Sclose mspace succeeded");
mspace_id = H5I_INVALID_HID;
VRFY((H5Sclose(fspace_id) >= 0), "H5Sclose fspace succeeded");
fspace_id = H5I_INVALID_HID;
VRFY((H5Dclose(dset_id) >= 0), "H5Dclose succeeded");
/* Re-open and read back on every rank independently, verifying both the
* chunk data and (via H5Pencode carrying the blob bytes inline) that
* every rank's H5HG_insert landed on a locator that reads back the
* exact bytes written -- the point of the every-rank-identical-insert
* protocol. */
dset_id = H5Dopen2(group_id, "dset", H5P_DEFAULT);
VRFY((dset_id >= 0), "H5Dopen2 succeeded");
rbuf = (C_DATATYPE *)calloc(1, nbytes);
VRFY((rbuf != NULL), "calloc rbuf succeeded");
fspace_id = H5Dget_space(dset_id);
VRFY((fspace_id >= 0), "H5Dget_space succeeded");
VRFY((H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, NULL, count, block) >= 0),
"H5Sselect_hyperslab read succeeded");
mspace_id = H5Screate_simple(2, block, NULL);
VRFY((mspace_id >= 0), "H5Screate_simple mspace succeeded");
ret = H5Dread(dset_id, HDF5_DATATYPE_NAME, mspace_id, fspace_id, dxpl_id, rbuf);
VRFY((ret >= 0), "H5Dread succeeded");
VRFY((H5Sclose(mspace_id) >= 0), "H5Sclose mspace succeeded");
VRFY((H5Sclose(fspace_id) >= 0), "H5Sclose fspace succeeded");
VRFY((memcmp(rbuf, wbuf, nbytes) == 0), "Data verification succeeded");
dcpl_out = H5Dget_create_plist(dset_id);
VRFY((dcpl_out >= 0), "H5Dget_create_plist succeeded");
VRFY((H5Pencode2(dcpl_out, NULL, &enc_size, H5P_DEFAULT) >= 0), "H5Pencode2 size-query succeeded");
enc_buf = malloc(enc_size);
VRFY((enc_buf != NULL), "malloc enc_buf succeeded");
VRFY((H5Pencode2(dcpl_out, enc_buf, &enc_size, H5P_DEFAULT) >= 0), "H5Pencode2 succeeded");
{
bool found = false;
unsigned char *hay = (unsigned char *)enc_buf;
for (size_t i = 0; i + PAR_BLOB_SIZE <= enc_size && !found; i++)
if (hay[i] == blob[0] && 0 == memcmp(hay + i, blob, PAR_BLOB_SIZE))
found = true;
VRFY(found, "encoded DCPL contains this rank's blob bytes verbatim");
}
free(enc_buf);
VRFY((H5Pclose(dcpl_out) >= 0), "H5Pclose dcpl_out succeeded");
free(wbuf);
free(rbuf);
free(blob);
VRFY((H5Dclose(dset_id) >= 0), "H5Dclose succeeded");
VRFY((H5Pclose(dxpl_id) >= 0), "H5Pclose DXPL succeeded");
VRFY((H5Pclose(dcpl_id) >= 0), "H5Pclose DCPL succeeded");
VRFY((H5Gclose(group_id) >= 0), "H5Gclose succeeded");
VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded");
VRFY((H5Zunregister(PAR_BLOB_FILTER_ID) >= 0), "H5Zunregister succeeded");
}
int
main(int argc, char **argv)
{
@@ -10707,6 +10898,7 @@ main(int argc, char **argv)
test_par_append_filter_error_propagation(fapl_id);
test_par_append_filter_rank_inconsistent_dcpl(fapl_id);
test_par_append_filter_builtin_string_pipeline(fapl_id);
test_par_append_filter_blob(fapl_id);
}
else {
if (MAINPROCESS)
tools/test/h5repack/expected/h5repack_layout.h5-plugin_test_cfg.ddl
+7 -7
View File
@@ -12,7 +12,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}
@@ -36,7 +36,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}
@@ -60,7 +60,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}
@@ -84,7 +84,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}
@@ -108,7 +108,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}
@@ -132,7 +132,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}
@@ -156,7 +156,7 @@ GROUP "/" {
FILTER_ID 261
COMMENT dynlib3
PARAMS { 35 }
PARAMS_STRING 'mode=rate,rate=3.5'
PARAMS_STRING 'mode="rate",rate=3.5'
DESCRIPTION "Test filter with string-based configuration (mode=rate)"
}
}