mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Enable data sieving for chunks that can't be cached (#6111)

Browse Source 
* Enable data sieving for chunks that can't be cached

Fixed an issue that prevented use of a data sieve buffer for I/O on dataset
chunks when those chunks couldn't be cached by the library. This issue
could result in worst-case behavior of I/O on a single data element at a
time when chunks are non-contiguous with respect to memory layout.

Added a test to attempt to catch performance regressions in I/O on dataset
chunks that are non-contiguous with respect to memory layout

Updated the External File List logic to set the data sieve buffer size to
the smaller of the dataset size and the size set in the FAPL, similar to
the logic elsewhere in the library
This commit is contained in:
jhendersonHDF
2026-01-23 10:58:11 -06:00
committed by GitHub
parent 9af6db11b6
commit ed5eceab58
9 changed files with 516 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
release_docs/CHANGELOG.md
+12
View File
@@ -112,6 +112,18 @@ We would like to thank the many HDF5 community members who contributed to this r
## Library
### Fixed a performance issue with chunked dataset I/O
When dataset chunks are unable to be placed in the dataset chunk cache (for example, if a chunk
is too large), the library falls back to an alternative approach for I/O on dataset chunks. An
issue with the logic in this approach prevented chunked dataset I/O from making use of the library's
data sieve buffer I/O optimization functionality. For chunk shapes that are non-contiguous with
the memory layout of a buffer, this could result in severely degraded I/O performance, with the
worst-case behavior causing I/O to be performed on a single data element at a time.
The data sieve buffer functionality has been extended to cover the case of uncached chunks and
will be used as long as the underlying Virtual File Driver supports data sieving.
## Java Library
## Configuration
src/H5Dchunk.c
+56 -1
View File
@@ -409,6 +409,9 @@ H5FL_BLK_DEFINE_STATIC(chunk);
/* Declare extern free list to manage the H5S_sel_iter_t struct */
H5FL_EXTERN(H5S_sel_iter_t);
/* Declare the external PQ free list for the sieve buffer information */
H5FL_BLK_EXTERN(sieve_buf);
/*-------------------------------------------------------------------------
* Function: H5D__chunk_direct_write
*
@@ -3105,6 +3108,16 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
ctg_io_info.dsets_info = &ctg_dset_info;
ctg_io_info.count = 1;
/* Ensure dataset I/O sieve buffer is properly setup for when the contiguous
* dataset I/O routines are used for chunks. Force reset of sieve buffer here
* on each I/O until more advanced use cases like H5Dset_extent between writes
* and subsequent writes or reads can be supported.
*/
dset_info->dset->shared->cache.sieve.sieve_buf_size = H5F_SIEVE_BUF_SIZE(dset_info->dset->oloc.file);
dset_info->dset->shared->cache.sieve.sieve_loc = HADDR_UNDEF;
dset_info->dset->shared->cache.sieve.sieve_size = 0;
assert(!dset_info->dset->shared->cache.sieve.sieve_dirty); /* Any writes should have been flushed */
/* Initialize temporary contiguous storage info */
ctg_store.contig.dset_size = dset_info->dset->shared->layout.u.chunk.size;
@@ -3201,6 +3214,14 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
} /* end else */
done:
/* Free dataset sieve buffer and reset cached fields */
if (dset_info->dset->shared->cache.sieve.sieve_buf) {
dset_info->dset->shared->cache.sieve.sieve_loc = HADDR_UNDEF;
dset_info->dset->shared->cache.sieve.sieve_size = 0;
dset_info->dset->shared->cache.sieve.sieve_buf =
(unsigned char *)H5FL_BLK_FREE(sieve_buf, dset_info->dset->shared->cache.sieve.sieve_buf);
}
/* Cleanup on failure */
if (ret_value < 0) {
if (chunk_mem_spaces != chunk_mem_spaces_local)
@@ -3263,6 +3284,15 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
ctg_io_info.dsets_info = &ctg_dset_info;
ctg_io_info.count = 1;
/* Ensure dataset I/O sieve buffer is properly setup for when the contiguous
* dataset I/O routines are used for chunks. Force reset of sieve buffer here
* on each I/O until more advanced use cases like H5Dset_extent between writes
* and subsequent writes or reads can be supported.
*/
dset_info->dset->shared->cache.sieve.sieve_buf_size = H5F_SIEVE_BUF_SIZE(dset_info->dset->oloc.file);
dset_info->dset->shared->cache.sieve.sieve_loc = HADDR_UNDEF;
dset_info->dset->shared->cache.sieve.sieve_size = 0;
/* Initialize temporary contiguous storage info */
ctg_store.contig.dset_size = dset_info->dset->shared->layout.u.chunk.size;
@@ -3599,9 +3629,30 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info)
/* Advance to next chunk in list */
chunk_node = H5D_CHUNK_GET_NEXT_NODE(dset_info, chunk_node);
} /* end while */
} /* end else */
/*
* Flush any data in sieve buffer - for now, don't keep data cached
* in sieve buffer between writes / writes and reads.
*/
if (H5D__flush_sieve_buf(dset_info->dset) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush sieve buffer");
} /* end else */
done:
/* Free dataset sieve buffer and reset cached fields */
if (dset_info->dset->shared->cache.sieve.sieve_buf) {
dset_info->dset->shared->cache.sieve.sieve_loc = HADDR_UNDEF;
dset_info->dset->shared->cache.sieve.sieve_size = 0;
/* Drop any unflushed sieve buffer data on failure - reads expect
* that sieve buffer will be clean
*/
dset_info->dset->shared->cache.sieve.sieve_dirty = false;
dset_info->dset->shared->cache.sieve.sieve_buf =
(unsigned char *)H5FL_BLK_FREE(sieve_buf, dset_info->dset->shared->cache.sieve.sieve_buf);
}
/* Cleanup on failure */
if (ret_value < 0) {
if (chunk_mem_spaces != chunk_mem_spaces_local)
@@ -3643,6 +3694,10 @@ H5D__chunk_flush(H5D_t *dset)
/* Sanity check */
assert(dset);
/* Flush any data in sieve buffer */
if (H5D__flush_sieve_buf(dset) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush sieve buffer");
/* Loop over all entries in the chunk cache */
for (ent = rdcc->head; ent; ent = next) {
next = ent->next;
src/H5Dcontig.c
+92 -93
View File
@@ -51,7 +51,7 @@
/* Callback info for sieve buffer readvv operation */
typedef struct H5D_contig_readvv_sieve_ud_t {
H5F_shared_t *f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig; /* Cached information about contiguous data */
H5D_sieve_buf_t *sieve_info; /* Information about dataset sieve buffer */
const H5D_contig_storage_t *store_contig; /* Contiguous storage info for this I/O operation */
unsigned char *rbuf; /* Pointer to buffer to fill */
} H5D_contig_readvv_sieve_ud_t;
@@ -66,7 +66,7 @@ typedef struct H5D_contig_readvv_ud_t {
/* Callback info for sieve buffer writevv operation */
typedef struct H5D_contig_writevv_sieve_ud_t {
H5F_shared_t *f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig; /* Cached information about contiguous data */
H5D_sieve_buf_t *sieve_info; /* Information about dataset sieve buffer */
const H5D_contig_storage_t *store_contig; /* Contiguous storage info for this I/O operation */
const unsigned char *wbuf; /* Pointer to buffer to write */
} H5D_contig_writevv_sieve_ud_t;
@@ -481,9 +481,9 @@ H5D__contig_construct(H5F_t *f, H5D_t *dset)
/* Adjust the sieve buffer size to the smaller one between the dataset size and the buffer size
* from the file access property. (SLU - 2012/3/30) */
if (tmp_size < tmp_sieve_buf_size)
dset->shared->cache.contig.sieve_buf_size = tmp_size;
dset->shared->cache.sieve.sieve_buf_size = tmp_size;
else
dset->shared->cache.contig.sieve_buf_size = tmp_sieve_buf_size;
dset->shared->cache.sieve.sieve_buf_size = tmp_sieve_buf_size;
/* If the layout is below version 3, upgrade to version 3 if allowed. Do not upgrade past version 3 since
* there is no benefit. */
@@ -557,9 +557,9 @@ H5D__contig_init(H5F_t *f, H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id, bool H5_AT
/* Adjust the sieve buffer size to the smaller one between the dataset size and the buffer size
* from the file access property. (SLU - 2012/3/30) */
if (dset->shared->layout.storage.u.contig.size < tmp_sieve_buf_size)
dset->shared->cache.contig.sieve_buf_size = dset->shared->layout.storage.u.contig.size;
dset->shared->cache.sieve.sieve_buf_size = dset->shared->layout.storage.u.contig.size;
else
dset->shared->cache.contig.sieve_buf_size = tmp_sieve_buf_size;
dset->shared->cache.sieve.sieve_buf_size = tmp_sieve_buf_size;
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -607,7 +607,7 @@ H5D__contig_is_data_cached(const H5D_shared_t *shared_dset)
/* Sanity checks */
assert(shared_dset);
FUNC_LEAVE_NOAPI(shared_dset->cache.contig.sieve_size > 0)
FUNC_LEAVE_NOAPI(shared_dset->cache.sieve.sieve_size > 0)
} /* end H5D__contig_is_data_cached() */
/*-------------------------------------------------------------------------
@@ -816,8 +816,8 @@ H5D__contig_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_SEL_IO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET;
}
else if ((op_type == H5D_IO_OP_READ && dataset->shared->cache.contig.sieve_dirty) ||
(op_type == H5D_IO_OP_WRITE && dataset->shared->cache.contig.sieve_buf)) {
else if ((op_type == H5D_IO_OP_READ && dataset->shared->cache.sieve.sieve_dirty) ||
(op_type == H5D_IO_OP_WRITE && dataset->shared->cache.sieve.sieve_buf)) {
io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF;
io_info->no_selection_io_cause |= H5D_SEL_IO_CONTIGUOUS_SIEVE_BUFFER;
}
@@ -1034,9 +1034,9 @@ static herr_t
H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata)
{
H5D_contig_readvv_sieve_ud_t *udata =
(H5D_contig_readvv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */
H5F_shared_t *f_sh = udata->f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig = udata->dset_contig; /* Cached information about contiguous data */
(H5D_contig_readvv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */
H5F_shared_t *f_sh = udata->f_sh; /* Shared file for dataset */
H5D_sieve_buf_t *sieve_info = udata->sieve_info; /* Information about dataset sieve buffer */
const H5D_contig_storage_t *store_contig =
udata->store_contig; /* Contiguous storage info for this I/O operation */
unsigned char *buf; /* Pointer to buffer to fill */
@@ -1052,9 +1052,9 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
FUNC_ENTER_PACKAGE
/* Stash local copies of these value */
if (dset_contig->sieve_buf != NULL) {
sieve_start = dset_contig->sieve_loc;
sieve_size = dset_contig->sieve_size;
if (sieve_info->sieve_buf != NULL) {
sieve_start = sieve_info->sieve_loc;
sieve_size = sieve_info->sieve_size;
sieve_end = sieve_start + sieve_size;
} /* end if */
@@ -1065,19 +1065,19 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
buf = udata->rbuf + src_off;
/* Check if the sieve buffer is allocated yet */
if (NULL == dset_contig->sieve_buf) {
if (NULL == sieve_info->sieve_buf) {
/* Check if we can actually hold the I/O request in the sieve buffer */
if (len > dset_contig->sieve_buf_size) {
if (len > sieve_info->sieve_buf_size) {
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed");
} /* end if */
else {
/* Allocate room for the data sieve buffer */
if (NULL == (dset_contig->sieve_buf = H5FL_BLK_CALLOC(sieve_buf, dset_contig->sieve_buf_size)))
if (NULL == (sieve_info->sieve_buf = H5FL_BLK_CALLOC(sieve_buf, sieve_info->sieve_buf_size)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed");
/* Determine the new sieve buffer size & location */
dset_contig->sieve_loc = addr;
sieve_info->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
@@ -1087,19 +1087,19 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
max_data = store_contig->dset_size - dst_off;
/* Compute the size of the sieve buffer */
min = MIN3(rel_eoa - dset_contig->sieve_loc, max_data, dset_contig->sieve_buf_size);
H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t);
min = MIN3(rel_eoa - sieve_info->sieve_loc, max_data, sieve_info->sieve_buf_size);
H5_CHECKED_ASSIGN(sieve_info->sieve_size, size_t, min, hsize_t);
/* Read the new sieve buffer */
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size,
dset_contig->sieve_buf) < 0)
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, sieve_info->sieve_loc, sieve_info->sieve_size,
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed");
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
H5MM_memcpy(buf, dset_contig->sieve_buf, len);
H5MM_memcpy(buf, sieve_info->sieve_buf, len);
/* Reset sieve buffer dirty flag */
dset_contig->sieve_dirty = false;
sieve_info->sieve_dirty = false;
} /* end else */
} /* end if */
else {
@@ -1108,7 +1108,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
/* If entire read is within the sieve buffer, read it from the buffer */
if (addr >= sieve_start && contig_end < sieve_end) {
unsigned char *base_sieve_buf = dset_contig->sieve_buf + (addr - sieve_start);
unsigned char *base_sieve_buf = sieve_info->sieve_buf + (addr - sieve_start);
/* Grab the data out of the buffer */
H5MM_memcpy(buf, base_sieve_buf, len);
@@ -1116,19 +1116,19 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
/* Entire request is not within this data sieve buffer */
else {
/* Check if we can actually hold the I/O request in the sieve buffer */
if (len > dset_contig->sieve_buf_size) {
if (len > sieve_info->sieve_buf_size) {
/* Check for any overlap with the current sieve buffer */
if ((sieve_start >= addr && sieve_start < (contig_end + 1)) ||
((sieve_end - 1) >= addr && (sieve_end - 1) < (contig_end + 1))) {
/* Flush the sieve buffer, if it's dirty */
if (dset_contig->sieve_dirty) {
if (sieve_info->sieve_dirty) {
/* Write to file */
if (H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size,
dset_contig->sieve_buf) < 0)
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed");
/* Reset sieve buffer dirty flag */
dset_contig->sieve_dirty = false;
sieve_info->sieve_dirty = false;
} /* end if */
} /* end if */
@@ -1139,18 +1139,18 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
/* Element size fits within the buffer size */
else {
/* Flush the sieve buffer if it's dirty */
if (dset_contig->sieve_dirty) {
if (sieve_info->sieve_dirty) {
/* Write to file */
if (H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size,
dset_contig->sieve_buf) < 0)
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed");
/* Reset sieve buffer dirty flag */
dset_contig->sieve_dirty = false;
sieve_info->sieve_dirty = false;
} /* end if */
/* Determine the new sieve buffer size & location */
dset_contig->sieve_loc = addr;
sieve_info->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
@@ -1164,19 +1164,19 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *
* the end of the data element, and don't read more than
* the buffer size.
*/
min = MIN3(rel_eoa - dset_contig->sieve_loc, max_data, dset_contig->sieve_buf_size);
H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t);
min = MIN3(rel_eoa - sieve_info->sieve_loc, max_data, sieve_info->sieve_buf_size);
H5_CHECKED_ASSIGN(sieve_info->sieve_size, size_t, min, hsize_t);
/* Read the new sieve buffer */
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc,
dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, sieve_info->sieve_loc, sieve_info->sieve_size,
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed");
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
H5MM_memcpy(buf, dset_contig->sieve_buf, len);
H5MM_memcpy(buf, sieve_info->sieve_buf, len);
/* Reset sieve buffer dirty flag */
dset_contig->sieve_dirty = false;
sieve_info->sieve_dirty = false;
} /* end else */
} /* end else */
} /* end else */
@@ -1251,7 +1251,7 @@ H5D__contig_readvv(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_
/* Set up user data for H5VM_opvv() */
udata.f_sh = io_info->f_sh;
udata.dset_contig = &(dset_info->dset->shared->cache.contig);
udata.sieve_info = &(dset_info->dset->shared->cache.sieve);
udata.store_contig = &(dset_info->store->contig);
udata.rbuf = (unsigned char *)dset_info->buf.vp;
@@ -1293,9 +1293,9 @@ static herr_t
H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata)
{
H5D_contig_writevv_sieve_ud_t *udata =
(H5D_contig_writevv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */
H5F_shared_t *f_sh = udata->f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig = udata->dset_contig; /* Cached information about contiguous data */
(H5D_contig_writevv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */
H5F_shared_t *f_sh = udata->f_sh; /* Shared file for dataset */
H5D_sieve_buf_t *sieve_info = udata->sieve_info; /* Information about dataset sieve buffer */
const H5D_contig_storage_t *store_contig =
udata->store_contig; /* Contiguous storage info for this I/O operation */
const unsigned char *buf; /* Pointer to buffer to fill */
@@ -1311,9 +1311,9 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void
FUNC_ENTER_PACKAGE
/* Stash local copies of these values */
if (dset_contig->sieve_buf != NULL) {
sieve_start = dset_contig->sieve_loc;
sieve_size = dset_contig->sieve_size;
if (sieve_info->sieve_buf != NULL) {
sieve_start = sieve_info->sieve_loc;
sieve_size = sieve_info->sieve_size;
sieve_end = sieve_start + sieve_size;
} /* end if */
@@ -1324,23 +1324,23 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void
buf = udata->wbuf + src_off;
/* No data sieve buffer yet, go allocate one */
if (NULL == dset_contig->sieve_buf) {
if (NULL == sieve_info->sieve_buf) {
/* Check if we can actually hold the I/O request in the sieve buffer */
if (len > dset_contig->sieve_buf_size) {
if (len > sieve_info->sieve_buf_size) {
if (H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed");
} /* end if */
else {
/* Allocate room for the data sieve buffer */
if (NULL == (dset_contig->sieve_buf = H5FL_BLK_CALLOC(sieve_buf, dset_contig->sieve_buf_size)))
if (NULL == (sieve_info->sieve_buf = H5FL_BLK_CALLOC(sieve_buf, sieve_info->sieve_buf_size)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed");
/* Clear memory */
if (dset_contig->sieve_size > len)
memset(dset_contig->sieve_buf + len, 0, (dset_contig->sieve_size - len));
if (sieve_info->sieve_size > len)
memset(sieve_info->sieve_buf + len, 0, (sieve_info->sieve_size - len));
/* Determine the new sieve buffer size & location */
dset_contig->sieve_loc = addr;
sieve_info->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
@@ -1350,26 +1350,26 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void
max_data = store_contig->dset_size - dst_off;
/* Compute the size of the sieve buffer */
min = MIN3(rel_eoa - dset_contig->sieve_loc, max_data, dset_contig->sieve_buf_size);
H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t);
min = MIN3(rel_eoa - sieve_info->sieve_loc, max_data, sieve_info->sieve_buf_size);
H5_CHECKED_ASSIGN(sieve_info->sieve_size, size_t, min, hsize_t);
/* Check if there is any point in reading the data from the file */
if (dset_contig->sieve_size > len) {
if (sieve_info->sieve_size > len) {
/* Read the new sieve buffer */
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc,
dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, sieve_info->sieve_loc, sieve_info->sieve_size,
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed");
} /* end if */
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
H5MM_memcpy(dset_contig->sieve_buf, buf, len);
H5MM_memcpy(sieve_info->sieve_buf, buf, len);
/* Set sieve buffer dirty flag */
dset_contig->sieve_dirty = true;
sieve_info->sieve_dirty = true;
/* Stash local copies of these values */
sieve_start = dset_contig->sieve_loc;
sieve_size = dset_contig->sieve_size;
sieve_start = sieve_info->sieve_loc;
sieve_size = sieve_info->sieve_size;
sieve_end = sieve_start + sieve_size;
} /* end else */
} /* end if */
@@ -1379,35 +1379,35 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void
/* If entire write is within the sieve buffer, write it to the buffer */
if (addr >= sieve_start && contig_end < sieve_end) {
unsigned char *base_sieve_buf = dset_contig->sieve_buf + (addr - sieve_start);
unsigned char *base_sieve_buf = sieve_info->sieve_buf + (addr - sieve_start);
/* Put the data into the sieve buffer */
H5MM_memcpy(base_sieve_buf, buf, len);
/* Set sieve buffer dirty flag */
dset_contig->sieve_dirty = true;
sieve_info->sieve_dirty = true;
} /* end if */
/* Entire request is not within this data sieve buffer */
else {
/* Check if we can actually hold the I/O request in the sieve buffer */
if (len > dset_contig->sieve_buf_size) {
if (len > sieve_info->sieve_buf_size) {
/* Check for any overlap with the current sieve buffer */
if ((sieve_start >= addr && sieve_start < (contig_end + 1)) ||
((sieve_end - 1) >= addr && (sieve_end - 1) < (contig_end + 1))) {
/* Flush the sieve buffer, if it's dirty */
if (dset_contig->sieve_dirty) {
if (sieve_info->sieve_dirty) {
/* Write to file */
if (H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size,
dset_contig->sieve_buf) < 0)
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed");
/* Reset sieve buffer dirty flag */
dset_contig->sieve_dirty = false;
sieve_info->sieve_dirty = false;
} /* end if */
/* Force the sieve buffer to be re-read the next time */
dset_contig->sieve_loc = HADDR_UNDEF;
dset_contig->sieve_size = 0;
sieve_info->sieve_loc = HADDR_UNDEF;
sieve_info->sieve_size = 0;
} /* end if */
/* Write directly from the user's buffer */
@@ -1418,44 +1418,43 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void
else {
/* Check if it is possible to (exactly) prepend or append to existing (dirty) sieve buffer */
if (((addr + len) == sieve_start || addr == sieve_end) &&
(len + sieve_size) <= dset_contig->sieve_buf_size && dset_contig->sieve_dirty) {
(len + sieve_size) <= sieve_info->sieve_buf_size && sieve_info->sieve_dirty) {
/* Prepend to existing sieve buffer */
if ((addr + len) == sieve_start) {
/* Move existing sieve information to correct location */
memmove(dset_contig->sieve_buf + len, dset_contig->sieve_buf,
dset_contig->sieve_size);
memmove(sieve_info->sieve_buf + len, sieve_info->sieve_buf, sieve_info->sieve_size);
/* Copy in new information (must be first in sieve buffer) */
H5MM_memcpy(dset_contig->sieve_buf, buf, len);
H5MM_memcpy(sieve_info->sieve_buf, buf, len);
/* Adjust sieve location */
dset_contig->sieve_loc = addr;
sieve_info->sieve_loc = addr;
} /* end if */
/* Append to existing sieve buffer */
else {
/* Copy in new information */
H5MM_memcpy(dset_contig->sieve_buf + sieve_size, buf, len);
H5MM_memcpy(sieve_info->sieve_buf + sieve_size, buf, len);
} /* end else */
/* Adjust sieve size */
dset_contig->sieve_size += len;
sieve_info->sieve_size += len;
} /* end if */
/* Can't add the new data onto the existing sieve buffer */
else {
/* Flush the sieve buffer if it's dirty */
if (dset_contig->sieve_dirty) {
if (sieve_info->sieve_dirty) {
/* Write to file */
if (H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size,
dset_contig->sieve_buf) < 0)
sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed");
/* Reset sieve buffer dirty flag */
dset_contig->sieve_dirty = false;
sieve_info->sieve_dirty = false;
} /* end if */
/* Determine the new sieve buffer size & location */
dset_contig->sieve_loc = addr;
sieve_info->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
@@ -1469,22 +1468,22 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len, void
* the end of the data element, and don't read more than
* the buffer size.
*/
min = MIN3(rel_eoa - dset_contig->sieve_loc, max_data, dset_contig->sieve_buf_size);
H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t);
min = MIN3(rel_eoa - sieve_info->sieve_loc, max_data, sieve_info->sieve_buf_size);
H5_CHECKED_ASSIGN(sieve_info->sieve_size, size_t, min, hsize_t);
/* Check if there is any point in reading the data from the file */
if (dset_contig->sieve_size > len) {
if (sieve_info->sieve_size > len) {
/* Read the new sieve buffer */
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc,
dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if (H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, sieve_info->sieve_loc,
sieve_info->sieve_size, sieve_info->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed");
} /* end if */
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
H5MM_memcpy(dset_contig->sieve_buf, buf, len);
H5MM_memcpy(sieve_info->sieve_buf, buf, len);
/* Set sieve buffer dirty flag */
dset_contig->sieve_dirty = true;
sieve_info->sieve_dirty = true;
} /* end else */
} /* end else */
} /* end else */
@@ -1561,7 +1560,7 @@ H5D__contig_writevv(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset
/* Set up user data for H5VM_opvv() */
udata.f_sh = io_info->f_sh;
udata.dset_contig = &(dset_info->dset->shared->cache.contig);
udata.sieve_info = &(dset_info->dset->shared->cache.sieve);
udata.store_contig = &(dset_info->store->contig);
udata.wbuf = (const unsigned char *)dset_info->buf.cvp;
@@ -1792,10 +1791,10 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src, H5F_t *f
/* If data sieving is enabled and the dataset is open in the file,
set up to copy data out of the sieve buffer if deemed possible later */
if (H5F_HAS_FEATURE(f_src, H5FD_FEAT_DATA_SIEVE) && shared_fo && shared_fo->cache.contig.sieve_buf) {
if (H5F_HAS_FEATURE(f_src, H5FD_FEAT_DATA_SIEVE) && shared_fo && shared_fo->cache.sieve.sieve_buf) {
try_sieve = true;
sieve_start = shared_fo->cache.contig.sieve_loc;
sieve_end = sieve_start + shared_fo->cache.contig.sieve_size;
sieve_start = shared_fo->cache.sieve.sieve_loc;
sieve_end = sieve_start + shared_fo->cache.sieve.sieve_size;
}
while (total_src_nbytes > 0) {
@@ -1825,7 +1824,7 @@ H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src, H5F_t *f
/* If the entire copy is within the sieve buffer, copy data from the sieve buffer */
if (try_sieve && (addr_src >= sieve_start) && ((addr_src + src_nbytes - 1) < sieve_end)) {
unsigned char *base_sieve_buf = shared_fo->cache.contig.sieve_buf + (addr_src - sieve_start);
unsigned char *base_sieve_buf = shared_fo->cache.sieve.sieve_buf + (addr_src - sieve_start);
H5MM_memcpy(buf, base_sieve_buf, src_nbytes);
}
src/H5Defl.c
+5 -2
View File
@@ -156,8 +156,11 @@ H5D__efl_construct(H5F_t *f, H5D_t *dset)
tmp_size = (hsize_t)stmp_size * dt_size;
H5_CHECKED_ASSIGN(dset->shared->layout.storage.u.contig.size, hsize_t, tmp_size, hssize_t);
/* Get the sieve buffer size for this dataset */
dset->shared->cache.contig.sieve_buf_size = H5F_SIEVE_BUF_SIZE(f);
/* Get the sieve buffer size for this dataset - the smaller of the dataset size and
* the sieve buffer size from the FAPL is used
*/
dset->shared->cache.sieve.sieve_buf_size =
MIN(dset->shared->layout.storage.u.contig.size, H5F_SIEVE_BUF_SIZE(f));
done:
FUNC_LEAVE_NOAPI(ret_value)
src/H5Dint.c
+17 -15
View File
@@ -1962,13 +1962,15 @@ H5D_close(H5D_t *dataset)
*/
dataset->shared->closing = true;
/* Free the data sieve buffer, if it's been allocated */
if (dataset->shared->cache.sieve.sieve_buf)
dataset->shared->cache.sieve.sieve_buf =
(unsigned char *)H5FL_BLK_FREE(sieve_buf, dataset->shared->cache.sieve.sieve_buf);
/* Free cached information for each kind of dataset */
switch (dataset->shared->layout.type) {
case H5D_CONTIGUOUS:
/* Free the data sieve buffer, if it's been allocated */
if (dataset->shared->cache.contig.sieve_buf)
dataset->shared->cache.contig.sieve_buf =
(unsigned char *)H5FL_BLK_FREE(sieve_buf, dataset->shared->cache.contig.sieve_buf);
/* Nothing special to do */
break;
case H5D_CHUNKED:
@@ -1991,6 +1993,7 @@ H5D_close(H5D_t *dataset)
H5FL_FREE(H5D_piece_info_t, dataset->shared->cache.chunk.single_piece_info);
dataset->shared->cache.chunk.single_piece_info = NULL;
} /* end if */
break;
case H5D_COMPACT:
@@ -2156,15 +2159,13 @@ H5D_mult_refresh_close(hid_t dset_id)
assert(dataset->shared->fo_count > 0);
if (dataset->shared->fo_count > 1) {
/* Free the data sieve buffer, if it's been allocated */
if (dataset->shared->cache.sieve.sieve_buf)
dataset->shared->cache.sieve.sieve_buf =
(unsigned char *)H5FL_BLK_FREE(sieve_buf, dataset->shared->cache.sieve.sieve_buf);
/* Free cached information for each kind of dataset */
switch (dataset->shared->layout.type) {
case H5D_CONTIGUOUS:
/* Free the data sieve buffer, if it's been allocated */
if (dataset->shared->cache.contig.sieve_buf)
dataset->shared->cache.contig.sieve_buf =
(unsigned char *)H5FL_BLK_FREE(sieve_buf, dataset->shared->cache.contig.sieve_buf);
break;
case H5D_CHUNKED:
/* Check for skip list for iterating over chunks during I/O to close */
if (dataset->shared->cache.chunk.sel_chunks) {
@@ -2187,6 +2188,7 @@ H5D_mult_refresh_close(hid_t dset_id)
} /* end if */
break;
case H5D_CONTIGUOUS:
case H5D_COMPACT:
case H5D_VIRTUAL:
/* Nothing special to do (info freed in the layout destroy) */
@@ -3250,18 +3252,18 @@ H5D__flush_sieve_buf(H5D_t *dataset)
assert(dataset);
/* Flush the raw data buffer, if we have a dirty one */
if (dataset->shared->cache.contig.sieve_buf && dataset->shared->cache.contig.sieve_dirty) {
if (dataset->shared->cache.sieve.sieve_buf && dataset->shared->cache.sieve.sieve_dirty) {
assert(dataset->shared->layout.type !=
H5D_COMPACT); /* We should never have a sieve buffer for compact storage */
/* Write dirty data sieve buffer to file */
if (H5F_shared_block_write(
H5F_SHARED(dataset->oloc.file), H5FD_MEM_DRAW, dataset->shared->cache.contig.sieve_loc,
dataset->shared->cache.contig.sieve_size, dataset->shared->cache.contig.sieve_buf) < 0)
H5F_SHARED(dataset->oloc.file), H5FD_MEM_DRAW, dataset->shared->cache.sieve.sieve_loc,
dataset->shared->cache.sieve.sieve_size, dataset->shared->cache.sieve.sieve_buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed");
/* Reset sieve buffer dirty flag */
dataset->shared->cache.contig.sieve_dirty = false;
dataset->shared->cache.sieve.sieve_dirty = false;
} /* end if */
done:
src/H5Dpkg.h
+5 -9
View File
@@ -562,14 +562,14 @@ typedef struct H5D_rdcc_t {
unsigned scaled_encode_bits[H5S_MAX_RANK]; /* The number of bits needed to encode the scaled dim sizes */
} H5D_rdcc_t;
/* The raw data contiguous data cache */
typedef struct H5D_rdcdc_t {
/* Information about the dataset sieve buffer */
typedef struct H5D_sieve_buf_t {
unsigned char *sieve_buf; /* Buffer to hold data sieve buffer */
haddr_t sieve_loc; /* File location (offset) of the data sieve buffer */
size_t sieve_size; /* Size of the data sieve buffer used (in bytes) */
size_t sieve_buf_size; /* Size of the data sieve buffer allocated (in bytes) */
bool sieve_dirty; /* Flag to indicate that the data sieve buffer is dirty */
} H5D_rdcdc_t;
} H5D_sieve_buf_t;
/*
* A dataset is made of two layers, an H5D_t struct that is unique to
@@ -598,12 +598,8 @@ struct H5D_shared_t {
/* Buffered/cached information for types of raw data storage*/
struct {
H5D_rdcdc_t contig; /* Information about contiguous data */
/* (Note that the "contig" cache
* information can be used by a chunked
* dataset in certain circumstances)
*/
H5D_rdcc_t chunk; /* Information about chunked data */
H5D_sieve_buf_t sieve; /* Information about dataset sieve buffer */
H5D_rdcc_t chunk; /* Information about chunked data */
} cache;
H5D_append_flush_t append_flush; /* Append flush property information */
src/H5Ppublic.h
+2 -2
View File
@@ -5405,8 +5405,8 @@ H5_DLL herr_t H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *
* the dataset being read in for hyperslab selections boosts
* performance by quite a bit.
*
* The default value is set to 64KB, indicating that file I/O for raw
* data reads and writes will occur in at least 64KB blocks. Setting
* The default value is set to 64KiB, indicating that file I/O for raw
* data reads and writes will occur in at least 64KiB blocks. Setting
* the value to zero (\TText{0}) with this API function will turn off
* the data sieving, even if the VFL driver attempts to use that
* strategy.
test/CMakeLists.txt
+1
View File
@@ -375,6 +375,7 @@ set (H5_EXPRESS_TESTS
rtree
objcopy_ref
objcopy
io_perf
)
set (H5_TESTS
test/io_perf.c
+326
View File
@@ -0,0 +1,326 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the LICENSE file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* This file contains tests which attempt to catch I/O performance regressions
* by scaling up problem sizes according to the current "TestExpress" setting.
* While not easy to do reliably, tests should generally be designed to try
* causing a timeout by running for an extraordinarily long duration when the
* "TestExpress" value is set to level 0 and the performance problem in question
* has been regressed on. When the performance problem has NOT been regressed
* on, tests should run reasonably fast for a "TestExpress" level of 0 in order
* to facilitate CI testing. For higher "TestExpress" values, tests should
* generally run quickly, even when the performance problem in question has been
* regressed on, so that the cumulative runtime of this test program is minimal.
* The duration that may elapse before a timeout occurs is currently determined
* by the CMake build system's CTEST_TEST_TIMEOUT / DART_TESTING_TIMEOUT
* variables (currently 1200 seconds by default).
*/
#include "testframe.h"
/*
* Test I/O on a dataset with chunks that are non-contiguous with respect to
* memory layout. This test attempts to catch an I/O performance issue where
* the library was skipping use of a sieve buffer and performing I/O on chunks
* element by element, resulting in very bad performance. The number of chunks
* being written and read scales up with lower values of TestExpress. At a
* TestExpress setting of 0 the number of chunks should result in a very long
* time spent in I/O if a sieve buffer isn't being used.
*/
#define FILE_NAME "chunk_non_contig_mem_io.h5"
#define DATASET_NAME "chunked_dataset"
#define DATASET_TYPE int
#define CHUNK_DIM_0 4194304
static void
chunk_non_contig_mem_io(void H5_ATTR_UNUSED *params)
{
hsize_t dims[2] = {0};
hsize_t mem_dims[2] = {0};
hsize_t chunk_dims[2] = {0};
hsize_t start[2] = {0};
hsize_t count[2] = {0};
hsize_t block[2] = {0};
hid_t file_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
hid_t space_id = H5I_INVALID_HID;
hid_t mem_space_id = H5I_INVALID_HID;
hid_t dcpl_id = H5I_INVALID_HID;
size_t num_chunks = 0;
size_t data_size = 0;
void *write_buf = NULL;
void *read_buf = NULL;
int TestExpress = GetTestExpress();
switch (TestExpress) {
case H5_TEST_EXPRESS_EXHAUSTIVE:
num_chunks = 1024;
break;
case H5_TEST_EXPRESS_FULL:
num_chunks = 512;
break;
case H5_TEST_EXPRESS_QUICK:
num_chunks = 128;
break;
case H5_TEST_EXPRESS_SMOKE_TEST:
default:
num_chunks = 64;
break;
}
MESSAGE(VERBO_NONE, ("Express test mode set to %d. Testing with %zu chunks\n", TestExpress, num_chunks));
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {
fprintf(stderr, "Failed to create FAPL\n");
goto error;
}
/* Disable dataset chunk caching */
if (H5Pset_cache(fapl_id, 0, 0, 0, 0.0) < 0) {
fprintf(stderr, "Failed to disable dataset chunk caching\n");
goto error;
}
/* Set sieve buffer size to size of a single chunk */
if (H5Pset_sieve_buf_size(fapl_id, (size_t)CHUNK_DIM_0 * sizeof(DATASET_TYPE)) < 0) {
fprintf(stderr, "Failed to set data sieve buffer size\n");
goto error;
}
if ((file_id = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) {
fprintf(stderr, "Failed to create file '%s'\n", FILE_NAME);
goto error;
}
dims[0] = (hsize_t)CHUNK_DIM_0;
dims[1] = (hsize_t)num_chunks;
if ((space_id = H5Screate_simple(2, dims, NULL)) < 0) {
fprintf(stderr, "Failed to create dataspace for dataset\n");
goto error;
}
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) {
fprintf(stderr, "Failed to create DCPL\n");
goto error;
}
if (H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER) < 0) {
fprintf(stderr, "Failed to set fill value writing time on DCPL\n");
goto error;
}
chunk_dims[0] = (hsize_t)CHUNK_DIM_0;
chunk_dims[1] = 1;
if (H5Pset_chunk(dcpl_id, 2, chunk_dims) < 0) {
fprintf(stderr, "Failed to set chunking on DCPL\n");
goto error;
}
if ((dset_id = H5Dcreate2(file_id, DATASET_NAME, H5T_NATIVE_INT, space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0) {
fprintf(stderr, "Failed to create dataset\n");
goto error;
}
data_size = (size_t)CHUNK_DIM_0 * sizeof(DATASET_TYPE);
if (NULL == (write_buf = malloc(data_size))) {
fprintf(stderr, "Failed to allocate write buffer\n");
goto error;
}
for (size_t i = 0; i < data_size / sizeof(DATASET_TYPE); i++)
((DATASET_TYPE *)write_buf)[i] = (DATASET_TYPE)1;
mem_dims[0] = 1;
mem_dims[1] = (hsize_t)CHUNK_DIM_0;
if ((mem_space_id = H5Screate_simple(2, mem_dims, NULL)) < 0) {
fprintf(stderr, "Failed to create memory dataspace\n");
goto error;
}
for (size_t chunk = 0; chunk < num_chunks; chunk++) {
MESSAGE(VERBO_DEF, ("Writing chunk %zu\n", chunk));
start[0] = 0;
start[1] = chunk;
count[0] = 1;
count[1] = 1;
block[0] = (hsize_t)CHUNK_DIM_0;
block[1] = 1;
if (H5Sselect_hyperslab(space_id, H5S_SELECT_SET, start, NULL, count, block) < 0) {
fprintf(stderr, "Failed to select hyperslab for chunk\n");
goto error;
}
if (H5Dwrite(dset_id, H5T_NATIVE_INT, mem_space_id, space_id, H5P_DEFAULT, write_buf) < 0) {
fprintf(stderr, "Failed to write to dataset\n");
goto error;
}
}
if (H5Sclose(mem_space_id) < 0) {
fprintf(stderr, "Failed to close dataspace\n");
goto error;
}
if (H5Dclose(dset_id) < 0) {
fprintf(stderr, "Failed to close dataset\n");
goto error;
}
free(write_buf);
write_buf = NULL;
if ((dset_id = H5Dopen2(file_id, DATASET_NAME, H5P_DEFAULT)) < 0) {
fprintf(stderr, "Failed to open dataset\n");
goto error;
}
if (NULL == (read_buf = malloc(2 * data_size))) {
fprintf(stderr, "Failed to allocate read buffer\n");
goto error;
}
/* Use same shape selection in memory buffer as file selection to
* avoid iterating element by element when setting up chunk dataspaces
* for selections
*/
mem_dims[0] = (hsize_t)CHUNK_DIM_0;
mem_dims[1] = 2;
if ((mem_space_id = H5Screate_simple(2, mem_dims, NULL)) < 0) {
fprintf(stderr, "Failed to create memory dataspace\n");
goto error;
}
start[0] = 0;
start[1] = 0;
count[0] = 1;
count[1] = 1;
block[0] = (hsize_t)CHUNK_DIM_0;
block[1] = 1;
if (H5Sselect_hyperslab(mem_space_id, H5S_SELECT_SET, start, NULL, count, block) < 0) {
fprintf(stderr, "Failed to select hyperslab in memory dataspace\n");
goto error;
}
for (size_t chunk = 0; chunk < num_chunks; chunk++) {
MESSAGE(VERBO_DEF, ("Reading chunk %zu\n", chunk));
start[0] = 0;
start[1] = chunk;
count[0] = 1;
count[1] = 1;
block[0] = (hsize_t)CHUNK_DIM_0;
block[1] = 1;
if (H5Sselect_hyperslab(space_id, H5S_SELECT_SET, start, NULL, count, block) < 0) {
fprintf(stderr, "Failed to select hyperslab for chunk\n");
goto error;
}
if (H5Dread(dset_id, H5T_NATIVE_INT, mem_space_id, space_id, H5P_DEFAULT, read_buf) < 0) {
fprintf(stderr, "Failed to read from dataset\n");
goto error;
}
}
free(read_buf);
read_buf = NULL;
if (H5Pclose(dcpl_id) < 0)
goto error;
if (H5Sclose(mem_space_id) < 0)
goto error;
if (H5Sclose(space_id) < 0)
goto error;
if (H5Dclose(dset_id) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
if (H5Fdelete(FILE_NAME, fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
return;
error:
IncTestNumErrs();
free(write_buf);
free(read_buf);
H5E_BEGIN_TRY
{
H5Pclose(dcpl_id);
H5Sclose(mem_space_id);
H5Sclose(space_id);
H5Dclose(dset_id);
H5Fclose(file_id);
H5Fdelete(FILE_NAME, fapl_id);
H5Pclose(fapl_id);
}
H5E_END_TRY
}
#undef FILE_NAME
#undef DATASET_NAME
#undef DATASET_TYPE
#undef CHUNK_DIM_0
int
main(int argc, char **argv)
{
/* Initialize testing framework */
if (TestInit(argv[0], NULL, NULL, NULL, NULL, 0) < 0) {
fprintf(stderr, "couldn't initialize testing framework\n");
exit(EXIT_FAILURE);
}
AddTest("chunk_non_contig_mem_io", chunk_non_contig_mem_io, NULL, NULL, NULL, 0,
"I/O on chunks that are non-contiguous with respect to memory layout");
/* Display testing information */
TestInfo(stdout);
/* Parse command line arguments */
if (TestParseCmdLine(argc, argv) < 0) {
fprintf(stderr, "couldn't parse command-line arguments\n");
TestShutdown();
exit(EXIT_FAILURE);
}
/* Perform requested testing */
if (PerformTests() < 0) {
fprintf(stderr, "couldn't run tests\n");
TestShutdown();
exit(EXIT_FAILURE);
}
/* Display test summary, if requested */
if (GetTestSummary())
TestSummary(stdout);
/* Release test infrastructure */
if (TestShutdown() < 0) {
fprintf(stderr, "couldn't shut down testing framework\n");
exit(EXIT_FAILURE);
}
/* Exit failure if errors encountered; else exit success. */
/* No need to print anything since PerformTests() already does. */
if (GetTestNumErrs() > 0)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}