mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
2952bb92169d25f2bd8ecd96f0d9b9373217461c
hdf5/test/io_perf.c
T
jhendersonHDF ed5eceab58 Enable data sieving for chunks that can't be cached (#6111) 
* Enable data sieving for chunks that can't be cached

Fixed an issue that prevented use of a data sieve buffer for I/O on dataset
chunks when those chunks couldn't be cached by the library. This issue
could result in worst-case behavior of I/O on a single data element at a
time when chunks are non-contiguous with respect to memory layout.

Added a test to attempt to catch performance regressions in I/O on dataset
chunks that are non-contiguous with respect to memory layout

Updated the External File List logic to set the data sieve buffer size to
the smaller of the dataset size and the size set in the FAPL, similar to
the logic elsewhere in the library
2026-01-23 10:58:11 -06:00

327 lines
11 KiB
C
Raw Blame History

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the LICENSE file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* This file contains tests which attempt to catch I/O performance regressions
* by scaling up problem sizes according to the current "TestExpress" setting.
* While not easy to do reliably, tests should generally be designed to try
* causing a timeout by running for an extraordinarily long duration when the
* "TestExpress" value is set to level 0 and the performance problem in question
* has been regressed on. When the performance problem has NOT been regressed
* on, tests should run reasonably fast for a "TestExpress" level of 0 in order
* to facilitate CI testing. For higher "TestExpress" values, tests should
* generally run quickly, even when the performance problem in question has been
* regressed on, so that the cumulative runtime of this test program is minimal.
* The duration that may elapse before a timeout occurs is currently determined
* by the CMake build system's CTEST_TEST_TIMEOUT / DART_TESTING_TIMEOUT
* variables (currently 1200 seconds by default).
*/
#include "testframe.h"
/*
* Test I/O on a dataset with chunks that are non-contiguous with respect to
* memory layout. This test attempts to catch an I/O performance issue where
* the library was skipping use of a sieve buffer and performing I/O on chunks
* element by element, resulting in very bad performance. The number of chunks
* being written and read scales up with lower values of TestExpress. At a
* TestExpress setting of 0 the number of chunks should result in a very long
* time spent in I/O if a sieve buffer isn't being used.
*/
#define FILE_NAME "chunk_non_contig_mem_io.h5"
#define DATASET_NAME "chunked_dataset"
#define DATASET_TYPE int
#define CHUNK_DIM_0 4194304
static void
chunk_non_contig_mem_io(void H5_ATTR_UNUSED *params)
{
hsize_t dims[2] = {0};
hsize_t mem_dims[2] = {0};
hsize_t chunk_dims[2] = {0};
hsize_t start[2] = {0};
hsize_t count[2] = {0};
hsize_t block[2] = {0};
hid_t file_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
hid_t space_id = H5I_INVALID_HID;
hid_t mem_space_id = H5I_INVALID_HID;
hid_t dcpl_id = H5I_INVALID_HID;
size_t num_chunks = 0;
size_t data_size = 0;
void *write_buf = NULL;
void *read_buf = NULL;
int TestExpress = GetTestExpress();
switch (TestExpress) {
case H5_TEST_EXPRESS_EXHAUSTIVE:
num_chunks = 1024;
break;
case H5_TEST_EXPRESS_FULL:
num_chunks = 512;
break;
case H5_TEST_EXPRESS_QUICK:
num_chunks = 128;
break;
case H5_TEST_EXPRESS_SMOKE_TEST:
default:
num_chunks = 64;
break;
}
MESSAGE(VERBO_NONE, ("Express test mode set to %d. Testing with %zu chunks\n", TestExpress, num_chunks));
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {
fprintf(stderr, "Failed to create FAPL\n");
goto error;
}
/* Disable dataset chunk caching */
if (H5Pset_cache(fapl_id, 0, 0, 0, 0.0) < 0) {
fprintf(stderr, "Failed to disable dataset chunk caching\n");
goto error;
}
/* Set sieve buffer size to size of a single chunk */
if (H5Pset_sieve_buf_size(fapl_id, (size_t)CHUNK_DIM_0 * sizeof(DATASET_TYPE)) < 0) {
fprintf(stderr, "Failed to set data sieve buffer size\n");
goto error;
}
if ((file_id = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) {
fprintf(stderr, "Failed to create file '%s'\n", FILE_NAME);
goto error;
}
dims[0] = (hsize_t)CHUNK_DIM_0;
dims[1] = (hsize_t)num_chunks;
if ((space_id = H5Screate_simple(2, dims, NULL)) < 0) {
fprintf(stderr, "Failed to create dataspace for dataset\n");
goto error;
}
if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) {
fprintf(stderr, "Failed to create DCPL\n");
goto error;
}
if (H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER) < 0) {
fprintf(stderr, "Failed to set fill value writing time on DCPL\n");
goto error;
}
chunk_dims[0] = (hsize_t)CHUNK_DIM_0;
chunk_dims[1] = 1;
if (H5Pset_chunk(dcpl_id, 2, chunk_dims) < 0) {
fprintf(stderr, "Failed to set chunking on DCPL\n");
goto error;
}
if ((dset_id = H5Dcreate2(file_id, DATASET_NAME, H5T_NATIVE_INT, space_id, H5P_DEFAULT, dcpl_id,
H5P_DEFAULT)) < 0) {
fprintf(stderr, "Failed to create dataset\n");
goto error;
}
data_size = (size_t)CHUNK_DIM_0 * sizeof(DATASET_TYPE);
if (NULL == (write_buf = malloc(data_size))) {
fprintf(stderr, "Failed to allocate write buffer\n");
goto error;
}
for (size_t i = 0; i < data_size / sizeof(DATASET_TYPE); i++)
((DATASET_TYPE *)write_buf)[i] = (DATASET_TYPE)1;
mem_dims[0] = 1;
mem_dims[1] = (hsize_t)CHUNK_DIM_0;
if ((mem_space_id = H5Screate_simple(2, mem_dims, NULL)) < 0) {
fprintf(stderr, "Failed to create memory dataspace\n");
goto error;
}
for (size_t chunk = 0; chunk < num_chunks; chunk++) {
MESSAGE(VERBO_DEF, ("Writing chunk %zu\n", chunk));
start[0] = 0;
start[1] = chunk;
count[0] = 1;
count[1] = 1;
block[0] = (hsize_t)CHUNK_DIM_0;
block[1] = 1;
if (H5Sselect_hyperslab(space_id, H5S_SELECT_SET, start, NULL, count, block) < 0) {
fprintf(stderr, "Failed to select hyperslab for chunk\n");
goto error;
}
if (H5Dwrite(dset_id, H5T_NATIVE_INT, mem_space_id, space_id, H5P_DEFAULT, write_buf) < 0) {
fprintf(stderr, "Failed to write to dataset\n");
goto error;
}
}
if (H5Sclose(mem_space_id) < 0) {
fprintf(stderr, "Failed to close dataspace\n");
goto error;
}
if (H5Dclose(dset_id) < 0) {
fprintf(stderr, "Failed to close dataset\n");
goto error;
}
free(write_buf);
write_buf = NULL;
if ((dset_id = H5Dopen2(file_id, DATASET_NAME, H5P_DEFAULT)) < 0) {
fprintf(stderr, "Failed to open dataset\n");
goto error;
}
if (NULL == (read_buf = malloc(2 * data_size))) {
fprintf(stderr, "Failed to allocate read buffer\n");
goto error;
}
/* Use same shape selection in memory buffer as file selection to
* avoid iterating element by element when setting up chunk dataspaces
* for selections
*/
mem_dims[0] = (hsize_t)CHUNK_DIM_0;
mem_dims[1] = 2;
if ((mem_space_id = H5Screate_simple(2, mem_dims, NULL)) < 0) {
fprintf(stderr, "Failed to create memory dataspace\n");
goto error;
}
start[0] = 0;
start[1] = 0;
count[0] = 1;
count[1] = 1;
block[0] = (hsize_t)CHUNK_DIM_0;
block[1] = 1;
if (H5Sselect_hyperslab(mem_space_id, H5S_SELECT_SET, start, NULL, count, block) < 0) {
fprintf(stderr, "Failed to select hyperslab in memory dataspace\n");
goto error;
}
for (size_t chunk = 0; chunk < num_chunks; chunk++) {
MESSAGE(VERBO_DEF, ("Reading chunk %zu\n", chunk));
start[0] = 0;
start[1] = chunk;
count[0] = 1;
count[1] = 1;
block[0] = (hsize_t)CHUNK_DIM_0;
block[1] = 1;
if (H5Sselect_hyperslab(space_id, H5S_SELECT_SET, start, NULL, count, block) < 0) {
fprintf(stderr, "Failed to select hyperslab for chunk\n");
goto error;
}
if (H5Dread(dset_id, H5T_NATIVE_INT, mem_space_id, space_id, H5P_DEFAULT, read_buf) < 0) {
fprintf(stderr, "Failed to read from dataset\n");
goto error;
}
}
free(read_buf);
read_buf = NULL;
if (H5Pclose(dcpl_id) < 0)
goto error;
if (H5Sclose(mem_space_id) < 0)
goto error;
if (H5Sclose(space_id) < 0)
goto error;
if (H5Dclose(dset_id) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
if (H5Fdelete(FILE_NAME, fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
return;
error:
IncTestNumErrs();
free(write_buf);
free(read_buf);
H5E_BEGIN_TRY
{
H5Pclose(dcpl_id);
H5Sclose(mem_space_id);
H5Sclose(space_id);
H5Dclose(dset_id);
H5Fclose(file_id);
H5Fdelete(FILE_NAME, fapl_id);
H5Pclose(fapl_id);
}
H5E_END_TRY
}
#undef FILE_NAME
#undef DATASET_NAME
#undef DATASET_TYPE
#undef CHUNK_DIM_0
int
main(int argc, char **argv)
{
/* Initialize testing framework */
if (TestInit(argv[0], NULL, NULL, NULL, NULL, 0) < 0) {
fprintf(stderr, "couldn't initialize testing framework\n");
exit(EXIT_FAILURE);
}
AddTest("chunk_non_contig_mem_io", chunk_non_contig_mem_io, NULL, NULL, NULL, 0,
"I/O on chunks that are non-contiguous with respect to memory layout");
/* Display testing information */
TestInfo(stdout);
/* Parse command line arguments */
if (TestParseCmdLine(argc, argv) < 0) {
fprintf(stderr, "couldn't parse command-line arguments\n");
TestShutdown();
exit(EXIT_FAILURE);
}
/* Perform requested testing */
if (PerformTests() < 0) {
fprintf(stderr, "couldn't run tests\n");
TestShutdown();
exit(EXIT_FAILURE);
}
/* Display test summary, if requested */
if (GetTestSummary())
TestSummary(stdout);
/* Release test infrastructure */
if (TestShutdown() < 0) {
fprintf(stderr, "couldn't shut down testing framework\n");
exit(EXIT_FAILURE);
}
/* Exit failure if errors encountered; else exit success. */
/* No need to print anything since PerformTests() already does. */
if (GetTestNumErrs() > 0)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}
Reference in New Issue View Git Blame Copy Permalink