mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

ROS3 VFD block cache feature (#6478)

Browse Source 
* ROS3 VFD block cache feature

Implements a minimal I/O block cache for files without paged allocation
enabled to better optimize I/O and reduce requests to S3

I/O is cached in fixed-size blocks (16MiB blocks by default) and are kept
in a simple LRU cache that evicts the oldest used block when a new block
needs to be cached

Reading and caching of the initial bytes of a file has been delayed
from file open to the first read for a file instead

API functions have been added for setting/getting the I/O block caching
parameters to be used

* Address comments from review

* Rename 'page' -> 'block'

* Change default block cache size and separate from page buffer logic

* Update CHANGELOG
This commit is contained in:
jhendersonHDF
2026-07-14 11:37:58 -05:00
committed by GitHub
parent 2f4ac20aa7
commit 1325d30b21
4 changed files with 960 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
release_docs/CHANGELOG.md
+6
View File
@@ -32,6 +32,8 @@ For releases prior to version 2.0.0, please see the release.txt file and for mor
## Performance Enhancements:
- Added an I/O block cache to the ROS3 VFD to reduce the number of requests to S3 for files not using paged allocation
- Improved the performance of several tools (h5dump, h5ls, h5diff, h5repack, h5stat and h5format_convert) for specific file structures where many objects are linked to with multiple hard links
## Significant Advancements:
@@ -98,6 +100,10 @@ We would like to thank the many HDF5 community members who contributed to this r
## Library
### Added an I/O block cache to the ROS3 VFD
Added an I/O block cache to the ROS3 VFD to reduce the number of requests to S3 for files that don't use paged allocation. This is a simple LRU cache that performs I/O in fixed-size blocks and serves I/O requests from the in-memory cached buffers. By default, the ROS3 VFD now performs I/O in 16 MiB (see new macro `HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE`) blocks, caching up to a total of 128 MiB (see new macro `HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE`) of data at a time. The new `H5Pset_fapl_ros3_block_caching()` / `H5Pget_fapl_ros3_block_caching()` API functions can be used to modify or retrieve the caching parameters set on a File Access Property List, respectively. Additionally, caching of the initial bytes of a file has been delayed from file open to the first read of a file instead to reduce the overhead of file opens.
### Added optional digital signature verification for dynamically loaded plugins
When built with `-DHDF5_REQUIRE_SIGNED_PLUGINS=ON` and OpenSSL, HDF5 will cryptographically verify each plugin before loading it. Plugins are signed with the new `h5sign` tool, which appends an RSA signature and a compact footer to the plugin binary. Verification uses a keystore directory of trusted public keys, configurable at compile time (`-DHDF5_PLUGIN_KEYSTORE_DIR=<path>`) or at runtime via the `HDF5_PLUGIN_KEYSTORE` environment variable. Individual signatures can be revoked without removing the entire public key by listing their SHA-256 hashes in a `revoked_signatures.txt` file in the keystore directory. Supported algorithms include SHA-256, SHA-384, and SHA-512 with both PKCS#1 v1.5 and PSS padding. See `docs/PLUGIN_SIGNATURE_README.md` for details.
src/H5FDros3.c
+688 -44
View File
@@ -34,9 +34,6 @@
/* Define to turn on stats collection and reporting */
/* #define ROS3_STATS */
/* Max size of the cache, in bytes */
#define ROS3_MAX_CACHE_SIZE 16777216
/* The driver identification number, initialized at runtime */
hid_t H5FD_ROS3_id_g = H5I_INVALID_HID;
@@ -51,9 +48,47 @@ static bool H5FD_ros3_init_s = false;
/* Endpoint URL property name */
#define ROS3_ENDPOINT_PROP_NAME "ros3_endpoint_prop"
/* I/O block caching parameters property name */
#define ROS3_BLOCK_CACHING_PARAMS_PROP_NAME "ros3_block_caching_params"
/* Default page buffer size */
#define ROS3_DEF_PAGE_BUF_SIZE ((size_t)64 * (size_t)1024 * (size_t)1024)
/* Insert entry at head of LRU linked list */
#define ROS3_BLOCK_CACHE_LRU_INSERT(file_ptr, block) \
do { \
if (!(file_ptr)->block_cache.LRU_head) { \
(file_ptr)->block_cache.LRU_head = (block); \
(file_ptr)->block_cache.LRU_tail = (block); \
} \
else { \
(file_ptr)->block_cache.LRU_head->prev = (block); \
(block)->next = (file_ptr)->block_cache.LRU_head; \
(file_ptr)->block_cache.LRU_head = (block); \
} \
} while (0)
#define ROS3_BLOCK_CACHE_LRU_REMOVE(file_ptr, block) \
do { \
if ((file_ptr)->block_cache.LRU_head == (block)) { \
(file_ptr)->block_cache.LRU_head = (block)->next; \
if ((file_ptr)->block_cache.LRU_head) \
(file_ptr)->block_cache.LRU_head->prev = NULL; \
} \
else \
(block)->prev->next = (block)->next; \
\
if ((file_ptr)->block_cache.LRU_tail == (block)) { \
(file_ptr)->block_cache.LRU_tail = (block)->prev; \
if ((file_ptr)->block_cache.LRU_tail) \
(file_ptr)->block_cache.LRU_tail->next = NULL; \
} \
else \
(block)->next->prev = (block)->prev; \
\
(block)->next = (block)->prev = NULL; \
} while (0)
#ifdef ROS3_STATS
/* The ros3 VFD can collect some simple I/O stats on a per-file basis. These
@@ -79,6 +114,34 @@ typedef struct H5FD_ros3_stats_bin {
#endif /* ROS3_STATS */
typedef struct H5FD_ros_block_hash_t {
UT_hash_handle hh; /* Hash table handle */
haddr_t addr; /* Block size-aligned file address for block */
size_t block_size; /* Size of block data buffer (buf) in bytes */
/* Fields for LRU eviction linked list */
struct H5FD_ros_block_hash_t *next;
struct H5FD_ros_block_hash_t *prev;
uint8_t buf[]; /* block data buffer; flexible array member */
} H5FD_ros_block_hash_t;
/* Structure for partitioning an I/O request into smaller requests
* along block size boundaries
*/
typedef struct H5FD_ros3_block_io_req_t {
haddr_t addr;
size_t io_size;
} H5FD_ros3_block_io_req_t;
/* Parameters for I/O block caching */
typedef struct H5FD_ros3_block_caching_params_t {
size_t block_size;
size_t block_cache_size;
bool lock_superblock;
} H5FD_ros3_block_caching_params_t;
/***************************************************************************
* Stores all information needed to maintain access to a single HDF5 file
* that has been stored as a S3 object.
@@ -105,14 +168,27 @@ typedef struct H5FD_ros3_stats_bin {
* Responsible for communicating with remote host and presenting file
* contents as indistinguishable from a file on the local filesystem.
*
* cache
* cache_size (in bytes)
* block_cache
*
* A simple cache of the first N bytes of the file. Especially useful
* at file open, when we perform several reads that would otherwise
* be uncached.
* Holds fields for implementing a simple I/O block cache used for
* optimizing I/O.
*
* *** present only if ROS3_SATS is set to enable stats collection ***
* hash_table - Pointer to the head node of a uthash hash table that
* is used for looking up cached I/O blocks by block size-aligned
* addresses.
*
* block_size - The size of each I/O block that is cached. Fixed upon
* file open.
*
* max_num_blocks - The maximum number of I/O blocks that can be cached
* before blocks need to be evicted to make space for others.
*
* lock_superblock - Boolean indicating whether the block which (usually)
* contains the superblock and nearby metadata should be locked in the
* cache to prevent its eviction. Note that a large userblock size
* could prevent this approach from being effective.
*
* *** present only if ROS3_STATS is set to enable stats collection ***
*
* `meta` (H5FD_ros3_stats_bin_t[])
* `raw` (H5FD_ros3_stats_bin_t[])
@@ -134,8 +210,18 @@ typedef struct H5FD_ros3_t {
haddr_t eoa;
H5FD_ros3_fapl_t fa;
s3r_t *s3r_handle;
uint8_t *cache;
size_t cache_size;
struct {
H5FD_ros_block_hash_t *hash_table;
size_t block_size;
size_t block_cache_size;
size_t max_num_blocks;
bool lock_superblock;
H5FD_ros_block_hash_t *LRU_head;
H5FD_ros_block_hash_t *LRU_tail;
bool disabled;
} block_cache;
#ifdef ROS3_STATS
H5FD_ros3_stats_bin_t meta[ROS3_STATS_BIN_COUNT + 1];
H5FD_ros3_stats_bin_t raw[ROS3_STATS_BIN_COUNT + 1];
@@ -173,6 +259,11 @@ static int H5FD__ros3_str_endpoint_cmp(const void *_value1, const void *_valu
static herr_t H5FD__ros3_str_endpoint_close(const char *name, size_t size, void *_value);
static herr_t H5FD__ros3_str_endpoint_delete(hid_t prop_id, const char *name, size_t size, void *_value);
static herr_t H5FD__ros3_init_block_cache(H5FD_ros3_t *file);
static herr_t H5FD__ros3_determine_io_reqs(H5FD_ros3_t *file, haddr_t addr, size_t io_size,
H5FD_ros3_block_io_req_t **io_reqs_out, size_t *num_io_reqs_out);
static herr_t H5FD__ros3_block_cache_make_space(H5FD_ros3_t *file);
#ifdef ROS3_STATS
static herr_t H5FD__ros3_reset_stats(H5FD_ros3_t *file);
static herr_t H5FD__ros3_log_read_stats(H5FD_ros3_t *file, H5FD_mem_t type, uint64_t size);
@@ -930,6 +1021,144 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_ros3_endpoint() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_block_caching_params_cmp
*
* Purpose: Compares two H5FD_ros3_block_caching_params_t structures
*
* Return: -1/0/1
*-------------------------------------------------------------------------
*/
static int
H5FD__ros3_block_caching_params_cmp(const void *value1, const void *value2, size_t H5_ATTR_UNUSED size)
{
const H5FD_ros3_block_caching_params_t *params1 = (const H5FD_ros3_block_caching_params_t *)value1;
const H5FD_ros3_block_caching_params_t *params2 = (const H5FD_ros3_block_caching_params_t *)value2;
if (params1->block_size != params2->block_size)
return (params1->block_size > params2->block_size) - (params1->block_size < params2->block_size);
if (params1->block_cache_size != params2->block_cache_size)
return (params1->block_cache_size > params2->block_cache_size) -
(params1->block_cache_size < params2->block_cache_size);
if (params1->lock_superblock != params2->lock_superblock)
return (params1->lock_superblock > params2->lock_superblock) -
(params1->lock_superblock < params2->lock_superblock);
return 0;
} /* end H5FD__ros3_block_caching_params_cmp() */
/*-------------------------------------------------------------------------
* Function: H5Pset_fapl_ros3_block_caching
*
* Purpose: Sets I/O block caching parameters for the ros3 VFD.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_fapl_ros3_block_caching(hid_t fapl_id, size_t block_size, size_t block_cache_size,
bool lock_superblock)
{
H5FD_ros3_block_caching_params_t block_caching_params = {0};
H5P_genplist_t *plist = NULL;
htri_t block_caching_params_exist;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
if (fapl_id == H5P_DEFAULT)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list");
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "ROS3 driver is not set on FAPL");
if ((block_caching_params_exist = H5P_exist_plist(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL,
"failed to check if I/O block caching parameters property exists in plist");
block_caching_params.block_size = block_size;
block_caching_params.block_cache_size = block_cache_size;
block_caching_params.lock_superblock = lock_superblock;
/* If block size is larger than block cache size, round block size down
* to block cache size so that block cache size is an upper limit but
* can still hold at least 1 block.
*/
if (block_caching_params.block_size != 0 && block_caching_params.block_cache_size != 0) {
if (block_caching_params.block_size > block_caching_params.block_cache_size)
block_caching_params.block_size = block_caching_params.block_cache_size;
}
if (block_caching_params_exist) {
if (H5P_set(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME, &block_caching_params) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set I/O block caching parameters");
}
else {
if (H5P_insert(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME, sizeof(H5FD_ros3_block_caching_params_t),
&block_caching_params, NULL, NULL, NULL, NULL, NULL, NULL,
H5FD__ros3_block_caching_params_cmp, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL,
"unable to register I/O block caching parameters property in plist");
}
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_fapl_ros3_block_caching() */
/*-------------------------------------------------------------------------
* Function: H5Pget_fapl_ros3_block_caching
*
* Purpose: Retrieves any I/O block caching parameters set for the ros3
* VFD.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_fapl_ros3_block_caching(hid_t fapl_id, size_t *block_size, size_t *block_cache_size,
bool *lock_superblock)
{
H5P_genplist_t *plist = NULL;
htri_t block_caching_params_exist;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list");
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "ROS3 driver is not set on FAPL");
if ((block_caching_params_exist = H5P_exist_plist(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL,
"failed to check if I/O block caching parameters property exists in plist");
if (block_caching_params_exist) {
H5FD_ros3_block_caching_params_t block_caching_params = {0};
if (H5P_get(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME, &block_caching_params) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get I/O block caching parameters");
if (block_size)
*block_size = block_caching_params.block_size;
if (block_cache_size)
*block_cache_size = block_caching_params.block_cache_size;
if (lock_superblock)
*lock_superblock = block_caching_params.lock_superblock;
}
else {
if (block_size)
*block_size = HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE;
if (block_cache_size)
*block_cache_size = ROS3_DEF_PAGE_BUF_SIZE;
if (lock_superblock)
*lock_superblock = true;
}
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_ros3_block_caching() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_open
*
@@ -952,14 +1181,15 @@ done:
static H5FD_t *
H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
H5FD_ros3_t *file = NULL;
s3r_t *handle = NULL;
const H5FD_ros3_fapl_t *fa = NULL;
H5P_genplist_t *plist = NULL;
char *fapl_token = NULL;
char *fapl_endpoint = NULL;
H5FD_t *ret_value = NULL;
htri_t endpt_exists = false;
H5FD_ros3_t *file = NULL;
s3r_t *handle = NULL;
const H5FD_ros3_fapl_t *fa = NULL;
H5P_genplist_t *plist = NULL;
char *fapl_token = NULL;
char *fapl_endpoint = NULL;
H5FD_t *ret_value = NULL;
htri_t endpt_exists = false;
htri_t block_caching_params_exist = false;
FUNC_ENTER_PACKAGE
@@ -1022,23 +1252,43 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
file->s3r_handle = handle;
H5MM_memcpy(&(file->fa), fa, sizeof(H5FD_ros3_fapl_t));
if ((block_caching_params_exist = H5P_exist_plist(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL,
"failed to check if I/O block caching parameters property exists in plist");
if (block_caching_params_exist) {
H5FD_ros3_block_caching_params_t block_caching_params = {0};
if (H5P_get(plist, ROS3_BLOCK_CACHING_PARAMS_PROP_NAME, &block_caching_params) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to get I/O block caching parameters");
file->block_cache.block_size = block_caching_params.block_size;
file->block_cache.block_cache_size = block_caching_params.block_cache_size;
file->block_cache.lock_superblock = block_caching_params.lock_superblock;
}
else {
file->block_cache.block_size = HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE;
file->block_cache.block_cache_size = HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE;
file->block_cache.lock_superblock = true;
}
if (file->block_cache.block_size == 0 || file->block_cache.block_cache_size == 0)
file->block_cache.disabled = true;
/* Determine the maximum number of blocks to keep around in the block cache */
if (!file->block_cache.disabled) {
/* final sanity check; the block size should never exceed the block cache size */
if (file->block_cache.block_size > file->block_cache.block_cache_size)
file->block_cache.block_size = file->block_cache.block_cache_size;
file->block_cache.max_num_blocks = file->block_cache.block_cache_size / file->block_cache.block_size;
assert(file->block_cache.max_num_blocks >= 1);
}
#ifdef ROS3_STATS
if (H5FD__ros3_reset_stats(file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics");
#endif
/* Cache the initial bytes of the file */
{
size_t filesize = H5FD__s3comms_s3r_get_filesize(file->s3r_handle);
file->cache_size = (filesize < ROS3_MAX_CACHE_SIZE) ? filesize : ROS3_MAX_CACHE_SIZE;
if (NULL == (file->cache = (uint8_t *)H5MM_calloc(file->cache_size)))
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate cache memory");
if (H5FD__s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache, file->cache_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, NULL, "unable to execute read");
}
ret_value = (H5FD_t *)file;
done:
@@ -1047,8 +1297,7 @@ done:
if (H5FD__s3comms_s3r_close(handle) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle");
if (file != NULL) {
file->cache = H5MM_xfree(file->cache);
file = H5FL_FREE(H5FD_ros3_t, file);
file = H5FL_FREE(H5FD_ros3_t, file);
}
}
@@ -1084,8 +1333,17 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle");
/* Release the file info */
file->cache = H5MM_xfree(file->cache);
file = H5FL_FREE(H5FD_ros3_t, file);
if (file->block_cache.hash_table) {
H5FD_ros_block_hash_t *p, *tmp;
HASH_ITER(hh, file->block_cache.hash_table, p, tmp)
{
HASH_DEL(file->block_cache.hash_table, p);
H5MM_free(p);
}
}
file = H5FL_FREE(H5FD_ros3_t, file);
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -1333,14 +1591,18 @@ static herr_t
H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr,
size_t size, void *buf)
{
H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
size_t filesize = 0;
herr_t ret_value = SUCCEED;
H5FD_ros3_block_io_req_t *io_reqs = NULL;
H5FD_ros_block_hash_t *new_block = NULL;
H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
uint8_t *buf_ptr = (uint8_t *)buf;
size_t filesize = 0;
size_t num_io_blocks = 0;
bool is_cached = false;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
assert(file);
assert(file->cache);
assert(file->s3r_handle);
assert(buf);
@@ -1349,13 +1611,68 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
if ((addr > filesize) || ((addr + size) > filesize))
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address");
/* Copy from the cache when accessing the first N bytes of the file.
* Saves network I/O operations when opening files.
/* If this is the first read for the file and block caching is not disabled,
* read an initial "block size" worth of bytes and add the block to the block
* cache. This initial block is mostly used to optimize locating the file's
* superblock, as well as general metadata reads following that process.
* Note that while reading the superblock of a file, it can't be determined
* at that time whether paged aggregation will be enabled or not, so the
* first "block size" bytes (rather than just an estimated amount of superblock
* bytes) are always cached for possible later use. Also note that it can't
* be determined at that time whether a userblock exists or how large it is,
* so a userblock size >= "block size" will effectively make this cache
* useless.
*/
if (addr + size <= file->cache_size) {
memcpy(buf, file->cache + addr, size);
if (!file->block_cache.disabled && !file->block_cache.hash_table) {
if (H5FD__ros3_init_block_cache(file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize I/O block cache");
assert(file->block_cache.hash_table);
}
else {
/* If the I/O request falls within the already-cached superblock block and
* either paged aggregation is enabled or the superblock block is locked
* in the cache, just serve the request from the cache. When paged aggregation
* is enabled, this can save a possible request to S3. When paged aggregation
* is not enabled, this saves a tiny bit of overhead from below since it's
* known that there will be only one I/O request block.
*
* Since no modifications are made to the block cache when paged aggregation
* is enabled other than caching the superblock block, the superblock block
* should always be available for use in this case. When paged aggregation
* isn't enabled, this optimization is only used when the superblock block
* is locked in the cache and thus should be found by a hash table lookup.
*/
/* clang-format off */
is_cached =
!file->block_cache.disabled && /* Block caching is enabled */
(addr + size <= file->block_cache.block_size) && /* Addr + size falls within cached block */
(file->pub.paged_aggr || file->block_cache.lock_superblock); /* Paged aggr. or locked cached block */
/* clang-format on */
if (is_cached) {
H5FD_ros_block_hash_t *super_block = NULL;
haddr_t super_block_addr = 0;
HASH_FIND(hh, file->block_cache.hash_table, &super_block_addr, sizeof(haddr_t), super_block);
if (!super_block)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to locate superblock block in block cache");
/* No need to promote block in LRU - if paged aggregation is enabled,
* block caching (and thus the LRU eviction list) is not enabled.
* Otherwise, the superblock block is locked in the cache and not part
* of the LRU eviction policy.
*/
memcpy(buf, super_block->buf + addr, size);
HGOTO_DONE(SUCCEED);
}
/* If block caching is disabled or if paged aggregation is enabled, just
* issue reads to S3 directly. When paged aggregation is enabled, it's
* assumed that higher layers are already performing caching and so
* additional caching is not needed here.
*/
if (file->block_cache.disabled || file->pub.paged_aggr) {
/*
* Note that the VFD interface doesn't specify the size of buf.
* Assume that the caller knows what they're doing.
@@ -1367,12 +1684,339 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
if (H5FD__ros3_log_read_stats(file, type, (uint64_t)size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "unable to log read stats");
#endif
HGOTO_DONE(SUCCEED);
}
/* If paged aggregation is not enabled, serve this I/O request from the block
* cache as able and otherwise issue reads to S3.
*/
/* Split I/O request among block boundaries as necessary */
if (H5FD__ros3_determine_io_reqs(file, addr, size, &io_reqs, &num_io_blocks) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to partition read request into I/O blocks");
for (size_t i = 0; i < num_io_blocks; i++) {
H5FD_ros_block_hash_t *io_block = NULL;
haddr_t block_addr = HADDR_UNDEF;
bool can_cache = true;
/* If block caching is enabled, check if the block is in the cache */
if (!file->block_cache.disabled) {
/* The first I/O request may not be "block size"-aligned; all others will be. */
if (i == 0)
block_addr = (io_reqs[i].addr / file->block_cache.block_size) * file->block_cache.block_size;
else
block_addr = io_reqs[i].addr;
HASH_FIND(hh, file->block_cache.hash_table, &block_addr, sizeof(haddr_t), io_block);
if (io_block) {
/* Serve read from the block cache if the block was found */
memcpy(buf_ptr, io_block->buf + (io_reqs[i].addr - io_block->addr), io_reqs[i].io_size);
buf_ptr += io_reqs[i].io_size;
/* If the block that was found is not the superblock block OR if it is
* the superblock block and the block isn't locked in the cache (can be
* evicted), promote the block to the head of the LRU eviction list
* (making it a less likely candidate for eviction) if it's not already
* at the head of the list.
*/
if (io_block->addr != 0 || !file->block_cache.lock_superblock) {
/* At this point, the block should already be in the LRU eviction list. */
assert(file->block_cache.LRU_head);
assert((file->block_cache.LRU_head == io_block) || io_block->prev);
if (file->block_cache.LRU_head != io_block) {
ROS3_BLOCK_CACHE_LRU_REMOVE(file, io_block);
ROS3_BLOCK_CACHE_LRU_INSERT(file, io_block);
}
}
continue;
}
}
/* If the I/O request wasn't served from the block cache, check if the
* request can be cached before issuing it. A lack of space in the
* cache isn't considered here, as old blocks will be evicted as necessary.
*/
if (file->block_cache.disabled)
can_cache = false;
else {
/* If the cache can only hold one block and the superblock block is
* locked in the cache, we cannot cache any more blocks.
*/
if (file->block_cache.lock_superblock && file->block_cache.max_num_blocks == 1) {
assert(HASH_COUNT(file->block_cache.hash_table) == 1); /* superblock block should be cached */
can_cache = false;
}
}
/* If the I/O request can be cached, allocate a new block for it and insert
* it into the block cache after the request is finished.
*/
if (can_cache) {
size_t alloc_size = sizeof(*new_block) + file->block_cache.block_size;
size_t read_size = 0;
/* Allocate a new block and read the entire block's bytes */
if (NULL == (new_block = H5MM_malloc(alloc_size)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate I/O block");
new_block->addr = block_addr;
new_block->block_size = file->block_cache.block_size;
new_block->next = NULL;
new_block->prev = NULL;
memset(&new_block->hh, 0, sizeof(UT_hash_handle));
read_size = MIN(new_block->block_size, filesize - new_block->addr);
if (H5FD__s3comms_s3r_read(file->s3r_handle, new_block->addr, read_size, new_block->buf,
new_block->block_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
#ifdef ROS3_STATS
if (H5FD__ros3_log_read_stats(file, type, (uint64_t)read_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "unable to log read stats");
#endif
memcpy(buf_ptr, new_block->buf + (io_reqs[i].addr - new_block->addr), io_reqs[i].io_size);
/* If block cache is full, evict oldest block before inserting a new block */
if (H5FD__ros3_block_cache_make_space(file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to make space in block cache");
/* Add block to block cache */
HASH_ADD(hh, file->block_cache.hash_table, addr, sizeof(haddr_t), new_block);
if (!new_block->hh.tbl)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to add I/O block to hash table");
/* Add block to head of LRU eviction list */
ROS3_BLOCK_CACHE_LRU_INSERT(file, new_block);
new_block = NULL; /* Now owned by hash table */
}
else {
/* Unable to cache this I/O request in the block cache. Just read
* from S3 directly. Note that the VFD interface doesn't specify
* the size of buf. Assume that the caller knows what they're doing.
*/
if (H5FD__s3comms_s3r_read(file->s3r_handle, io_reqs[i].addr, io_reqs[i].io_size, buf_ptr,
io_reqs[i].io_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
#ifdef ROS3_STATS
if (H5FD__ros3_log_read_stats(file, type, (uint64_t)io_reqs[i].io_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "unable to log read stats");
#endif
}
buf_ptr += io_reqs[i].io_size;
}
done:
H5MM_free(new_block);
H5MM_free(io_reqs);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__ros3_read() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_init_block_cache
*
* Purpose: Initializes an I/O block cache for optimizing I/O. Reads
* the first "block size" bytes of the file (or the entire
* file if it's smaller than "block size") and stores the
* block in a hash table for quick lookups by
* "block size"-aligned addresses.
*
* If the 'lock_superblock' field in the file struct is false,
* this initial block will be added to an LRU list of blocks
* which can be evicted when attempting to add another block
* to the block cache. Otherwise, this block will be omitted
* from that list, preventing it from being evicted until file
* close.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_init_block_cache(H5FD_ros3_t *file)
{
H5FD_ros_block_hash_t *super_block = NULL;
size_t file_size = 0;
size_t alloc_size = 0;
size_t read_size = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
assert(file);
assert(!file->block_cache.disabled);
assert(file->block_cache.block_size > 0);
/* Already initialized */
if (file->block_cache.hash_table)
HGOTO_DONE(SUCCEED);
alloc_size = sizeof(*super_block) + file->block_cache.block_size;
if (NULL == (super_block = H5MM_calloc(alloc_size)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate I/O block");
super_block->addr = 0;
super_block->block_size = file->block_cache.block_size;
/* Read either the entire file or "block size" bytes, whichever is smaller.
* If the file size is smaller than the "block size", the remaining bytes
* in the block will be zeroes.
*/
file_size = H5FD__s3comms_s3r_get_filesize(file->s3r_handle);
read_size = (file_size < file->block_cache.block_size) ? file_size : file->block_cache.block_size;
if (H5FD__s3comms_s3r_read(file->s3r_handle, 0, read_size, super_block->buf,
file->block_cache.block_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
/* Add block to block cache */
HASH_ADD(hh, file->block_cache.hash_table, addr, sizeof(haddr_t), super_block);
if (!file->block_cache.hash_table)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate I/O block hash table");
if (!super_block->hh.tbl)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to add I/O block to hash table");
/* Add the superblock block to the list of blocks that can be
* evicted if it isn't setup to be locked in the cache.
*/
if (!file->block_cache.lock_superblock)
ROS3_BLOCK_CACHE_LRU_INSERT(file, super_block);
super_block = NULL; /* Now owned by hash table */
done:
if (ret_value < 0) {
H5MM_free(super_block);
}
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__ros3_init_block_cache() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_determine_io_reqs
*
* Purpose: Given an offset and length for an I/O request, partitions
* the I/O request among "block size"-aligned boundaries and
* returns a new array of smaller I/O requests.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_determine_io_reqs(H5FD_ros3_t *file, haddr_t addr, size_t io_size,
H5FD_ros3_block_io_req_t **io_reqs_out, size_t *num_io_reqs_out)
{
H5FD_ros3_block_io_req_t *io_reqs = NULL;
haddr_t cur_addr = HADDR_UNDEF;
haddr_t first_block_addr = HADDR_UNDEF;
haddr_t last_block_addr = HADDR_UNDEF;
size_t block_size = 0;
size_t num_blocks = 0;
size_t num_blocks_left = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
assert(file);
assert(!file->block_cache.disabled);
assert(file->block_cache.block_size > 0);
assert(io_reqs_out);
assert(num_io_reqs_out);
if (io_size == 0) {
*io_reqs_out = NULL;
*num_io_reqs_out = 0;
HGOTO_DONE(SUCCEED);
}
block_size = file->block_cache.block_size;
first_block_addr = ((addr / block_size) * block_size);
last_block_addr = ((addr + io_size - 1) / block_size) * block_size;
num_blocks = (last_block_addr / block_size + 1) - (first_block_addr / block_size);
assert(num_blocks > 0);
if (NULL == (io_reqs = H5MM_malloc(num_blocks * sizeof(*io_reqs))))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate array of I/O requests");
/* Setup I/O request to first block */
io_reqs[0].addr = addr;
io_reqs[0].io_size = MIN(io_size, block_size - (size_t)(addr - first_block_addr));
assert(io_reqs[0].io_size <= io_size);
io_size -= io_reqs[0].io_size;
num_blocks_left = num_blocks - 1;
cur_addr = addr + io_reqs[0].io_size;
/* Setup I/O requests for any blocks between first and last */
if (num_blocks_left > 1) {
for (size_t i = 1; i < num_blocks - 1; i++) {
io_reqs[i].addr = cur_addr;
io_reqs[i].io_size = block_size;
cur_addr += block_size;
assert(io_reqs[i].io_size <= io_size);
io_size -= io_reqs[i].io_size;
num_blocks_left--;
}
}
/* Setup I/O request for last block, if applicable */
if (num_blocks_left) {
assert(num_blocks_left == 1);
io_reqs[num_blocks - 1].addr = cur_addr;
io_reqs[num_blocks - 1].io_size = io_size;
io_size -= io_reqs[num_blocks - 1].io_size;
}
assert(io_size == 0);
*io_reqs_out = io_reqs;
*num_io_reqs_out = num_blocks;
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__ros3_determine_io_reqs() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_block_cache_make_space
*
* Purpose: If necessary, evicts the oldest block in the block cache
* (by LRU policy) to make space for a new block.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_block_cache_make_space(H5FD_ros3_t *file)
{
H5FD_ros_block_hash_t *oldest_block = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
assert(file);
/* If there's space in the cache, no need to evict a block */
if (HASH_COUNT(file->block_cache.hash_table) < file->block_cache.max_num_blocks)
HGOTO_DONE(SUCCEED);
oldest_block = file->block_cache.LRU_tail;
ROS3_BLOCK_CACHE_LRU_REMOVE(file, oldest_block);
HASH_DEL(file->block_cache.hash_table, oldest_block);
H5MM_free(oldest_block);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__ros3_block_cache_make_space() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_write
*
src/H5FDros3.h
+132
View File
@@ -147,6 +147,30 @@
* \since 2.0.0
*/
#define HDF5_ROS3_VFD_FORCE_PATH_STYLE "HDF5_ROS3_VFD_FORCE_PATH_STYLE"
/**
* \def HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE
* The default size, in bytes, of a cached I/O block. By default, the
* #H5FD_ROS3 driver tries to reduce requests to S3 by performing I/O in
* fixed-size blocks and caching these blocks in an I/O block cache. This
* value may be specified for the \p block_size parameter to
* H5Pset_fapl_ros3_block_caching() in order to set the default I/O block
* size.
*
* \since 2.2.0
*/
#define HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE 16777216
/**
* \def HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE
* The default size, in bytes, of the #H5FD_ROS3 driver's I/O block cache.
* By default, the #H5FD_ROS3 driver tries to reduce requests to S3 by
* performing I/O in fixed-size blocks and caching these blocks in an I/O
* block cache. This value may be specified for the \p block_cache_size
* parameter to H5Pset_fapl_ros3_block_caching() in order to set the default
* I/O block cache size.
*
* \since 2.2.0
*/
#define HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE 134217728
/**
* \struct H5FD_ros3_fapl_t
@@ -341,6 +365,114 @@ H5_DLL herr_t H5Pget_fapl_ros3_endpoint(hid_t fapl_id, size_t size, char *endpoi
*/
H5_DLL herr_t H5Pset_fapl_ros3_endpoint(hid_t fapl_id, const char *endpoint);
/**
* \ingroup FAPL
*
* \brief Queries a File Access Property List for #H5FD_ROS3 I/O block caching parameters.
*
* \fapl_id
* \param[out] block_size Pointer for returning the currently set size, in bytes, of a cached
* I/O block. May be \c NULL, in which case no value is returned.
* \param[out] block_cache_size Pointer for returning the currently set maximum size, in bytes,
* of the #H5FD_ROS3 I/O block cache. This is an upper limit of
* the amount of bytes which will be allocated for caching I/O
* blocks, excluding a small amount of additional metadata bytes
* allocated for each block. May be \c NULL, in which case no value
* is returned.
* \param[out] lock_superblock Pointer for returning the currently set boolean value for whether
* the I/O block containing a file's superblock metadata should be
* locked in the I/O block cache. This will prevent that block from
* being evicted from the block cache when trying to make space for
* other I/O blocks. May be \c NULL, in which case no value is
* returned.
* \returns \herr_t
*
* \since 2.2.0
*/
H5_DLL herr_t H5Pget_fapl_ros3_block_caching(hid_t fapl_id, size_t *block_size, size_t *block_cache_size,
bool *lock_superblock);
/**
* \ingroup FAPL
*
* \brief Modifies the specified File Access Property List to set I/O block caching
* parameters for the #H5FD_ROS3 driver.
*
* \fapl_id
* \param[in] block_size Specifies the size, in bytes, of a cached I/O block. The default
* value is #HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE.
* \param[in] block_cache_size Specifies the total size, in bytes, of the I/O block cache.
* The default value is #HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE.
* \param[in] lock_superblock Specifies whether or not to keep the I/O block containing a
* file's superblock metadata locked in the I/O block cache and
* unable to be evicted. The default value is \c true.
*
* \details H5Pset_fapl_ros3_block_caching() sets parameters that control how the
* #H5FD_ROS3 driver caches I/O to reduce requests to S3. By default, the
* #H5FD_ROS3 driver performs I/O in large, fixed-size
* (#HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE bytes, by default) blocks which are cached
* in memory and used to serve I/O requests. This function can be used to modify
* the default parameters to better suit a particular I/O pattern.
*
* For \p block_size, the macro #HDF5_ROS3_VFD_DEFAULT_BLOCK_SIZE may be used
* to specify that the default block size is desired. For \p block_cache_size,
* the macro #HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE may be used to specify that
* the default block cache size is desired. Setting either \p block_size or
* \p block_cache_size to 0 will disable I/O block caching in the #H5FD_ROS3
* driver. This can be useful for reading small amounts of data from a file when
* the default \p block_size is much larger than the amount of data being read.
*
* \p block_cache_size is an upper limit of the amount of bytes which will be
* allocated for caching I/O blocks, excluding a small amount of additional
* metadata bytes allocated for each block. The block cache will only keep whole
* blocks, so \p block_cache_size should be set to some multiple of \p block_size.
* If \p block_cache_size is specified as a value smaller than \p block_size,
* \p block_size will be adjusted down to be equal to \p block_cache_size and
* the block cache will only hold a single cached I/O block.
*
* \p lock_superblock determines whether the I/O block containing a file's
* superblock metadata should be locked in the block cache and prevented from
* being evicted when trying to make space for other I/O blocks. Depending on
* the layout of a file, this can be useful for keeping specific often-used
* metadata in the block cache.
*
* \parblock
* \note The #H5FD_ROS3 driver only performs I/O block caching when a file does
* <strong>NOT</strong> use paged file space allocation. If a file uses paged file
* space allocation, the library's existing page buffering mechanism will be used
* instead.
* \endparblock
*
* \parblock
* \remark The value chosen for \p block_size involves a tradeoff between the number of
* S3 requests the #H5FD_ROS3 driver might issue and how many bytes may be
* transferred as a result of those requests. Larger \p block_size values may
* result in less S3 requests being made, while potentially increasing the total
* number of bytes transferred if more data than necessary is cached. If
* \p block_cache_size is too small, I/O blocks may be evicted from the block
* cache and re-cached later, possibly resulting in the transferring of
* significantly more bytes than the size of the file.
*
* \remark The value chosen for \p block_cache_size involves a tradeoff between performance
* and memory usage, with larger \p block_cache_size values causing more memory to
* be used to cache more I/O blocks.
* \endparblock
*
* \parblock
* \warning When \p lock_superblock is specified as \c true, the first I/O block cached when
* attempting to locate a file's superblock is assumed to contain the superblock
* metadata. If a file contains a user block that is as large as, or larger than,
* \p block_size, this will cause the I/O block containing the file's superblock
* metadata to <strong>NOT</strong> be locked in the I/O block cache. For this case,
* \p block_size should be adjusted accordingly to be at least as large as the user
* block plus some bytes for the superblock metadata.
* \endparblock
*
* \since 2.2.0
*/
H5_DLL herr_t H5Pset_fapl_ros3_block_caching(hid_t fapl_id, size_t block_size, size_t block_cache_size,
bool lock_superblock);
#ifdef __cplusplus
}
#endif
test/ros3.c
+134
View File
@@ -1144,6 +1144,139 @@ error:
return 1;
} /* end test_hive_style_object_key() */
/*---------------------------------------------------------------------------
* Function: test_ros3_block_caching_apis
*
* Purpose: Tests the API functions for setting I/O block caching
* parameters for the ros3 VFD.
*
* Return: PASS : 0
* FAIL : 1
*---------------------------------------------------------------------------
*/
static int
test_ros3_block_caching_apis(void)
{
size_t block_size;
size_t block_cache_size;
hid_t fid = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
bool lock_superblock;
TESTING("ros3 I/O block caching parameter APIs");
if (s3_test_credentials_loaded == 0) {
SKIPPED();
puts(" s3 credentials are not loaded");
fflush(stdout);
return 0;
}
if (false == s3_test_bucket_defined) {
SKIPPED();
puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
fflush(stdout);
return 0;
}
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &anonymous_fa) < 0)
TEST_ERROR;
if (*s3_test_aws_session_token != '\0')
if (H5Pset_fapl_ros3_token(fapl_id, s3_test_aws_session_token) < 0)
TEST_ERROR;
/* Set block size to 0 - should disable block caching */
if (H5Pset_fapl_ros3_block_caching(fapl_id, 0, HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE, true) < 0)
TEST_ERROR;
if (H5Pget_fapl_ros3_block_caching(fapl_id, &block_size, &block_cache_size, &lock_superblock) < 0)
TEST_ERROR;
if (block_size != 0)
TEST_ERROR;
if (block_cache_size != HDF5_ROS3_VFD_DEFAULT_BLOCK_CACHE_SIZE)
TEST_ERROR;
if (!lock_superblock)
TEST_ERROR;
/* Check that parameters are accepted - no validation performed since H5FD_ros3_t fields are internal */
if ((fid = H5Fopen(url_h5_public, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if (H5Fclose(fid) < 0)
TEST_ERROR;
/* Set block cache size to 0 - should disable block caching */
if (H5Pset_fapl_ros3_block_caching(fapl_id, 1048576, 0, true) < 0)
TEST_ERROR;
if (H5Pget_fapl_ros3_block_caching(fapl_id, &block_size, &block_cache_size, &lock_superblock) < 0)
TEST_ERROR;
if (block_size != 1048576)
TEST_ERROR;
if (block_cache_size != 0)
TEST_ERROR;
if (!lock_superblock)
TEST_ERROR;
/* Check that parameters are accepted - no validation performed since H5FD_ros3_t fields are internal */
if ((fid = H5Fopen(url_h5_public, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if (H5Fclose(fid) < 0)
TEST_ERROR;
/* Set block size to slightly larger than block cache size - should round block size down */
if (H5Pset_fapl_ros3_block_caching(fapl_id, 1048580, 1048576, true) < 0)
TEST_ERROR;
if (H5Pget_fapl_ros3_block_caching(fapl_id, &block_size, &block_cache_size, &lock_superblock) < 0)
TEST_ERROR;
if (block_size != 1048576)
TEST_ERROR;
if (block_cache_size != 1048576)
TEST_ERROR;
if (!lock_superblock)
TEST_ERROR;
/* Check that parameters are accepted - no validation performed since H5FD_ros3_t fields are internal */
if ((fid = H5Fopen(url_h5_public, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if (H5Fclose(fid) < 0)
TEST_ERROR;
/* Disable locking of the superblock block into the block cache */
if (H5Pset_fapl_ros3_block_caching(fapl_id, 1048576, 4194304, false) < 0)
TEST_ERROR;
if (H5Pget_fapl_ros3_block_caching(fapl_id, &block_size, &block_cache_size, &lock_superblock) < 0)
TEST_ERROR;
if (block_size != 1048576)
TEST_ERROR;
if (block_cache_size != 4194304)
TEST_ERROR;
if (lock_superblock)
TEST_ERROR;
/* Check that parameters are accepted - no validation performed since H5FD_ros3_t fields are internal */
if ((fid = H5Fopen(url_h5_public, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if (H5Fclose(fid) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
PASSED();
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(fapl_id);
H5Fclose(fid);
}
H5E_END_TRY
return 1;
}
#endif /* H5_HAVE_ROS3_VFD */
/*-------------------------------------------------------------------------
@@ -1315,6 +1448,7 @@ main(void)
nerrors += test_cmp();
nerrors += test_ros3_access_modes();
nerrors += test_hive_style_object_key();
nerrors += test_ros3_block_caching_apis();
}
if (H5FD__s3comms_term() < 0) {