Merge pull request #6046 from vchoi-hdfgroup/feature/sparse_data

Feature/sparse data
This commit is contained in:
vchoi-hdfgroup
2025-11-20 14:11:34 -06:00
committed by GitHub
8 changed files with 97 additions and 99 deletions
+11 -22
View File
@@ -1834,10 +1834,10 @@ H5D__bt2_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* allowing for an extra byte, in case the structured chunk
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)layout->size) + storage->offset_size) / storage->offset_size);
if (chunk_size_len > storage->offset_size)
chunk_size_len = storage->offset_size;
/* General parameters */
if (idx_info->stc_pline->tot_filt_nsects > 0) {
@@ -1853,8 +1853,8 @@ H5D__bt2_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
*/
bt2_cparam.rrec_size =
(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len + (layout->ndims - 1) * 8 +
(uint8_t)((storage->nsects - 1) * H5O_STRUCT_CHUNK_OFFSET_SIZE) +
(uint8_t)(storage->nsects * H5O_STRUCT_CHUNK_OFFSET_SIZE) + (uint8_t)(4 * storage->nsects));
(uint8_t)((storage->nsects - 1) * storage->offset_size) +
(uint8_t)(storage->nsects * storage->offset_size) + (uint8_t)(4 * storage->nsects));
bt2_cparam.cls = H5D_BT2_FILT_STRUCT_CHUNK;
}
@@ -1867,7 +1867,7 @@ H5D__bt2_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* size of offsets for (n - 1) sections
*/
bt2_cparam.rrec_size = (H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len + (layout->ndims - 1) * 8 +
(uint8_t)((storage->nsects - 1) * H5O_STRUCT_CHUNK_OFFSET_SIZE));
(uint8_t)((storage->nsects - 1) * storage->offset_size));
bt2_cparam.cls = H5D_BT2_STRUCT_CHUNK;
}
@@ -1945,10 +1945,10 @@ H5D__bt2_stc_idx_open(const H5D_chk_idx_info_t *idx_info)
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + idx_info->stc_storage->offset_size) /
idx_info->stc_storage->offset_size);
if (chunk_size_len > idx_info->stc_storage->offset_size)
chunk_size_len = idx_info->stc_storage->offset_size;
/* Set up the user data */
u_ctx.f = idx_info->f;
@@ -2866,17 +2866,6 @@ H5D__bt2_stc_crt_context(void *_udata)
H5MM_memcpy(my_dim, udata->dim, H5O_LAYOUT_NDIMS * sizeof(uint32_t));
ctx->dim = my_dim;
#ifdef out
/*
* Compute the size required for encoding the size of a chunk,
* allowing for an extra byte, in case the filter makes the chunk larger.
*/
ctx->chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)udata->chunk_size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (ctx->chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
ctx->chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
#endif
/* Set return value */
ret_value = ctx;
+13 -13
View File
@@ -2143,10 +2143,10 @@ H5D__earray_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* allowing for an extra byte, in case the structured chunk
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)layout->size) + storage->offset_size) / storage->offset_size);
if (chunk_size_len > storage->offset_size)
chunk_size_len = storage->offset_size;
/* General parameters */
if (idx_info->stc_pline->tot_filt_nsects > 0) {
@@ -2159,10 +2159,10 @@ H5D__earray_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* size of unfiltered size for n sections +
* size of filtered mask for n sections
*/
cparam.raw_elmt_size = (uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len +
(uint8_t)((storage->nsects - 1) * H5O_STRUCT_CHUNK_OFFSET_SIZE) +
(uint8_t)(storage->nsects * H5O_STRUCT_CHUNK_OFFSET_SIZE) +
(uint8_t)(4 * storage->nsects));
cparam.raw_elmt_size =
(uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len +
(uint8_t)((storage->nsects - 1) * storage->offset_size) +
(uint8_t)(storage->nsects * storage->offset_size) + (uint8_t)(4 * storage->nsects));
}
else {
cparam.cls = H5EA_CLS_STRUCT_CHUNK;
@@ -2172,7 +2172,7 @@ H5D__earray_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* size of offsets for (n - 1) sections
*/
cparam.raw_elmt_size = (uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len +
(uint8_t)((storage->nsects - 1) * H5O_STRUCT_CHUNK_OFFSET_SIZE));
(uint8_t)((storage->nsects - 1) * storage->offset_size));
}
cparam.max_nelmts_bits = layout->u.earray.cparam.max_nelmts_bits;
@@ -2249,10 +2249,10 @@ H5D__earray_stc_idx_open(const H5D_chk_idx_info_t *idx_info)
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + idx_info->stc_storage->offset_size) /
idx_info->stc_storage->offset_size);
if (chunk_size_len > idx_info->stc_storage->offset_size)
chunk_size_len = idx_info->stc_storage->offset_size;
/* Set up the user data */
udata.f = idx_info->f;
+13 -13
View File
@@ -1958,10 +1958,10 @@ H5D__farray_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* allowing for an extra byte, in case the structured chunk
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)layout->size) + storage->offset_size) / storage->offset_size);
if (chunk_size_len > storage->offset_size)
chunk_size_len = storage->offset_size;
/* General parameters */
if (idx_info->stc_pline->tot_filt_nsects > 0) {
@@ -1974,10 +1974,10 @@ H5D__farray_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* size of unfiltered size for n sections +
* size of filtered mask for n sections
*/
cparam.raw_elmt_size = (uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len +
(uint8_t)((storage->nsects - 1) * H5O_STRUCT_CHUNK_OFFSET_SIZE) +
(uint8_t)(storage->nsects * H5O_STRUCT_CHUNK_OFFSET_SIZE) +
(uint8_t)(4 * storage->nsects));
cparam.raw_elmt_size =
(uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len +
(uint8_t)((storage->nsects - 1) * storage->offset_size) +
(uint8_t)(storage->nsects * storage->offset_size) + (uint8_t)(4 * storage->nsects));
}
else {
cparam.cls = H5FA_CLS_STRUCT_CHUNK;
@@ -1987,7 +1987,7 @@ H5D__farray_stc_idx_create(const H5D_chk_idx_info_t *idx_info)
* size of offsets for (n - 1) sections
*/
cparam.raw_elmt_size = (uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + chunk_size_len +
(uint8_t)((storage->nsects - 1) * H5O_STRUCT_CHUNK_OFFSET_SIZE));
(uint8_t)((storage->nsects - 1) * storage->offset_size));
}
cparam.max_dblk_page_nelmts_bits = layout->u.farray.cparam.max_dblk_page_nelmts_bits;
@@ -2056,10 +2056,10 @@ H5D__farray_stc_idx_open(const H5D_chk_idx_info_t *idx_info)
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + idx_info->stc_storage->offset_size) /
idx_info->stc_storage->offset_size);
if (chunk_size_len > idx_info->stc_storage->offset_size)
chunk_size_len = idx_info->stc_storage->offset_size;
/* Set up the user data */
udata.f = idx_info->f;
+1 -1
View File
@@ -328,7 +328,7 @@ H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, bool include_c
case H5D_CHUNK_IDX_SINGLE:
/* chunk size */
ret_value += H5O_STRUCT_CHUNK_OFFSET_SIZE;
ret_value += layout->storage.u.struct_chunk.offset_size;
/* offsets for n - 1 sections */
ret_value += ((layout->storage.u.struct_chunk.nsects - 1) * 8);
+4 -4
View File
@@ -687,10 +687,10 @@ H5D__single_stc_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len =
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
1 + ((H5VM_log2_gen((uint64_t)idx_info->stc_layout->size) + idx_info->stc_storage->offset_size) /
idx_info->stc_storage->offset_size);
if (chunk_size_len > idx_info->stc_storage->offset_size)
chunk_size_len = idx_info->stc_storage->offset_size;
idx_info->stc_storage->u.single.chunk_size_len = chunk_size_len;
+35 -44
View File
@@ -590,6 +590,24 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
for (unsigned u = 1; u < mesg->u.struct_chunk.ndims; u++)
mesg->u.struct_chunk.size *= mesg->u.struct_chunk.dim[u];
/* Structured chunk composition */
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
mesg->storage.u.struct_chunk.offset_size = *p++;
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
mesg->storage.u.struct_chunk.nsects = *p++;
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
mesg->storage.u.struct_chunk.nsects_md = *p++;
if (H5_IS_BUFFER_OVERFLOW(p, mesg->storage.u.struct_chunk.nsects_md, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
for (unsigned i = 0; i < mesg->storage.u.struct_chunk.nsects_md; i++)
mesg->storage.u.struct_chunk.seq_sects_md[i] = *p++;
/* Chunk index type */
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
@@ -599,9 +617,6 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "unknown chunk index type");
mesg->storage.u.struct_chunk.idx_type = mesg->u.struct_chunk.idx_type;
/* NOTE HERE */
/* TBD: probably put structured chunk composition here after idx type */
switch (mesg->u.struct_chunk.idx_type) {
case H5D_CHUNK_IDX_BTREE:
HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL,
@@ -621,10 +636,10 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
* size (encoded selection + data) make the chunk larger.
*/
chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)mesg->u.struct_chunk.size) +
H5O_STRUCT_CHUNK_OFFSET_SIZE) /
H5O_STRUCT_CHUNK_OFFSET_SIZE);
if (chunk_size_len > H5O_STRUCT_CHUNK_OFFSET_SIZE)
chunk_size_len = H5O_STRUCT_CHUNK_OFFSET_SIZE;
mesg->storage.u.struct_chunk.offset_size) /
mesg->storage.u.struct_chunk.offset_size);
if (chunk_size_len > mesg->storage.u.struct_chunk.offset_size)
chunk_size_len = mesg->storage.u.struct_chunk.offset_size;
if (H5_IS_BUFFER_OVERFLOW(p, chunk_size_len, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
@@ -633,28 +648,24 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
/* chunk size */
UINT64DECODE_VAR(p, mesg->storage.u.struct_chunk.u.single.chunk_size, chunk_size_len);
/* NOTE HERE */
/* TBD: composition information (nsects, offset_size) is not available yet at this
* point */
if (H5_IS_BUFFER_OVERFLOW(p, (H5O_SPARSE_NSECTS - 1) * 8, p_end))
if (H5_IS_BUFFER_OVERFLOW(p, (mesg->storage.u.struct_chunk.nsects - 1) * 8, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
for (unsigned i = 1; i < H5O_SPARSE_NSECTS; i++)
for (unsigned i = 1; i < mesg->storage.u.struct_chunk.nsects; i++)
UINT64DECODE(p, mesg->storage.u.struct_chunk.u.single.offset[i]);
if (mesg->u.struct_chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) {
if (H5_IS_BUFFER_OVERFLOW(p, (H5O_SPARSE_NSECTS * 8), p_end))
if (H5_IS_BUFFER_OVERFLOW(p, (mesg->storage.u.struct_chunk.nsects * 8), p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
for (unsigned i = 0; i < H5O_SPARSE_NSECTS; i++)
for (unsigned i = 0; i < mesg->storage.u.struct_chunk.nsects; i++)
UINT64DECODE(p, mesg->storage.u.struct_chunk.u.single.unfilt_size[i]);
if (H5_IS_BUFFER_OVERFLOW(p, (H5O_SPARSE_NSECTS * 4), p_end))
if (H5_IS_BUFFER_OVERFLOW(p, (mesg->storage.u.struct_chunk.nsects * 4), p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL,
"ran off end of input buffer while decoding");
for (unsigned i = 0; i < H5O_SPARSE_NSECTS; i++)
for (unsigned i = 0; i < mesg->storage.u.struct_chunk.nsects; i++)
UINT32DECODE(p, mesg->storage.u.struct_chunk.u.single.filt_mask[i]);
}
@@ -772,26 +783,6 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
H5F_addr_decode(f, &p, &(mesg->storage.u.struct_chunk.idx_addr));
/* NOTE HERE */
/* Structured chunk composition */
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
mesg->storage.u.struct_chunk.offset_size = *p++;
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
mesg->storage.u.struct_chunk.nsects = *p++;
if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
mesg->storage.u.struct_chunk.nsects_md = *p++;
if (H5_IS_BUFFER_OVERFLOW(p, mesg->storage.u.struct_chunk.nsects_md, p_end))
HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
for (unsigned i = 0; i < mesg->storage.u.struct_chunk.nsects_md; i++)
mesg->storage.u.struct_chunk.seq_sects_md[i] = *p++;
/* Set the layout operations */
mesg->ops = H5D_LOPS_STRUCT_CHUNK;
/* Set the shared Chunk Cache layout operations */
@@ -1223,6 +1214,13 @@ H5O__layout_encode(H5F_t *f, bool H5_ATTR_UNUSED disable_shared, size_t H5_ATTR_
for (u = 0; u < mesg->u.struct_chunk.ndims; u++)
UINT64ENCODE_VAR(p, mesg->u.struct_chunk.dim[u], mesg->u.struct_chunk.enc_bytes_per_dim);
/* structured chunk composition */
*p++ = (uint8_t)mesg->storage.u.struct_chunk.offset_size;
*p++ = (uint8_t)mesg->storage.u.struct_chunk.nsects;
*p++ = (uint8_t)mesg->storage.u.struct_chunk.nsects_md;
for (unsigned i = 0; i < mesg->storage.u.struct_chunk.nsects_md; i++)
*p++ = (uint8_t)mesg->storage.u.struct_chunk.seq_sects_md[i];
/* Chunk index type */
*p++ = (uint8_t)mesg->u.struct_chunk.idx_type;
@@ -1287,13 +1285,6 @@ H5O__layout_encode(H5F_t *f, bool H5_ATTR_UNUSED disable_shared, size_t H5_ATTR_
*/
H5F_addr_encode(f, &p, mesg->storage.u.struct_chunk.idx_addr);
*p++ = (uint8_t)mesg->storage.u.struct_chunk.offset_size;
*p++ = (uint8_t)mesg->storage.u.struct_chunk.nsects;
*p++ = (uint8_t)mesg->storage.u.struct_chunk.nsects_md;
for (unsigned i = 0; i < mesg->storage.u.struct_chunk.nsects_md; i++)
*p++ = (uint8_t)mesg->storage.u.struct_chunk.seq_sects_md[i];
break;
case H5D_VIRTUAL:
+20 -2
View File
@@ -69,8 +69,8 @@
#define H5D_DEF_STORAGE_STRUCT_CHUNK_INIT \
{ \
H5D_CHUNK_IDX_FARRAY, HADDR_UNDEF, H5D_COPS_STRUCT_CHUNK_FARRAY, H5O_STRUCT_CHUNK_OFFSET_SIZE, 0, 0, \
{0, 0, 0}, \
H5D_CHUNK_IDX_FARRAY, HADDR_UNDEF, H5D_COPS_STRUCT_CHUNK_FARRAY, H5O_STRUCT_CHUNK_OFFSET_SIZE, \
H5O_SPARSE_NSECTS, 1, {0, 0, 0}, \
{ \
{ \
HADDR_UNDEF, NULL \
@@ -3649,6 +3649,24 @@ H5Pset_struct_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/], uns
if (!dim)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no chunk dimensions specified");
/*
* The structured chunk composition fields are initialized as:
* --offset_size:
* --number of bytes used to store offset
* --H5O_STRUCT_CHUNK_OFFSET_SIZE which is 8 bytes
* --nsects:
* --number of sections
* --H5O_SPARSE_NSECTS which is 2 for sparse chunk
* --nsects_md:
* --number of sections containing metadata
* --1 for sparse chunk
* --seq_sects_md[]:
* --sequence number of the ith section that contains metadata
* --seq_sects_md[0] = 0 (sequence number for selection section)
*
* The above initializations are implied by the structured chunk type, which is
* H5D_SPARSE_CHUNK type for now. This is work in progress and may change later.
*/
H5MM_memcpy(&layout, &H5D_def_layout_struct_chunk_g, sizeof(H5D_def_layout_struct_chunk_g));
layout.u.struct_chunk.stc_type = (uint8_t)flag;
Binary file not shown.