mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Onion VFD (#1953)

Browse Source 
* Onion VFD feature

* Fixes onion VFD errors with non-sec2 backing store VFDs

* Disables the onion VFD tests w/ ph5diff

* Disables non-sec2 VFDs as onion VFD backing stores

* Committing clang-format changes

* Formatted source

* Typo

* Adds onion VFD tools tests to CMake

* Fixes for v16 API compatibility

* Memset structs to avoid bad frees on errors

* H5Dwrite() calls now use H5T_NATIVE_INT as the memory type vs LE

* Properly decodes checksums on BE machines

* Be more careful about uint64_t to haddr_t/hsize_t conversions

* Another fix for BE data comparison

* Removed double underscores from onion constants

* Replace hard-coded onion header string w/ constant

* Fixes cleanup paths in H5FD__onion_ingest_history()

* Fixed use of size_t revision numbers

* Fix h5dump revision count format string

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Dana Robinson
2022-08-02 12:54:40 -07:00
committed by GitHub
co-authored by github-actions
parent ea13de1bb0
commit fcf41b3cd6
55 changed files with 10632 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/trace
+1
View File
@@ -200,6 +200,7 @@ $Source = "";
"H5FD_t" => "#",
"H5FD_hdfs_fapl_t" => "#",
"H5FD_mirror_fapl_t" => "#",
"H5FD_onion_fapl_t" => "#",
"H5FD_ros3_fapl_t" => "#",
"H5FD_splitter_vfd_config_t" => "#",
"H5L_class_t" => "#",
src/CMakeLists.txt
+9
View File
@@ -240,6 +240,10 @@ set (H5FD_SOURCES
${HDF5_SRC_DIR}/H5FDmpi.c
${HDF5_SRC_DIR}/H5FDmpio.c
${HDF5_SRC_DIR}/H5FDmulti.c
${HDF5_SRC_DIR}/H5FDonion.c
${HDF5_SRC_DIR}/H5FDonion_header.c
${HDF5_SRC_DIR}/H5FDonion_history.c
${HDF5_SRC_DIR}/H5FDonion_index.c
${HDF5_SRC_DIR}/H5FDperform.c
${HDF5_SRC_DIR}/H5FDros3.c
${HDF5_SRC_DIR}/H5FDs3comms.c
@@ -262,6 +266,7 @@ set (H5FD_HDRS
${HDF5_SRC_DIR}/H5FDmpi.h
${HDF5_SRC_DIR}/H5FDmpio.h
${HDF5_SRC_DIR}/H5FDmulti.h
${HDF5_SRC_DIR}/H5FDonion.h
${HDF5_SRC_DIR}/H5FDpublic.h
${HDF5_SRC_DIR}/H5FDros3.h
${HDF5_SRC_DIR}/H5FDs3comms.h
@@ -902,6 +907,10 @@ set (H5_PRIVATE_HEADERS
${HDF5_SRC_DIR}/H5FAprivate.h
${HDF5_SRC_DIR}/H5FDmirror_priv.h
${HDF5_SRC_DIR}/H5FDonion_header.h
${HDF5_SRC_DIR}/H5FDonion_history.h
${HDF5_SRC_DIR}/H5FDonion_index.h
${HDF5_SRC_DIR}/H5FDonion_priv.h
${HDF5_SRC_DIR}/H5FDpkg.h
${HDF5_SRC_DIR}/H5FDprivate.h
src/H5FDonion.c
+1762
View File
@@ -0,0 +1,1762 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Provide in-file provenance and revision/version control.
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fprivate.h" /* Files */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
#include "H5FDsec2.h" /* Sec2 file driver */
#include "H5FLprivate.h" /* Free Lists */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
/* The driver identification number, initialized at runtime */
static hid_t H5FD_ONION_g = 0;
/******************************************************************************
*
* Structure: H5FD_onion_t
*
* Purpose: Store information required to manage an onionized file.
* This structure is created when such a file is "opened" and
* discarded when it is "closed".
*
* pu
*
* Instance of H5FD_t which contains fields common to all VFDs.
* It must be the first item in this structure, since at higher levels,
* this structure will be treated as an instance of H5FD_t.
*
* fa
*
* Instance of `H5FD_onion_fapl_info_t` containing the configuration data
* needed to "open" the HDF5 file.
*
* original_file
*
* VFD handle for the original HDF5 file.
*
* onion_file
*
* VFD handle for the onion file.
* NULL if not set to use the single, separate storage target.
*
* recovery_file
*
* VFD handle for the history recovery file. This file is a backup of
* the existing history when an existing onion file is opened in RW mode.
*
* recovery_file_name
*
* String allocated and populated on file-open in write mode and freed on
* file-close, stores the path/name of the 'recovery' file. The file
* created at this location is to be removed upon successful file-close
* from write mode.
*
* is_open_rw
*
* Remember whether the file was opened in a read-write mode.
*
* align_history_on_pages
*
* Remember whether onion-writes must be aligned to page boundaries.
*
* header
*
* In-memory copy of the onion history data header.
*
* history
*
* In-memory copy of the onion history.
*
* curr_rev_record
*
* Record for the currently open revision.
*
* rev_index
*
* Index for maintaining modified pages (RW mode only).
* Pointer is NULL when the file is not opened in write mode.
* Pointer is allocated on open and must be freed on close.
* Contents must be merged with the revision record's archival index prior
* to commitment of history to backing store.
*
* onion_eof
*
* Last byte in the onion file.
*
* origin_eof
*
* Size of the original HDF5 file.
*
* logical_eoa
*
* Address of first byte past addressed space in the logical 'file'
* presented by this VFD.
*
* logical_eof
*
* Address of first byte past last byte in the logical 'file' presented
* by this VFD.
* Must be copied into the revision record on close to write onion data.
*
******************************************************************************
*/
typedef struct H5FD_onion_t {
H5FD_t pub;
H5FD_onion_fapl_info_t fa;
hbool_t is_open_rw;
hbool_t align_history_on_pages;
/* Onion-related files */
H5FD_t *original_file;
H5FD_t *onion_file;
H5FD_t *recovery_file;
char *recovery_file_name;
/* Onion data structures */
H5FD_onion_header_t header;
H5FD_onion_history_t history;
H5FD_onion_revision_record_t curr_rev_record;
H5FD_onion_revision_index_t *rev_index;
/* End of addresses and files */
haddr_t onion_eof;
haddr_t origin_eof;
haddr_t logical_eoa;
haddr_t logical_eof;
} H5FD_onion_t;
H5FL_DEFINE_STATIC(H5FD_onion_t);
#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1)
#define H5FD_CTL_GET_NUM_REVISIONS 20001
/* Prototypes */
static herr_t H5FD__onion_close(H5FD_t *);
static haddr_t H5FD__onion_get_eoa(const H5FD_t *, H5FD_mem_t);
static haddr_t H5FD__onion_get_eof(const H5FD_t *, H5FD_mem_t);
static H5FD_t *H5FD__onion_open(const char *, unsigned int, hid_t, haddr_t);
static herr_t H5FD__onion_read(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, void *);
static herr_t H5FD__onion_set_eoa(H5FD_t *, H5FD_mem_t, haddr_t);
static herr_t H5FD__onion_term(void);
static herr_t H5FD__onion_write(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, const void *);
static herr_t H5FD__onion_open_rw(H5FD_onion_t *, unsigned int, haddr_t, bool new_open);
static herr_t H5FD__onion_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/);
static herr_t H5FD__onion_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf);
static hsize_t H5FD__onion_sb_size(H5FD_t *_file);
static herr_t H5FD__onion_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags,
const void H5_ATTR_UNUSED *input, void H5_ATTR_UNUSED **output);
static herr_t H5FD__get_onion_revision_count(H5FD_t *file, uint64_t *revision_count);
/* Temporary */
H5_DLL herr_t H5FD__onion_write_final_history(H5FD_onion_t *file);
static const H5FD_class_t H5FD_onion_g = {
H5FD_CLASS_VERSION, /* struct version */
H5FD_ONION_VALUE, /* value */
"onion", /* name */
MAXADDR, /* maxaddr */
H5F_CLOSE_WEAK, /* fc_degree */
H5FD__onion_term, /* terminate */
H5FD__onion_sb_size, /* sb_size */
H5FD__onion_sb_encode, /* sb_encode */
H5FD__onion_sb_decode, /* sb_decode */
sizeof(H5FD_onion_fapl_info_t), /* fapl_size */
NULL, /* fapl_get */
NULL, /* fapl_copy */
NULL, /* fapl_free */
0, /* dxpl_size */
NULL, /* dxpl_copy */
NULL, /* dxpl_free */
H5FD__onion_open, /* open */
H5FD__onion_close, /* close */
NULL, /* cmp */
NULL, /* query */
NULL, /* get_type_map */
NULL, /* alloc */
NULL, /* free */
H5FD__onion_get_eoa, /* get_eoa */
H5FD__onion_set_eoa, /* set_eoa */
H5FD__onion_get_eof, /* get_eof */
NULL, /* get_handle */
H5FD__onion_read, /* read */
H5FD__onion_write, /* write */
NULL, /* read_vector */
NULL, /* write_vector */
NULL, /* read_selection */
NULL, /* write_selection */
NULL, /* flush */
NULL, /* truncate */
NULL, /* lock */
NULL, /* unlock */
NULL, /* del */
H5FD__onion_ctl, /* ctl */
H5FD_FLMAP_DICHOTOMY /* fl_map */
};
/*-----------------------------------------------------------------------------
* Function: H5FD_onion_init
*
* Purpose: Initialize this driver by registering the driver with the
* library.
*
* Return: Success: The driver ID for the onion driver.
* Failure: Negative
*-----------------------------------------------------------------------------
*/
hid_t
H5FD_onion_init(void)
{
hid_t ret_value = H5I_INVALID_HID;
FUNC_ENTER_NOAPI_NOERR
if (H5I_VFL != H5I_get_type(H5FD_ONION_g))
H5FD_ONION_g = H5FD_register(&H5FD_onion_g, sizeof(H5FD_class_t), FALSE);
/* Set return value */
ret_value = H5FD_ONION_g;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_onion_init() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_term
*
* Purpose: Shut down the Onion VFD.
*
* Returns: SUCCEED (Can't fail)
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_term(void)
{
FUNC_ENTER_PACKAGE_NOERR
/* Reset VFL ID */
H5FD_ONION_g = 0;
FUNC_LEAVE_NOAPI(SUCCEED);
} /* end H5FD__onion_term() */
/*-----------------------------------------------------------------------------
* Function: H5Pget_fapl_onion
*
* Purpose: Copy the Onion configuration information from the FAPL at
* `fapl_id` to the destination pointer `fa_out`.
*
* Return: Success: Non-negative value (SUCCEED).
* Failure: Negative value (FAIL).
*-----------------------------------------------------------------------------
*/
herr_t
H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out)
{
const H5FD_onion_fapl_info_t *info_ptr = NULL;
H5P_genplist_t *plist = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "i*!", fapl_id, fa_out);
if (NULL == fa_out)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info-out pointer")
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID")
if (H5FD_ONION != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Incorrect VFL driver")
if (NULL == (info_ptr = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad VFL driver info")
HDmemcpy(fa_out, info_ptr, sizeof(H5FD_onion_fapl_info_t));
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_onion() */
/*-----------------------------------------------------------------------------
* Function: H5Pset_fapl_onion
*
* Purpose Set the file access property list at `fapl_id` to use the
* Onion virtual file driver with the given configuration.
* The info structure may be modified or deleted after this call,
* as its contents are copied into the FAPL.
*
* Return: Success: Non-negative value (SUCCEED).
* Failure: Negative value (FAIL).
*-----------------------------------------------------------------------------
*/
herr_t
H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa)
{
H5P_genplist_t *fapl = NULL;
H5P_genplist_t *backing_fapl = NULL;
hid_t backing_vfd_id = H5I_INVALID_HID;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
H5TRACE2("e", "i*!", fapl_id, fa);
if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID");
if (NULL == fa)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info pointer");
if (H5FD_ONION_FAPL_INFO_VERSION_CURR != fa->version)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info version");
if (!POWER_OF_TWO(fa->page_size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size");
if (fa->page_size < 1)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size");
if (H5P_DEFAULT == fa->backing_fapl_id) {
if (NULL == (backing_fapl = H5P_object_verify(H5P_FILE_ACCESS_DEFAULT, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id");
}
else {
if (NULL == (backing_fapl = H5P_object_verify(fa->backing_fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id");
}
/* The only backing fapl that is currently supported is sec2 */
if ((backing_vfd_id = H5P_peek_driver(backing_fapl)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "Can't get VFD from fapl");
if (backing_vfd_id != H5FD_SEC2)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Onion VFD only supports sec2 backing store");
if (H5P_set_driver(fapl, H5FD_ONION, (const void *)fa, NULL) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "Can't set the onion VFD");
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_fapl_onion() */
/*-------------------------------------------------------------------------
* Function: H5FD__onion_sb_size
*
* Purpose: Returns the size of the private information to be stored in
* the superblock.
*
* Return: Success: The super block driver data size
* Failure: never fails
*-------------------------------------------------------------------------
*/
static hsize_t
H5FD__onion_sb_size(H5FD_t *_file)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
hsize_t ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR
/* Sanity check */
HDassert(file);
HDassert(file->original_file);
if (file->original_file)
ret_value = H5FD_sb_size(file->original_file);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__onion_sb_size */
/*-------------------------------------------------------------------------
* Function: H5FD__onion_sb_encode
*
* Purpose: Encodes the superblock information for this driver
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Sanity check */
HDassert(file);
HDassert(file->original_file);
if (file->original_file && H5FD_sb_encode(file->original_file, name, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTENCODE, FAIL, "unable to encode the superblock in R/W file")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__onion_sb_encode */
/*-------------------------------------------------------------------------
* Function: H5FD__onion_sb_decode
*
* Purpose: Decodes the superblock information for this driver
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* Sanity check */
HDassert(file);
HDassert(file->original_file);
if (H5FD_sb_load(file->original_file, name, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "unable to decode the superblock in R/W file")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__onion_sb_decode */
/*-----------------------------------------------------------------------------
* Write in-memory revision record to appropriate backing file.
* Update information in other in-memory components.
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_commit_new_revision_record(H5FD_onion_t *file)
{
uint32_t checksum = 0; /* required */
size_t size = 0;
haddr_t phys_addr = 0; /* offset in history file to record start */
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
H5FD_onion_revision_record_t *rec = &file->curr_rev_record;
H5FD_onion_history_t *history = &file->history;
H5FD_onion_record_loc_t *new_list = NULL;
time_t rawtime;
struct tm *info;
FUNC_ENTER_PACKAGE
HDtime(&rawtime);
info = HDgmtime(&rawtime);
HDstrftime(rec->time_of_creation, sizeof(rec->time_of_creation), "%Y%m%dT%H%M%SZ", info);
rec->logical_eof = file->logical_eof;
if ((TRUE == file->is_open_rw) && (H5FD__onion_merge_revision_index_into_archival_index(
file->rev_index, &file->curr_rev_record.archival_index) < 0))
HGOTO_ERROR(H5E_VFL, H5E_INTERNAL, FAIL, "unable to update index to write")
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_REVISION_RECORD + (size_t)rec->comment_size +
(H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * rec->archival_index.n_entries))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for encoded revision record")
if (0 == (size = H5FD__onion_revision_record_encode(rec, buf, &checksum)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding revision record")
phys_addr = file->onion_eof;
if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, phys_addr + size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA for new revision record")
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, phys_addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write new revision record")
file->onion_eof = phys_addr + size;
if (TRUE == file->align_history_on_pages)
file->onion_eof = (file->onion_eof + (file->header.page_size - 1)) & (~(file->header.page_size - 1));
/* Update history info to accommodate new revision */
if (history->n_revisions == 0) {
unsigned char *ptr = buf; /* re-use buffer space to compute checksum */
HDassert(history->record_locs == NULL);
history->n_revisions = 1;
if (NULL == (history->record_locs = H5MM_calloc(sizeof(H5FD_onion_record_loc_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate temporary record pointer list")
history->record_locs[0].phys_addr = phys_addr;
history->record_locs[0].record_size = size;
UINT64ENCODE(ptr, phys_addr);
UINT64ENCODE(ptr, size);
history->record_locs[0].checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
/* TODO: size-reset belongs where? */
file->header.history_size += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER;
} /* end if no extant revisions in history */
else {
unsigned char *ptr = buf; /* re-use buffer space to compute checksum */
HDassert(history->record_locs != NULL);
if (NULL == (new_list = H5MM_calloc((history->n_revisions + 1) * sizeof(H5FD_onion_record_loc_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to resize record pointer list")
HDmemcpy(new_list, history->record_locs, sizeof(H5FD_onion_record_loc_t) * history->n_revisions);
H5MM_xfree(history->record_locs);
history->record_locs = new_list;
new_list = NULL;
history->record_locs[history->n_revisions].phys_addr = phys_addr;
history->record_locs[history->n_revisions].record_size = size;
UINT64ENCODE(ptr, phys_addr);
UINT64ENCODE(ptr, size);
history->record_locs[history->n_revisions].checksum =
H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
file->header.history_size += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER;
history->n_revisions += 1;
} /* end if one or more revisions present in history */
file->header.history_addr = file->onion_eof;
done:
H5MM_xfree(buf);
H5MM_xfree(new_list);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_commit_new_revision_record() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_close
*
* Purpose: Close an onionized file
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_close(H5FD_t *_file)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
HDassert(file);
if (H5FD_ONION_STORE_TARGET_ONION == file->fa.store_target) {
HDassert(file->onion_file);
if (file->is_open_rw) {
HDassert(file->recovery_file);
if (H5FD__onion_commit_new_revision_record(file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write revision record to backing store")
if (H5FD__onion_write_final_history(file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write history to backing store")
/* Unset write-lock flag and write header */
if (file->is_open_rw)
file->header.flags &= (uint32_t)~H5FD_ONION_HEADER_FLAG_WRITE_LOCK;
if (H5FD__onion_write_header(&(file->header), file->onion_file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write updated header to backing store")
}
}
else if (H5FD_ONION_STORE_TARGET_H5 == file->fa.store_target)
HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "hdf5 store-target not supported")
else
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid history target")
done:
/* Destroy things as best we can, even if there were earlier errors */
if (file->original_file)
if (H5FD_close(file->original_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing canon file")
if (file->onion_file)
if (H5FD_close(file->onion_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing onion file")
if (file->recovery_file) {
if (H5FD_close(file->recovery_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing recovery file")
/* TODO: Use the VFD's del callback instead of remove (this requires
* storing a copy of the fapl that was used to open it)
*/
HDremove(file->recovery_file_name);
}
if (file->rev_index)
if (H5FD__onion_revision_index_destroy(file->rev_index) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close revision index")
H5MM_xfree(file->recovery_file_name);
H5MM_xfree(file->history.record_locs);
H5MM_xfree(file->curr_rev_record.comment);
H5MM_xfree(file->curr_rev_record.archival_index.list);
file = H5FL_FREE(H5FD_onion_t, file);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__onion_close() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_get_eoa
*
* Purpose: Get end-of-address address.
*
* Return: Address of first byte past the addressed space
*-----------------------------------------------------------------------------
*/
static haddr_t
H5FD__onion_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
{
const H5FD_onion_t *file = (const H5FD_onion_t *)_file;
FUNC_ENTER_PACKAGE_NOERR;
FUNC_LEAVE_NOAPI(file->logical_eoa)
} /* end H5FD__onion_get_eoa() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_get_eof
*
* Purpose: Get end-of-file address.
*
* Return: Address of first byte past the file-end.
*-----------------------------------------------------------------------------
*/
static haddr_t
H5FD__onion_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
{
const H5FD_onion_t *file = (const H5FD_onion_t *)_file;
FUNC_ENTER_PACKAGE_NOERR;
FUNC_LEAVE_NOAPI(file->logical_eof)
} /* end H5FD__onion_get_eof() */
/*-----------------------------------------------------------------------------
* Sanitize the backing FAPL ID
*-----------------------------------------------------------------------------
*/
static inline hid_t
H5FD__onion_get_legit_fapl_id(hid_t fapl_id)
{
if (H5P_DEFAULT == fapl_id)
return H5P_FILE_ACCESS_DEFAULT;
else if (TRUE == H5P_isa_class(fapl_id, H5P_FILE_ACCESS))
return fapl_id;
else
return H5I_INVALID_HID;
}
/*-----------------------------------------------------------------------------
* Function: H5FD_onion_create_truncate_onion
*
* Purpose: Create/truncate HDF5 and onion data for a fresh file
*
* Special open operation required to instantiate the canonical file and
* history simultaneously. If successful, the required backing files are
* craeated and given initial population on the backing store, and the Onion
* virtual file handle is set; open effects a write-mode open.
*
* Cannot create 'template' history and proceed with normal write-mode open,
* as this would in effect create an empty first revision, making the history
* unintuitive. (create file -> initialize and commit empty first revision
* (revision 0); any data written to file during the 'create' open, as seen by
* the user, would be in the second revision (revision 1).)
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_create_truncate_onion(H5FD_onion_t *file, const char *filename, const char *name_onion,
const char *recovery_file_nameery, unsigned int flags, haddr_t maxaddr)
{
hid_t backing_fapl_id = H5I_INVALID_HID;
H5FD_onion_header_t *hdr = NULL;
H5FD_onion_history_t *history = NULL;
H5FD_onion_revision_record_t *rec = NULL;
unsigned char *buf = NULL;
size_t size = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(file != NULL);
hdr = &file->header;
history = &file->history;
rec = &file->curr_rev_record;
hdr->flags = H5FD_ONION_HEADER_FLAG_WRITE_LOCK;
if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY & file->fa.creation_flags)
hdr->flags |= H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY;
if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & file->fa.creation_flags)
hdr->flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT;
hdr->origin_eof = 0;
backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id);
if (H5I_INVALID_HID == backing_fapl_id)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid backing FAPL ID")
/* Create backing files for onion history */
if (NULL == (file->original_file = H5FD_open(filename, flags, backing_fapl_id, maxaddr)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "cannot open the backing file")
if (NULL == (file->onion_file = H5FD_open(name_onion, flags, backing_fapl_id, maxaddr)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "cannot open the backing onion file")
if (NULL == (file->recovery_file = H5FD_open(recovery_file_nameery, flags, backing_fapl_id, maxaddr)))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "cannot open the backing file")
/* Write "empty" .h5 file contents (signature ONIONEOF) */
if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, 8) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA")
if (H5FD_write(file->original_file, H5FD_MEM_DRAW, 0, 8, "ONIONEOF") < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "cannot write header to the backing h5 file")
/* Write nascent history (with no revisions) to "recovery" */
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer")
size = H5FD__onion_history_encode(history, buf, &history->checksum);
if (H5FD_ONION_ENCODED_SIZE_HISTORY != size)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history")
if (H5FD_set_eoa(file->recovery_file, H5FD_MEM_DRAW, size) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA")
if (H5FD_write(file->recovery_file, H5FD_MEM_DRAW, 0, size, buf) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "cannot write history to the backing recovery file")
hdr->history_size = size; /* record for later use */
H5MM_xfree(buf);
buf = NULL;
/* Write history header with "no" history.
* Size of the "recovery" history recorded for later use on close.
*/
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer")
size = H5FD__onion_header_encode(hdr, buf, &hdr->checksum);
if (H5FD_ONION_ENCODED_SIZE_HEADER != size)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history header")
if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, size) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA")
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, size, buf) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "cannot write header to the backing onion file")
file->onion_eof = (haddr_t)size;
if (TRUE == file->align_history_on_pages)
file->onion_eof = (file->onion_eof + (hdr->page_size - 1)) & (~(hdr->page_size - 1));
rec->archival_index.list = NULL;
if (NULL == (file->rev_index = H5FD__onion_revision_index_init(file->fa.page_size)))
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize revision index")
done:
H5MM_xfree(buf);
if (FAIL == ret_value)
HDremove(recovery_file_nameery); /* destroy new temp file, if 'twas created */
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_create_truncate_onion() */
static herr_t
H5FD__onion_remove_unused_symbols(char *s)
{
char *d = s;
FUNC_ENTER_PACKAGE_NOERR;
do {
while (*d == '{' || *d == '}' || *d == ' ') {
++d;
}
} while ((*s++ = *d++));
FUNC_LEAVE_NOAPI(SUCCEED);
}
static herr_t
H5FD__onion_parse_config_str(const char *config_str, H5FD_onion_fapl_info_t *fa)
{
char *config_str_copy = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
if (!HDstrcmp(config_str, ""))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "configure string can't be empty")
/* Initialize to the default values */
fa->version = H5FD_ONION_FAPL_INFO_VERSION_CURR;
fa->backing_fapl_id = H5P_DEFAULT;
fa->page_size = 4;
fa->store_target = H5FD_ONION_STORE_TARGET_ONION;
fa->revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
fa->force_write_open = 0;
fa->creation_flags = 0;
HDstrcpy(fa->comment, "initial comment");
/* If a single integer is passed in as a string, it's a shortcut for the tools
* (h5repack, h5diff, h5dump). Otherwise, the string should have curly brackets,
* e.g. {revision_num: 2; page_size: 4;}
*/
if (config_str[0] != '{')
fa->revision_num = (uint64_t)HDstrtoull(config_str, NULL, 10);
else {
char *token1 = NULL, *token2 = NULL;
/* Duplicate the configure string since strtok will mess with it */
if (NULL == (config_str_copy = H5MM_strdup(config_str)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't duplicate configure string")
/* Remove the curly brackets and space from the configure string */
H5FD__onion_remove_unused_symbols(config_str_copy);
/* The configure string can't be empty after removing the curly brackets */
if (!HDstrcmp(config_str_copy, ""))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "configure string can't be empty")
token1 = HDstrtok(config_str_copy, ":");
token2 = HDstrtok(NULL, ";");
do {
if (token1 && token2) {
if (!HDstrcmp(token1, "version")) {
if (!HDstrcmp(token2, "H5FD_ONION_FAPL_INFO_VERSION_CURR"))
fa->version = H5FD_ONION_FAPL_INFO_VERSION_CURR;
}
else if (!HDstrcmp(token1, "backing_fapl_id")) {
if (!HDstrcmp(token2, "H5P_DEFAULT"))
fa->backing_fapl_id = H5P_DEFAULT;
else if (!strcmp(token2, "H5I_INVALID_HID"))
fa->backing_fapl_id = H5I_INVALID_HID;
else
fa->backing_fapl_id = HDstrtoll(token2, NULL, 10);
}
else if (!HDstrcmp(token1, "page_size")) {
fa->page_size = (uint32_t)HDstrtoul(token2, NULL, 10);
}
else if (!HDstrcmp(token1, "revision_num")) {
if (!HDstrcmp(token2, "H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST"))
fa->revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
else
fa->revision_num = (uint64_t)HDstrtoull(token2, NULL, 10);
}
else if (!HDstrcmp(token1, "force_write_open")) {
fa->force_write_open = (uint8_t)HDstrtoul(token2, NULL, 10);
}
else if (!HDstrcmp(token1, "creation_flags")) {
fa->creation_flags = (uint8_t)HDstrtoul(token2, NULL, 10);
}
else if (!HDstrcmp(token1, "comment")) {
HDstrcpy(fa->comment, token2);
}
else
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unknown token in the configure string: %s",
token1)
}
token1 = HDstrtok(NULL, ":");
token2 = HDstrtok(NULL, ";");
} while (token1);
}
if (H5P_DEFAULT == fa->backing_fapl_id || H5I_INVALID_HID == fa->backing_fapl_id) {
H5P_genclass_t *pclass; /* Property list class to modify */
if (NULL == (pclass = (H5P_genclass_t *)H5I_object_verify(H5P_FILE_ACCESS, H5I_GENPROP_CLS)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a property list class");
/* Create the new property list */
if ((fa->backing_fapl_id = H5P_create_id(pclass, TRUE)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "unable to create property list");
}
done:
H5MM_free(config_str_copy);
FUNC_LEAVE_NOAPI(ret_value);
}
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_open
*
* Purpose: Open an onionized file
*
* Return: Success: A pointer to a new file data structure
* Failure: NULL
*-----------------------------------------------------------------------------
*/
static H5FD_t *
H5FD__onion_open(const char *filename, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
H5P_genplist_t *plist = NULL;
H5FD_onion_t *file = NULL;
const H5FD_onion_fapl_info_t *fa = NULL;
H5FD_onion_fapl_info_t *new_fa = NULL;
const char *config_str = NULL;
hid_t backing_fapl_id = H5I_INVALID_HID;
char *name_onion = NULL;
char *recovery_file_nameery = NULL;
bool new_open = false;
haddr_t canon_eof = 0;
H5FD_t *ret_value = NULL;
FUNC_ENTER_PACKAGE
/* Check arguments */
if (!filename || !*filename)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name")
if (0 == maxaddr || HADDR_UNDEF == maxaddr)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr")
HDassert(H5P_DEFAULT != fapl_id);
if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
/* This VFD can be invoked by either H5Pset_fapl_onion() or
* H5Pset_driver_by_name(). When invoked by the former, there will be
* driver info to peek at.
*/
fa = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist);
if (NULL == fa) {
if (NULL == (config_str = H5P_peek_driver_config_str(plist)))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "missing VFL driver configure string")
/* Allocate a new onion fapl info struct and fill it from the
* configuration string
*/
if (NULL == (new_fa = H5MM_calloc(sizeof(H5FD_onion_fapl_info_t))))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate memory for onion fapl info struct")
if (H5FD__onion_parse_config_str(config_str, new_fa) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "failed to parse configure string")
fa = new_fa;
}
/* Check for unsupported target values */
if (H5FD_ONION_STORE_TARGET_H5 == fa->store_target)
HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "same-file storage not implemented")
else if (H5FD_ONION_STORE_TARGET_ONION != fa->store_target)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid store target")
/* Allocate space for the file struct */
if (NULL == (file = H5FL_CALLOC(H5FD_onion_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate file struct")
/* Allocate space for onion VFD file names */
if (NULL == (name_onion = H5MM_malloc(sizeof(char) * (HDstrlen(filename) + 7))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate onion name string")
HDsnprintf(name_onion, HDstrlen(filename) + 7, "%s.onion", filename);
if (NULL == (recovery_file_nameery = H5MM_malloc(sizeof(char) * (HDstrlen(name_onion) + 10))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate recovery name string")
HDsnprintf(recovery_file_nameery, HDstrlen(name_onion) + 10, "%s.recovery", name_onion);
file->recovery_file_name = recovery_file_nameery;
if (NULL == (file->recovery_file_name = H5MM_malloc(sizeof(char) * (HDstrlen(name_onion) + 10))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate recovery name string")
HDsnprintf(file->recovery_file_name, HDstrlen(name_onion) + 10, "%s.recovery", name_onion);
/* Translate H5P_DEFAULT to a a real fapl ID, if necessary */
backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id);
if (H5I_INVALID_HID == backing_fapl_id)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID");
/* Initialize file structure fields */
HDmemcpy(&(file->fa), fa, sizeof(H5FD_onion_fapl_info_t));
file->header.version = H5FD_ONION_HEADER_VERSION_CURR;
file->header.page_size = file->fa.page_size; /* guarded on FAPL-set */
file->history.version = H5FD_ONION_HISTORY_VERSION_CURR;
file->curr_rev_record.version = H5FD_ONION_REVISION_RECORD_VERSION_CURR;
file->curr_rev_record.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR;
/* Check that the page size is a power of two */
if ((fa->page_size == 0) || ((fa->page_size & (fa->page_size - 1)) != 0))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "page size is not a power of two")
/* Assign the page size */
double log2_page_size = HDlog2((double)(fa->page_size));
file->curr_rev_record.archival_index.page_size_log2 = (uint32_t)log2_page_size;
/* Proceed with open. */
if ((H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) {
/* Create a new onion file from scratch */
/* Set flags */
if (fa->creation_flags & H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT) {
file->header.flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT;
file->align_history_on_pages = TRUE;
}
/* Truncate and create everything as necessary */
if (H5FD__onion_create_truncate_onion(file, filename, name_onion, file->recovery_file_name, flags,
maxaddr) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to create/truncate onionized files")
file->is_open_rw = TRUE;
}
else {
/* Opening an existing onion file */
/* Open the existing file using the specified fapl */
if (NULL == (file->original_file = H5FD_open(filename, flags, backing_fapl_id, maxaddr)))
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open canonical file (does not exist?)")
/* Try to open any existing onion file */
H5E_BEGIN_TRY
{
file->onion_file = H5FD_open(name_onion, flags, backing_fapl_id, maxaddr);
}
H5E_END_TRY;
/* If that didn't work, create a new onion file */
/* TODO: Move to a new function */
if (NULL == file->onion_file) {
if (H5F_ACC_RDWR & flags) {
H5FD_onion_header_t *hdr = NULL;
H5FD_onion_history_t *history = NULL;
H5FD_onion_revision_record_t *rec = NULL;
unsigned char *head_buf = NULL;
unsigned char *hist_buf = NULL;
size_t size = 0;
size_t saved_size = 0;
HDassert(file != NULL);
hdr = &file->header;
history = &file->history;
rec = &file->curr_rev_record;
new_open = true;
if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY & file->fa.creation_flags)
hdr->flags |= H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY;
if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & file->fa.creation_flags) {
hdr->flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT;
file->align_history_on_pages = TRUE;
}
if (HADDR_UNDEF == (canon_eof = H5FD_get_eof(file->original_file, H5FD_MEM_DEFAULT))) {
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "cannot get size of canonical file")
}
if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, canon_eof) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't extend EOA")
hdr->origin_eof = canon_eof;
file->logical_eof = canon_eof;
backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id);
if (H5I_INVALID_HID == backing_fapl_id)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID")
/* Create backing files for onion history */
if ((file->onion_file = H5FD_open(name_onion, (H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC),
backing_fapl_id, maxaddr)) == NULL) {
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "cannot open the backing onion file")
}
/* Write history header with "no" history */
hdr->history_size = H5FD_ONION_ENCODED_SIZE_HISTORY; /* record for later use */
hdr->history_addr =
H5FD_ONION_ENCODED_SIZE_HEADER + 1; /* TODO: comment these 2 or do some other way */
head_buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER);
if (NULL == head_buf)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate buffer")
size = H5FD__onion_header_encode(hdr, head_buf, &hdr->checksum);
if (H5FD_ONION_ENCODED_SIZE_HEADER != size)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't encode history header")
hist_buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY);
if (NULL == hist_buf)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate buffer")
saved_size = size;
history->n_revisions = 0;
size = H5FD__onion_history_encode(history, hist_buf, &history->checksum);
file->header.history_size = size; /* record for later use */
if (H5FD_ONION_ENCODED_SIZE_HISTORY != size) {
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't encode history")
}
if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, saved_size + size + 1) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't extend EOA")
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, saved_size, head_buf) < 0) {
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, NULL,
"cannot write header to the backing onion file")
}
file->onion_eof = (haddr_t)saved_size;
if (TRUE == file->align_history_on_pages)
file->onion_eof = (file->onion_eof + (hdr->page_size - 1)) & (~(hdr->page_size - 1));
rec->archival_index.list = NULL;
file->header.history_addr = file->onion_eof;
/* Write nascent history (with no revisions) to the backing onion file */
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, saved_size + 1, size, hist_buf) < 0) {
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, NULL,
"cannot write history to the backing onion file")
}
file->header.history_size = size; /* record for later use */
H5MM_xfree(head_buf);
H5MM_xfree(hist_buf);
}
else {
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open onion file (does not exist?).")
}
}
if (HADDR_UNDEF == (canon_eof = H5FD_get_eof(file->original_file, H5FD_MEM_DEFAULT))) {
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "cannot get size of canonical file")
}
if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, canon_eof) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't extend EOA")
/* Get the history header from the onion file */
if (H5FD__onion_ingest_header(&file->header, file->onion_file, 0) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get history header from backing store")
file->align_history_on_pages =
(file->header.flags & H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT) ? TRUE : FALSE;
if (H5FD_ONION_HEADER_FLAG_WRITE_LOCK & file->header.flags) {
/* Opening a file twice in write mode is an error */
HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Can't open file already opened in write-mode")
}
else {
/* Read in the history from the onion file */
if (H5FD__onion_ingest_history(&file->history, file->onion_file, file->header.history_addr,
file->header.history_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get history from backing store")
/* Sanity check on revision ID */
if (fa->revision_num > file->history.n_revisions &&
fa->revision_num != H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "target revision ID out of range")
if (fa->revision_num == 0) {
file->curr_rev_record.logical_eof = canon_eof;
}
else if (file->history.n_revisions > 0 &&
H5FD__onion_ingest_revision_record(
&file->curr_rev_record, file->onion_file, &file->history,
MIN(fa->revision_num - 1, (file->history.n_revisions - 1))) < 0) {
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get revision record from backing store")
}
if (H5F_ACC_RDWR & flags)
if (H5FD__onion_open_rw(file, flags, maxaddr, new_open) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "can't write-open write-locked file")
}
} /* End if opening existing file */
/* Copy comment from FAPL info, if one is given */
if ((H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) {
if (fa->comment) {
/* Free the old comment */
file->curr_rev_record.comment = H5MM_xfree(file->curr_rev_record.comment);
/* TODO: Lengths of strings should be size_t */
file->curr_rev_record.comment_size = (uint32_t)HDstrlen(fa->comment) + 1;
if (NULL == (file->curr_rev_record.comment = H5MM_xstrdup(fa->comment)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate comment string")
}
}
file->origin_eof = file->header.origin_eof;
file->logical_eof = MAX(file->curr_rev_record.logical_eof, file->logical_eof);
file->logical_eoa = 0;
file->onion_eof = H5FD_get_eoa(file->onion_file, H5FD_MEM_DRAW);
if (TRUE == file->align_history_on_pages)
file->onion_eof = (file->onion_eof + (file->header.page_size - 1)) & (~(file->header.page_size - 1));
ret_value = (H5FD_t *)file;
done:
H5MM_xfree(name_onion);
H5MM_xfree(recovery_file_nameery);
if (config_str && new_fa)
if (fa && fa->backing_fapl_id)
if (H5I_GENPROP_LST == H5I_get_type(fa->backing_fapl_id))
H5I_dec_app_ref(fa->backing_fapl_id);
if ((NULL == ret_value) && file) {
if (file->original_file)
if (H5FD_close(file->original_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing canon")
if (file->onion_file)
if (H5FD_close(file->onion_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing onion")
if (file->recovery_file)
if (H5FD_close(file->recovery_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing recov")
if (file->rev_index)
if (H5FD__onion_revision_index_destroy(file->rev_index) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy revision index")
H5MM_xfree(file->history.record_locs);
H5MM_xfree(file->recovery_file_name);
H5MM_xfree(file->curr_rev_record.comment);
H5FL_FREE(H5FD_onion_t, file);
}
H5MM_xfree(new_fa);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_open() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_open_rw
*
* Purpose: Complete onion file-open, handling process for write mode.
*
* Creates recovery file if one does not exist.
* Initializes 'live' revision index.
* Force write-open is not yet supported (recovery provision) TODO
* Establishes write-lock in history header (sets lock flag).
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_open_rw(H5FD_onion_t *file, unsigned int flags, haddr_t maxaddr, bool new_open)
{
unsigned char *buf = NULL;
size_t size = 0;
uint32_t checksum = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
/* Guard against simultaneous write-open.
* TODO: support recovery open with force-write-open flag in FAPL info.
*/
if (file->header.flags & H5FD_ONION_HEADER_FLAG_WRITE_LOCK)
HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "can't write-open write-locked file")
/* Copy history to recovery file */
if (NULL ==
(file->recovery_file = H5FD_open(file->recovery_file_name, (flags | H5F_ACC_CREAT | H5F_ACC_TRUNC),
file->fa.backing_fapl_id, maxaddr)))
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "unable to create recovery file")
if (0 == (size = H5FD__onion_write_history(&file->history, file->recovery_file, 0, 0)))
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write history to recovery file")
if (size != file->header.history_size)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "written history differed from expected size")
/* Set write-lock flag in onion header */
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for encoded buffer")
file->header.flags |= H5FD_ONION_HEADER_FLAG_WRITE_LOCK;
if (0 == (size = H5FD__onion_header_encode(&file->header, buf, &checksum)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding history header")
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write updated history header")
/* Prepare revision index and finalize write-mode open */
if (NULL == (file->rev_index = H5FD__onion_revision_index_init(file->fa.page_size)))
HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize revision index")
file->curr_rev_record.parent_revision_num = file->curr_rev_record.revision_num;
if (!new_open)
file->curr_rev_record.revision_num += 1;
file->is_open_rw = TRUE;
done:
if (FAIL == ret_value) {
if (file->recovery_file != NULL) {
if (H5FD_close(file->recovery_file) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close recovery file")
file->recovery_file = NULL;
}
if (file->rev_index != NULL) {
if (H5FD__onion_revision_index_destroy(file->rev_index) < 0)
HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't destroy revision index")
file->rev_index = NULL;
}
}
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_open_rw() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_read
*
* Purpose: Read bytes from an onionized file
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len,
void *_buf_out)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
uint64_t page_0 = 0;
size_t n_pages = 0;
uint32_t page_size = 0;
uint32_t page_size_log2 = 0;
size_t bytes_to_read = len;
unsigned char *buf_out = (unsigned char *)_buf_out;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
HDassert(file != NULL);
HDassert(buf_out != NULL);
if ((uint64_t)(offset + len) > file->logical_eoa)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Read extends beyond addressed space")
if (0 == len)
goto done;
page_size = file->header.page_size;
page_size_log2 = file->curr_rev_record.archival_index.page_size_log2;
page_0 = offset >> page_size_log2;
n_pages = (len + page_size - 1) >> page_size_log2;
/* Read, page-by-page */
for (size_t i = 0; i < n_pages; i++) {
const H5FD_onion_index_entry_t *entry_out = NULL;
haddr_t page_gap_head = 0; /* start of page to start of buffer */
haddr_t page_gap_tail = 0; /* end of buffer to end of page */
size_t page_readsize = 0;
uint64_t page_i = page_0 + i;
if (0 == i) {
page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1);
/* Check if we need to add an additional page to make up for the page_gap_head */
if (page_gap_head > 0 &&
(page_gap_head + (bytes_to_read % page_size) > page_size || bytes_to_read % page_size == 0)) {
n_pages++;
}
}
if (n_pages - 1 == i)
page_gap_tail = page_size - bytes_to_read - page_gap_head;
page_readsize = (size_t)page_size - page_gap_head - page_gap_tail;
if (TRUE == file->is_open_rw && file->fa.revision_num != 0 &&
H5FD__onion_revision_index_find(file->rev_index, page_i, &entry_out)) {
/* Page exists in 'live' revision index */
if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr + page_gap_head,
page_readsize, buf_out) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get working file data")
}
else if (file->fa.revision_num != 0 &&
H5FD__onion_archival_index_find(&file->curr_rev_record.archival_index, page_i, &entry_out)) {
/* Page exists in archival index */
if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr + page_gap_head,
page_readsize, buf_out) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get previously-amended file data")
}
else {
/* Page does not exist in either index */
/* Casts prevent truncation */
haddr_t addr_start = (haddr_t)page_i * (haddr_t)page_size + (haddr_t)page_gap_head;
haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 : file->origin_eof - addr_start;
haddr_t read_size = MIN(overlap_size, page_readsize);
/* Get all original bytes in page range */
if ((read_size > 0) && H5FD_read(file->original_file, type, addr_start, read_size, buf_out) < 0) {
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get original file data")
}
/* Fill with 0s any gaps after end of original bytes
* and before end of page.
*/
for (size_t j = read_size; j < page_readsize; j++)
buf_out[j] = 0;
}
buf_out += page_readsize;
bytes_to_read -= page_readsize;
} /* end for each page in range */
HDassert(0 == bytes_to_read);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_read() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_set_eoa
*
* Purpose: Set end-of-address marker of the logical file.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
FUNC_ENTER_PACKAGE_NOERR;
file->logical_eoa = addr;
FUNC_LEAVE_NOAPI(SUCCEED);
} /* end H5FD__onion_set_eoa() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_write
*
* Purpose: Write bytes to an onionized file
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len,
const void *_buf)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
uint64_t page_0 = 0;
size_t n_pages = 0;
unsigned char *page_buf = NULL;
uint32_t page_size = 0;
uint32_t page_size_log2 = 0;
size_t bytes_to_write = len;
const unsigned char *buf = (const unsigned char *)_buf;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
HDassert(file != NULL);
HDassert(buf != NULL);
HDassert(file->rev_index != NULL);
HDassert((uint64_t)(offset + len) <= file->logical_eoa);
if (FALSE == file->is_open_rw)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Write not allowed if file not opened in write mode")
if (0 == len)
goto done;
page_size = file->header.page_size;
page_size_log2 = file->curr_rev_record.archival_index.page_size_log2;
page_0 = offset >> page_size_log2;
n_pages = (len + page_size - 1) >> page_size_log2;
if (NULL == (page_buf = H5MM_calloc(page_size * sizeof(unsigned char))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate temporary buffer")
/* Write, page-by-page */
for (size_t i = 0; i < n_pages; i++) {
const unsigned char *write_buf = buf;
H5FD_onion_index_entry_t new_entry;
const H5FD_onion_index_entry_t *entry_out = NULL;
haddr_t page_gap_head = 0; /* start of page to start of buffer */
haddr_t page_gap_tail = 0; /* end of buffer to end of page */
size_t page_n_used = 0; /* nbytes from buffer for this page-write */
uint64_t page_i = page_0 + i;
if (0 == i) {
page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1);
/* If we have a page_gap_head and the number of bytes to write is
* evenly divisible by the page size we need to add an additional
* page to make up for the page_gap_head
*/
if (page_gap_head > 0 && (page_gap_head + (bytes_to_write % page_size) > page_size ||
bytes_to_write % page_size == 0)) {
n_pages++;
}
}
if (n_pages - 1 == i)
page_gap_tail = page_size - bytes_to_write - page_gap_head;
page_n_used = page_size - page_gap_head - page_gap_tail;
/* Modify page in revision index, if present */
if (H5FD__onion_revision_index_find(file->rev_index, page_i, &entry_out)) {
if (page_gap_head | page_gap_tail) {
/* Copy existing page verbatim. */
if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, page_buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get working file data")
/* Overlay delta from input buffer onto page buffer. */
HDmemcpy(page_buf + page_gap_head, buf, page_n_used);
write_buf = page_buf;
} /* end if partial page */
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, write_buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write amended page data to backing file")
buf += page_n_used; /* overflow never touched */
bytes_to_write -= page_n_used;
continue;
} /* end if page exists in 'live' revision index */
if (page_gap_head || page_gap_tail) {
/* Fill gaps with existing data or zeroes. */
if (H5FD__onion_archival_index_find(&file->curr_rev_record.archival_index, page_i, &entry_out)) {
/* Page exists in archival index */
/* Copy existing page verbatim */
if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, page_buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get previously-amended data")
}
else {
haddr_t addr_start = (haddr_t)(page_i * page_size);
haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 : file->origin_eof - addr_start;
haddr_t read_size = MIN(overlap_size, page_size);
/* Get all original bytes in page range */
if ((read_size > 0) &&
H5FD_read(file->original_file, type, addr_start, read_size, page_buf) < 0) {
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get original file data")
}
/* Fill with 0s any gaps after end of original bytes
* or start of page and before start of new data.
*/
for (size_t j = read_size; j < page_gap_head; j++)
page_buf[j] = 0;
/* Fill with 0s any gaps after end of original bytes
* or end of new data and before end of page.
*/
for (size_t j = MAX(read_size, page_size - page_gap_tail); j < page_size; j++)
page_buf[j] = 0;
} /* end if page exists in neither index */
/* Copy input buffer to temporary page buffer */
HDassert((page_size - page_gap_head) >= page_n_used);
HDmemcpy(page_buf + page_gap_head, buf, page_n_used);
write_buf = page_buf;
} /* end if data range does not span entire page */
new_entry.logical_page = page_i;
new_entry.phys_addr = file->onion_eof;
if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, file->onion_eof + page_size) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA for new page amendment")
if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, file->onion_eof, page_size, write_buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write amended page data to backing file")
if (H5FD__onion_revision_index_insert(file->rev_index, &new_entry) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, "can't insert new index entry into revision index")
file->onion_eof += page_size;
buf += page_n_used; /* possible overflow never touched */
bytes_to_write -= page_n_used;
} /* end for each page to write */
HDassert(0 == bytes_to_write);
file->logical_eof = MAX(file->logical_eof, (offset + len));
done:
H5MM_xfree(page_buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_write() */
/*-------------------------------------------------------------------------
* Function: H5FD__onion_ctl
*
* Purpose: Onion VFD version of the ctl callback.
*
* The desired operation is specified by the op_code
* parameter.
*
* The flags parameter controls management of op_codes that
* are unknown to the callback
*
* The input and output parameters allow op_code specific
* input and output
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
void **output)
{
H5FD_onion_t *file = (H5FD_onion_t *)_file;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
/* Sanity checks */
HDassert(file);
switch (op_code) {
case H5FD_CTL_GET_NUM_REVISIONS:
if (!output || !*output)
HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "the output parameter is null")
**((uint64_t **)output) = file->history.n_revisions;
break;
/* Unknown op code */
default:
if (flags & H5FD_CTL_FAIL_IF_UNKNOWN_FLAG)
HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
break;
}
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__onion_ctl() */
/*-------------------------------------------------------------------------
* Function: H5FDget_onion_revision_count
*
* Purpose: Get the number of revisions in an onion file
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
herr_t
H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revision_count /*out*/)
{
H5P_genplist_t *plist = NULL;
H5FD_t *file = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
H5TRACE3("e", "*six", filename, fapl_id, revision_count);
/* Check args */
if (!filename || !HDstrcmp(filename, ""))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid file name")
if (!revision_count)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "revision count can't be null")
/* Make sure using the correct driver */
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid FAPL ID")
if (H5FD_ONION != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a Onion VFL driver")
/* Open the file with the driver */
if (NULL == (file = H5FD_open(filename, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "unable to open file with onion driver")
/* Call the private function */
if (H5FD__get_onion_revision_count(file, revision_count) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "failed to get the number of revisions")
done:
/* Close H5FD_t structure pointer */
if (file && H5FD_close(file) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file")
FUNC_LEAVE_API(ret_value)
}
/*-------------------------------------------------------------------------
* Function: H5FD__get_onion_revision_count
*
* Purpose: Private version of H5FDget_onion_revision_count()
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__get_onion_revision_count(H5FD_t *file, uint64_t *revision_count)
{
uint64_t op_code;
uint64_t flags;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
HDassert(file);
HDassert(revision_count);
op_code = H5FD_CTL_GET_NUM_REVISIONS;
flags = H5FD_CTL_FAIL_IF_UNKNOWN_FLAG;
/* Get the number of revisions via the ctl callback */
if (H5FD_ctl(file, op_code, flags, NULL, (void **)&revision_count) < 0)
HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
}
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_write_final_history
*
* Purpose: Write final history to appropriate backing file on file close
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_write_final_history(H5FD_onion_t *file)
{
size_t size = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
/* TODO: history EOF may not be correct (under what circumstances?) */
if (0 == (size = H5FD__onion_write_history(&(file->history), file->onion_file, file->onion_eof,
file->onion_eof)))
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write final history")
if (size != file->header.history_size)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "written history differed from expected size")
/* Is last write operation to history file; no need to extend to page
* boundary if set to page-align.
*/
file->onion_eof += size;
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_write_final_history() */
src/H5FDonion.h
+200
View File
@@ -0,0 +1,200 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: The public header file for the Onion VFD.
*/
#ifndef H5FDonion_H
#define H5FDonion_H
#define H5FD_ONION (H5FDperform_init(H5FD_onion_init))
#define H5FD_ONION_VALUE H5_VFD_ONION
/* Current version of the fapl info struct */
#define H5FD_ONION_FAPL_INFO_VERSION_CURR 1
/* Flag to open a file that has a locked header (after crashes, for example) */
#define H5FD_ONION_FAPL_INFO_FLAG_FORCE_OPEN 1
/* Flag to enable opening older revisions in write mode, creating a tree */
#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY 0x1
/* Flag to require page alignment of onion revision data */
#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT 0x2
/* Max length of a comment */
#define H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN 255
/* Indicates that you want the latest revision
* TODO: Does this work?
*/
#define H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST UINT64_MAX
typedef enum H5FD_onion_target_file_constant_t {
H5FD_ONION_STORE_TARGET_H5, /* Onion history as part of HDF5 file */
H5FD_ONION_STORE_TARGET_ONION, /* Separate, single "onion" file */
} H5FD_onion_target_file_constant_t;
/*-----------------------------------------------------------------------------
* Structure H5FD_onion_fapl_info_t
*
* Purpose: Encapsulate info for the Onion driver FAPL entry.
*
* version: Future-proofing identifier. Informs struct membership.
* Must equal H5FD_ONION_FAPL_VERSION_CURR to be considered valid.
*
* backing_fapl_id:
* Backing or 'child' FAPL ID to handle I/O with the
* underlying backing store. If the onion data is stored as a
* separate file, it must use the same backing driver as the
* original file.
*
* page_size: Size of the amended data pages. If opening an existing file,
* must equal the existing page size or zero. If creating a new
* file or an initial revision of an existing file, must be a
* power of 2.
*
* store_target:
* Enumerated/defined value identifying where the history data is
* stored, either in the same file (appended to HDF5 data) or a
* separate file. Other options may be added in later versions.
*
* + H5FD_ONION_FAPL_STORE_MODE_SEPARATE_SINGLE (1)
* Onion history is stored in a single, separate "onion
* file". Shares filename and path as hdf5 file (if any),
* with only a different filename extension.
*
* revision_num: Which revision to open. Must be 0 (the original file) or the
* revision number of an existing revision.
* Revision ID -1 is reserved to open the most recently-created
* revision in history.
*
* force_write_open:
* Flag to ignore the write-lock flag in the onion data
* and attempt to open the file write-only anyway.
* This may be relevant if, for example, the library crashed
* while the file was open in write mode and the write-lock
* flag was not cleared.
* Must equal H5FD_ONION_FAPL_FLAG_FORCE_OPEN to enable.
*
* creation_flags:
* Flag used only when instantiating an Onion file.
* If the relevant bit is set to a nonzero value, its feature
* will be enabled.
*
* + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY
* (1, bit 1)
* User will be allowed to open arbitrary revisions
* in write mode.
* If disabled (0), only the most recent revision may be
* opened for amendment.
*
* + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT (2, bit 2)
* Onion history metadata will align to page_size.
* Partial pages of unused space will occur in the file,
* but may improve read performance from the backing store
* on some systems.
* If disabled (0), padding will not be inserted to align
* to page boundaries.
*
* + <Remaining bits reserved>
*
* comment: User-supplied NULL-terminated comment for a revision to be
* written.
* Cannot be longer than H5FD_ONION_FAPL_COMMENT_MAX_LEN.
* Ignored if part of a FAPL used to open in read mode.
*
* The comment for a revision may be modified prior to committing
* to the revision (closing the file and writing the record)
* with a call to H5FDfctl().
* This H5FDfctl overwrite may be used to exceed constraints of
* maximum string length and the NULL-terminator requirement.
*
*-----------------------------------------------------------------------------
*/
typedef struct H5FD_onion_fapl_info_t {
uint8_t version;
hid_t backing_fapl_id;
uint32_t page_size;
H5FD_onion_target_file_constant_t store_target;
uint64_t revision_num;
uint8_t force_write_open;
uint8_t creation_flags;
char comment[H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN + 1];
} H5FD_onion_fapl_info_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL hid_t H5FD_onion_init(void);
/**
* --------------------------------------------------------------------------
* \ingroup H5P
*
* \brief get the onion info from the file access property list
*
* \param[in] fapl_id The ID of the file access property list
* \param[out] fa_out The pointer to the structure H5FD_onion_fapl_info_t
*
* \return \herr_t
*
* \details H5Pget_fapl_onion() retrieves the structure H5FD_onion_fapl_info_t
* from the file access property list that is set for the onion VFD
* driver.
*/
H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out);
/**
* --------------------------------------------------------------------------
* \ingroup H5P
*
* \brief set the onion info for the file access property list
*
* \param[in] fapl_id The ID of the file access property list
* \param[in] fa The pointer to the structure H5FD_onion_fapl_info_t
*
* \return \herr_t
*
* \details H5Pset_fapl_onion() sets the structure H5FD_onion_fapl_info_t
* for the file access property list that is set for the onion VFD
* driver.
*/
H5_DLL herr_t H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa);
/**
* --------------------------------------------------------------------------
* \ingroup H5FD
*
* \brief get the number of revisions
*
* \param[in] filename The name of the onion file
* \param[in] fapl_id The ID of the file access property list
* \param[out] revision_count The number of revisions
*
* \return \herr_t
*
* \details H5FDonion_get_revision_count() returns the number of revisions
* for an onion file. It takes the file name and file access property
* list that is set for the onion VFD driver.
*
*/
H5_DLL herr_t H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revision_count);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_H */
src/H5FDonion_header.c
+231
View File
@@ -0,0 +1,231 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Code for the onion file's header
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/*-----------------------------------------------------------------------------
* Function: H5FD_ingest_header
*
* Purpose: Read and decode the history header information from `raw_file`
* at `addr`, and store the decoded information in the structure
* at `hdr_out`.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_ingest_header(H5FD_onion_header_t *hdr_out, H5FD_t *raw_file, haddr_t addr)
{
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
haddr_t size = (haddr_t)H5FD_ONION_ENCODED_SIZE_HEADER;
uint32_t sum = 0;
FUNC_ENTER_PACKAGE;
if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "header indicates history beyond EOF")
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space")
if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA")
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read history header from file")
if (H5FD__onion_header_decode(buf, hdr_out) == 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history header")
sum = H5_checksum_fletcher32(buf, size - 4);
if (hdr_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored")
done:
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_ingest_header() */
/*-------------------------------------------------------------------------
* Function: H5FD__onion_write_header
*
* Purpose: Write in-memory history header to appropriate backing file.
* Overwrites existing header data.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
herr_t
H5FD__onion_write_header(H5FD_onion_header_t *header, H5FD_t *file)
{
uint32_t sum = 0; /* Not used, but required by the encoder */
uint64_t size = 0;
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer for updated history header")
if (0 == (size = H5FD__onion_header_encode(header, buf, &sum)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding updated history header")
if (H5FD_write(file, H5FD_MEM_DRAW, 0, (haddr_t)size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write updated history header")
done:
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_write_header()*/
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_header_decode
*
* Purpose: Attempt to read a buffer and store it as a history-header
* structure.
*
* Implementation must correspond with
* H5FD__onion_header_encode().
*
* Return: Success: Number of bytes read from buffer
* Failure: 0
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_header_decode(unsigned char *buf, H5FD_onion_header_t *header)
{
uint32_t ui32 = 0;
uint32_t sum = 0;
uint64_t ui64 = 0;
uint8_t *ui8p = NULL;
unsigned char *ptr = NULL;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE;
HDassert(buf != NULL);
HDassert(header != NULL);
HDassert(H5FD_ONION_HEADER_VERSION_CURR == header->version);
if (HDstrncmp((const char *)buf, H5FD_ONION_HEADER_SIGNATURE, 4))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header signature")
if (buf[4] != H5FD_ONION_HEADER_VERSION_CURR)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header version")
ptr = buf + 5;
ui32 = 0;
HDmemcpy(&ui32, ptr, 3);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, header->flags);
ptr += 3;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, header->page_size);
ptr += 4;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT32DECODE(ui8p, header->origin_eof);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT32DECODE(ui8p, header->history_addr);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT32DECODE(ui8p, header->history_size);
ptr += 8;
sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, header->checksum);
ptr += 4;
if (sum != header->checksum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch")
ret_value = (size_t)(ptr - buf);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_header_decode() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_header_encode
*
* Purpose: Write history-header structure to the given buffer.
* All multi-byte elements are stored in little-endian word order.
*
* Implementation must correspond with
* H5FD__onion_header_decode().
*
* The destination buffer must be sufficiently large to hold the
* encoded contents (H5FD_ONION_ENCODED_SIZE_HEADER).
*
* Return: Number of bytes written to buffer.
* The checksum of the generated buffer contents (excluding the
* checksum itself) is stored in the pointer `checksum`).
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_header_encode(H5FD_onion_header_t *header, unsigned char *buf, uint32_t *checksum /*out*/)
{
unsigned char *ptr = buf;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(buf != NULL);
HDassert(checksum != NULL);
HDassert(header != NULL);
HDassert(H5FD_ONION_HEADER_VERSION_CURR == header->version);
HDassert(0 == (header->flags & 0xFF000000)); /* max three bits long */
HDmemcpy(ptr, H5FD_ONION_HEADER_SIGNATURE, 4);
ptr += 4;
HDmemcpy(ptr, (unsigned char *)&header->version, 1);
ptr += 1;
UINT32ENCODE(ptr, header->flags);
ptr -= 1; /* truncate to three bytes */
UINT32ENCODE(ptr, header->page_size);
UINT64ENCODE(ptr, header->origin_eof);
UINT64ENCODE(ptr, header->history_addr);
UINT64ENCODE(ptr, header->history_size);
*checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
UINT32ENCODE(ptr, *checksum);
ret_value = (size_t)(ptr - buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_header_encode() */
src/H5FDonion_header.h
+56
View File
@@ -0,0 +1,56 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Interface for the onion file's header
*/
#ifndef H5FDonion_header_H
#define H5FDonion_header_H
/* Number of bytes to encode fixed-size components */
#define H5FD_ONION_ENCODED_SIZE_HEADER 40
/* Flags must align exactly one per bit, up to 24 bits */
#define H5FD_ONION_HEADER_FLAG_WRITE_LOCK 0x1
#define H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY 0x2
#define H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT 0x4
#define H5FD_ONION_HEADER_SIGNATURE "OHDH"
#define H5FD_ONION_HEADER_VERSION_CURR 1
/* In-memory representation of the on-store onion history file header.
*/
typedef struct H5FD_onion_header_t {
uint8_t version;
uint32_t flags; /* At most three bytes used! */
uint32_t page_size;
uint64_t origin_eof; /* Size of the 'original' canonical file */
uint64_t history_addr;
uint64_t history_size;
uint32_t checksum;
} H5FD_onion_header_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL herr_t H5FD__onion_ingest_header(H5FD_onion_header_t *hdr_out, H5FD_t *raw_file, haddr_t addr);
H5_DLL herr_t H5FD__onion_write_header(H5FD_onion_header_t *header, H5FD_t *file);
H5_DLL size_t H5FD__onion_header_decode(unsigned char *buf, H5FD_onion_header_t *header);
H5_DLL size_t H5FD__onion_header_encode(H5FD_onion_header_t *header, unsigned char *buf, uint32_t *checksum);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_header_H */
src/H5FDonion_history.c
+305
View File
@@ -0,0 +1,305 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Code for the onion file's history
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_write_history
*
* Purpose: Read and decode the history information from `raw_file` at
* `addr` .. `addr + size` (taken from history header), and store
* the decoded information in the structure at `history_out`.
*
* Returns: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_ingest_history(H5FD_onion_history_t *history_out, H5FD_t *raw_file, haddr_t addr, haddr_t size)
{
unsigned char *buf = NULL;
uint32_t sum = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(history_out);
HDassert(raw_file);
/* Set early so we can clean up properly on errors */
history_out->record_locs = NULL;
if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "header indicates history beyond EOF");
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer space");
if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA");
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read history from file");
if (H5FD__onion_history_decode(buf, history_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history (initial)");
sum = H5_checksum_fletcher32(buf, size - 4);
if (history_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored");
if (history_out->n_revisions > 0)
if (NULL == (history_out->record_locs =
H5MM_calloc(history_out->n_revisions * sizeof(H5FD_onion_record_loc_t))))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate record pointer list");
if (H5FD__onion_history_decode(buf, history_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history (final)");
done:
H5MM_xfree(buf);
if (ret_value < 0)
H5MM_xfree(history_out->record_locs);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_ingest_history() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_write_history
*
* Purpose: Encode and write history to file at the given address.
*
* Returns: Success: Number of bytes written to destination file (always non-zero)
* Failure: 0
*-----------------------------------------------------------------------------
*/
uint64_t
H5FD__onion_write_history(H5FD_onion_history_t *history, H5FD_t *file, haddr_t off_start,
haddr_t filesize_curr)
{
uint32_t _sum = 0; /* Required by the API call but unused here */
uint64_t size = 0;
unsigned char *buf = NULL;
uint64_t ret_value = 0;
FUNC_ENTER_PACKAGE;
if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY +
(H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * history->n_revisions))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, 0, "can't allocate buffer for updated history")
if (0 == (size = H5FD__onion_history_encode(history, buf, &_sum)))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, 0, "problem encoding updated history")
if ((size + off_start > filesize_curr) && (H5FD_set_eoa(file, H5FD_MEM_DRAW, off_start + size) < 0))
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, 0, "can't modify EOA for updated history")
if (H5FD_write(file, H5FD_MEM_DRAW, off_start, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, 0, "can't write history as intended")
ret_value = size;
done:
H5MM_xfree(buf);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_write_history() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_history_decode
*
* Purpose: Attempt to read a buffer and store it as a history
* structure.
*
* Implementation must correspond with
* H5FD__onion_history_encode().
*
* MUST BE CALLED TWICE:
* On the first call, n_records in the destination structure must
* be zero, and record_locs be NULL.
*
* If the buffer is well-formed, the destination structure is
* tentatively populated with fixed-size values, and the number of
* bytes read are returned.
*
* Prior to the second call, the user must allocate space for
* record_locs to hold n_records record-pointer structs.
*
* Then the decode operation is called a second time, and all
* components will be populated (and again number of bytes read is
* returned).
*
* Return: Success: Number of bytes read from buffer
* Failure: 0
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_history_decode(unsigned char *buf, H5FD_onion_history_t *history)
{
uint32_t ui32 = 0;
uint32_t sum = 0;
uint64_t ui64 = 0;
uint64_t n_revisions = 0;
uint8_t *ui8p = NULL;
unsigned char *ptr = NULL;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE;
HDassert(buf != NULL);
HDassert(history != NULL);
HDassert(H5FD_ONION_HISTORY_VERSION_CURR == history->version);
if (HDstrncmp((const char *)buf, H5FD_ONION_HISTORY_SIGNATURE, 4))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature")
if (H5FD_ONION_HISTORY_VERSION_CURR != buf[4])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid version")
ptr = buf + 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, n_revisions);
ptr += 8;
if (0 == history->n_revisions) {
history->n_revisions = n_revisions;
ptr += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * n_revisions;
}
else {
if (history->n_revisions != n_revisions)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0,
"history argument suggests different revision count than encoded buffer")
if (NULL == history->record_locs)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "list is NULL -- cannot populate")
for (uint64_t i = 0; i < n_revisions; i++) {
H5FD_onion_record_loc_t *rloc = &history->record_locs[i];
/* Decode into appropriately sized types, then do a checked
* assignment to the struct value. We don't have access to
* the H5F_t struct for this file, so we can't use the
* offset/length macros in H5Fprivate.h.
*/
uint64_t record_size;
uint64_t phys_addr;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, phys_addr);
H5_CHECKED_ASSIGN(rloc->phys_addr, haddr_t, phys_addr, uint64_t);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record_size);
H5_CHECKED_ASSIGN(rloc->record_size, hsize_t, record_size, uint64_t);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, rloc->checksum);
ptr += 4;
}
}
sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, history->checksum);
ptr += 4;
if (sum != history->checksum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch")
ret_value = (size_t)(ptr - buf);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_history_decode() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_history_encode
*
* Purpose: Write history structure to the given buffer.
* All multi-byte elements are stored in little-endian word order.
*
* Implementation must correspond with
* H5FD__onion_history_decode().
*
* The destination buffer must be sufficiently large to hold the
* encoded contents.
* (Hint: `sizeof(history struct) +
* sizeof(record-pointer-struct) * n_records)` guarantees
* ample/excess space.)
*
* Return: Number of bytes written to buffer.
* The checksum of the generated buffer contents (excluding the
* checksum itself) is stored in the pointer `checksum`).
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_history_encode(H5FD_onion_history_t *history, unsigned char *buf, uint32_t *checksum)
{
unsigned char *ptr = buf;
size_t vers_u32 = (uint32_t)history->version; /* pad out unused bytes */
FUNC_ENTER_PACKAGE_NOERR;
HDassert(history != NULL);
HDassert(H5FD_ONION_HISTORY_VERSION_CURR == history->version);
HDassert(buf != NULL);
HDassert(checksum != NULL);
HDmemcpy(ptr, H5FD_ONION_HISTORY_SIGNATURE, 4);
ptr += 4;
UINT32ENCODE(ptr, vers_u32);
UINT64ENCODE(ptr, history->n_revisions);
if (history->n_revisions > 0) {
HDassert(history->record_locs != NULL);
for (uint64_t i = 0; i < history->n_revisions; i++) {
H5FD_onion_record_loc_t *rloc = &history->record_locs[i];
/* Do a checked assignment from the struct value into appropriately
* sized types. We don't have access to the H5F_t struct for this
* file, so we can't use the offset/length macros in H5Fprivate.h.
*/
uint64_t phys_addr;
uint64_t record_size;
H5_CHECKED_ASSIGN(phys_addr, uint64_t, rloc->phys_addr, haddr_t);
H5_CHECKED_ASSIGN(record_size, uint64_t, rloc->record_size, hsize_t);
UINT64ENCODE(ptr, phys_addr);
UINT64ENCODE(ptr, record_size);
UINT32ENCODE(ptr, rloc->checksum);
}
}
*checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
UINT32ENCODE(ptr, *checksum);
FUNC_LEAVE_NOAPI((size_t)(ptr - buf));
} /* end H5FD__onion_history_encode() */
src/H5FDonion_history.h
+63
View File
@@ -0,0 +1,63 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Interface for the onion file's history
*/
#ifndef H5FDonion_history_H
#define H5FDonion_history_H
/* Number of bytes to encode fixed-size components */
#define H5FD_ONION_ENCODED_SIZE_HISTORY 20
#define H5FD_ONION_HISTORY_SIGNATURE "OWHS"
#define H5FD_ONION_HISTORY_VERSION_CURR 1
/* In-memory representation of the on-store revision record.
* Used in the history.
*/
typedef struct H5FD_onion_record_loc_t {
haddr_t phys_addr;
hsize_t record_size;
uint32_t checksum;
} H5FD_onion_record_loc_t;
/* In-memory representation of the on-store history record/summary.
*/
typedef struct H5FD_onion_history_t {
uint8_t version;
uint64_t n_revisions;
H5FD_onion_record_loc_t *record_locs;
uint32_t checksum;
} H5FD_onion_history_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL herr_t H5FD__onion_ingest_history(H5FD_onion_history_t *history_out, H5FD_t *raw_file, haddr_t addr,
haddr_t size);
H5_DLL uint64_t H5FD__onion_write_history(H5FD_onion_history_t *history, H5FD_t *file, haddr_t off_start,
haddr_t filesize_curr);
H5_DLL size_t H5FD__onion_history_decode(unsigned char *buf, H5FD_onion_history_t *history);
H5_DLL size_t H5FD__onion_history_encode(H5FD_onion_history_t *history, unsigned char *buf,
uint32_t *checksum);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_history_H */
src/H5FDonion_index.c
+935
View File
@@ -0,0 +1,935 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Code for the archival and revision indexes
*/
/* This source code file is part of the H5FD driver module */
#include "H5FDdrvr_module.h"
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDonion.h" /* Onion file driver */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */
#define U64_EXP2(n) ((uint64_t)1 << (n))
static int H5FD__onion_archival_index_list_sort_cmp(const void *, const void *);
static herr_t H5FD__onion_revision_index_resize(H5FD_onion_revision_index_t *rix);
/*-----------------------------------------------------------------------------
* Read and decode the revision_record information from `raw_file` at
* `addr` .. `addr + size` (taken from history), and store the decoded
* information in the structure at `r_out`.
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_ingest_revision_record(H5FD_onion_revision_record_t *r_out, H5FD_t *raw_file,
const H5FD_onion_history_t *history, uint64_t revision_num)
{
unsigned char *buf = NULL;
herr_t ret_value = SUCCEED;
uint64_t n = 0;
uint64_t high = 0;
uint64_t low = 0;
uint64_t range = 0;
uint32_t sum = 0;
haddr_t addr = 0;
size_t size = 0;
FUNC_ENTER_PACKAGE;
HDassert(r_out);
HDassert(raw_file);
HDassert(history);
HDassert(history->record_locs);
HDassert(history->n_revisions > 0);
high = history->n_revisions - 1;
range = high;
addr = history->record_locs[high].phys_addr;
size = history->record_locs[high].record_size;
/* Initialize r_out
*
* TODO: This function should completely initialize r_out. Relying on
* other code to some of the work while we just paste over parts
* of the struct here is completely bananas.
*/
r_out->comment = H5MM_xfree(r_out->comment);
r_out->archival_index.list = H5MM_xfree(r_out->archival_index.list);
if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "at least one record extends beyond EOF")
/* recovery-open may have EOA below revision record */
if ((H5FD_get_eoa(raw_file, H5FD_MEM_DRAW) < (addr + size)) &&
(H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)) {
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA");
}
/* Perform binary search on records to find target revision by ID.
* As IDs are added sequentially, they are guaranteed to be sorted.
*/
while (range > 0) {
n = (range / 2) + low;
addr = history->record_locs[n].phys_addr;
size = history->record_locs[n].record_size;
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space")
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read revision record from file")
if (H5FD__onion_revision_record_decode(buf, r_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (initial)")
sum = H5_checksum_fletcher32(buf, size - 4);
if (r_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored")
if (revision_num == r_out->revision_num)
break;
H5MM_xfree(buf);
buf = NULL;
r_out->archival_index.n_entries = 0;
r_out->comment_size = 0;
if (r_out->revision_num < revision_num)
low = (n == high) ? high : n + 1;
else
high = (n == low) ? low : n - 1;
range = high - low;
} /* end while 'non-leaf' binary search */
if (range == 0) {
n = low;
addr = history->record_locs[n].phys_addr;
size = history->record_locs[n].record_size;
if (NULL == (buf = H5MM_malloc(sizeof(char) * size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space")
if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read revision record from file")
if (H5FD__onion_revision_record_decode(buf, r_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (initial)")
sum = H5_checksum_fletcher32(buf, size - 4);
if (r_out->checksum != sum)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored")
if (revision_num != r_out->revision_num)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL,
"could not find target revision!") /* TODO: corrupted? */
} /* end if revision ID at 'leaf' in binary search */
if (r_out->comment_size > 0)
if (NULL == (r_out->comment = H5MM_malloc(sizeof(char) * r_out->comment_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate comment space")
if (r_out->archival_index.n_entries > 0)
if (NULL == (r_out->archival_index.list =
H5MM_calloc(r_out->archival_index.n_entries * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate index entry list")
if (H5FD__onion_revision_record_decode(buf, r_out) != size)
HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (final)")
done:
H5MM_xfree(buf);
if (ret_value == FAIL) {
H5MM_xfree(r_out->comment);
H5MM_xfree(r_out->archival_index.list);
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_ingest_revision_record() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_archival_index_is_valid
*
* Purpose: Determine whether an archival index structure is valid.
*
* + Verify page size (power of two).
* + Verify list exists.
* + Verify list contents:
* + Sorted by increasing logical address (no duplicates)
* + Logical addresses are multiples of page size.
*
* Return: TRUE/FALSE
*-----------------------------------------------------------------------------
*/
hbool_t
H5FD__onion_archival_index_is_valid(const H5FD_onion_archival_index_t *aix)
{
hbool_t ret_value = TRUE;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(aix);
if (H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR != aix->version)
HGOTO_DONE(FALSE)
if (NULL == aix->list)
HGOTO_DONE(FALSE)
/* Ensure list is sorted on logical_page field */
if (aix->n_entries > 1)
for (uint64_t i = 1; i < aix->n_entries - 1; i++)
if (aix->list[i + 1].logical_page <= aix->list[i].logical_page)
HGOTO_DONE(FALSE)
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_archival_index_is_valid() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_archival_index_find
*
* Purpose: Retrieve the archival index entry by logical page ID.
*
* The archival index pointer must point to a valid index entry.
* The entry out pointer-pointer cannot be null.
*
* Return: Success: Positive value (1) -- entry found.
* Entry out pointer-pointer is set to point to entry.
* Failure: Zero (0) -- entry not found.
* Entry out pointer-pointer is unmodified.
*-----------------------------------------------------------------------------
*/
int
H5FD__onion_archival_index_find(const H5FD_onion_archival_index_t *aix, uint64_t logical_page,
const H5FD_onion_index_entry_t **entry_out)
{
uint64_t low = 0;
uint64_t high = 0;
uint64_t n = 0;
uint64_t range = 0;
H5FD_onion_index_entry_t *x = NULL;
int ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(aix);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version);
HDassert(entry_out);
if (aix->n_entries != 0)
HDassert(aix->list);
high = aix->n_entries - 1;
range = high;
/* Trivial cases */
if (aix->n_entries == 0 || logical_page > aix->list[high].logical_page ||
logical_page < aix->list[0].logical_page)
HGOTO_DONE(0)
/*
* Binary search on sorted list
*/
/* Winnow down to first of found or one element */
while (range > 0) {
HDassert(high < aix->n_entries);
n = low + (range / 2);
x = &(aix->list[n]);
if (x->logical_page == logical_page) {
*entry_out = x; /* element found at fence */
ret_value = 1;
goto done;
}
else if (x->logical_page < logical_page) {
low = (n == high) ? high : n + 1;
}
else {
high = (n == low) ? low : n - 1;
}
range = high - low;
}
HDassert(high == low); /* one element */
/* n == low/high check because we may have tested it already above */
if ((n != low || n != high) && (aix->list[low].logical_page == logical_page)) {
*entry_out = &aix->list[low];
ret_value = 1;
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_archival_index_find() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_destroy
*
* Purpose: Release all resources of a revision index.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_revision_index_destroy(H5FD_onion_revision_index_t *rix)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
for (size_t i = 0; 0 < rix->_hash_table_n_keys_populated && i < rix->_hash_table_size; i++) {
H5FD_onion_revision_index_hash_chain_node_t *next = NULL;
H5FD_onion_revision_index_hash_chain_node_t *node = rix->_hash_table[i];
if (node != NULL)
rix->_hash_table_n_keys_populated -= 1;
while (node != NULL) {
HDassert(H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR == node->version);
next = node->next;
H5MM_xfree(node);
node = next;
}
}
H5MM_xfree(rix->_hash_table);
H5MM_xfree(rix);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_destroy() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_init
*
* Purpose: Initialize a revision index structure with a default starting
* size. A new structure is allocated and populated with initial
* values.
*
* Return: Success: Pointer to newly-allocated structure
* Failure: NULL
*-----------------------------------------------------------------------------
*/
H5FD_onion_revision_index_t *
H5FD__onion_revision_index_init(uint32_t page_size)
{
uint64_t table_size = U64_EXP2(H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2);
H5FD_onion_revision_index_t *rix = NULL;
H5FD_onion_revision_index_t *ret_value = NULL;
FUNC_ENTER_PACKAGE;
HDassert(0 != page_size);
HDassert(POWER_OF_TWO(page_size));
if (NULL == (rix = H5MM_calloc(sizeof(H5FD_onion_revision_index_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "cannot allocate index")
if (NULL ==
(rix->_hash_table = H5MM_calloc(table_size * sizeof(H5FD_onion_revision_index_hash_chain_node_t *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "cannot allocate hash table")
rix->version = H5FD_ONION_REVISION_INDEX_VERSION_CURR;
rix->n_entries = 0;
/* Compute and store log2(page_size) */
for (rix->page_size_log2 = 0; (((uint32_t)1 << rix->page_size_log2) & page_size) == 0;
rix->page_size_log2++)
;
rix->_hash_table_size = table_size;
rix->_hash_table_size_log2 = H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2;
rix->_hash_table_n_keys_populated = 0;
ret_value = rix;
done:
if (NULL == ret_value)
H5MM_xfree(rix);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_init() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_resize()
*
* Purpose: Replace the hash table in the revision index.
*
* Doubles the available number of keys, re-hashes table contents,
* and updates relevant components in the index structure.
*
* Fails if unable to allocate space for larger hash table.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
static herr_t
H5FD__onion_revision_index_resize(H5FD_onion_revision_index_t *rix)
{
H5FD_onion_revision_index_hash_chain_node_t **new_table = NULL;
uint64_t new_size_log2 = rix->_hash_table_size_log2 + 1;
uint64_t new_size = U64_EXP2(new_size_log2);
uint64_t new_n_keys_populated = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(rix->_hash_table);
if (NULL == (new_table = H5MM_calloc(new_size * sizeof(H5FD_onion_revision_index_hash_chain_node_t *))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate new hash table")
for (uint64_t i = 0; i < rix->_hash_table_size; i++) {
while (rix->_hash_table[i] != NULL) {
H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
uint64_t key = 0;
/* Pop entry off of bucket stack and re-hash */
node = rix->_hash_table[i];
rix->_hash_table[i] = node->next;
node->next = NULL;
key = node->entry_data.logical_page & (new_size - 1);
if (NULL == new_table[key]) {
new_table[key] = node;
new_n_keys_populated++;
}
else {
node->next = new_table[i];
new_table[i] = node;
}
}
}
H5MM_xfree(rix->_hash_table);
rix->_hash_table_size = new_size;
rix->_hash_table_size_log2 = new_size_log2;
rix->_hash_table_n_keys_populated = new_n_keys_populated;
rix->_hash_table = new_table;
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_resize() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_insert()
*
* Purpose: Add an entry to the revision index, or update an existing
* entry. Must be used to update entries as well as add --
* checksum value will change.
*
* Entry data is copied into separate memory region; user pointer
* can be safley re-used or discarded after operation.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_revision_index_insert(H5FD_onion_revision_index_t *rix, const H5FD_onion_index_entry_t *entry)
{
uint64_t key = 0;
H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
H5FD_onion_revision_index_hash_chain_node_t **append_dest = NULL;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(entry);
/* Resize and re-hash table if necessary */
if (rix->n_entries >= (rix->_hash_table_size * 2) ||
rix->_hash_table_n_keys_populated >= (rix->_hash_table_size / 2)) {
if (H5FD__onion_revision_index_resize(rix) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NONE_MINOR, FAIL, "unable to resize and hash table")
}
key = entry->logical_page & (rix->_hash_table_size - 1);
HDassert(key < rix->_hash_table_size);
if (NULL == rix->_hash_table[key]) {
/* Key maps to empty bucket */
append_dest = &rix->_hash_table[key];
rix->_hash_table_n_keys_populated++;
}
else {
/* Key maps to populated bucket */
for (node = rix->_hash_table[key]; node != NULL; node = node->next) {
append_dest = &node->next; /* look for bucket tail */
if (entry->logical_page == node->entry_data.logical_page) {
if (entry->phys_addr != node->entry_data.phys_addr) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "physical address mismatch");
}
HDmemcpy(&node->entry_data, entry, sizeof(H5FD_onion_index_entry_t));
append_dest = NULL; /* Node updated, do not append */
break;
}
}
}
/* Add new entry to bucket chain */
if (append_dest != NULL) {
if (NULL == (node = H5MM_malloc(sizeof(H5FD_onion_revision_index_hash_chain_node_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate new ash chain node")
node->version = H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR;
node->next = NULL;
HDmemcpy(&node->entry_data, entry, sizeof(H5FD_onion_index_entry_t));
*append_dest = node;
rix->n_entries++;
}
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_insert() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_index_find()
*
*
* Purpose: Get pointer to revision index entry with the given page number,
* if it exists in the index.
*
* Return: Success: Positive value (1) -- entry found.
* Entry out pointer-pointer is set to point to entry.
* Failure: Zero (0) -- entry not found.
* Entry out pointer-pointer is unmodified.
*-----------------------------------------------------------------------------
*/
int
H5FD__onion_revision_index_find(const H5FD_onion_revision_index_t *rix, uint64_t logical_page,
const H5FD_onion_index_entry_t **entry_out)
{
uint64_t key = 0;
int ret_value = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(rix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(rix->_hash_table);
HDassert(entry_out);
key = logical_page & (rix->_hash_table_size - 1);
HDassert(key < rix->_hash_table_size);
if (rix->_hash_table[key] != NULL) {
H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
for (node = rix->_hash_table[key]; node != NULL; node = node->next) {
if (logical_page == node->entry_data.logical_page) {
*entry_out = &node->entry_data;
ret_value = 1;
break;
}
}
}
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_index_find() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_record_decode
*
* Purpose: Attempt to read a buffer and store it as a revision record
* structure.
*
* Implementation must correspond with
* H5FD__onion_revision_record_encode().
*
* MUST BE CALLED TWICE:
* On the first call, n_entries and comment_size in the
* destination structure must all all be zero, and their
* respective variable-length components (index_entry_list,
* comment) must all be NULL.
*
* If the buffer is well-formed, the destination structure is
* tentatively populated with fixed-size values, and the number of
* bytes read are returned.
*
* Prior to the second call, the user must allocate space for the
* variable-length components, in accordance with the associated
* indicators (array of index-entry structures for
* index_entry_list, of size n_entries; character arrays for
* comment, allocated with the *_size number of bytes -- space
* for NULL-terminator is included in _size).
*
* Then the decode operation is called a second time, and all
* components will be populated (and again number of bytes read is
* returned).
*
* Return: Success: Number of bytes read from buffer
* Failure: 0
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_revision_record_decode(unsigned char *buf, H5FD_onion_revision_record_t *record)
{
uint32_t ui32 = 0;
uint32_t page_size = 0;
uint32_t sum = 0;
uint64_t ui64 = 0;
uint64_t n_entries = 0;
uint32_t comment_size = 0;
uint8_t *ui8p = NULL;
unsigned char *ptr = NULL;
size_t ret_value = 0;
FUNC_ENTER_PACKAGE;
HDassert(buf != NULL);
HDassert(record != NULL);
HDassert(H5FD_ONION_REVISION_RECORD_VERSION_CURR == record->version);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version);
if (HDstrncmp((const char *)buf, H5FD_ONION_REVISION_RECORD_SIGNATURE, 4))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature")
if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != buf[4])
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid record version")
ptr = buf + 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record->revision_num);
ptr += 8;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record->parent_revision_num);
ptr += 8;
HDmemcpy(record->time_of_creation, ptr, 16);
ptr += 16;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, record->logical_eof);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, page_size);
ptr += 4;
if (page_size == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size is zero")
if (!POWER_OF_TWO(page_size))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size not power of two")
for (record->archival_index.page_size_log2 = 0;
(((uint32_t)1 << record->archival_index.page_size_log2) & page_size) == 0;
record->archival_index.page_size_log2++)
;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, n_entries);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, comment_size);
ptr += 4;
if (record->archival_index.n_entries == 0) {
record->archival_index.n_entries = n_entries;
ptr += H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * n_entries;
}
else if (n_entries != record->archival_index.n_entries) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "n_entries in archival index does not match decoded")
}
else {
H5FD_onion_index_entry_t *entry = NULL;
if (record->archival_index.list == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no archival index entry list")
for (size_t i = 0; i < n_entries; i++) {
entry = &record->archival_index.list[i];
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, entry->logical_page);
ptr += 8;
/* logical_page actually encoded as address; check and convert */
if (entry->logical_page & (page_size - 1))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "logical address does not align with page size")
entry->logical_page = entry->logical_page >> record->archival_index.page_size_log2;
HDmemcpy(&ui64, ptr, 8);
ui8p = (uint8_t *)&ui64;
UINT64DECODE(ui8p, entry->phys_addr);
ptr += 8;
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, sum);
ptr += 4;
ui32 = H5_checksum_fletcher32((ptr - 20), 16);
if (ui32 != sum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "index entry checksum mismatch")
}
}
if (record->comment_size == 0) {
if (record->comment != NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "comment pointer prematurely allocated")
record->comment_size = comment_size;
}
else {
if (record->comment == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no comment pointer")
HDmemcpy(record->comment, ptr, comment_size);
}
ptr += comment_size;
sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
HDmemcpy(&ui32, ptr, 4);
ui8p = (uint8_t *)&ui32;
UINT32DECODE(ui8p, record->checksum);
ptr += 4;
if (sum != record->checksum)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch")
ret_value = (size_t)(ptr - buf);
done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_revision_record_decode() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_revision_record_encode
*
* Purpose: Write revision-record structure to the given buffer.
* All multi-byte elements are stored in little-endian word order.
*
* Implementation must correspond with
* H5FD__onion_revision_record_decode().
*
* The destination buffer must be sufficiently large to hold the
* encoded contents.
* (Hint: `sizeof(revision-record-struct) + comment-size +
* sizeof(index-entry-struct) * n_entries)`
* guarantees ample/excess space.)
*
* Return: Number of bytes written to buffer.
* The checksum of the generated buffer contents (excluding the
* checksum itself) is stored in the pointer `checksum`).
*-----------------------------------------------------------------------------
*/
size_t
H5FD__onion_revision_record_encode(H5FD_onion_revision_record_t *record, unsigned char *buf,
uint32_t *checksum)
{
unsigned char *ptr = buf; /* original pointer */
uint32_t vers_u32 = (uint32_t)record->version; /* pad out unused bytes */
uint32_t page_size = 0;
FUNC_ENTER_PACKAGE_NOERR;
HDassert(checksum != NULL);
HDassert(buf != NULL);
HDassert(record != NULL);
HDassert(vers_u32 < 0x100);
HDassert(H5FD_ONION_REVISION_RECORD_VERSION_CURR == record->version);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version);
page_size = (uint32_t)(1 << record->archival_index.page_size_log2);
HDmemcpy(ptr, H5FD_ONION_REVISION_RECORD_SIGNATURE, 4);
ptr += 4;
UINT32ENCODE(ptr, vers_u32);
UINT64ENCODE(ptr, record->revision_num);
UINT64ENCODE(ptr, record->parent_revision_num);
HDmemcpy(ptr, record->time_of_creation, 16);
ptr += 16;
UINT64ENCODE(ptr, record->logical_eof);
UINT32ENCODE(ptr, page_size);
UINT64ENCODE(ptr, record->archival_index.n_entries);
UINT32ENCODE(ptr, record->comment_size);
if (record->archival_index.n_entries > 0) {
uint64_t page_size_log2 = record->archival_index.page_size_log2;
HDassert(record->archival_index.list != NULL);
for (uint64_t i = 0; i < record->archival_index.n_entries; i++) {
uint32_t sum = 0;
H5FD_onion_index_entry_t *entry = NULL;
uint64_t logi_addr = 0;
entry = &record->archival_index.list[i];
logi_addr = entry->logical_page << page_size_log2;
UINT64ENCODE(ptr, logi_addr);
UINT64ENCODE(ptr, entry->phys_addr);
sum = H5_checksum_fletcher32((ptr - 16), 16);
UINT32ENCODE(ptr, sum);
}
}
if (record->comment_size > 0) {
HDassert(record->comment != NULL && *record->comment != '\0');
HDmemcpy(ptr, record->comment, record->comment_size);
ptr += record->comment_size;
}
*checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf));
UINT32ENCODE(ptr, *checksum);
FUNC_LEAVE_NOAPI((size_t)(ptr - buf));
} /* end H5FD__onion_revision_record_encode() */
/*-----------------------------------------------------------------------------
* Callback for comparisons in sorting archival index entries by logical_page.
*-----------------------------------------------------------------------------
*/
static int
H5FD__onion_archival_index_list_sort_cmp(const void *_a, const void *_b)
{
const H5FD_onion_index_entry_t *a = (const H5FD_onion_index_entry_t *)_a;
const H5FD_onion_index_entry_t *b = (const H5FD_onion_index_entry_t *)_b;
if (a->logical_page < b->logical_page)
return -1;
else if (a->logical_page > b->logical_page)
return 1;
return 0;
} /* end H5FD__onion_archival_index_list_sort_cmp() */
/*-----------------------------------------------------------------------------
* Function: H5FD__onion_merge_revision_index_into_archival_index
*
* Purpose: Merge index entries from revision index into archival index.
*
* If successful, the archival index is expanded 'behind the
* scenes' and new entries from the revision index are inserted.
* The archival index remains sorted in ascending order of logical
* address.
*
* The conversion to archival index changes logical pages in
* revision index entries to their logical addresses in-file.
*
* Return: SUCCEED/FAIL
*-----------------------------------------------------------------------------
*/
herr_t
H5FD__onion_merge_revision_index_into_archival_index(const H5FD_onion_revision_index_t *rix,
H5FD_onion_archival_index_t *aix)
{
uint64_t n_kept = 0;
H5FD_onion_index_entry_t *kept_list = NULL;
H5FD_onion_archival_index_t new_aix = {
H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, 0, /* page_size_log2 tbd */
0, /* n_entries */
NULL, /* list pointer (allocated later) */
};
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE;
HDassert(rix);
HDassert(aix);
HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version);
HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version);
HDassert(aix->page_size_log2 == rix->page_size_log2);
/* If the revision index is empty there is nothing to archive */
if (rix->n_entries == 0)
goto done;
/* Add all revision index entries to new archival list */
new_aix.page_size_log2 = aix->page_size_log2;
if (NULL == (new_aix.list = H5MM_calloc(rix->n_entries * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate new archival index list")
for (uint64_t i = 0; i < rix->_hash_table_size; i++) {
const H5FD_onion_revision_index_hash_chain_node_t *node = NULL;
for (node = rix->_hash_table[i]; node != NULL; node = node->next) {
HDmemcpy(&new_aix.list[new_aix.n_entries], &node->entry_data, sizeof(H5FD_onion_index_entry_t));
new_aix.n_entries++;
}
}
/* Sort the new archival list */
HDqsort(new_aix.list, new_aix.n_entries, sizeof(H5FD_onion_index_entry_t),
H5FD__onion_archival_index_list_sort_cmp);
/* Add the old archival index entries to a 'kept' list containing the
* old archival list entries that are not also included in the revision
* list.
*
* Note that kept_list will be NULL if there are no entries in the passed-in
* archival list.
*/
if (aix->n_entries > 0)
if (NULL == (kept_list = H5MM_calloc(aix->n_entries * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate larger archival index list")
for (uint64_t i = 0; i < aix->n_entries; i++) {
const H5FD_onion_index_entry_t *entry = NULL;
/* Add only if page not already added from revision index */
if (H5FD__onion_archival_index_find(&new_aix, aix->list[i].logical_page, &entry) == 0) {
HDmemcpy(&kept_list[n_kept], &aix->list[i], sizeof(H5FD_onion_index_entry_t));
n_kept++;
}
}
/* Destroy the old archival list and replace with a list big enough to hold
* the revision list entries and the kept list entries
*/
H5MM_xfree(aix->list);
if (NULL == (aix->list = H5MM_calloc((new_aix.n_entries + n_kept) * sizeof(H5FD_onion_index_entry_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate exact-size archival index list")
/* Copy (new) revision list entries to replacement list */
HDmemcpy(aix->list, new_aix.list, sizeof(H5FD_onion_index_entry_t) * new_aix.n_entries);
aix->n_entries = new_aix.n_entries;
/* Copy (old) kept archival list entries to replacement list */
if (n_kept > 0) {
HDmemcpy(&aix->list[aix->n_entries], kept_list, sizeof(H5FD_onion_index_entry_t) * n_kept);
aix->n_entries += n_kept;
}
/* Sort this list */
HDqsort(aix->list, aix->n_entries, sizeof(H5FD_onion_index_entry_t),
H5FD__onion_archival_index_list_sort_cmp);
done:
/* Free the temporary lists */
H5MM_xfree(kept_list);
H5MM_xfree(new_aix.list);
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD__onion_merge_revision_index_into_archival_index() */
src/H5FDonion_index.h
+150
View File
@@ -0,0 +1,150 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef H5FDonion_index_H
#define H5FDonion_index_H
#define H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR 1
/* Number of bytes to encode fixed-size components */
#define H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY 20
#define H5FD_ONION_ENCODED_SIZE_RECORD_POINTER 20
#define H5FD_ONION_ENCODED_SIZE_REVISION_RECORD 68
#define H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR 1
#define H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2 10 /* 2^n slots */
#define H5FD_ONION_REVISION_INDEX_VERSION_CURR 1
#define H5FD_ONION_REVISION_RECORD_SIGNATURE "ORRS"
#define H5FD_ONION_REVISION_RECORD_VERSION_CURR 1
/*
* Onion Virtual File Driver (VFD)
*
* Purpose: Interface for the archival and revision indexes
*/
/*-----------------------------------------------------------------------------
*
* Structure H5FD__onion_index_entry
*
* Purpose: Map a page in the logical file to a 'physical address' in the
* onion file.
*
* logical_page:
*
* Page 'id' in the logical file.
*
* phys_addr:
*
* Address/offset of start of page in the onion file.
*
*-----------------------------------------------------------------------------
*/
typedef struct H5FD_onion_index_entry_t {
uint64_t logical_page;
haddr_t phys_addr;
} H5FD_onion_index_entry_t;
/*-----------------------------------------------------------------------------
*
* Structure H5FD__onion_archival_index
*
* Purpose: Encapsulate archival index and associated data.
* Convenience structure with sanity-checking components.
*
* version: Future-proofing identifier. Informs struct membership.
* Must equal H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR to be
* considered valid.
*
* page_size: Interval to which the `logical_page` component of each list
* entry must align.
* Value is taken from the onion history data; must not change
* following onionization or file or creation of onion file.
*
* n_entries: Number of entries in the list.
*
* list: Pointer to array of archival index entries.
* Cannot be NULL.
* Entries must be sorted by `logical_page_id` in ascending order.
*
*-----------------------------------------------------------------------------
*/
typedef struct H5FD_onion_archival_index_t {
uint8_t version;
uint32_t page_size_log2;
uint64_t n_entries;
H5FD_onion_index_entry_t *list;
} H5FD_onion_archival_index_t;
/* data structure for storing index entries at a hash key collision */
/* version 1 implements a singly-linked list */
typedef struct H5FD_onion_revision_index_hash_chain_node_t H5FD_onion_revision_index_hash_chain_node_t;
struct H5FD_onion_revision_index_hash_chain_node_t {
uint8_t version;
H5FD_onion_index_entry_t entry_data;
H5FD_onion_revision_index_hash_chain_node_t *next;
};
typedef struct H5FD_onion_revision_index_t {
uint8_t version;
uint32_t page_size_log2;
uint64_t n_entries; /* count of all entries in table */
uint64_t _hash_table_size; /* 'slots' in hash table */
uint64_t _hash_table_size_log2; /* 2^(n) -> 'slots' in hash table */
uint64_t _hash_table_n_keys_populated; /* count of slots not NULL */
H5FD_onion_revision_index_hash_chain_node_t **_hash_table;
} H5FD_onion_revision_index_t;
/* In-memory representation of the on-store revision record.
*/
typedef struct H5FD_onion_revision_record_t {
uint8_t version;
uint64_t revision_num;
uint64_t parent_revision_num;
char time_of_creation[16];
uint64_t logical_eof;
H5FD_onion_archival_index_t archival_index;
uint32_t comment_size;
char *comment;
uint32_t checksum;
} H5FD_onion_revision_record_t;
#ifdef __cplusplus
extern "C" {
#endif
H5_DLL herr_t H5FD__onion_ingest_revision_record(H5FD_onion_revision_record_t *r_out, H5FD_t *raw_file,
const H5FD_onion_history_t *history, uint64_t revision_num);
H5_DLL hbool_t H5FD__onion_archival_index_is_valid(const H5FD_onion_archival_index_t *);
H5_DLL int H5FD__onion_archival_index_find(const H5FD_onion_archival_index_t *, uint64_t,
const H5FD_onion_index_entry_t **);
H5_DLL H5FD_onion_revision_index_t *H5FD__onion_revision_index_init(uint32_t page_size);
H5_DLL herr_t H5FD__onion_revision_index_destroy(H5FD_onion_revision_index_t *);
H5_DLL herr_t H5FD__onion_revision_index_insert(H5FD_onion_revision_index_t *,
const H5FD_onion_index_entry_t *);
H5_DLL int H5FD__onion_revision_index_find(const H5FD_onion_revision_index_t *, uint64_t,
const H5FD_onion_index_entry_t **);
H5_DLL herr_t H5FD__onion_merge_revision_index_into_archival_index(const H5FD_onion_revision_index_t *,
H5FD_onion_archival_index_t *);
H5_DLL size_t H5FD__onion_revision_record_decode(unsigned char *buf, H5FD_onion_revision_record_t *record);
H5_DLL size_t H5FD__onion_revision_record_encode(H5FD_onion_revision_record_t *record, unsigned char *buf,
uint32_t *checksum);
#ifdef __cplusplus
}
#endif
#endif /* H5FDonion_index_H */
src/H5FDonion_priv.h
+28
View File
@@ -0,0 +1,28 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD) Internals.
*
* Purpose: The private header file for the Onion VFD.
* Contains definitions and declarations used internallay and by
* tests.
*/
#ifndef H5FDonion_priv_H
#define H5FDonion_priv_H
#include "H5FDonion_header.h"
#include "H5FDonion_history.h"
#include "H5FDonion_index.h"
#endif /* H5FDonion_priv_H */
src/H5FDpublic.h
+1
View File
@@ -46,6 +46,7 @@
#define H5_VFD_ROS3 ((H5FD_class_value_t)(11))
#define H5_VFD_SUBFILING ((H5FD_class_value_t)(12))
#define H5_VFD_IOC ((H5FD_class_value_t)(13))
#define H5_VFD_ONION ((H5FD_class_value_t)(14))
/* VFD IDs below this value are reserved for library use. */
#define H5_VFD_RESERVED 256
src/H5private.h
+3
View File
@@ -1068,6 +1068,9 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation);
#ifndef HDlog
#define HDlog(X) log(X)
#endif
#ifndef HDlog2
#define HDlog2(X) log2(X)
#endif
#ifndef HDlog10
#define HDlog10(X) log10(X)
#endif
src/H5trace.c
+3
View File
@@ -1098,6 +1098,9 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap)
H5RS_acat(rs, "H5_VFD_ROS3");
break;
#endif
case H5_VFD_ONION:
H5RS_acat(rs, "H5_VFD_ONION");
break;
default:
H5RS_asprintf_cat(rs, "%ld", (long)class_val);
break;
src/Makefile.am
+7 -5
View File
@@ -61,8 +61,9 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \
H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \
H5FAint.c H5FAstat.c H5FAtest.c \
H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c \
H5FDmulti.c H5FDperform.c H5FDsec2.c H5FDspace.c \
H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmulti.c \
H5FDonion.c H5FDonion_header.c H5FDonion_history.c H5FDonion_index.c \
H5FDperform.c H5FDsec2.c H5FDspace.c \
H5FDsplitter.c H5FDstdio.c H5FDtest.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \
H5FSstat.c H5FStest.c \
@@ -154,9 +155,10 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers
H5Cpublic.h H5Dpublic.h \
H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \
H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \
H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \
H5FDsec2.h H5FDsplitter.h \
H5FDstdio.h H5FDwindows.h H5FDsubfiling/H5FDsubfiling.h H5FDsubfiling/H5FDioc.h \
H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h \
H5FDonion.h H5FDros3.h H5FDsec2.h H5FDsplitter.h \
H5FDstdio.h H5FDsubfiling/H5FDsubfiling.h H5FDsubfiling/H5FDioc.h \
H5FDwindows.h \
H5Gpublic.h H5Ipublic.h H5Lpublic.h \
H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \
H5PLextern.h H5PLpublic.h \
src/hdf5.h
+1
View File
@@ -64,6 +64,7 @@
#include "H5FDmirror.h" /* Mirror VFD and IPC definitions */
#include "H5FDmpi.h" /* MPI-based file drivers */
#include "H5FDmulti.h" /* Usage-partitioned file family */
#include "H5FDonion.h" /* Onion file I/O */
#include "H5FDros3.h" /* R/O S3 "file" I/O */
#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
#include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */
test/CMakeLists.txt
+1
View File
@@ -381,6 +381,7 @@ set (H5_TESTS
timer
cmpd_dtransform
event_set
onion
)
if (HDF5_BUILD_UTILS)
set (H5_TESTS ${H5_TESTS} mirror_vfd)
test/Makefile.am
+2 -1
View File
@@ -70,7 +70,8 @@ TEST_PROG= testhdf5 \
flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \
enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \
dangle dtransform reserved cross_read freespace mf vds file_image \
unregister cache_logging cork swmr thread_id vol timer event_set
unregister cache_logging cork swmr thread_id vol timer event_set \
onion
# List programs to be built when testing here
#
test/onion.c
+4966
View File
@@ -0,0 +1,4966 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Onion Virtual File Driver (VFD)
*
* Purpose:
*
* Verify Onion VFD behavior that is not involved with operations on the
* backing store.
*/
#include "h5test.h"
#include "H5Fprivate.h" /* encode/decode macros */
#include "H5FDonion.h" /* This file driver's utilities */
#include "H5FDonion_priv.h" /* Onion file driver internals */
/* The Onion VFD uses H5MM calls internally, so any tests that allocate
* or free memory for said internal structures (e.g., the revision lists)
* will need to allocate memory using H5MM calls.
*/
#include "H5MMprivate.h" /* Memory management */
/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */
#define U64_EXP2(n) ((uint64_t)1 << (n))
#define ONION_TEST_PAGE_SIZE_1 4
#define ONION_TEST_PAGE_SIZE_5 32
#define ONION_TEST_BASENAME_SIZE 32
#define ONION_TEST_FIXNAME_SIZE 1024
#define ONION_TEST_EXPECTED_HISTORY_REVISIONS_MAX 16
#define ONION_TEST_REV_REV_WRITES_MAX 8
#define ONE_DIM_SIZE 1024
/* Structure to collect the onion filepaths in one place. */
struct onion_filepaths {
char *canon;
char *onion;
char *recovery;
};
struct expected_revision {
uint64_t revision_num;
uint64_t parent_revision_num;
uint64_t logical_eof;
uint64_t n_index_entries;
const char *comment;
};
struct expected_history {
uint64_t page_size;
uint64_t n_revisions;
uint64_t origin_eof;
struct expected_revision revisions[ONION_TEST_EXPECTED_HISTORY_REVISIONS_MAX];
};
struct write_info {
haddr_t offset;
haddr_t size;
const unsigned char *buf;
};
struct revise_revision {
hbool_t truncate; /* onion-create, truncating any existing data */
uint64_t revision_num;
size_t n_writes;
struct write_info writes[ONION_TEST_REV_REV_WRITES_MAX];
const char *comment;
};
static int compare_file_bytes_exactly(const char *filepath, hid_t fapl_id, size_t nbytes,
const unsigned char *exp);
static int do_onion_open_and_writes(const char *filename, H5FD_onion_fapl_info_t *onion_info_p, size_t n_ops,
struct revise_revision *about);
static void onion_filepaths_destroy(struct onion_filepaths *paths);
static struct onion_filepaths *onion_filepaths_init(const char *basename);
/* set at runtime in main() */
static unsigned int flags_create_s = 0;
/* NOTE: b_list must be longer than a_list.
* Sizes must match respective buffer lengths.
*/
/* twenty-six four-character words beginning with 'a' -> 104 bytes */
const unsigned char *a_list_s =
(const unsigned char *)"abetableacedacesacheacidacneadzeafaragedagesaidsairsajarallyalum"
"amokantsapesarcsareaartsasksaspsavidaxes";
uint64_t a_list_size_s = 104;
/* fifty-three four-character words beginning with 'b' -> 212 bytes */
const unsigned char *b_list_s =
(const unsigned char *)"badebailbaitbalebanebarebaskbeambeanbearbeenbeerbeltbentbestbide"
"bikebilebindbirdbiteblipblueboarboatbobsbodyboilboldbollboltbond"
"boneboobboorboosbootbradbragbratbraybrewbritbrowbuckbudsbunkbunt"
"buoyburnburybustbuys";
uint64_t b_list_size_s = 212;
/* Allocate and populate filepaths with h5_fixname'd strings as appropriate.
* Should be released with onion_filepaths_destroy() when done.
*/
static struct onion_filepaths *
onion_filepaths_init(const char *basename)
{
struct onion_filepaths *paths = NULL;
if (NULL == (paths = HDmalloc(sizeof(struct onion_filepaths))))
TEST_ERROR;
paths->canon = NULL;
paths->onion = NULL;
paths->recovery = NULL;
if (NULL == (paths->canon = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
TEST_ERROR;
HDsnprintf(paths->canon, ONION_TEST_FIXNAME_SIZE, "%s", basename);
if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
TEST_ERROR;
HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon);
if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
TEST_ERROR;
HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon);
return paths;
error:
if (paths != NULL) {
HDfree(paths->canon);
HDfree(paths->onion);
HDfree(paths->recovery);
}
HDfree(paths);
return NULL;
}
/* Free onion file paths */
static void
onion_filepaths_destroy(struct onion_filepaths *paths)
{
HDfree(paths->canon);
HDfree(paths->onion);
HDfree(paths->recovery);
HDfree(paths);
}
/*-----------------------------------------------------------------------------
* Function: test_archival_index()
*
* Purpose: Unit-test mechanisms for the onion archival index.
* Specifies and verifies index-validation and -search routines.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_archival_index(void)
{
/* We can ignore each entry's physical address and checksum values */
H5FD_onion_index_entry_t e0 = {1, 474};
H5FD_onion_index_entry_t e1 = {4, 558};
H5FD_onion_index_entry_t e2 = {5, 306};
H5FD_onion_index_entry_t e3 = {9, 515};
H5FD_onion_index_entry_t e4 = {14, 386};
H5FD_onion_index_entry_t e5 = {18, 90};
H5FD_onion_index_entry_t e6 = {19, 94};
H5FD_onion_index_entry_t e7 = {20, 509};
H5FD_onion_index_entry_t sorted[8] = {e0, e1, e2, e3, e4, e5, e6, e7};
H5FD_onion_index_entry_t sorted_duplicates[8] = {e0, e1, e2, e2, e4, e5, e6, e7};
H5FD_onion_index_entry_t sorted_incomplete[8] = {e1, e3, e4, e5};
/* Partially-sorted list also aligned to 2 * page-size */
H5FD_onion_index_entry_t sorted_partial[8] = {e1, e4, e5, e7, e0, e6, e2, e3}; /* 0..3 sorted */
H5FD_onion_index_entry_t unsorted[8] = {e3, e1, e4, e5, e0, e6, e2, e7};
H5FD_onion_archival_index_t aix = {
H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, 1, /* page_size_log2 */
8, /* list must be populated and sorted through 0 .. (count-1) */
sorted, /* list */
};
const H5FD_onion_index_entry_t *entry_out_p = NULL;
TESTING("archival index");
/*
* Failing validity checks
*/
/* Invalid version should fail */
aix.version++;
if (H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/* Invalid version should fail */
aix.version = 0;
if (H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
aix.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR;
/* NULL list should fail */
aix.list = NULL;
if (H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/* List not full should fail */
aix.list = sorted_incomplete;
if (H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/* Unsorted list should fail */
aix.list = unsorted;
if (H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/* List with duplicates should fail */
aix.list = sorted_duplicates;
if (H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/*
* Passing validity checks
*/
/* Sorted list should pass */
aix.list = sorted;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/* Extra elements ignored (should pass) */
aix.list = sorted_partial;
aix.n_entries = 4;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
/*
* Archival index search routine
*/
aix.list = sorted;
aix.n_entries = 8;
/* Check that address not in array returns zero */
if (H5FD__onion_archival_index_find(&aix, 3, &entry_out_p) != 0)
TEST_ERROR;
/* Pointer should remain unset */
if (entry_out_p != NULL)
TEST_ERROR;
/* Address found should return 1 */
if (H5FD__onion_archival_index_find(&aix, 4, &entry_out_p) != 1)
TEST_ERROR;
/* Pointer should be set */
if (NULL == entry_out_p)
TEST_ERROR;
/* Incorrect address recorded */
if (558 != entry_out_p->phys_addr)
TEST_ERROR;
/*
* Test search edge cases
*/
aix.list = sorted_incomplete;
aix.n_entries = 4;
/* Address not in array should return 0 */
if (H5FD__onion_archival_index_find(&aix, 1, &entry_out_p) != 0)
TEST_ERROR;
/* Address not in array should return 0 */
if (H5FD__onion_archival_index_find(&aix, 101, &entry_out_p) != 0)
TEST_ERROR;
/*
* Empty archival index
*/
entry_out_p = NULL;
aix.n_entries = 0; /* actually populated list is irrelevant */
/* Address not in array should return 0 */
if (H5FD__onion_archival_index_find(&aix, 3, &entry_out_p) != 0)
TEST_ERROR;
/* Pointer should remain unset */
if (entry_out_p != NULL)
TEST_ERROR;
PASSED();
return 0;
error:
return -1;
} /* end test_archival_index() */
/*-----------------------------------------------------------------------------
* Function: test_revision_index()
*
* Purpose: Test revision index functionality
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_revision_index(void)
{
H5FD_onion_revision_index_t *rix_p = NULL;
H5FD_onion_index_entry_t entry = {
42, /* logical_page */
111112, /* phys_addr */
};
const H5FD_onion_index_entry_t *entry_out_p = NULL;
TESTING("revision index");
/* Test index creation */
if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5)))
TEST_ERROR;
if (H5FD_ONION_REVISION_INDEX_VERSION_CURR != rix_p->version)
TEST_ERROR;
if (0 != rix_p->n_entries)
TEST_ERROR;
/* Test missed search */
if (H5FD__onion_revision_index_find(rix_p, entry.logical_page, &entry_out_p) != 0)
TEST_ERROR;
/* Test successful insertion and lookup */
/* Insertion failed */
if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0)
TEST_ERROR;
if (1 != rix_p->n_entries)
TEST_ERROR;
/* Lookup failed */
if (H5FD__onion_revision_index_find(rix_p, entry.logical_page, &entry_out_p) < 0)
TEST_ERROR;
/* Failure to set output parameter */
if (NULL == entry_out_p)
TEST_ERROR;
if (entry.logical_page != entry_out_p->logical_page)
TEST_ERROR;
/* Seeking missing page should miss */
if (H5FD__onion_revision_index_find(rix_p, entry.logical_page + 1, &entry_out_p) != 0)
TEST_ERROR;
/* Test / demonstrate stored entry independent of user object */
entry.logical_page = 100;
entry.phys_addr = 101;
if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0)
TEST_ERROR;
if (2 != rix_p->n_entries)
TEST_ERROR;
entry.logical_page = 500;
entry.phys_addr = 501;
if (H5FD__onion_revision_index_find(rix_p, 100, &entry_out_p) < 0)
TEST_ERROR;
if (100 != entry_out_p->logical_page || 101 != entry_out_p->phys_addr)
TEST_ERROR;
/* Demonstrate updating an entry */
/* Error cases */
entry.logical_page = 100; /* phys_addr still 501, checksum bbbbbbbb */
if (H5FD__onion_revision_index_insert(rix_p, &entry) >= 0)
TEST_ERROR; /* all components but sum must match */
entry.phys_addr = 101;
/* Successful update */
entry.logical_page = 100;
entry.phys_addr = 101;
if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0)
TEST_ERROR;
/* Should still be two unique entries, not three */
if (2 != rix_p->n_entries)
TEST_ERROR;
if (H5FD__onion_revision_index_find(rix_p, 100, &entry_out_p) < 0)
TEST_ERROR;
if (100 != entry_out_p->logical_page || 101 != entry_out_p->phys_addr)
TEST_ERROR;
if (H5FD__onion_revision_index_destroy(rix_p) < 0)
TEST_ERROR;
PASSED();
return 0;
error:
if (rix_p != NULL)
(void)H5FD__onion_revision_index_destroy(rix_p);
return -1;
} /* end test_revision_index() */
/*-----------------------------------------------------------------------------
* Function: test_revision_index_collisions()
*
* Purpose: With knowledge of the revision index implementation, test
* hash key collisions.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_revision_index_collisions(void)
{
H5FD_onion_revision_index_t *rix_p = NULL;
H5FD_onion_index_entry_t entry = {
0, /* logical_page */
0, /* phys_addr */
};
const H5FD_onion_index_entry_t *entry_out_p = NULL;
const uint64_t n_insert = 40;
const uint64_t offset_from_power = 5;
TESTING("revision index collisions");
if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5)))
TEST_ERROR;
for (uint64_t i = 0; i < n_insert; i++) {
entry.phys_addr = i;
entry.logical_page = U64_EXP2(i) + offset_from_power;
if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0)
TEST_ERROR;
}
if (n_insert != rix_p->n_entries)
TEST_ERROR;
for (uint64_t i = 0; i < n_insert; i++) {
uint64_t page_id = U64_EXP2(i) + offset_from_power;
if (H5FD__onion_revision_index_find(rix_p, page_id, &entry_out_p) != 1)
TEST_ERROR;
if (entry_out_p->phys_addr != i)
TEST_ERROR;
}
if (H5FD__onion_revision_index_destroy(rix_p) < 0)
TEST_ERROR;
PASSED();
return 0;
error:
if (rix_p != NULL)
(void)H5FD__onion_revision_index_destroy(rix_p);
return -1;
} /* end test_revision_index_collisions() */
/*-----------------------------------------------------------------------------
*
* Function: test_revision_index_resizing()
*
* Purpose: With knowledge of the revision index implementation, test
* one or more resizig of the index.
*
* Return: PASSED : 0
* FAILED : -1
*
*-----------------------------------------------------------------------------
*/
static int
test_revision_index_resizing(void)
{
H5FD_onion_revision_index_t *rix_p = NULL;
H5FD_onion_index_entry_t entry = {
0, /* logical_page */
0, /* phys_addr */
};
const H5FD_onion_index_entry_t *entry_out_p = NULL;
const uint64_t n_insert = U64_EXP2((H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2 + 3));
TESTING("revision index resizing");
if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5)))
TEST_ERROR;
for (uint64_t i = 0; i < n_insert; i++) {
entry.logical_page = i;
entry.phys_addr = ((uint64_t)(-1) - i);
if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0)
TEST_ERROR;
}
if (n_insert != rix_p->n_entries)
TEST_ERROR;
for (uint64_t i = 0; i < n_insert; i++) {
uint64_t page_id = i;
if (H5FD__onion_revision_index_find(rix_p, page_id, &entry_out_p) != 1)
TEST_ERROR;
if (entry_out_p->phys_addr != ((uint64_t)(-1) - i))
TEST_ERROR;
}
if (H5FD__onion_revision_index_destroy(rix_p) < 0)
TEST_ERROR;
PASSED();
return 0;
error:
if (rix_p != NULL)
(void)H5FD__onion_revision_index_destroy(rix_p);
return -1;
} /* end test_revision_index_resizing() */
/*-----------------------------------------------------------------------------
* Function: test_revision_index_to_archival_index()
*
* Purpose: Verify to_archival_index().
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_revision_index_to_archival_index(void)
{
H5FD_onion_revision_index_t *rix_p = NULL;
H5FD_onion_index_entry_t rix_entry = {
0, /* logical_page */
0, /* phys_addr */
};
H5FD_onion_archival_index_t aix = {
H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR,
5, /* page_size_log2 */
0, /* n_entries to be set */
NULL,
};
const uint64_t n_insert = 10;
TESTING("revision index to archival index");
/*
* SETUP
*/
if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5)))
TEST_ERROR;
/* Add scattered entries in reverse order. */
for (uint64_t i = 0; i < n_insert; i++) {
uint64_t n = 2003 * (n_insert - i) + 47;
rix_entry.logical_page = n;
rix_entry.phys_addr = n * 13;
if (H5FD__onion_revision_index_insert(rix_p, &rix_entry) < 0)
TEST_ERROR;
}
if (n_insert != rix_p->n_entries)
TEST_ERROR;
aix.list = NULL;
aix.n_entries = 0;
/* Successful merge into empty archival index */
if (H5FD__onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0)
TEST_ERROR;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
if (n_insert != aix.n_entries)
TEST_ERROR;
for (uint64_t i = 0; i < n_insert; i++) {
const H5FD_onion_index_entry_t *aix_entry_p = NULL;
uint64_t n = 2003 * (i + 1) + 47;
aix_entry_p = &aix.list[i];
if (aix_entry_p->logical_page != n)
TEST_ERROR;
if (aix_entry_p->phys_addr != (n * 13))
TEST_ERROR;
}
/* Successful merge into populated archival index */
H5MM_xfree(aix.list);
aix.list = NULL;
if (NULL == (aix.list = H5MM_malloc(sizeof(H5FD_onion_index_entry_t) * 2)))
TEST_ERROR;
aix.list[0].logical_page = 47;
aix.list[0].phys_addr = 47 * 13;
aix.list[1].logical_page = (2003 * (n_insert + 1) + 47);
aix.list[1].phys_addr = (2003 * (n_insert + 1) + 47) * 13;
aix.n_entries = 2;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
if (H5FD__onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0)
TEST_ERROR;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
if (n_insert + 2 != aix.n_entries)
TEST_ERROR;
for (uint64_t i = 0; i < (n_insert + 2); i++) {
const H5FD_onion_index_entry_t *aix_entry_p = NULL;
uint64_t n = 2003 * i + 47;
aix_entry_p = &aix.list[i];
if (aix_entry_p->logical_page != n)
TEST_ERROR;
if (aix_entry_p->phys_addr != (n * 13))
TEST_ERROR;
}
/* Merged enties from revision index replace existing entries */
H5MM_xfree(aix.list);
aix.list = NULL;
if (NULL == (aix.list = H5MM_malloc(sizeof(H5FD_onion_index_entry_t) * 2)))
TEST_ERROR;
aix.list[0].logical_page = 2003 * (n_insert / 2) + 47;
aix.list[0].phys_addr = 103;
aix.list[1].logical_page = 2003 * (n_insert / 2 + 1) + 47;
aix.list[1].phys_addr = 101;
aix.n_entries = 2;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
if (H5FD__onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0)
TEST_ERROR;
if (!H5FD__onion_archival_index_is_valid(&aix))
TEST_ERROR;
if (n_insert != aix.n_entries)
TEST_ERROR;
for (uint64_t i = 0; i < n_insert; i++) {
const H5FD_onion_index_entry_t *aix_entry_p = NULL;
uint64_t n = 2003 * (i + 1) + 47;
aix_entry_p = &aix.list[i];
if (aix_entry_p->logical_page != n)
TEST_ERROR;
if (aix_entry_p->phys_addr != (n * 13))
TEST_ERROR;
}
/* CLEANUP */
if (H5FD__onion_revision_index_destroy(rix_p) < 0)
TEST_ERROR;
H5MM_xfree(aix.list);
PASSED();
return 0;
error:
if (rix_p)
(void)H5FD__onion_revision_index_destroy(rix_p);
H5MM_xfree(aix.list);
return -1;
} /* end test_revision_index_to_archival_index() */
/*-----------------------------------------------------------------------------
* Function: test_fapl()
*
* Purpose: Verify H5Pget and set behavior, and data-consistency checks.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_fapl(void)
{
H5FD_onion_fapl_info_t info_in = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_1, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"indoor speaking voices", /* comment */
};
H5FD_onion_fapl_info_t info_out;
hid_t dxpl_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
hid_t fapl_id_sec2 = H5I_INVALID_HID;
herr_t ret = FAIL;
TESTING("file access property list");
if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0)
TEST_ERROR;
if ((fapl_id_sec2 = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_sec2(fapl_id_sec2))
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
/* Set FAPL */
/* Invalid fapl should fail */
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(H5I_INVALID_HID, &info_in);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* NULL info pointer should fail */
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(fapl_id, NULL);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* Invalid version should fail */
info_in.version++;
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(fapl_id, &info_in);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
info_in.version--;
/* Page size not a power of 2 should fail */
info_in.page_size = 7;
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(fapl_id, &info_in);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* Page size <=0 should fail */
info_in.page_size = 0;
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(fapl_id, &info_in);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
info_in.page_size = ONION_TEST_PAGE_SIZE_1;
/* Invalid backing fapl should fail */
info_in.backing_fapl_id = H5I_INVALID_HID;
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(fapl_id, &info_in);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* Backing fapl not a fapl should fail */
info_in.backing_fapl_id = dxpl_id;
H5E_BEGIN_TRY
{
ret = H5Pset_fapl_onion(fapl_id, &info_in);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
info_in.backing_fapl_id = H5P_DEFAULT;
if (H5Pset_fapl_onion(fapl_id, &info_in) < 0)
TEST_ERROR;
/* Get onion fapl info */
/* NULL info_out pointer should fail */
H5E_BEGIN_TRY
{
ret = H5Pget_fapl_onion(fapl_id, NULL);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* Invalid fapl should fail */
H5E_BEGIN_TRY
{
ret = H5Pget_fapl_onion(H5I_INVALID_HID, &info_out);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* Non-onion fapl ID should fail */
H5E_BEGIN_TRY
{
ret = H5Pget_fapl_onion(fapl_id_sec2, &info_out);
}
H5E_END_TRY;
if (SUCCEED == ret)
TEST_ERROR;
/* Normal case */
if (H5Pget_fapl_onion(fapl_id, &info_out) < 0)
TEST_ERROR;
if (H5FD_ONION_FAPL_INFO_VERSION_CURR != info_out.version)
TEST_ERROR;
if (H5P_DEFAULT != info_out.backing_fapl_id)
TEST_ERROR;
if (ONION_TEST_PAGE_SIZE_1 != info_out.page_size)
TEST_ERROR;
if (H5FD_ONION_STORE_TARGET_ONION != info_out.store_target)
TEST_ERROR;
if (H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST != info_out.revision_num)
TEST_ERROR;
if (0 != info_out.creation_flags)
TEST_ERROR;
if (0 != info_out.force_write_open)
TEST_ERROR;
if (HDstrcmp(info_in.comment, info_out.comment))
TEST_ERROR;
/* Cleanup */
if (H5Pclose(dxpl_id) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id_sec2) < 0)
TEST_ERROR;
PASSED();
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(dxpl_id);
H5Pclose(fapl_id);
H5Pclose(fapl_id_sec2);
}
H5E_END_TRY
return -1;
} /* end test_fapl() */
/*-----------------------------------------------------------------------------
* Function: test_header_encode_decode()
*
* Purpose: Verify onion header encoding and decoding behavior.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_header_encode_decode(void)
{
unsigned char buf[64];
unsigned char exp[64] = {
/* bogus but unique values */
'O', 'H', 'D', 'H', /* NOTE: match signature define in onion_priv.h */
1, 12, 0, 0, /* NOTE: update version w/ "current" as needed */
0, 16, 0, 0, 0x11, 0x00, 0, 0, 0x02, 0, 0, 0, /* origin_eof */
0x40, 0xe2, 0x01, 0, 0, 0, 0, 0, /* history_addr */
88, 0, 0, 0, 0, 0, 0, 0, /* history_size */
0, 0, 0, 0 /* sum populated below */
};
unsigned char *ptr = NULL;
uint32_t checksum = 0;
uint32_t checksum_out = 0;
size_t i = 0;
uint64_t size_ret = 0;
H5FD_onion_header_t hdr;
H5FD_onion_header_t hdr_out;
TESTING("encode/decode history header");
checksum = H5_checksum_fletcher32(exp, H5FD_ONION_ENCODED_SIZE_HEADER - 4);
ptr = exp + H5FD_ONION_ENCODED_SIZE_HEADER - 4;
UINT32ENCODE(ptr, checksum);
hdr.version = H5FD_ONION_HEADER_VERSION_CURR;
hdr.flags = 12;
hdr.origin_eof = 8589934609ull, hdr.page_size = 4096;
hdr.history_addr = 123456;
hdr.history_size = 88;
if (H5FD__onion_header_encode(&hdr, buf, &checksum_out) != H5FD_ONION_ENCODED_SIZE_HEADER)
TEST_ERROR;
if (checksum != checksum_out)
TEST_ERROR;
for (i = 0; i < H5FD_ONION_ENCODED_SIZE_HEADER; i++) {
if (exp[i] != buf[i]) {
HDprintf("first mismatched byte at %zu: %02x %02x\n", i, exp[i], buf[i]);
TEST_ERROR;
}
}
hdr_out.version = H5FD_ONION_HEADER_VERSION_CURR;
hdr_out.flags = 0;
hdr_out.page_size = 0;
hdr_out.history_addr = 0;
hdr_out.history_size = 0;
/* Invalid header signature prevents decoding.
*/
exp[3] = 'X'; /* invalidate encoded signature */
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_header_decode(exp, &hdr_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[3] = 'H'; /* reset */
/* Invalid header version prevents decoding.
*/
exp[4] = 0; /* encoded version 0?!? */
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_header_decode(exp, &hdr_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[4] = H5FD_ONION_HEADER_VERSION_CURR + 1; /* encoded super-version?! */
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_header_decode(exp, &hdr_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
/* Reset */
exp[4] = H5FD_ONION_HEADER_VERSION_CURR;
/* Valid header can be decoded */
if (H5FD__onion_header_decode(buf, &hdr_out) != H5FD_ONION_ENCODED_SIZE_HEADER)
TEST_ERROR;
if (H5FD_ONION_HEADER_VERSION_CURR != hdr_out.version)
TEST_ERROR;
if (hdr.flags != hdr_out.flags)
TEST_ERROR;
if (hdr.page_size != hdr_out.page_size)
TEST_ERROR;
if (hdr.history_addr != hdr_out.history_addr)
TEST_ERROR;
if (hdr.history_size != hdr_out.history_size)
TEST_ERROR;
PASSED();
return 0;
error:
return -1;
} /* end test_header_encode_decode() */
/*-----------------------------------------------------------------------------
* Function: test_history_encode_decode_empty()
*
* Purpose: Verify onion history encoding and decoding behavior.
* Tests the case of the "empty" history.
* Verifies behavior in standard error cases.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_history_encode_decode_empty(void)
{
unsigned char buf[32];
unsigned char exp[32] = {
'O', 'W', 'H', 'S', /* NOTE: match signature define in onion_priv.h */
1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* sum populated below */
};
unsigned char *ptr = NULL;
uint32_t checksum = 0;
uint32_t checksum_out = 0;
size_t i = 0;
uint64_t size_ret = 0;
H5FD_onion_history_t history = {
H5FD_ONION_HISTORY_VERSION_CURR, 0, /* n_revisions */
NULL, /* list */
0, /* checksum */
};
H5FD_onion_history_t history_out = {
H5FD_ONION_HISTORY_VERSION_CURR, 0, /* n_revisions */
NULL, /* list */
0, /* checksum */
};
TESTING("encode/decode history (empty and failures)");
/* Generage checksum but don't store it yet */
checksum = H5_checksum_fletcher32(exp, H5FD_ONION_ENCODED_SIZE_HISTORY - 4);
ptr = exp + H5FD_ONION_ENCODED_SIZE_HISTORY - 4;
UINT32ENCODE(ptr, checksum);
if (H5FD__onion_history_encode(&history, buf, &checksum_out) != H5FD_ONION_ENCODED_SIZE_HISTORY)
TEST_ERROR;
for (i = 0; i < 20; i++) {
if (exp[i] != buf[i]) {
HDprintf("first mismatched byte at %zu: %02x %02x\n", i, exp[i], buf[i]);
TEST_ERROR;
}
}
if (checksum != checksum_out)
TEST_ERROR;
history.checksum = checksum; /* set to compare later */
/* Invalid signature prevents decoding */
exp[3] = 'X'; /* invalidate encoded signature */
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_history_decode(exp, &history_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[3] = 'H'; /* reset */
/* Invalid version prevents decoding */
exp[4] = 0; /* encoded version 0?!? */
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_history_decode(exp, &history_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[4] = H5FD_ONION_HISTORY_VERSION_CURR + 1;
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_history_decode(exp, &history_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[4] = H5FD_ONION_HISTORY_VERSION_CURR; /* reset */
/* Valid summary can be decoded */
if (H5FD__onion_history_decode(buf, &history_out) != H5FD_ONION_ENCODED_SIZE_HISTORY)
TEST_ERROR;
if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version)
TEST_ERROR;
if (history.n_revisions != history_out.n_revisions)
TEST_ERROR;
if (history.checksum != history_out.checksum)
TEST_ERROR;
if (NULL != history_out.record_locs)
TEST_ERROR;
PASSED();
return 0;
error:
return -1;
} /* end test_history_encode_decode_empty() */
/*-----------------------------------------------------------------------------
* Function: test_history_encode_decode()
*
* Purpose: Verify onion history encoding and decoding behavior.
* Encode/decode with some set of revision record pointers.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_history_encode_decode(void)
{
unsigned char *buf = NULL;
unsigned char exp[80] = {
'O', 'W', 'H', 'S', /* NOTE: match signature define in onion_priv.h */
1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */
3, 0, 0, 0, 0, 0, 0, 0,
/* rev0 pointer */
56, 2, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* sum populated below */
/* rev1 pointer */
121, 173, 3, 0, 0, 0, 0, 0, 203, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* sum populated below */
/* rev2 pointer */
96, 158, 52, 198, 213, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* sum populated below */
/* final checksum */
0, 0, 0, 0 /* sum populated below */
};
unsigned char *buf_p = NULL;
uint32_t checksum_out = 0;
size_t i = 0;
H5FD_onion_history_t history = {
H5FD_ONION_HISTORY_VERSION_CURR, 3, /* n_revisions */
NULL, /* list set below */
0, /* checksum not set by us */
};
H5FD_onion_history_t history_out = {
H5FD_ONION_HISTORY_VERSION_CURR, 0, /* n_revisions must start as zero */
NULL, /* list */
0, /* checksum */
};
size_t exp_size =
H5FD_ONION_ENCODED_SIZE_HISTORY + H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * history.n_revisions;
TESTING("encode/decode history");
if (80 != exp_size)
TEST_ERROR;
history.record_locs = HDcalloc(history.n_revisions, sizeof(H5FD_onion_record_loc_t));
if (NULL == history.record_locs)
TEST_ERROR;
/* Must match values in exp */
history.record_locs[0].phys_addr = 568ull;
history.record_locs[0].record_size = 238ull;
history.record_locs[1].phys_addr = 241017ull;
history.record_locs[1].record_size = 4555ull;
history.record_locs[2].phys_addr = 918153371232ull;
history.record_locs[2].record_size = 240ull;
/* Populate revision pointer sums in exp */
for (i = 0; i < history.n_revisions; i++) {
uint64_t history_pre = H5FD_ONION_ENCODED_SIZE_HISTORY - 4;
uint64_t ptr_pre = H5FD_ONION_ENCODED_SIZE_RECORD_POINTER - 4;
uint64_t ptr_size = H5FD_ONION_ENCODED_SIZE_RECORD_POINTER;
buf_p = exp + history_pre + ptr_size * i;
history.record_locs[i].checksum = H5_checksum_fletcher32(buf_p, ptr_pre);
buf_p += ptr_pre;
UINT32ENCODE(buf_p, history.record_locs[i].checksum);
}
/* Compute, populate, and store exp final sum */
history.checksum = H5_checksum_fletcher32(exp, exp_size - 4);
buf_p = exp + exp_size - 4;
UINT32ENCODE(buf_p, history.checksum);
if (NULL == (buf = HDmalloc(exp_size)))
TEST_ERROR;
if (H5FD__onion_history_encode(&history, buf, &checksum_out) != exp_size)
TEST_ERROR;
for (i = 0; i < exp_size; i++) {
if (exp[i] != buf[i])
TEST_ERROR;
}
if (history.checksum != checksum_out)
TEST_ERROR;
/* Initial decode, gets always-present components */
history_out.n_revisions = 0; /* must be initialized to 0 */
if (H5FD__onion_history_decode(exp, &history_out) != exp_size)
TEST_ERROR;
if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version)
TEST_ERROR;
if (history.n_revisions != history_out.n_revisions)
TEST_ERROR;
/* Must be created by us */
if (NULL != history_out.record_locs)
TEST_ERROR;
/* True decode requires allocating space for record pointers */
history_out.record_locs = HDcalloc(history_out.n_revisions, sizeof(H5FD_onion_record_loc_t));
if (NULL == history_out.record_locs)
TEST_ERROR;
if (H5FD__onion_history_decode(exp, &history_out) != exp_size)
TEST_ERROR;
if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version)
TEST_ERROR;
if (history.n_revisions != history_out.n_revisions)
TEST_ERROR;
if (history.checksum != history_out.checksum)
TEST_ERROR;
if (NULL == history_out.record_locs)
TEST_ERROR;
for (i = 0; i < history.n_revisions; i++) {
H5FD_onion_record_loc_t exp_rp = history.record_locs[i];
H5FD_onion_record_loc_t act_rp = history_out.record_locs[i];
if (exp_rp.phys_addr != act_rp.phys_addr)
TEST_ERROR;
if (exp_rp.record_size != act_rp.record_size)
TEST_ERROR;
if (exp_rp.checksum != act_rp.checksum)
TEST_ERROR;
}
HDfree(history_out.record_locs);
HDfree(buf);
HDfree(history.record_locs);
PASSED();
return 0;
error:
HDfree(history_out.record_locs);
HDfree(buf);
HDfree(history.record_locs);
return -1;
} /* end test_history_encode_decode() */
/*-----------------------------------------------------------------------------
*
* Function: test_revision_record_encode_decode()
*
* Purpose: Verify onion revision-record encoding and decoding behavior.
*
* Return: PASSED : 0
* FAILED : -1
*
*-----------------------------------------------------------------------------
*/
static int
test_revision_record_encode_decode(void)
{
/* clang-format off */
/* Byte array of expected values (FRAGILE!) */
unsigned char exp[173] = {
'O', 'R', 'R', 'S', /* Bytes 000-003: signature */
1, 0, 0, 0, /* Bytes 004-007: version */
5, 0, 0, 0, 0, 0, 0, 0, /* Bytes 008-015: revision ID */
2, 0, 0, 0, 0, 0, 0, 0, /* Bytes 016-023: parent revision ID */
'1', '9', '4', '1', '1', '2', '0', '7', /* Bytes 024-039: time of creation */
'T', '1', '9', '0', '6', '4', '3', 'Z', /* ... */
0x11, 0x00, 0, 0, 0x02, 0, 0, 0, /* Bytes 040-047: logical file size */
0, 16, 0, 0, /* Bytes 048-051: page size */
4, 0, 0, 0, 0, 0, 0, 0, /* Bytes 052-059: # entries */
25, 0, 0, 0, /* Bytes 060-063: comment size */
/* ENTRY 0 */
0, 0xB0, 0x1E, 0, 0, 0, 0, 0, /* Bytes 064-071: entry 0: logical offset */
0x4B, 0x02, 0, 0, 0, 0, 0, 0, /* Bytes 072-079: entry 0: physical address */
0, 0, 0, 0, /* Bytes 080-083: checksum (populated below) */
/* ENTRY 1 */
0, 0xF0, 0x2E, 0, 0, 0, 0, 0, /* Bytes 084-091: entry 1: logical offset */
0xA7, 0, 0, 0, 0, 0, 0, 0, /* Bytes 092-099: entry 1: physical address */
0, 0, 0, 0, /* Bytes 100-103: checksum (populated below) */
/* ENTRY 2 */
0, 0x50, 0x15, 0, 0, 0x20, 0, 0, /* Bytes 104-111: entry 2: logical offset */
0x11, 0, 0, 0, 0x02, 0, 0, 0, /* Bytes 112-119: entry 2: physical address */
0, 0, 0, 0, /* Bytes 120-123: checksum (populated below) */
/* ENTRY 3 */
0, 0xE0, 0x24, 0, 0, 0, 0, 0, /* Bytes 124-131: entry 3: logical offset */
0xB1, 0x01, 0, 0, 0, 0, 0, 0, /* Bytes 132-139: entry 3: physical address */
0, 0, 0, 0, /* Bytes 140-143: checksum (populated below) */
'E', 'x', 'a', 'm', 'p', 'l', 'e', ' ', /* Bytes 144-168: comment */
'c', 'o', 'm', 'm', 'e', 'n', 't', ' ', /* ... */
'm', 'e', 's', 's', 'a', 'g', 'e', '.', /* ... */
'\0', /* ... */
0, 0, 0, 0 /* Bytes 169-172: final checksum (populated below) */
};
/* clang-format on */
unsigned char *buf = NULL;
unsigned char *buf_p = NULL;
size_t i = 0;
uint64_t size_ret;
H5FD_onion_revision_record_t r_out;
uint32_t checksum = 0;
char comment[25] = "Example comment message.";
H5FD_onion_revision_record_t record = {
H5FD_ONION_REVISION_RECORD_VERSION_CURR,
5, /* revision ID */
2, /* parent revision ID */
{'\0'}, /* time of creation - populated below */
8589934609ull, /* logical file size */
{
H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, /* version */
12, /* page_size_log2 */
4, /* n_entries */
NULL, /* list - populated below */
}, /* archival index struct */
25, /* comment size */
comment, /* comment */
0, /* checksum (computed later) */
};
size_t exp_size = H5FD_ONION_ENCODED_SIZE_REVISION_RECORD +
(H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * record.archival_index.n_entries) +
HDstrlen("Example comment message.") + 1;
r_out.archival_index.list = NULL;
r_out.comment = NULL;
TESTING("encode/decode revision record");
HDmemcpy(record.time_of_creation, "19411207T190643Z", 16);
record.archival_index.list = HDcalloc(record.archival_index.n_entries, sizeof(H5FD_onion_index_entry_t));
if (NULL == record.archival_index.list)
TEST_ERROR;
/* Convert logical_page and should match address in expected buffer */
record.archival_index.list[0].logical_page = 491ull;
record.archival_index.list[0].phys_addr = 587ull;
record.archival_index.list[1].logical_page = 751ull;
record.archival_index.list[1].phys_addr = 167ull;
record.archival_index.list[2].logical_page = 8589934933ull;
record.archival_index.list[2].phys_addr = 8589934609ull;
record.archival_index.list[3].logical_page = 590ull;
record.archival_index.list[3].phys_addr = 433ull;
/* Set expected checksum for each archival index entry in buffer */
for (i = 0; i < record.archival_index.n_entries; i++) {
uint64_t rec_pre = H5FD_ONION_ENCODED_SIZE_REVISION_RECORD - 4;
uint64_t idx_pre = H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY - 4;
uint64_t idx_size = H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY;
buf_p = exp + rec_pre + idx_size * i;
checksum = H5_checksum_fletcher32(buf_p, idx_pre);
buf_p += idx_pre;
UINT32ENCODE(buf_p, checksum);
}
checksum = 0;
record.checksum = H5_checksum_fletcher32(exp, exp_size - 4);
buf_p = exp + exp_size - 4;
UINT32ENCODE(buf_p, record.checksum);
/* Required initialization for record-out structure */
r_out.version = H5FD_ONION_REVISION_RECORD_VERSION_CURR;
r_out.comment_size = 0;
r_out.comment = NULL;
r_out.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR;
r_out.archival_index.n_entries = 0;
r_out.archival_index.list = NULL;
if (NULL == (buf = HDmalloc(sizeof(unsigned char) * exp_size)))
TEST_ERROR;
/* Test encode */
if (H5FD__onion_revision_record_encode(&record, buf, &checksum) != exp_size)
TEST_ERROR;
hbool_t badness = FALSE;
for (i = 0; i < exp_size; i++) {
if (exp[i] != buf[i]) {
badness = TRUE;
HDprintf("Bad encoded record - Index %zu: expected 0x%02X but got 0x%02X\n", i,
(unsigned int)exp[i], (unsigned int)buf[i]);
}
}
if (badness) {
/* If this fragile test breaks, this information is helpful... */
HDprintf("INDEX\n");
for (i = 0; i < exp_size; i++)
HDprintf("%4zu ", i);
HDprintf("\n");
HDprintf("EXPECTED\n");
for (i = 0; i < exp_size; i++)
HDprintf("0x%02X ", (unsigned int)exp[i]);
HDprintf("\n");
HDprintf("ACTUAL\n");
for (i = 0; i < exp_size; i++)
HDprintf("0x%02X ", (unsigned int)buf[i]);
HDprintf("\n");
}
if (badness)
TEST_ERROR;
if (record.checksum != checksum)
TEST_ERROR;
/* Test decode (malformed encoding) */
/* Invalid signature */
exp[2] = 'Y';
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_revision_record_decode(exp, &r_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[2] = 'R'; /* reset */
/* Zero version */
exp[4] = 0;
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_revision_record_decode(exp, &r_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
/* Advance version */
exp[4] = H5FD_ONION_REVISION_RECORD_VERSION_CURR + 1;
H5E_BEGIN_TRY
{
size_ret = H5FD__onion_revision_record_decode(exp, &r_out);
}
H5E_END_TRY;
if (0 != size_ret)
TEST_ERROR;
exp[4] = H5FD_ONION_REVISION_RECORD_VERSION_CURR; /* reset */
/* Test successful decode */
/* Initial decode; get variable-length component sizes */
if (H5FD__onion_revision_record_decode(exp, &r_out) != exp_size)
TEST_ERROR;
if (record.comment_size != r_out.comment_size)
TEST_ERROR;
if (record.archival_index.n_entries != r_out.archival_index.n_entries)
TEST_ERROR;
/* Allocate variable-length components */
r_out.comment = HDcalloc(r_out.comment_size, sizeof(char));
if (NULL == r_out.comment)
TEST_ERROR;
r_out.archival_index.list = HDcalloc(r_out.archival_index.n_entries, sizeof(H5FD_onion_index_entry_t));
if (NULL == r_out.archival_index.list)
TEST_ERROR;
/* Decode into all components */
if (H5FD__onion_revision_record_decode(exp, &r_out) != exp_size)
TEST_ERROR;
if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != r_out.version)
TEST_ERROR;
if (record.revision_num != r_out.revision_num)
TEST_ERROR;
if (record.parent_revision_num != r_out.parent_revision_num)
TEST_ERROR;
if (record.parent_revision_num != r_out.parent_revision_num)
TEST_ERROR;
if (record.checksum != r_out.checksum)
TEST_ERROR;
if (HDstrncmp(record.time_of_creation, r_out.time_of_creation, 16) != 0)
TEST_ERROR;
if (record.comment_size != r_out.comment_size)
TEST_ERROR;
if (record.comment_size != HDstrlen(r_out.comment) + 1)
TEST_ERROR;
if (HDstrlen(record.comment) != HDstrlen(r_out.comment))
TEST_ERROR;
if (HDstrcmp(record.comment, r_out.comment) != 0)
TEST_ERROR;
if (H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR != r_out.archival_index.version)
TEST_ERROR;
if (record.archival_index.page_size_log2 != r_out.archival_index.page_size_log2)
TEST_ERROR;
if (record.archival_index.n_entries != r_out.archival_index.n_entries)
TEST_ERROR;
for (i = 0; i < record.archival_index.n_entries; i++) {
H5FD_onion_index_entry_t *ep = &record.archival_index.list[i];
H5FD_onion_index_entry_t *ap = &r_out.archival_index.list[i];
if (ep->phys_addr != ap->phys_addr)
TEST_ERROR;
if (ep->logical_page != ap->logical_page)
TEST_ERROR;
}
/* Cleanup */
HDfree(r_out.archival_index.list);
HDfree(r_out.comment);
HDfree(buf);
HDfree(record.archival_index.list);
PASSED();
return 0;
error:
HDfree(r_out.archival_index.list);
HDfree(r_out.comment);
HDfree(buf);
HDfree(record.archival_index.list);
return -1;
} /* end test_revision_record_encode_decode() */
/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Use VFL to open target file and check that its bytes exactly match those
* of given buffer 'exp'[ected].
*
* Returns 0 if successful, -1 if error or mismatch.
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
*/
static int
compare_file_bytes_exactly(const char *filepath, hid_t fapl_id, size_t nbytes, const unsigned char *exp)
{
H5FD_t *raw_vfile = NULL; /* virtual file to look at raw file contents */
unsigned char *act_buf = NULL; /* allocated area for actual file bytes */
uint64_t filesize = 0;
if (NULL == (raw_vfile = H5FDopen(filepath, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
TEST_ERROR;
/* filesize is wrong w/ stdio - it's zero instead of 40 or whatnot */
filesize = (uint64_t)H5FDget_eof(raw_vfile, H5FD_MEM_DRAW);
if ((uint64_t)nbytes != filesize) {
HDfprintf(stderr, "\nSizes not the same - nbytes: %zu, filesize: %" PRIu64 "\n", nbytes, filesize);
TEST_ERROR;
}
if (NULL == (act_buf = HDmalloc(nbytes)))
TEST_ERROR;
/* Fill buffer with bogus UCHAR_MAX values */
for (size_t i = 0; i < nbytes; i++)
act_buf[i] = UCHAR_MAX;
if (H5FDset_eoa(raw_vfile, H5FD_MEM_DRAW, nbytes) < 0)
TEST_ERROR;
if (H5FDread(raw_vfile, H5FD_MEM_DRAW, H5P_DEFAULT, 0, nbytes, act_buf) < 0)
TEST_ERROR;
/* Compare raw bytes data */
for (size_t i = 0; i < nbytes; i++) {
if (exp[i] != act_buf[i]) {
HDprintf("first mismatched byte %zu: expected 0x%02X was 0x%02X\n", i, exp[i], act_buf[i]);
TEST_ERROR;
}
}
if (H5FDclose(raw_vfile) < 0)
TEST_ERROR;
HDfree(act_buf);
return 0;
error:
if (raw_vfile != NULL)
H5FDclose(raw_vfile);
HDfree(act_buf);
return -1;
} /* end compare_file_bytes_exactly() */
/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Do a manual read of the onion history (separate, single "Onion" file).
* Verify that the history data is well-formed and matches the expected state.
*
* Inspect file contents on backing store.
* Return -1 on problem, 0 if okay.
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
*/
static int
verify_history_as_expected_onion(H5FD_t *raw_file, struct expected_history *filter)
{
unsigned char *buf = NULL; /* allocated area for actual file bytes */
H5FD_onion_header_t hdr_out;
H5FD_onion_history_t history_out;
H5FD_onion_revision_record_t rev_out;
uint64_t filesize = 0;
uint64_t readsize = 0;
uint8_t *ui8p = NULL;
uint32_t buf_checksum = 0;
/* memset to avoid bad frees on errors */
HDmemset(&rev_out, 0, sizeof(H5FD_onion_revision_record_t));
HDmemset(&history_out, 0, sizeof(H5FD_onion_history_t));
hdr_out.version = H5FD_ONION_HEADER_VERSION_CURR;
history_out.version = H5FD_ONION_HISTORY_VERSION_CURR;
history_out.n_revisions = 0;
history_out.record_locs = NULL;
rev_out.version = H5FD_ONION_REVISION_RECORD_VERSION_CURR;
rev_out.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR;
filesize = (uint64_t)H5FDget_eof(raw_file, H5FD_MEM_DRAW);
if (H5FDset_eoa(raw_file, H5FD_MEM_DRAW, filesize) < 0)
TEST_ERROR;
/* Ingest onion header */
readsize = MIN(filesize, H5FD_ONION_ENCODED_SIZE_HEADER);
if (NULL == (buf = HDmalloc(readsize * sizeof(unsigned char))))
TEST_ERROR;
if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, readsize, buf) < 0)
TEST_ERROR;
readsize = H5FD__onion_header_decode(buf, &hdr_out);
if (0 == readsize)
TEST_ERROR;
if (H5FD_ONION_HEADER_VERSION_CURR != hdr_out.version)
TEST_ERROR;
/* Decode from the buffer to we can compare on BE systems */
ui8p = (uint8_t *)(&buf[readsize - 4]);
UINT32DECODE(ui8p, buf_checksum);
if (hdr_out.checksum != buf_checksum)
TEST_ERROR;
if (hdr_out.checksum != H5_checksum_fletcher32(buf, readsize - 4))
TEST_ERROR;
if (filter->page_size != hdr_out.page_size)
TEST_ERROR;
if (hdr_out.history_addr + hdr_out.history_size != filesize)
TEST_ERROR;
if (filter->origin_eof != hdr_out.origin_eof)
TEST_ERROR;
HDfree(buf);
buf = NULL;
/* Ingest history */
readsize = hdr_out.history_size;
if (NULL == (buf = HDmalloc(readsize * sizeof(unsigned char))))
TEST_ERROR;
if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, hdr_out.history_addr, readsize, buf) < 0)
TEST_ERROR;
/* Initial read, get count of revisions */
readsize = H5FD__onion_history_decode(buf, &history_out);
if (0 == readsize)
TEST_ERROR;
if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version)
TEST_ERROR;
/* Decode from the buffer to we can compare on BE systems */
ui8p = (uint8_t *)(&buf[readsize - 4]);
UINT32DECODE(ui8p, buf_checksum);
if (history_out.checksum != buf_checksum)
TEST_ERROR;
if (history_out.checksum != H5_checksum_fletcher32(buf, readsize - 4))
TEST_ERROR;
if (filter->n_revisions != history_out.n_revisions)
TEST_ERROR;
/* Final read, populate pointers to revision records */
history_out.record_locs = HDcalloc(history_out.n_revisions, sizeof(H5FD_onion_record_loc_t));
if (NULL == history_out.record_locs)
TEST_ERROR;
if (H5FD__onion_history_decode(buf, &history_out) != readsize)
TEST_ERROR;
/* Re-use buffer space to sanity-check checksum for record pointer(s). */
HDassert(readsize >= sizeof(H5FD_onion_record_loc_t));
for (size_t i = 0; i < history_out.n_revisions; i++) {
uint64_t phys_addr;
uint64_t record_size;
/* Do a checked assignment from the struct value into appropriately
* sized types. We don't have access to the H5F_t struct for this
* file, so we can't use the offset/length macros in H5Fprivate.h.
*
* Have to do an encode to get these values so the test passes on BE
* systems.
*/
H5_CHECKED_ASSIGN(phys_addr, uint64_t, history_out.record_locs[i].phys_addr, haddr_t);
H5_CHECKED_ASSIGN(record_size, uint64_t, history_out.record_locs[i].record_size, hsize_t);
ui8p = (uint8_t *)buf;
UINT64ENCODE(ui8p, phys_addr);
ui8p = (uint8_t *)(buf + 8);
UINT64ENCODE(ui8p, record_size);
if (history_out.record_locs[i].checksum != H5_checksum_fletcher32(buf, 16))
TEST_ERROR;
}
HDfree(buf);
buf = NULL;
/* Ingest revision(s) */
for (size_t i = 0; i < history_out.n_revisions; i++) {
H5FD_onion_record_loc_t *rpp = &history_out.record_locs[i];
struct expected_revision *erp = &filter->revisions[i];
rev_out.archival_index.list = NULL;
rev_out.archival_index.n_entries = 0;
rev_out.archival_index.page_size_log2 = 0;
rev_out.comment_size = 0;
rev_out.comment = NULL;
readsize = rpp->record_size;
if (NULL == (buf = HDmalloc((size_t)rpp->record_size)))
TEST_ERROR;
if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, rpp->phys_addr, rpp->record_size, buf) < 0)
TEST_ERROR;
/* Initial revision read -- get fixed components */
readsize = H5FD__onion_revision_record_decode(buf, &rev_out);
if (0 == readsize)
TEST_ERROR;
if (rpp->record_size != readsize)
TEST_ERROR;
if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != rev_out.version)
TEST_ERROR;
/* Decode from the buffer to we can compare on BE systems */
ui8p = (uint8_t *)(&buf[readsize - 4]);
UINT32DECODE(ui8p, buf_checksum);
if (rev_out.checksum != buf_checksum)
TEST_ERROR;
if (rev_out.checksum != H5_checksum_fletcher32(buf, readsize - 4))
TEST_ERROR;
if (erp->revision_num != rev_out.revision_num)
TEST_ERROR;
if (erp->parent_revision_num != rev_out.parent_revision_num)
TEST_ERROR;
if (erp->logical_eof != rev_out.logical_eof)
TEST_ERROR;
/* Final read, get variable-length data */
if (NULL == (rev_out.comment = HDmalloc((size_t)rev_out.comment_size)))
TEST_ERROR;
rev_out.archival_index.list =
HDcalloc(rev_out.archival_index.n_entries, sizeof(H5FD_onion_index_entry_t));
if (NULL == rev_out.archival_index.list)
TEST_ERROR;
readsize = H5FD__onion_revision_record_decode(buf, &rev_out);
if (rpp->record_size != readsize)
TEST_ERROR;
/* Compare revision info with expected filter */
if (erp->comment == NULL) {
if (rev_out.comment_size != 0)
TEST_ERROR;
}
else {
if (HDstrlen(rev_out.comment) != HDstrlen(erp->comment))
TEST_ERROR;
if (HDstrcmp(rev_out.comment, erp->comment) != 0)
TEST_ERROR;
}
if (erp->n_index_entries != (uint64_t)(-1) &&
erp->n_index_entries != rev_out.archival_index.n_entries)
TEST_ERROR;
HDfree(buf);
HDfree(rev_out.comment);
HDfree(rev_out.archival_index.list);
}
HDfree(history_out.record_locs);
history_out.record_locs = NULL;
return 0;
error:
HDfree(buf);
HDfree(rev_out.comment);
HDfree(rev_out.archival_index.list);
HDfree(history_out.record_locs);
return -1;
} /* end verify_history_as_expected_onion() */
/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
* Verify file bytes on the backing store
* + onion storage target
* + create from nothing
* + stage 0 (initializing)
* + open (not yet written)
* + "Empty" .h5 file created
* + .onion file created w/ only header (0 whole-hist addr)
* + .onion.recovery created w/ "empty" history
* + Cannot open onionized canonical file (incomplete history, no rev)
*
* Inspect file contents on backing store.
* Return -1 on problem, 0 if okay.
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
*/
static int
verify_stored_onion_create_0_open(struct onion_filepaths *paths, H5FD_onion_fapl_info_t *onion_info)
{
H5FD_t *file = NULL; /* virtual file to look at raw file contents */
unsigned char *act_buf = NULL; /* allocated area for actual file bytes */
hid_t fapl_id = onion_info->backing_fapl_id;
herr_t err_ret = FAIL;
unsigned char hdr_exp_bytes[] = {
'O', 'H', 'D', 'H', 1, 1, 0, 0, 0, 0, 0, 0, /* page-size encoded below */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* checksum encoded below */
};
size_t history_exp_bytes_size = 20;
unsigned char history_exp_bytes[] = {
'O', 'W', 'H', 'S', 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* checksum encoded below */
};
unsigned char *ptr = NULL;
uint32_t checksum = 0;
/* Finish populating expected header bytes */
ptr = hdr_exp_bytes + 8; /* WARNING: must match format */
UINT32ENCODE(ptr, onion_info->page_size);
checksum = H5_checksum_fletcher32(hdr_exp_bytes, H5FD_ONION_ENCODED_SIZE_HEADER - 4);
ptr = hdr_exp_bytes + H5FD_ONION_ENCODED_SIZE_HEADER - 4;
UINT32ENCODE(ptr, checksum);
ptr = NULL;
/* Finish populating expected history bytes */
checksum = H5_checksum_fletcher32(history_exp_bytes, H5FD_ONION_ENCODED_SIZE_HISTORY - 4);
ptr = history_exp_bytes + H5FD_ONION_ENCODED_SIZE_HISTORY - 4;
UINT32ENCODE(ptr, checksum);
ptr = NULL;
/* Look at h5 file: should have zero bytes */
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
/* Size here is arbitrary */
if (NULL == (act_buf = HDcalloc(1, 8)))
TEST_ERROR;
/* Should fail when reading from an empty file */
H5E_BEGIN_TRY
{
err_ret = H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 1, act_buf);
}
H5E_END_TRY;
if (err_ret != FAIL)
TEST_ERROR;
HDfree(act_buf);
act_buf = NULL;
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
/* Look at onion file: should have header */
if (compare_file_bytes_exactly(paths->onion, fapl_id, H5FD_ONION_ENCODED_SIZE_HEADER, hdr_exp_bytes) < 0)
TEST_ERROR;
/* Look at history backing file: should have nascent history */
if (compare_file_bytes_exactly(paths->recovery, fapl_id, history_exp_bytes_size, history_exp_bytes) < 0)
TEST_ERROR;
/* Inspect .h5 file contents */
if (compare_file_bytes_exactly(paths->canon, fapl_id, 8, (const unsigned char *)"ONIONEOF") < 0)
TEST_ERROR;
return 0;
error:
if (file != NULL)
(void)H5FDclose(file);
HDfree(act_buf);
return -1;
} /* end verify_stored_onion_create_0_open() */
/*-----------------------------------------------------------------------------
* Function: test_create_oniontarget()
*
* Purpose: Test the ability of the Onion VFD to create a valid
* 'onionized' file.
*
* When `truncate_canonical` is FALSE, the canonical file is
* nonexistent on the backing store on onion-creation.
* When `truncate_canonical` is TRUE, a canonical file is created
* on the backing store with known contents, which are to be
* truncated on onion-creation.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_create_oniontarget(hbool_t truncate_canonical, hbool_t with_initial_data)
{
const char *basename = "somesuch";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"initial commit" /* comment */
};
H5FD_t *vfile_raw = NULL; /* virtual file to look at raw file contents */
H5FD_t *vfile_rw = NULL; /* Onion virtual file for read/write */
H5FD_t *vfile_ro = NULL; /* Onion virtual file for read-only */
struct expected_history filter;
char *buf = NULL;
if (TRUE == truncate_canonical && TRUE == with_initial_data)
TESTING("onion creation; truncate extant canonical; w/ initial data");
else if (TRUE == truncate_canonical)
TESTING("onion creation; truncate extant canonical; no initial data");
else if (TRUE == with_initial_data)
TESTING("onion creation; no extant canonical; w/ initial data");
else
TESTING("onion creation; no extant canonical; no initial data");
/*********
* SETUP *
*********/
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/* Create canonical file to be truncated */
if (TRUE == truncate_canonical) {
/* Create canonical file. */
vfile_raw = H5FDopen(paths->canon, flags_create_s, onion_info.backing_fapl_id, HADDR_UNDEF);
if (NULL == vfile_raw)
TEST_ERROR;
if (H5FDset_eoa(vfile_raw, H5FD_MEM_DRAW, b_list_size_s) < 0)
TEST_ERROR;
if (H5FDwrite(vfile_raw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, b_list_size_s, b_list_s) < 0)
TEST_ERROR;
if (H5FDclose(vfile_raw) < 0)
TEST_ERROR;
vfile_raw = NULL;
H5E_BEGIN_TRY
{
vfile_raw = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
}
H5E_END_TRY;
/* Check if onion history for onion-open created file */
if (NULL != vfile_raw)
TEST_ERROR;
/* Create "existing onion file". */
vfile_raw = H5FDopen(paths->onion, flags_create_s, onion_info.backing_fapl_id, HADDR_UNDEF);
if (NULL == vfile_raw)
TEST_ERROR;
if (H5FDset_eoa(vfile_raw, H5FD_MEM_DRAW, b_list_size_s) < 0)
TEST_ERROR;
if (H5FDwrite(vfile_raw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 23, "prior history stand-in") < 0)
TEST_ERROR;
if (H5FDclose(vfile_raw) < 0)
TEST_ERROR;
vfile_raw = NULL;
} /* end if to create canonical file for truncation */
/*
* OPENED
*/
/* Begin creation of onionized file from nothing */
vfile_rw = H5FDopen(paths->canon, flags_create_s, fapl_id, HADDR_UNDEF);
if (NULL == vfile_rw)
TEST_ERROR;
if (verify_stored_onion_create_0_open(paths, &onion_info) < 0)
TEST_ERROR;
H5E_BEGIN_TRY
{
vfile_ro = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
}
H5E_END_TRY;
/* check if onionization (creation) not complete; nothing to open */
if (vfile_ro != NULL)
TEST_ERROR;
/*
* WRITING
*/
if (TRUE == with_initial_data) {
haddr_t half_size = 0;
haddr_t buf_size = 0;
/* Write the sub-page entry at addr 0 */
if (4 >= onion_info.page_size)
TEST_ERROR;
if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, 4) < 0)
TEST_ERROR;
if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 4, a_list_s) < 0) {
TEST_ERROR;
}
/* Verify logical file contents. */
if (NULL == (buf = HDmalloc(4 * sizeof(char))))
TEST_ERROR;
if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 4, buf) < 0)
TEST_ERROR;
if (HDmemcmp(a_list_s, buf, 4) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
/* Write the latter half of buffer at addr 0 (more than one page) */
half_size = a_list_size_s / 2;
buf_size = a_list_size_s - half_size;
if (buf_size <= onion_info.page_size)
TEST_ERROR;
if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, buf_size) < 0)
TEST_ERROR;
if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, buf_size, a_list_s + half_size) < 0)
TEST_ERROR;
/* Verify logical file contents. */
if (NULL == (buf = HDmalloc(buf_size * sizeof(char))))
TEST_ERROR;
if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, buf_size, buf) < 0)
TEST_ERROR;
if (HDmemcmp(a_list_s + half_size, buf, buf_size) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
/* Overwrite existing data with entire buffer at addr 0 */
if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, a_list_size_s) < 0)
TEST_ERROR;
if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, a_list_s) < 0)
TEST_ERROR;
/* Verify logical file contents. */
if (NULL == (buf = HDmalloc(a_list_size_s * sizeof(char))))
TEST_ERROR;
if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, buf) < 0)
TEST_ERROR;
if (HDmemcmp(a_list_s, buf, a_list_size_s) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
} /* end if writing data to logical file */
/*
* CLOSED
*/
if (H5FDclose(vfile_rw) < 0)
TEST_ERROR;
vfile_rw = NULL;
/* Look at h5 file: should be known-empty */
if (compare_file_bytes_exactly(paths->canon, onion_info.backing_fapl_id, 8,
(const unsigned char *)"ONIONEOF") < 0)
TEST_ERROR;
/* Look at recovery file: should be gone */
H5E_BEGIN_TRY
{
vfile_raw = H5FDopen(paths->recovery, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF);
}
H5E_END_TRY;
if (NULL != vfile_raw)
TEST_ERROR;
/* Inspect onion file */
vfile_raw = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF);
if (NULL == vfile_raw)
TEST_ERROR;
filter.page_size = onion_info.page_size;
filter.n_revisions = 1;
filter.origin_eof = 0;
filter.revisions[0].comment = onion_info.comment;
filter.revisions[0].n_index_entries = (uint64_t)(-1); /* don't care */
filter.revisions[0].revision_num = 0;
filter.revisions[0].parent_revision_num = 0;
filter.revisions[0].logical_eof = (TRUE == with_initial_data) ? a_list_size_s : 0;
if (verify_history_as_expected_onion(vfile_raw, &filter) < 0)
TEST_ERROR;
if (H5FDclose(vfile_raw) < 0)
TEST_ERROR;
vfile_raw = NULL;
/* R/O open the file with Onion VFD; inspect logial file */
vfile_ro = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == vfile_ro)
TEST_ERROR;
if (TRUE == with_initial_data) {
/* Initial revision contains data */
if (H5FDget_eof(vfile_ro, H5FD_MEM_DRAW) != a_list_size_s)
TEST_ERROR;
if (H5FDget_eoa(vfile_ro, H5FD_MEM_DRAW) != 0)
TEST_ERROR;
if (H5FDset_eoa(vfile_ro, H5FD_MEM_DRAW, a_list_size_s) < 0)
TEST_ERROR;
if (NULL == (buf = HDmalloc(a_list_size_s * 64 * sizeof(char))))
TEST_ERROR;
if (H5FDread(vfile_ro, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, buf) < 0)
TEST_ERROR;
if (HDmemcmp(a_list_s, buf, a_list_size_s) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
}
else {
/* Initial revision has no data */
if (H5FDget_eoa(vfile_ro, H5FD_MEM_DRAW) != 0)
TEST_ERROR;
if (H5FDget_eof(vfile_ro, H5FD_MEM_DRAW) != 0)
TEST_ERROR;
}
if (H5FDclose(vfile_ro) < 0)
TEST_ERROR;
vfile_ro = NULL;
/*
* CLEANUP
*/
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
HDfree(buf);
if (vfile_raw != NULL)
(void)H5FDclose(vfile_raw);
if (vfile_rw != NULL)
(void)H5FDclose(vfile_rw);
if (vfile_ro != NULL)
(void)H5FDclose(vfile_ro);
H5E_BEGIN_TRY
{
H5Pclose(fapl_id);
}
H5E_END_TRY;
return -1;
} /* end test_create_oniontarget() */
/*-----------------------------------------------------------------------------
*
* Function: test_several_revisions_with_logical_gaps()
*
* Purpose: Test the ability of the Onion VFD to create a valid
* 'onionized' file.
*
* When `truncate_canonical` is FALSE, the canonical file is
* nonexistent on the backing store on onion-creation.
* When `truncate_canonical` is TRUE, a canonical file is created
* on the backing store with known contents, which are to be
* truncated on onion-creation.
*
* Return: PASSED : 0
* FAILED : -1
*
*-----------------------------------------------------------------------------
*/
static int
test_several_revisions_with_logical_gaps(void)
{
const char *basename = "somesuch";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* flags */
"first" /* comment */
};
H5FD_t *file = NULL; /* Onion virtual file for read/write */
struct expected_history filter;
unsigned char *buf = NULL;
struct revise_revision about[4];
H5FD_onion_history_t history_out;
size_t i = 0;
haddr_t size = 0;
uint64_t a_off = ONION_TEST_PAGE_SIZE_5 + 7; /* 39 */
uint64_t b_off = (((a_off + a_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5) << 5) +
ONION_TEST_PAGE_SIZE_5 + 7; /* full page between */
TESTING("multiple revisions with gaps and overlap");
/*********
* SETUP *
*********/
history_out.version = H5FD_ONION_HISTORY_VERSION_CURR;
history_out.n_revisions = 0;
history_out.record_locs = NULL;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/* Empty first revision */
about[0].truncate = TRUE;
about[0].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
about[0].comment = "first";
about[0].n_writes = 0;
about[1].truncate = FALSE;
about[1].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
about[1].comment = "second";
about[1].n_writes = 1;
about[1].writes[0].offset = a_off;
about[1].writes[0].size = a_list_size_s;
about[1].writes[0].buf = a_list_s;
about[2].truncate = FALSE;
about[2].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
about[2].comment = "third";
about[2].n_writes = 1; /* TODO: several writes */
about[2].writes[0].offset = b_off;
about[2].writes[0].size = b_list_size_s;
about[2].writes[0].buf = b_list_s;
about[3].truncate = FALSE;
about[3].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
about[3].comment = "fourth";
about[3].n_writes = 1;
about[3].writes[0].offset = 0;
about[3].writes[0].size = a_list_size_s;
about[3].writes[0].buf = a_list_s;
if (do_onion_open_and_writes(paths->canon, &onion_info, 4, about) < 0)
TEST_ERROR;
/* Inspect logical file */
/* THIS IS THE INITIAL FILE, SHOULD ONLY HAVE 8 BYTES */
onion_info.revision_num = 0;
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (H5I_INVALID_HID == fapl_id)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
if (8 != H5FDget_eof(file, H5FD_MEM_DRAW)) {
HDprintf("\nEOF is not zero, it is: %" PRIuHADDR "\n", H5FDget_eof(file, H5FD_MEM_DRAW));
TEST_ERROR;
}
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/* Empty first revision */
onion_info.revision_num = 1;
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (H5I_INVALID_HID == fapl_id)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
if (0 != H5FDget_eof(file, H5FD_MEM_DRAW)) {
HDprintf("\nEOF is not zero, it is: %" PRIuHADDR "\n", H5FDget_eof(file, H5FD_MEM_DRAW));
TEST_ERROR;
}
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/* One offset block in second revision */
onion_info.revision_num = 2;
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (H5I_INVALID_HID == fapl_id)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
size = a_off + a_list_size_s;
if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) {
HDprintf("\nEOF is not %" PRIuHADDR ", it is: %" PRIuHADDR "\n", size,
H5FDget_eof(file, H5FD_MEM_DRAW));
TEST_ERROR;
}
if (NULL == (buf = HDmalloc(size * sizeof(unsigned char))))
TEST_ERROR;
if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0)
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0)
TEST_ERROR;
for (i = 0; i < a_off; i++) {
if (0 != buf[i])
TEST_ERROR;
}
if (HDmemcmp(buf + a_off, a_list_s, a_list_size_s) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
/* Repeat read at page offset; test possible read offset error */
if (NULL == (buf = HDmalloc(ONION_TEST_PAGE_SIZE_5 * sizeof(unsigned char))))
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, ONION_TEST_PAGE_SIZE_5, ONION_TEST_PAGE_SIZE_5, buf) < 0)
TEST_ERROR;
size = a_off - ONION_TEST_PAGE_SIZE_5;
for (i = 0; i < size; i++) {
if (0 != buf[i])
TEST_ERROR;
}
if (HDmemcmp(buf + size, a_list_s, ONION_TEST_PAGE_SIZE_5 - size) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/* Two offset blocks in third revision */
onion_info.revision_num = 3;
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (H5I_INVALID_HID == fapl_id)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
size = b_off + b_list_size_s;
if (size != H5FDget_eof(file, H5FD_MEM_DRAW))
TEST_ERROR;
if (NULL == (buf = HDmalloc(size * sizeof(unsigned char))))
TEST_ERROR;
if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0)
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0)
TEST_ERROR;
for (i = 0; i < a_off; i++) {
if (0 != buf[i])
TEST_ERROR;
}
if (HDmemcmp(buf + a_off, a_list_s, a_list_size_s) != 0)
TEST_ERROR;
for (i = a_off + a_list_size_s; i < b_off; i++) {
if (0 != buf[i])
TEST_ERROR;
}
if (HDmemcmp(buf + b_off, b_list_s, b_list_size_s) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/* From start and partial overwrite in fourth revision */
onion_info.revision_num = 4;
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (H5I_INVALID_HID == fapl_id)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
size = b_off + b_list_size_s;
if (size != H5FDget_eof(file, H5FD_MEM_DRAW))
TEST_ERROR;
buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * size);
if (NULL == buf)
TEST_ERROR;
if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0)
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0)
TEST_ERROR;
if (HDmemcmp(buf, a_list_s, a_list_size_s) != 0)
TEST_ERROR;
if (HDmemcmp(buf + a_list_size_s, a_list_s + a_list_size_s - a_off, a_off) != 0)
TEST_ERROR;
for (i = a_off + a_list_size_s; i < b_off; i++) {
if (0 != buf[i])
TEST_ERROR;
}
if (HDmemcmp(buf + b_off, b_list_s, b_list_size_s) != 0)
TEST_ERROR;
HDfree(buf);
buf = NULL;
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/* No fifth revision */
/* Inspect history construction */
file = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
filter.page_size = onion_info.page_size;
filter.n_revisions = 4;
filter.origin_eof = 0;
filter.revisions[0].comment = "first";
filter.revisions[0].n_index_entries = 0;
filter.revisions[0].revision_num = 0;
filter.revisions[0].parent_revision_num = 0;
filter.revisions[0].logical_eof = 0;
filter.revisions[1].comment = "second";
filter.revisions[1].n_index_entries = (a_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5;
filter.revisions[1].revision_num = 1;
filter.revisions[1].parent_revision_num = 0;
filter.revisions[1].logical_eof = a_off + a_list_size_s;
filter.revisions[2].comment = "third";
filter.revisions[2].n_index_entries =
filter.revisions[1].n_index_entries + ((b_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5);
filter.revisions[2].revision_num = 2;
filter.revisions[2].parent_revision_num = 1;
filter.revisions[2].logical_eof = b_off + b_list_size_s;
filter.revisions[3].comment = "fourth";
filter.revisions[3].n_index_entries = filter.revisions[2].n_index_entries + 1;
filter.revisions[3].revision_num = 3;
filter.revisions[3].parent_revision_num = 2;
filter.revisions[3].logical_eof = b_off + b_list_size_s;
if (verify_history_as_expected_onion(file, &filter) < 0)
TEST_ERROR;
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
/* CLEANUP */
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
HDfree(history_out.record_locs);
HDfree(buf);
if (file != NULL)
(void)H5FDclose(file);
H5E_BEGIN_TRY
{
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* end test_several_revisions_with_logical_gaps() */
/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
*
* Function: do_onion_open_and_writes
*
* Purpose: Automate the process of creating/opening a file and performing
* a series of writes.
*
* Return: Success : 0
* Failure : -1
*
*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
*/
// TODO: Modify to create initial file without onion
static int
do_onion_open_and_writes(const char *filename, H5FD_onion_fapl_info_t *onion_info_p, size_t n_ops,
struct revise_revision *about)
{
hid_t fapl_id = H5I_INVALID_HID;
H5FD_t *file = NULL; /* Onion virtual file for read/write */
unsigned char *buf_vfy = NULL;
size_t i = 0;
for (i = 0; i < n_ops; i++) {
size_t j = 0;
unsigned int flags = H5F_ACC_RDWR;
if (i != 0 && about[i].truncate == TRUE)
goto error;
if (TRUE == about[i].truncate)
flags |= H5F_ACC_CREAT | H5F_ACC_TRUNC;
onion_info_p->revision_num = about[i].revision_num;
if (about[i].comment != NULL) {
j = MIN(HDstrlen(about[i].comment), H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN);
HDmemcpy(onion_info_p->comment, about[i].comment, j);
}
onion_info_p->comment[j] = '\0';
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if (H5I_INVALID_HID == fapl_id)
goto error;
if (H5Pset_fapl_onion(fapl_id, onion_info_p) < 0)
goto error;
file = H5FDopen(filename, flags, fapl_id, HADDR_UNDEF);
if (NULL == file)
goto error;
for (j = 0; j < about[i].n_writes; j++) {
struct write_info *wi = &about[i].writes[j];
/* Write to file */
if (H5FDget_eoa(file, H5FD_MEM_DRAW) < wi->offset + wi->size &&
H5FDset_eoa(file, H5FD_MEM_DRAW, wi->offset + wi->size) < 0)
TEST_ERROR;
if (H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, wi->offset, wi->size, wi->buf) < 0)
TEST_ERROR;
/* Verify write as expected */
if (NULL == (buf_vfy = HDmalloc(wi->size * sizeof(unsigned char))))
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, wi->offset, wi->size, buf_vfy) < 0)
TEST_ERROR;
if (HDmemcmp(buf_vfy, wi->buf, wi->size) != 0) {
const unsigned char *_buf = wi->buf;
size_t z = 0;
HDputs("i exp act");
for (z = 0; z < wi->size; z++)
HDprintf("%02zx %c %c\n", z, _buf[z], buf_vfy[z]);
HDfflush(stdout);
TEST_ERROR;
}
HDfree(buf_vfy);
buf_vfy = NULL;
} /* end for each write */
if (H5FDclose(file) < 0)
goto error;
file = NULL;
if (H5Pclose(fapl_id) < 0)
goto error;
fapl_id = H5I_INVALID_HID;
} /* end for each open-close cycle */
return 0;
error:
if (file != NULL)
(void)H5FDclose(file);
H5E_BEGIN_TRY
{
H5Pclose(fapl_id);
}
H5E_END_TRY;
HDfree(buf_vfy);
return -1;
} /* end do_onion_open_and_writes() */
/*-----------------------------------------------------------------------------
*
* Function: test_page_aligned_history_create()
*
* Purpose: Verify that, when specified in FAPL on onionization/creation,
* All history writes are aligned to page-size boundaries.
*
* Return: PASSED : 0
* FAILED : -1
*
*-----------------------------------------------------------------------------
*/
static int
test_page_aligned_history_create(void)
{
const char *basename = "somesuch";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT,
"initial commit" /* comment */
};
H5FD_t *file = NULL; /* Onion virtual file for read/write */
unsigned char *buf = NULL;
struct revise_revision about[2];
H5FD_onion_header_t hdr_out;
H5FD_onion_history_t history_out;
size_t i = 0;
uint64_t a_off = b_list_size_s - a_list_size_s;
TESTING("page-aligned history on onion-created file");
/*********
* SETUP *
*********/
hdr_out.version = H5FD_ONION_HEADER_VERSION_CURR;
history_out.version = H5FD_ONION_HISTORY_VERSION_CURR;
history_out.n_revisions = 0;
history_out.record_locs = NULL;
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
about[0].truncate = TRUE;
about[0].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
about[0].comment = "initial_commit";
about[0].n_writes = 1;
about[0].writes[0].offset = 0;
about[0].writes[0].size = b_list_size_s;
about[0].writes[0].buf = b_list_s;
about[1].truncate = FALSE;
about[1].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST;
about[1].comment = "second";
about[1].n_writes = 1;
about[1].writes[0].offset = a_off;
about[1].writes[0].size = a_list_size_s;
about[1].writes[0].buf = a_list_s;
if (do_onion_open_and_writes(paths->canon, &onion_info, 2, about) < 0)
TEST_ERROR;
/* Inspect logical file */
if (NULL == (buf = HDmalloc(b_list_size_s * sizeof(unsigned char))))
TEST_ERROR;
file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF);
if (NULL == file)
TEST_ERROR;
if (b_list_size_s != H5FDget_eof(file, H5FD_MEM_DRAW))
TEST_ERROR;
if (H5FDset_eoa(file, H5FD_MEM_DRAW, b_list_size_s) < 0)
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, b_list_size_s, buf) < 0)
TEST_ERROR;
if (HDmemcmp(a_list_s, buf + a_off, a_list_size_s) != 0) {
size_t k;
HDprintf("aoff: %" PRIu64 "\n", a_off);
HDputs("i exp act");
for (k = 0; k < b_list_size_s; k++) {
HDprintf("%3zu:: %c : %c\n", k, (k < a_off) ? ' ' : a_list_s[k - a_off], buf[k]);
}
HDfflush(stdout);
TEST_ERROR;
}
if (HDmemcmp(b_list_s, buf, a_off) != 0) {
size_t k;
HDprintf("aoff: %" PRIu64 "\n", a_off);
HDputs("i exp act");
for (k = 0; k < b_list_size_s; k++) {
HDprintf("%3zu:: %c : %c\n", k, (k < a_off) ? b_list_s[k] : ' ', buf[k]);
}
HDfflush(stdout);
TEST_ERROR;
}
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
HDfree(buf);
buf = NULL;
/* Inspect history construction */
if (NULL == (file = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF)))
TEST_ERROR;
if (H5FDset_eoa(file, H5FD_MEM_DRAW, H5FDget_eof(file, H5FD_MEM_DRAW)) < 0)
TEST_ERROR;
if (NULL == (buf = HDmalloc(H5FD_ONION_ENCODED_SIZE_HEADER)))
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, H5FD_ONION_ENCODED_SIZE_HEADER, buf) < 0)
TEST_ERROR;
if (H5FD__onion_header_decode(buf, &hdr_out) != H5FD_ONION_ENCODED_SIZE_HEADER)
TEST_ERROR;
if (hdr_out.history_addr & ((1 << 5) - 1)) /* 5::PAGE_SIZE_5 */
TEST_ERROR;
HDfree(buf);
buf = NULL;
if (NULL == (buf = HDmalloc(hdr_out.history_size)))
TEST_ERROR;
if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, hdr_out.history_addr, hdr_out.history_size, buf) < 0)
TEST_ERROR;
if (H5FD__onion_history_decode(buf, &history_out) != hdr_out.history_size)
TEST_ERROR;
if (history_out.n_revisions != 2)
TEST_ERROR;
history_out.record_locs = HDcalloc(history_out.n_revisions, sizeof(H5FD_onion_record_loc_t));
if (NULL == history_out.record_locs)
TEST_ERROR;
if (H5FD__onion_history_decode(buf, &history_out) != hdr_out.history_size)
TEST_ERROR;
HDfree(buf);
buf = NULL;
for (i = 0; i < history_out.n_revisions; i++) {
H5FD_onion_record_loc_t *rloc = &history_out.record_locs[i];
if (rloc->phys_addr & ((1 << 5) - 1)) /* 5::PAGE_SIZE_5 */
TEST_ERROR;
/* TODO: check phys_addr of each page entry? */
}
HDfree(history_out.record_locs);
history_out.record_locs = NULL;
if (H5FDclose(file) < 0)
TEST_ERROR;
file = NULL;
/* CLEANUP */
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
HDfree(buf);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
HDfree(history_out.record_locs);
HDfree(buf);
if (file != NULL)
(void)H5FDclose(file);
H5E_BEGIN_TRY
{
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* end test_page_aligned_history_create() */
/*-----------------------------------------------------------------------------
* Function: test_integration_create()
*
* Purpose: Create and make multiple revisions in an HDF5 file.
*
* Return: PASSED : 0
* FAILED : -1
*-----------------------------------------------------------------------------
*/
static int
test_integration_create(void)
{
const char *basename = "integration_2d.h5";
hid_t fapl_id = H5I_INVALID_HID;
hid_t file_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hsize_t dims[2] = {128, 256};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
int fillval;
struct {
int arr[128][256];
} *wdata = NULL;
struct {
int arr[128][256];
} *rdata = NULL;
struct {
int arr[128][256];
} *dset_data = NULL;
TESTING("onion-created two dimensional HDF5 file with revisions");
/* SETUP */
if (NULL == (wdata = HDcalloc(1, sizeof(*wdata))))
TEST_ERROR;
if (NULL == (rdata = HDcalloc(1, sizeof(*rdata))))
TEST_ERROR;
if (NULL == (dset_data = HDcalloc(1, sizeof(*dset_data))))
TEST_ERROR;
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 128; i++)
for (int j = 0; j < 256; j++)
wdata->arr[i][j] = i * j - j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
TEST_ERROR;
/* Create the dataset creation property list, and set the chunk
* size
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
TEST_ERROR;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
TEST_ERROR;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
/* Close and release resources */
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < 128; i++)
for (int j = 0; j < 256; j++)
dset_data->arr[i][j] = i * 6 + j + 1;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < 128; i++)
for (int j = 0; j < 256; j++)
dset_data->arr[i][j] = i * 3 + j + 5;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
/* CLEANUP */
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Start to verify the revision
*----------------------------------------------------------------------
*/
/*----------------------------------------------------------------------
* Verify the original file
*----------------------------------------------------------------------
*/
onion_info.revision_num = 0;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < 128; i++) {
for (int j = 0; j < 256; j++) {
int expected = i * j - j;
if (rdata->arr[i][j] != expected) {
HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i][j]);
HDfflush(stdout);
TEST_ERROR;
}
}
}
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Verify the first revision
*----------------------------------------------------------------------
*/
onion_info.revision_num = 1;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < 128; i++) {
for (int j = 0; j < 256; j++) {
int expected = i * 6 + j + 1;
if (rdata->arr[i][j] != expected) {
HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i][j]);
HDfflush(stdout);
TEST_ERROR;
}
}
}
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Verify the second revision
*----------------------------------------------------------------------
*/
onion_info.revision_num = 2;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < 128; i++) {
for (int j = 0; j < 256; j++) {
int expected = i * 3 + j + 5;
if (rdata->arr[i][j] != expected) {
HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i][j]);
HDfflush(stdout);
TEST_ERROR;
}
else {
HDfflush(stdout);
}
}
}
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
HDfree(wdata);
HDfree(rdata);
HDfree(dset_data);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file_id);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
HDfree(wdata);
HDfree(rdata);
HDfree(dset_data);
return -1;
} /* end test_integration_create() */
static int
test_integration_create_simple(void)
{
const char *basename = "integration_1d.h5";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t file_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {1, ONE_DIM_SIZE};
hsize_t maxdims[2] = {1, ONE_DIM_SIZE};
int fillval;
struct {
int arr[ONE_DIM_SIZE];
} *wdata = NULL;
struct {
int arr[ONE_DIM_SIZE];
} *rdata = NULL;
struct {
int arr[ONE_DIM_SIZE];
} *dset_data = NULL;
TESTING("onion-created one-dimensional HDF5 file with revisions");
/* Setup */
if (NULL == (wdata = HDcalloc(1, sizeof(*wdata))))
TEST_ERROR;
if (NULL == (rdata = HDcalloc(1, sizeof(*rdata))))
TEST_ERROR;
if (NULL == (dset_data = HDcalloc(1, sizeof(*dset_data))))
TEST_ERROR;
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < ONE_DIM_SIZE; i++)
wdata->arr[i] = i;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
TEST_ERROR;
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
TEST_ERROR;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
/* Close everything */
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data->arr[i] = i + ONE_DIM_SIZE;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data->arr[i] = i + 2048;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
/* CLEANUP */
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i += 20)
dset_data->arr[i] = i + 3072;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
/* CLEANUP */
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Start to verify the revision
*----------------------------------------------------------------------
*/
/*----------------------------------------------------------------------
* Verify the second revision
*----------------------------------------------------------------------
*/
onion_info.revision_num = 2;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i += 20) {
int expected = i + 2048;
if (rdata->arr[i] != expected) {
HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i]);
TEST_ERROR;
}
}
/* Close everything */
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
HDfree(wdata);
HDfree(rdata);
HDfree(dset_data);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file_id);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
HDfree(wdata);
HDfree(rdata);
HDfree(dset_data);
return -1;
} /* end test_integration_create_simple() */
static int
test_integration_create_delete_objects(void)
{
const char *basename = "integration_objs.h5";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t group_id = H5I_INVALID_HID;
hid_t attr_space_id = H5I_INVALID_HID;
hid_t attr_id = H5I_INVALID_HID;
hsize_t attr_dim[1] = {4};
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {4, 4}, maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}, chunk[2] = {4, 4};
int wdata[4][4], /* Write buffer */
fillval, i, j;
TESTING("onion-created HDF5 file with revisions testing addition and deletion of objects");
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (i = 0; i < 4; i++)
for (j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
TEST_ERROR;
/* Create the dataset creation property list, and set the chunk
* size
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
TEST_ERROR;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
TEST_ERROR;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2) to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and remove the dataset (DS2),
* which was added during the first revision.
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and add an attribute to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Create dataspace for attribute */
if ((attr_space_id = H5Screate_simple(1, attr_dim, NULL)) < 0)
TEST_ERROR;
if ((attr_id =
H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Sclose(attr_space_id) < 0)
TEST_ERROR;
if (H5Aclose(attr_id) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fourth revision: open the file with Onion VFD and delete the attribute
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if (H5Adelete(file, "file_attribute") < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fifth revision: open the file with Onion VFD and add a group to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Gclose(group_id) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Sixth revision: open the file with Onion VFD and delete the newly added group
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Start to verify the revision
*----------------------------------------------------------------------
*/
/*----------------------------------------------------------------------
* Verify the first revision: it should have the second dataset (DS2)
*----------------------------------------------------------------------
*/
onion_info.revision_num = 1;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* The second dataset (DS2) should exist */
if (H5Lexists(file, "DS2", H5P_DEFAULT) <= 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*------------------------------------------------------------------------
* Verify the second revision: the second dataset (DS2) should be removed
*------------------------------------------------------------------------
*/
onion_info.revision_num = 2;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* The second dataset (DS2) shouldn't exist */
if (H5Lexists(file, "DS2", H5P_DEFAULT) > 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*-------------------------------------------------------------------------
* Verify the third revision: the file attribute (file_attribute) should exist
*-------------------------------------------------------------------------
*/
onion_info.revision_num = 3;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* The file attribute should exist */
if (H5Aexists(file, "file_attribute") <= 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*-------------------------------------------------------------------------
* Verify the fourth revision: the file attribute (file_attribute) should be removed
*-------------------------------------------------------------------------
*/
onion_info.revision_num = 4;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* The file attribute should be removed */
if (H5Aexists(file, "file_attribute") > 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*-------------------------------------------------------------------------
* Verify the fifth revision: the group (new_group) should exist
*-------------------------------------------------------------------------
*/
onion_info.revision_num = 5;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* The new group should exist */
if (H5Lexists(file, "new_group", H5P_DEFAULT) <= 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*-------------------------------------------------------------------------
* Verify the sixth revision: the group (new_group) should be removed
*-------------------------------------------------------------------------
*/
onion_info.revision_num = 6;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* The new group should exist */
if (H5Lexists(file, "new_group", H5P_DEFAULT) > 0)
TEST_ERROR;
/* Close everything */
if (H5Fclose(file) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Sclose(space);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* end test_integration_create_delete_objects */
static int
test_integration_dset_extension(void)
{
const char *basename = "integration_dset_ext.h5";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset_space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
hsize_t size[2];
hsize_t offset[2];
int wdata[4][4]; /* Write buffer */
int fillval;
int rdata[4][4]; /* Read buffer */
TESTING("onion-created HDF5 file with revisions testing dataset extension");
/* Setup */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
TEST_ERROR;
/* Create the dataset creation property list, and set the chunk
* size
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
TEST_ERROR;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
TEST_ERROR;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and extend the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Extend the dataset and double the rows */
size[0] = 2 * dims[0];
size[1] = dims[1];
if (H5Dset_extent(dset, size) < 0)
TEST_ERROR;
if ((dset_space = H5Dget_space(dset)) < 0)
TEST_ERROR;
offset[0] = dims[0];
offset[1] = 0;
if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0)
TEST_ERROR;
/* Write the data to the dataset. */
if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Sclose(dset_space) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and shrink the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Extend the dataset and shrink back the size */
if (H5Dset_extent(dset, dims) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Start to verify the revision
*----------------------------------------------------------------------
*/
/*----------------------------------------------------------------------
* Verify the first revision: it should have the extended data
*----------------------------------------------------------------------
*/
onion_info.revision_num = 1;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((dset_space = H5Dget_space(dset)) < 0)
TEST_ERROR;
offset[0] = dims[0];
offset[1] = 0;
if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
if (rdata[i][j] != wdata[i][j])
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Verify the second revision: it should have the original data
*----------------------------------------------------------------------
*/
onion_info.revision_num = 2;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Open the dataset */
dset = H5Dopen2(file, "DS1", H5P_DEFAULT);
if ((dset_space = H5Dget_space(dset)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
if (rdata[i][j] != wdata[i][j])
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/* Close and release resources. */
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* end test_integration_dset_extension */
static int
test_integration_ctl(void)
{
const char *basename = "integration_ctl.h5";
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
int wdata[4][4]; /* Write buffer */
int fillval;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
uint64_t revision_count;
TESTING("onion-created HDF5 file with revisions testing H5FDctl");
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
TEST_ERROR;
/* Create the dataset creation property list, and set the chunk size */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
TEST_ERROR;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
TEST_ERROR;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2) to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and remove the dataset (DS2),
* which was added during the first revision.
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Start to verify the number of revisions
*----------------------------------------------------------------------
*/
if (H5FDonion_get_revision_count(basename, fapl_id, &revision_count) < 0)
TEST_ERROR;
if (2 != revision_count)
TEST_ERROR;
/* Close and release resources */
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
}
static int
test_integration_reference(void)
{
const char *basename = "integration_refer.h5";
hid_t file = H5I_INVALID_HID;
hid_t group = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t space2 = H5I_INVALID_HID;
hid_t space_ref = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dset2 = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
hsize_t dims[2] = {4, 4};
hsize_t dim_ref[1] = {2};
int wdata[4][4]; /* Write buffer */
int rdata[4][4]; /* Read buffer */
H5R_ref_t wbuf[2];
H5R_ref_t rbuf[2];
H5O_type_t obj_type;
hsize_t start[2];
hsize_t stride[2];
hsize_t count[2];
hsize_t block[2];
hsize_t coord1[4][2]; /* Coordinates for point selection */
hssize_t nelmts;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE_5, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
TESTING("onion-created HDF5 file with revisions testing references");
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace */
if ((space = H5Screate_simple(2, dims, NULL)) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2)
* of object references
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions */
if ((space_ref = H5Screate_simple(1, dim_ref, NULL)) < 0)
TEST_ERROR;
/* Create the dataset of object references */
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_REF, space_ref, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create reference to dataset */
if (H5Rcreate_object(file, "DS1", H5P_DEFAULT, &wbuf[0]) < 0)
TEST_ERROR;
if (H5Rget_obj_type3(&wbuf[0], H5P_DEFAULT, &obj_type) < 0)
TEST_ERROR;
if (obj_type != H5O_TYPE_DATASET)
TEST_ERROR;
/* Create reference to the root group */
if (H5Rcreate_object(file, "/", H5P_DEFAULT, &wbuf[1]) < 0)
TEST_ERROR;
if (H5Rget_obj_type3(&wbuf[1], H5P_DEFAULT, &obj_type) < 0)
TEST_ERROR;
if (obj_type != H5O_TYPE_GROUP)
TEST_ERROR;
/* Write the object reference data to the dataset */
if (H5Dwrite(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
for (int i = 0; i < 2; i++)
if (H5Rdestroy(&wbuf[i]) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and add a dataset (DS3)
* of region references
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
/* Create the dataset of region references */
if ((dset = H5Dcreate2(file, "DS3", H5T_STD_REF, space_ref, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Select 2x4 hyperslab for first reference */
start[0] = 0;
start[1] = 0;
stride[0] = 1;
stride[1] = 1;
count[0] = 1;
count[1] = 1;
block[0] = 2;
block[1] = 4;
/* Make a hyperslab selection of 2x4 elements */
if (H5Sselect_hyperslab(space, H5S_SELECT_SET, start, stride, count, block) < 0)
TEST_ERROR;
/* Verify the number of selection */
if ((nelmts = H5Sget_select_npoints(space)) != 8) {
HDprintf("Number of selected elements is supposed to be 8, but got %" PRIuHSIZE "\n", nelmts);
TEST_ERROR;
}
/* Store first data region */
if (H5Rcreate_region(file, "/DS1", space, H5P_DEFAULT, &wbuf[0]) < 0)
TEST_ERROR;
if (H5Rget_obj_type3(&wbuf[0], H5P_DEFAULT, &obj_type) < 0)
TEST_ERROR;
if (obj_type != H5O_TYPE_DATASET)
TEST_ERROR;
/* Select the sequence of four points for the second reference */
coord1[0][0] = 0;
coord1[0][1] = 0;
coord1[1][0] = 1;
coord1[1][1] = 1;
coord1[2][0] = 2;
coord1[2][1] = 2;
coord1[3][0] = 3;
coord1[3][1] = 3;
if (H5Sselect_elements(space, H5S_SELECT_SET, 4, (const hsize_t *)coord1) < 0)
TEST_ERROR;
/* Store the second data region */
if (H5Rcreate_region(file, "/DS1", space, H5P_DEFAULT, &wbuf[1]) < 0)
TEST_ERROR;
if (H5Rget_obj_type3(&wbuf[1], H5P_DEFAULT, &obj_type) < 0)
TEST_ERROR;
if (obj_type != H5O_TYPE_DATASET)
TEST_ERROR;
/* Write the region reference data to the dataset */
if (H5Dwrite(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
for (int i = 0; i < 2; i++)
if (H5Rdestroy(&wbuf[i]) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* Start to verify the revisions
*----------------------------------------------------------------------
*/
/*----------------------------------------------------------------------
* Verify the first revision: it should have the object references
*----------------------------------------------------------------------
*/
onion_info.revision_num = 1;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
/* Open the dataset of the object references */
if ((dset = H5Dopen2(file, "DS2", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0)
TEST_ERROR;
/* Open the referenced dataset and check the data */
if ((dset2 = H5Ropen_object(&rbuf[0], H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < 4; i++) {
for (int j = 0; j < 4; j++) {
int expected = i + j;
if (rdata[i][j] != expected) {
HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata[i][j]);
TEST_ERROR;
}
}
}
/* Open the referenced group and make sure it's a group object */
if ((group = H5Ropen_object(&rbuf[1], H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5I_GROUP != H5Iget_type(group))
TEST_ERROR;
if (H5Gclose(group) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Dclose(dset2) < 0)
TEST_ERROR;
dset2 = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
for (int i = 0; i < 2; i++)
if (H5Rdestroy(&rbuf[i]) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* Verify the second revision: it should have the region references
*----------------------------------------------------------------------
*/
onion_info.revision_num = 2;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
TEST_ERROR;
if ((file = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
/* Open the dataset of the region reference */
if ((dset = H5Dopen2(file, "DS3", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0)
TEST_ERROR;
/* Get the hyperslab selection and check the referenced region of the dataset */
if ((space2 = H5Ropen_region(&rbuf[0], H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((nelmts = H5Sget_select_npoints(space2)) != 8) {
HDprintf("Number of selected elements is supposed to be 8, but got %" PRIuHSIZE "\n", nelmts);
TEST_ERROR;
}
if (H5Sclose(space2) < 0)
TEST_ERROR;
space2 = H5I_INVALID_HID;
/* Get the element selection and check the referenced region of the dataset */
if ((space2 = H5Ropen_region(&rbuf[1], H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
if ((nelmts = H5Sget_select_npoints(space2)) != 4) {
HDprintf("Number of selected elements is supposed to be 4, but got %" PRIuHSIZE "\n", nelmts);
TEST_ERROR;
}
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Sclose(space2) < 0)
TEST_ERROR;
space2 = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
TEST_ERROR;
file = H5I_INVALID_HID;
for (int i = 0; i < 2; i++)
if (H5Rdestroy(&rbuf[i]) < 0)
TEST_ERROR;
/* Close and release resources */
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Sclose(space_ref) < 0)
TEST_ERROR;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
}
static int
test_integration_create_by_name(void)
{
const char *basename = "integration_by_name.h5";
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
hid_t file_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {1, ONE_DIM_SIZE};
hsize_t maxdims[2] = {1, ONE_DIM_SIZE};
int fillval;
struct {
int arr[ONE_DIM_SIZE];
} *wdata = NULL;
struct {
int arr[ONE_DIM_SIZE];
} *rdata = NULL;
struct {
int arr[ONE_DIM_SIZE];
} *dset_data = NULL;
TESTING("H5Pset_driver_by_name");
/* Setup */
if (NULL == (wdata = HDcalloc(1, sizeof(*wdata))))
TEST_ERROR;
if (NULL == (rdata = HDcalloc(1, sizeof(*rdata))))
TEST_ERROR;
if (NULL == (dset_data = HDcalloc(1, sizeof(*dset_data))))
TEST_ERROR;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
/* Use H5Pset_driver_by_name to enable the Onion VFD */
if (H5Pset_driver_by_name(fapl_id, "onion", "{revision_num: H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST}") <
0)
TEST_ERROR;
if (NULL == (paths = onion_filepaths_init(basename)))
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < ONE_DIM_SIZE; i++)
wdata->arr[i] = i;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Create dataspace with unlimited dimensions*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
TEST_ERROR;
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
TEST_ERROR;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
TEST_ERROR;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
TEST_ERROR;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
TEST_ERROR;
/* Close everything */
if (H5Pclose(dcpl) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Sclose(space) < 0)
TEST_ERROR;
if (H5Fclose(file) < 0)
TEST_ERROR;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data->arr[i] = i + ONE_DIM_SIZE;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data->arr[i] = i + 2048;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
/* CLEANUP */
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i += 20)
dset_data->arr[i] = i + 3072;
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
TEST_ERROR;
/* CLEANUP */
if (H5Dclose(dset) < 0)
TEST_ERROR;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
file_id = H5I_INVALID_HID;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
fapl_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Start to verify the revision with H5Pset_driver_by_name
*----------------------------------------------------------------------
*/
/*----------------------------------------------------------------------
* Verify the second revision
*----------------------------------------------------------------------
*/
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
if (H5Pset_driver_by_name(fapl_id, "onion", "{revision_num: 2; page_size: 4; }") < 0)
TEST_ERROR;
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0)
TEST_ERROR;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
TEST_ERROR;
if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0)
TEST_ERROR;
for (int i = 0; i < ONE_DIM_SIZE; i += 20) {
int expected = i + 2048;
if (rdata->arr[i] != expected) {
HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i]);
TEST_ERROR;
}
}
/* Close everything */
if (H5Dclose(dset) < 0)
TEST_ERROR;
if (H5Fclose(file_id) < 0)
TEST_ERROR;
if (H5Pclose(fapl_id) < 0)
TEST_ERROR;
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
HDfree(wdata);
HDfree(rdata);
HDfree(dset_data);
PASSED();
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file_id);
H5Pclose(fapl_id);
}
H5E_END_TRY;
HDfree(wdata);
HDfree(rdata);
HDfree(dset_data);
return -1;
} /* end test_integration_create_simple() */
/*-----------------------------------------------------------------------------
*
* Function: main()
*
* Purpose: Perform unit tests on for the Onion VFD.
*
*-----------------------------------------------------------------------------
*/
int
main(void)
{
const char *env_h5_drvr = NULL; /* VFD value from environment */
int nerrors = 0;
HDprintf("Testing Onion VFD functionality.\n");
h5_reset();
/* The onion VFD only supports the sec2 VFD under the hood, so skip this
* test when the environment variable has been set to something else
*/
env_h5_drvr = HDgetenv(HDF5_DRIVER);
if (env_h5_drvr == NULL)
env_h5_drvr = "nomatch";
if ((0 != HDstrcmp(env_h5_drvr, "nomatch")) && (0 != HDstrcmp(env_h5_drvr, "sec2"))) {
SKIPPED();
HDputs("Onion VFD test skipped due to non-sec2 default VFD");
HDexit(EXIT_SUCCESS);
}
/* Initialize */
flags_create_s = H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC;
/* Run tests. Return values on error are negative. */
nerrors -= test_archival_index();
nerrors -= test_revision_index();
nerrors -= test_revision_index_collisions();
nerrors -= test_revision_index_resizing();
nerrors -= test_revision_index_to_archival_index();
nerrors -= test_fapl();
nerrors -= test_header_encode_decode();
nerrors -= test_history_encode_decode_empty();
nerrors -= test_history_encode_decode();
nerrors -= test_revision_record_encode_decode();
nerrors -= test_create_oniontarget(FALSE, FALSE);
nerrors -= test_create_oniontarget(TRUE, FALSE);
nerrors -= test_create_oniontarget(FALSE, TRUE);
nerrors -= test_create_oniontarget(TRUE, TRUE);
nerrors -= test_several_revisions_with_logical_gaps();
nerrors -= test_page_aligned_history_create();
nerrors -= test_integration_create();
nerrors -= test_integration_create_simple();
nerrors -= test_integration_create_delete_objects();
nerrors -= test_integration_dset_extension();
nerrors -= test_integration_ctl();
nerrors -= test_integration_reference();
nerrors -= test_integration_create_by_name();
if (nerrors > 0) {
HDprintf("***** %d Onion TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : "");
return EXIT_FAILURE;
}
HDprintf("All Onion tests passed.\n");
return EXIT_SUCCESS;
} /* end main() */
tools/lib/h5diff.c
+15 -2
View File
@@ -648,8 +648,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
*-------------------------------------------------------------------------
*/
/* open file 1 */
if (opts->vfd_info[0].u.name) {
if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[0]))) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
}
}
if (opts->custom_vol[0] || opts->custom_vfd[0]) {
if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL,
if ((fapl1_id = h5tools_get_fapl(fapl1_id, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL,
opts->custom_vfd[0] ? &(opts->vfd_info[0]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
@@ -664,9 +671,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
H5TOOLS_DEBUG("file1_id = %s", fname1);
/* open file 2 */
if (opts->vfd_info[1].u.name) {
if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[1]))) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
}
}
if (opts->custom_vol[1] || opts->custom_vfd[1]) {
if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL,
if ((fapl2_id = h5tools_get_fapl(fapl2_id, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL,
opts->custom_vfd[1] ? &(opts->vfd_info[1]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
tools/lib/h5tools.c
+8
View File
@@ -85,6 +85,7 @@ const char *drivernames[] = {
[WINDOWS_VFD_IDX] = "windows", [STDIO_VFD_IDX] = "stdio", [CORE_VFD_IDX] = "core",
[FAMILY_VFD_IDX] = "family", [SPLIT_VFD_IDX] = "split", [MULTI_VFD_IDX] = "multi",
[MPIO_VFD_IDX] = "mpio", [ROS3_VFD_IDX] = "ros3", [HDFS_VFD_IDX] = "hdfs",
[ONION_VFD_IDX] = "onion",
};
#define NUM_VOLS (sizeof(volnames) / sizeof(volnames[0]))
@@ -573,6 +574,13 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
H5TOOLS_GOTO_ERROR(FAIL, "The HDFS VFD is not enabled");
#endif
}
else if (!HDstrcmp(vfd_info->u.name, drivernames[ONION_VFD_IDX])) {
/* Onion driver */
if (!vfd_info->info)
H5TOOLS_GOTO_ERROR(FAIL, "Onion VFD info is invalid");
if (H5Pset_fapl_onion(fapl_id, (const H5FD_onion_fapl_info_t *)vfd_info->info) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_onion() failed");
}
else {
/*
* Try to load VFD plugin.
tools/lib/h5tools.h
+1
View File
@@ -600,6 +600,7 @@ typedef enum {
MPIO_VFD_IDX,
ROS3_VFD_IDX,
HDFS_VFD_IDX,
ONION_VFD_IDX,
} driver_idx;
/* The following include, h5tools_str.h, must be after the
tools/src/h5diff/h5diff_common.c
+57 -1
View File
@@ -56,6 +56,28 @@ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'},
{"vfd-info-2", require_arg, 'Z'},
{NULL, 0, '\0'}};
static H5FD_onion_fapl_info_t onion_fa_g_1 = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"first input file", /* comment */
};
static H5FD_onion_fapl_info_t onion_fa_g_2 = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"second input file", /* comment */
};
/*-------------------------------------------------------------------------
* Function: check_options
*
@@ -451,7 +473,7 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
case '8':
opts->vfd_info[0].type = VFD_BY_NAME;
opts->vfd_info[0].u.name = H5_optarg;
opts->custom_vol[0] = TRUE;
opts->custom_vfd[0] = TRUE;
break;
case '9':
@@ -476,6 +498,40 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
}
}
/* If file 1 uses the onion VFD, get the revision number */
if (opts->vfd_info[0].u.name && !HDstrcmp(opts->vfd_info[0].u.name, "onion")) {
if (opts->vfd_info[0].info) {
errno = 0;
onion_fa_g_1.revision_num = HDstrtoull(opts->vfd_info[0].info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified for file 1\n");
usage();
h5diff_exit(EXIT_FAILURE);
}
}
else
onion_fa_g_1.revision_num = 0;
opts->vfd_info[0].info = &onion_fa_g_1;
}
/* If file 2 uses the onion VFD, get the revision number */
if (opts->vfd_info[1].u.name && !HDstrcmp(opts->vfd_info[1].u.name, "onion")) {
if (opts->vfd_info[1].info) {
errno = 0;
onion_fa_g_2.revision_num = HDstrtoull(opts->vfd_info[1].info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified for file 2\n");
usage();
h5diff_exit(EXIT_FAILURE);
}
}
else
onion_fa_g_2.revision_num = 0;
opts->vfd_info[1].info = &onion_fa_g_2;
}
/* check options */
check_options(opts);
tools/src/h5dump/h5dump.c
+56 -5
View File
@@ -23,10 +23,13 @@ static hbool_t doxml_g = FALSE;
static hbool_t useschema_g = TRUE;
static const char *xml_dtd_uri_g = NULL;
static hbool_t use_custom_vol_g = FALSE;
static hbool_t use_custom_vfd_g = FALSE;
static h5tools_vol_info_t vol_info_g = {0};
static h5tools_vfd_info_t vfd_info_g = {0};
static hbool_t use_custom_vol_g = FALSE;
static hbool_t use_custom_vfd_g = FALSE;
static h5tools_vol_info_t vol_info_g = {0};
static h5tools_vfd_info_t vfd_info_g = {0};
static hbool_t get_onion_revision_count = FALSE;
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
@@ -51,6 +54,17 @@ static H5FD_hdfs_fapl_t hdfs_fa_g = {
};
#endif /* H5_HAVE_LIBHDFS */
static H5FD_onion_fapl_info_t onion_fa_g = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"input file", /* comment */
};
/* module-scoped variables for XML option */
#define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
#define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@@ -1305,6 +1319,29 @@ end_collect:
}
}
/* If the file uses the onion VFD, get the revision number */
if (vfd_info_g.u.name && !HDstrcmp(vfd_info_g.u.name, "onion")) {
if (vfd_info_g.info) {
if (!HDstrcmp(vfd_info_g.info, "revision_count"))
get_onion_revision_count = TRUE;
else {
errno = 0;
onion_fa_g.revision_num = HDstrtoull(vfd_info_g.info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified\n");
goto error;
}
HDprintf("Using revision %" PRIu64 "\n", onion_fa_g.revision_num);
}
}
else
onion_fa_g.revision_num = 0;
vfd_info_g.info = &onion_fa_g;
}
parse_end:
/* check for file name to be processed */
if (argc <= H5_optind) {
@@ -1424,7 +1461,21 @@ main(int argc, char *argv[])
while (H5_optind < argc) {
fname = HDstrdup(argv[H5_optind++]);
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
/* A short cut to get the revision count of an onion file without opening the file */
if (get_onion_revision_count && H5FD_ONION == H5Pget_driver(fapl_id)) {
uint64_t revision_count = 0;
if (H5FDonion_get_revision_count(fname, fapl_id, &revision_count) < 0) {
error_msg("unable to create FAPL for file access\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
HDprintf("The number of revisions for the onion file is %" PRIu64 "\n", revision_count);
goto done;
}
else
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
tools/src/h5repack/h5repack_main.c
+28
View File
@@ -76,6 +76,17 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
{"dst-vfd-info", require_arg, 'Z'},
{NULL, 0, '\0'}};
static H5FD_onion_fapl_info_t onion_fa_in_g = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5P_DEFAULT, /* backing_fapl_id */
32, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation_flags */
"input file", /* comment */
};
/*-------------------------------------------------------------------------
* Function: usage
*
@@ -875,6 +886,23 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options)
ret_value = -1;
}
/* If the input file uses the onion VFD, get the revision number */
if (in_vfd_info.u.name && !HDstrcmp(in_vfd_info.u.name, "onion")) {
if (in_vfd_info.info) {
errno = 0;
onion_fa_in_g.revision_num = HDstrtoull(in_vfd_info.info, NULL, 10);
if (errno == ERANGE) {
HDprintf("Invalid onion revision specified for the input file\n");
usage(h5tools_getprogname());
exit(EXIT_FAILURE);
}
}
else
onion_fa_in_g.revision_num = 0;
in_vfd_info.info = &onion_fa_in_g;
}
/* Setup FAPL for input and output file accesses */
if (custom_in_vol || custom_in_vfd) {
if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL,
tools/test/h5diff/CMakeTests.cmake
+23
View File
@@ -70,6 +70,13 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_strings2.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_eps1.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_eps2.h5
# onion VFD files
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_objs.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_objs.h5.onion
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_ext.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_ext.h5.onion
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_1d.h5
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_1d.h5.onion
# tools/testfiles/vds
${HDF5_TOOLS_DIR}/testfiles/vds/1_a.h5
${HDF5_TOOLS_DIR}/testfiles/vds/1_b.h5
@@ -295,6 +302,9 @@
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_801.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_830.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_90.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_900.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_901.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_902.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8625.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8639.txt
${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_reg.txt
@@ -932,6 +942,12 @@
h5diff_8639.out.err
h5diff_90.out
h5diff_90.out.err
h5diff_900.out
h5diff_900.out.err
h5diff_901.out
h5diff_901.out.err
h5diff_902.out
h5diff_902.out.err
h5diff_v1.out
h5diff_v1.out.err
h5diff_v2.out
@@ -1563,6 +1579,13 @@ ADD_H5_TEST (h5diff_v1 0 -v ${FILEV1} ${FILEV2})
ADD_H5_TEST (h5diff_v2 0 -r ${FILEV1} ${FILEV2})
ADD_H5_TEST (h5diff_v3 0 -c ${FILEV1} ${FILEV2})
# ##############################################################################
# # onion VFD tests (serial only)
# ##############################################################################
ADD_SH5_TEST (h5diff_900 1 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_objs.h5 h5diff_onion_objs.h5)
ADD_SH5_TEST (h5diff_901 0 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_ext.h5 h5diff_onion_dset_ext.h5)
ADD_SH5_TEST (h5diff_902 1 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_1d.h5 h5diff_onion_dset_1d.h5)
##############################################################################
### P L U G I N T E S T S
##############################################################################
tools/test/h5diff/h5diffgentest.c
+705 -24
View File
@@ -38,30 +38,34 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024);
*-------------------------------------------------------------------------
*/
#define FILE1 "h5diff_basic1.h5"
#define FILE2 "h5diff_basic2.h5"
#define FILE3 "h5diff_types.h5"
#define FILE4 "h5diff_dtypes.h5"
#define FILE5 "h5diff_attr1.h5"
#define FILE6 "h5diff_attr2.h5"
#define FILE6a "h5diff_attr3.h5"
#define FILE7 "h5diff_dset1.h5"
#define FILE8 "h5diff_dset2.h5"
#define FILE8A "h5diff_dset3.h5"
#define FILE9 "h5diff_hyper1.h5"
#define FILE10 "h5diff_hyper2.h5"
#define FILE11 "h5diff_empty.h5"
#define FILE12 "h5diff_links.h5"
#define FILE13 "h5diff_softlinks.h5"
#define FILE14 "h5diff_linked_softlink.h5"
#define FILE15 "h5diff_extlink_src.h5"
#define FILE16 "h5diff_extlink_trg.h5"
#define FILE17 "h5diff_ext2softlink_src.h5"
#define FILE18 "h5diff_ext2softlink_trg.h5"
#define FILE19 "h5diff_dset_zero_dim_size1.h5"
#define FILE20 "h5diff_dset_zero_dim_size2.h5"
#define FILE21 "h5diff_dset_idx1.h5"
#define FILE22 "h5diff_dset_idx2.h5"
#define FILE1 "h5diff_basic1.h5"
#define FILE2 "h5diff_basic2.h5"
#define FILE3 "h5diff_types.h5"
#define FILE4 "h5diff_dtypes.h5"
#define FILE5 "h5diff_attr1.h5"
#define FILE6 "h5diff_attr2.h5"
#define FILE6a "h5diff_attr3.h5"
#define FILE7 "h5diff_dset1.h5"
#define FILE8 "h5diff_dset2.h5"
#define FILE8A "h5diff_dset3.h5"
#define FILE9 "h5diff_hyper1.h5"
#define FILE10 "h5diff_hyper2.h5"
#define FILE11 "h5diff_empty.h5"
#define FILE12 "h5diff_links.h5"
#define FILE13 "h5diff_softlinks.h5"
#define FILE14 "h5diff_linked_softlink.h5"
#define FILE15 "h5diff_extlink_src.h5"
#define FILE16 "h5diff_extlink_trg.h5"
#define FILE17 "h5diff_ext2softlink_src.h5"
#define FILE18 "h5diff_ext2softlink_trg.h5"
#define FILE19 "h5diff_dset_zero_dim_size1.h5"
#define FILE20 "h5diff_dset_zero_dim_size2.h5"
#define FILE21 "h5diff_dset_idx1.h5"
#define FILE22 "h5diff_dset_idx2.h5"
#define FILE23 "h5diff_onion_dset_1d.h5"
#define FILE24 "h5diff_onion_objs.h5"
#define FILE25 "h5diff_onion_dset_ext.h5"
#define DANGLE_LINK_FILE1 "h5diff_danglelinks1.h5"
#define DANGLE_LINK_FILE2 "h5diff_danglelinks2.h5"
#define GRP_RECURSE_FILE1 "h5diff_grp_recurse1.h5"
@@ -111,6 +115,11 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024);
#define SPACE1_DIM1 0
#define SPACE1_DIM2 0
/* For Onion VFD */
#define ONION_TEST_FIXNAME_SIZE 1024
#define ONION_TEST_PAGE_SIZE (uint32_t)32
#define ONE_DIM_SIZE 16
/* Error macros */
#define AT() HDprintf("ERROR at %s:%d in %s()...\n", __FILE__, __LINE__, __func__);
#define PROGRAM_ERROR \
@@ -179,6 +188,11 @@ static void test_objs_nocomparables(const char *fname1, const char *fname2);
static void test_objs_strings(const char *fname, const char *fname2);
static void test_double_epsilon(const char *fname1, const char *fname2);
/* Generate the files for testing Onion VFD */
static int test_onion_1d_dset(const char *fname);
static int test_onion_create_delete_objects(const char *fname);
static int test_onion_dset_extension(const char *fname);
/* called by test_attributes() and test_datasets() */
static void write_attr_strings(hid_t loc_id, const char *dset_name, hid_t fid, int make_diffs);
static void write_attr_in(hid_t loc_id, const char *dset_name, hid_t fid, int make_diffs);
@@ -298,9 +312,676 @@ main(void)
/* double dataset and epsilion. HDFFV-10897 */
test_double_epsilon(DIFF_EPS1, DIFF_EPS2);
/* Generate the files for testing Onion VFD */
test_onion_1d_dset(FILE23);
test_onion_create_delete_objects(FILE24);
test_onion_dset_extension(FILE25);
return EXIT_SUCCESS;
}
/* Structure to collect the onion filepaths in one place. */
struct onion_filepaths {
char *canon;
char *onion;
char *recovery;
};
/* Allocate and populate filepaths with h5_fixname'd strings as appropriate.
* Should be released with onion_filepaths_destroy() when done.
*/
static struct onion_filepaths *
onion_filepaths_init(const char *basename)
{
struct onion_filepaths *paths = NULL;
if (NULL == (paths = HDcalloc(1, sizeof(struct onion_filepaths))))
goto error;
if (NULL == (paths->canon = HDstrdup(basename)))
goto error;
if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon);
if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon);
return paths;
error:
if (paths != NULL) {
HDfree(paths->canon);
HDfree(paths->onion);
HDfree(paths->recovery);
HDfree(paths);
}
return NULL;
}
static void
onion_filepaths_destroy(struct onion_filepaths *s)
{
if (s) {
HDfree(s->canon);
HDfree(s->onion);
HDfree(s->recovery);
HDfree(s);
}
}
static int
test_onion_1d_dset(const char *fname)
{
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {1, ONE_DIM_SIZE}, maxdims[2] = {1, ONE_DIM_SIZE};
int wdata[1][ONE_DIM_SIZE], /* Write buffer */
fillval;
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t file_id = H5I_INVALID_HID;
/* Setup */
onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(fname)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < ONE_DIM_SIZE; i++)
wdata[0][i] = i;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]) < 0)
goto error;
/* Close and release resources */
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
int dset_data[1][ONE_DIM_SIZE];
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + ONE_DIM_SIZE;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + 2048;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
goto error;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i += 20)
dset_data[0][i] = i + 3072;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(onion_info.backing_fapl_id);
H5Pclose(fapl_id);
H5Dclose(dset);
H5Sclose(space);
H5Fclose(file_id);
}
H5E_END_TRY;
return -1;
} /* test_onion_1d_dset */
static int
test_onion_create_delete_objects(const char *fname)
{
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t fapl_id = H5I_INVALID_HID;
hid_t group_id = H5I_INVALID_HID;
hid_t attr_space_id = H5I_INVALID_HID;
hid_t attr_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t attr_dim[1] = {4};
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
int wdata[4][4]; /* Write buffer */
int fillval;
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(fname)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/*
* Initialize data.
*/
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/*
* Create a new file using the default properties.
*/
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/*
* Create dataspace with unlimited dimensions.
*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/*
* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/*
* Set the fill value for the dataset.
*/
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/*
* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2) to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and remove the dataset (DS2),
* which was added during the first revision.
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and add an attribute to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Create dataspace for attribute */
attr_space_id = H5Screate_simple(1, attr_dim, NULL);
if ((attr_id =
H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Sclose(attr_space_id) < 0)
goto error;
if (H5Aclose(attr_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fourth revision: open the file with Onion VFD and delete the attribute
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Adelete(file, "file_attribute") < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fifth revision: open the file with Onion VFD and add a group to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Gclose(group_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Sixth revision: open the file with Onion VFD and delete the newly added group
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*
* Close and release resources.
*/
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
if (dset != H5I_INVALID_HID)
(void)H5Dclose(dset);
if (file != H5I_INVALID_HID)
(void)H5Fclose(file);
if (fapl_id != H5I_INVALID_HID)
(void)H5Pclose(fapl_id);
if (onion_info.backing_fapl_id != H5I_INVALID_HID)
H5Pclose(onion_info.backing_fapl_id);
return -1;
} /* test_onion_create_delete_objects */
static int
test_onion_dset_extension(const char *fname)
{
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset_space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t dims[2] = {4, 4}, maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}, chunk[2] = {4, 4};
hsize_t size[2], offset[2];
int wdata[4][4], /* Write buffer */
fillval;
/* Setup */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(fname)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and extend the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and double the rows */
size[0] = 2 * dims[0];
size[1] = dims[1];
if (H5Dset_extent(dset, size) < 0)
goto error;
if ((dset_space = H5Dget_space(dset)) < 0)
goto error;
offset[0] = dims[0];
offset[1] = 0;
if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0)
goto error;
/* Write the data to the dataset. */
if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Sclose(dset_space) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and shrink the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and shrink back the size */
if (H5Dset_extent(dset, dims) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/* Close and release resources. */
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* test_onion_dset_extension */
/*-------------------------------------------------------------------------
* Function: test_basic
*
tools/test/h5diff/testfiles/h5diff_900.txt
+12
View File
@@ -0,0 +1,12 @@
file1 file2
---------------------------------------
x x /
x x /DS1
x /DS2
group : </> and </>
0 differences found
dataset: </DS1> and </DS1>
0 differences found
EXIT CODE: 1
tools/test/h5diff/testfiles/h5diff_901.txt
+17
View File
@@ -0,0 +1,17 @@
file1 file2
---------------------------------------
x x /
x x /DS1
group : </> and </>
0 differences found
dataset: </DS1> and </DS1>
Not comparable: </DS1> has rank 2, dimensions [4x4], max dimensions [18446744073709551615x18446744073709551615]
and </DS1> has rank 2, dimensions [8x4], max dimensions [18446744073709551615x18446744073709551615]
0 differences found
--------------------------------
Some objects are not comparable
--------------------------------
Use -c for a list of objects without details of differences.
EXIT CODE: 0
tools/test/h5diff/testfiles/h5diff_902.txt
+30
View File
@@ -0,0 +1,30 @@
file1 file2
---------------------------------------
x x /
x x /DS1
group : </> and </>
0 differences found
dataset: </DS1> and </DS1>
size: [1x16] [1x16]
position DS1 DS1 difference
------------------------------------------------------------
[ 0 0 ] 0 16 16
[ 0 1 ] 1 17 16
[ 0 2 ] 2 18 16
[ 0 3 ] 3 19 16
[ 0 4 ] 4 20 16
[ 0 5 ] 5 21 16
[ 0 6 ] 6 22 16
[ 0 7 ] 7 23 16
[ 0 8 ] 8 24 16
[ 0 9 ] 9 25 16
[ 0 10 ] 10 26 16
[ 0 11 ] 11 27 16
[ 0 12 ] 12 28 16
[ 0 13 ] 13 29 16
[ 0 14 ] 14 30 16
[ 0 15 ] 15 31 16
16 differences found
EXIT CODE: 1
tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5
BIN
View File
Binary file not shown.
tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5.onion
BIN
View File
Binary file not shown.
tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5
BIN
View File
Binary file not shown.
tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5.onion
BIN
View File
Binary file not shown.
tools/test/h5diff/testfiles/h5diff_onion_objs.h5
BIN
View File
Binary file not shown.
tools/test/h5diff/testfiles/h5diff_onion_objs.h5.onion
BIN
View File
Binary file not shown.
tools/test/h5diff/testh5diff.sh.in
+18
View File
@@ -124,6 +124,12 @@ $SRC_H5DIFF_TESTFILES/h5diff_strings1.h5
$SRC_H5DIFF_TESTFILES/h5diff_strings2.h5
$SRC_H5DIFF_TESTFILES/h5diff_eps1.h5
$SRC_H5DIFF_TESTFILES/h5diff_eps2.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_objs.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_objs.h5.onion
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_ext.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_ext.h5.onion
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_1d.h5
$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_1d.h5.onion
$SRC_TOOLS_TESTFILES/tvlstr.h5
"
@@ -356,6 +362,9 @@ $SRC_H5DIFF_TESTFILES/h5diff_800.txt
$SRC_H5DIFF_TESTFILES/h5diff_801.txt
$SRC_H5DIFF_TESTFILES/h5diff_830.txt
$SRC_H5DIFF_TESTFILES/h5diff_90.txt
$SRC_H5DIFF_TESTFILES/h5diff_900.txt
$SRC_H5DIFF_TESTFILES/h5diff_901.txt
$SRC_H5DIFF_TESTFILES/h5diff_902.txt
$SRC_H5DIFF_TESTFILES/h5diff_8625.txt
$SRC_H5DIFF_TESTFILES/h5diff_8639.txt
$SRC_H5DIFF_TESTFILES/h5diff_reg.txt
@@ -1214,6 +1223,15 @@ TOOLTEST h5diff_v1.txt -v 1_vds.h5 2_vds.h5
TOOLTEST h5diff_v2.txt -r 1_vds.h5 2_vds.h5
TOOLTEST h5diff_v3.txt -c 1_vds.h5 2_vds.h5
# ##############################################################################
# Onion VFD tests
# ##############################################################################
# These tests won't pass under ph5diff
if test -z "$pmode"; then
TOOLTEST h5diff_900.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_objs.h5 h5diff_onion_objs.h5
TOOLTEST h5diff_901.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_ext.h5 h5diff_onion_dset_ext.h5
TOOLTEST h5diff_902.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_1d.h5 h5diff_onion_dset_1d.h5
fi
# ##############################################################################
# # END
tools/test/h5dump/CMakeTests.cmake
+19 -1
View File
@@ -211,6 +211,11 @@
${HDF5_TOOLS_DIR}/testfiles/trefer_paramR.ddl
${HDF5_TOOLS_DIR}/testfiles/trefer_reg_1dR.ddl
${HDF5_TOOLS_DIR}/testfiles/trefer_regR.ddl
# Onion VFD files
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.ddl
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.ddl
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.ddl
${HDF5_TOOLS_DIR}/testfiles/tst_onion_revision_count.ddl
)
set (HDF5_N_REFERENCE_FILES
tall-3
@@ -350,7 +355,13 @@
${HDF5_TOOLS_DIR}/testfiles/trefer_param.h5
${HDF5_TOOLS_DIR}/testfiles/trefer_reg_1d.h5
${HDF5_TOOLS_DIR}/testfiles/trefer_reg.h5
# Onion VFD files
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion
)
set (HDF5_ERROR_REFERENCE_TEST_FILES
${PROJECT_SOURCE_DIR}/errfiles/filter_fail.err
@@ -1187,6 +1198,13 @@
ADD_H5_TEST (tCVE_2018_11206_fill_old 1 tCVE_2018_11206_fill_old.h5)
ADD_H5_TEST (tCVE_2018_11206_fill_new 1 tCVE_2018_11206_fill_new.h5)
# onion VFD tests
ADD_H5_TEST (tst_onion_objs 0 --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5)
ADD_H5_TEST (tst_onion_dset_ext 0 --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_ext.h5)
ADD_H5_TEST (tst_onion_dset_1d 0 --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_1d.h5)
ADD_H5_TEST (tst_onion_revision_count 0 --enable-error-stack --vfd-name onion --vfd-info revision_count tst_onion_objs.h5)
##############################################################################
### P L U G I N T E S T S
##############################################################################
tools/test/h5dump/h5dumpgentest.c
+677
View File
@@ -117,6 +117,13 @@
#define FILE87 "tintsnodata.h5"
#define FILE88 "tldouble_scalar.h5"
#define FILE89 "tfloatsattrs.h5"
#define FILE90 "tst_onion_dset_1d.h5"
#define FILE91 "tst_onion_objs.h5"
#define FILE92 "tst_onion_dset_ext.h5"
#define ONION_TEST_FIXNAME_SIZE 1024
#define ONION_TEST_PAGE_SIZE (uint32_t)32
#define ONE_DIM_SIZE 16
/*-------------------------------------------------------------------------
* prototypes
@@ -11350,6 +11357,671 @@ error:
H5E_END_TRY;
} /* gen_err_attr_dspace() */
/* Structure to collect the onion filepaths in one place. */
struct onion_filepaths {
char *canon;
char *onion;
char *recovery;
};
/* Allocate and populate filepaths with h5_fixname'd strings as appropriate.
* Should be released with onion_filepaths_destroy() when done.
*/
static struct onion_filepaths *
onion_filepaths_init(const char *basename)
{
struct onion_filepaths *paths = NULL;
if (NULL == (paths = HDcalloc(1, sizeof(struct onion_filepaths))))
goto error;
if (NULL == (paths->canon = HDstrdup(basename)))
goto error;
if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon);
if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE)))
goto error;
HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon);
return paths;
error:
if (paths != NULL) {
HDfree(paths->canon);
HDfree(paths->onion);
HDfree(paths->recovery);
HDfree(paths);
}
return NULL;
}
static void
onion_filepaths_destroy(struct onion_filepaths *s)
{
if (s) {
HDfree(s->canon);
HDfree(s->onion);
HDfree(s->recovery);
HDfree(s);
}
}
static int
gent_onion_1d_dset(void)
{
hid_t file_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hsize_t dims[2] = {1, ONE_DIM_SIZE};
hsize_t maxdims[2] = {1, ONE_DIM_SIZE};
int wdata[1][ONE_DIM_SIZE];
int fillval;
/* Setup */
onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS);
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(FILE90)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < ONE_DIM_SIZE; i++)
wdata[0][i] = i;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list */
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]) < 0)
goto error;
/* Close and release resources */
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
int dset_data[1][ONE_DIM_SIZE];
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + ONE_DIM_SIZE;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i++)
dset_data[0][i] = i + 2048;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file_id) < 0)
goto error;
file_id = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and change the data
*----------------------------------------------------------------------
*/
if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0)
goto error;
for (int i = 0; i < ONE_DIM_SIZE; i += 20)
dset_data[0][i] = i + 3072;
if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0)
goto error;
/* CLEANUP */
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
H5E_BEGIN_TRY
{
H5Pclose(onion_info.backing_fapl_id);
H5Pclose(fapl_id);
H5Dclose(dset);
H5Sclose(space);
H5Fclose(file_id);
}
H5E_END_TRY;
return -1;
} /* gent_onion_1d_dset */
static int
gent_onion_create_delete_objects(void)
{
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hid_t fapl_id = H5I_INVALID_HID;
hid_t group_id = H5I_INVALID_HID;
hid_t attr_space_id = H5I_INVALID_HID;
hid_t attr_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
hsize_t attr_dim[1] = {4};
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
int wdata[4][4];
int fillval;
/* Set up */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(FILE91)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/*
* Initialize data.
*/
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/*
* Create a new file using the default properties.
*/
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/*
* Create dataspace with unlimited dimensions.
*/
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/*
* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/*
* Set the fill value for the dataset.
*/
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/*
* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and add a dataset (DS2) to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/*
* Create the dataset using the dataset creation property list.
*/
if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/*
* Write the data to the dataset.
*/
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and remove the dataset (DS2),
* which was added during the first revision.
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Third revision: open the file with Onion VFD and add an attribute to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Create dataspace for attribute */
attr_space_id = H5Screate_simple(1, attr_dim, NULL);
if ((attr_id =
H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Sclose(attr_space_id) < 0)
goto error;
if (H5Aclose(attr_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fourth revision: open the file with Onion VFD and delete the attribute
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Adelete(file, "file_attribute") < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Fifth revision: open the file with Onion VFD and add a group to the file
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
if (H5Gclose(group_id) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Sixth revision: open the file with Onion VFD and delete the newly added group
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*
* Close and release resources.
*/
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
if (dset != H5I_INVALID_HID)
(void)H5Dclose(dset);
if (file != H5I_INVALID_HID)
(void)H5Fclose(file);
if (fapl_id != H5I_INVALID_HID)
(void)H5Pclose(fapl_id);
if (onion_info.backing_fapl_id != H5I_INVALID_HID)
H5Pclose(onion_info.backing_fapl_id);
return -1;
} /* gent_onion_create_delete_objects */
static int
gent_onion_dset_extension(void)
{
hid_t fapl_id = H5I_INVALID_HID;
hid_t file = H5I_INVALID_HID;
hid_t space = H5I_INVALID_HID;
hid_t dset_space = H5I_INVALID_HID;
hid_t dset = H5I_INVALID_HID;
hid_t dcpl = H5I_INVALID_HID;
struct onion_filepaths *paths = NULL;
H5FD_onion_fapl_info_t onion_info = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
H5I_INVALID_HID, /* backing_fapl_id */
ONION_TEST_PAGE_SIZE, /* page_size */
H5FD_ONION_STORE_TARGET_ONION, /* store_target */
H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
0, /* force_write_open */
0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */
"initial commit" /* comment */
};
hsize_t dims[2] = {4, 4};
hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
hsize_t chunk[2] = {4, 4};
hsize_t size[2];
hsize_t offset[2];
int wdata[4][4]; /* Write buffer */
int fillval;
/* Setup */
if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto error;
if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0)
goto error;
if ((paths = onion_filepaths_init(FILE92)) == NULL)
goto error;
/*----------------------------------------------------------------------
* Create the skeleton file (create the file without Onion VFD)
*----------------------------------------------------------------------
*/
/* Initialize data */
for (int i = 0; i < 4; i++)
for (int j = 0; j < 4; j++)
wdata[i][j] = i + j;
/* Create a new file using the default properties */
if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
goto error;
/* Create dataspace with unlimited dimensions */
if ((space = H5Screate_simple(2, dims, maxdims)) < 0)
goto error;
/* Create the dataset creation property list, and set the chunk
* size.
*/
if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto error;
if (H5Pset_chunk(dcpl, 2, chunk) < 0)
goto error;
/* Set the fill value for the dataset */
fillval = 99;
if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0)
goto error;
/* Set the allocation time to "early". This way we can be sure
* that reading from the dataset immediately after creation will
* return the fill value.
*/
if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0)
goto error;
/* Create the dataset using the dataset creation property list */
if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
goto error;
/* Write the data to the dataset */
if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
if (H5Fclose(file) < 0)
goto error;
/*----------------------------------------------------------------------
* First revision: open the file with Onion VFD and extend the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and double the rows */
size[0] = 2 * dims[0];
size[1] = dims[1];
if (H5Dset_extent(dset, size) < 0)
goto error;
if ((dset_space = H5Dget_space(dset)) < 0)
goto error;
offset[0] = dims[0];
offset[1] = 0;
if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0)
goto error;
/* Write the data to the dataset. */
if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0)
goto error;
if (H5Sclose(dset_space) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/*----------------------------------------------------------------------
* Second revision: open the file with Onion VFD and shrink the dataset
*----------------------------------------------------------------------
*/
if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0)
goto error;
/* Open the dataset */
if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0)
goto error;
/* Extend the dataset and shrink back the size */
if (H5Dset_extent(dset, dims) < 0)
goto error;
if (H5Dclose(dset) < 0)
goto error;
dset = H5I_INVALID_HID;
if (H5Fclose(file) < 0)
goto error;
file = H5I_INVALID_HID;
/* Close and release resources. */
if (H5Pclose(onion_info.backing_fapl_id) < 0)
goto error;
if (H5Pclose(fapl_id) < 0)
goto error;
if (H5Pclose(dcpl) < 0)
goto error;
if (H5Sclose(space) < 0)
goto error;
onion_filepaths_destroy(paths);
return 0;
error:
if (paths != NULL) {
HDremove(paths->canon);
HDremove(paths->onion);
HDremove(paths->recovery);
onion_filepaths_destroy(paths);
}
H5E_BEGIN_TRY
{
H5Dclose(dset);
H5Fclose(file);
H5Pclose(fapl_id);
H5Pclose(onion_info.backing_fapl_id);
}
H5E_END_TRY;
return -1;
} /* gent_onion_dset_extension */
int
main(void)
{
@@ -11449,5 +12121,10 @@ main(void)
gent_err_attr_dspace();
/* Generate the files for testing Onion VFD */
gent_onion_1d_dset();
gent_onion_create_delete_objects();
gent_onion_dset_extension();
return 0;
}
tools/test/h5dump/testh5dump.sh.in
+16
View File
@@ -184,6 +184,12 @@ $SRC_H5DUMP_TESTFILES/tvms.h5
$SRC_H5DUMP_TESTFILES/err_attr_dspace.h5
$SRC_H5DUMP_TESTFILES/tCVE_2018_11206_fill_old.h5
$SRC_H5DUMP_TESTFILES/tCVE_2018_11206_fill_new.h5
$SRC_H5DUMP_TESTFILES/tst_onion_objs.h5
$SRC_H5DUMP_TESTFILES/tst_onion_objs.h5.onion
$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5
$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5.onion
$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5
$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5.onion
"
LIST_OTHER_TEST_FILES="
@@ -372,6 +378,10 @@ $SRC_H5DUMP_TESTFILES/h5dump-help.txt
$SRC_H5DUMP_TESTFILES/out3.h5import
$SRC_H5DUMP_TESTFILES/tbinregR.exp
$SRC_H5DUMP_TESTFILES/err_attr_dspace.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_objs.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.ddl
$SRC_H5DUMP_TESTFILES/tst_onion_revision_count.ddl
"
LIST_ERROR_TEST_FILES="
@@ -1486,6 +1496,12 @@ TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5
TOOLTEST_FAIL tCVE_2018_11206_fill_old.h5
TOOLTEST_FAIL tCVE_2018_11206_fill_new.h5
# test Onion VFD
TOOLTEST tst_onion_objs.ddl --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5
TOOLTEST tst_onion_dset_ext.ddl --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_ext.h5
TOOLTEST tst_onion_dset_1d.ddl --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_1d.h5
TOOLTEST tst_onion_revision_count.ddl --enable-error-stack --vfd-name onion --vfd-info revision_count tst_onion_objs.h5
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR
tools/test/h5repack/CMakeTests.cmake
+55
View File
@@ -122,6 +122,13 @@
${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5
${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5
${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5
# tools/testfiles onion VFD files
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5
${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion
)
set (LIST_OTHER_TEST_FILES
@@ -446,6 +453,50 @@
endif ()
endmacro ()
macro (ADD_H5_DMP_NO_OPT_TEST testname testtype resultcode resultfile)
if ("${testtype}" STREQUAL "SKIP")
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_DMP-${testname}
COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}"
)
set_property(TEST H5REPACK_DMP-${testname} PROPERTY DISABLED)
endif ()
else ()
add_test (
NAME H5REPACK_DMP-${testname}-clear-objects
COMMAND ${CMAKE_COMMAND} -E remove testfiles/out-${testname}.${resultfile}
)
set_tests_properties (H5REPACK_DMP-${testname}-clear-objects PROPERTIES
FIXTURES_REQUIRED clear_h5repack
)
add_test (
NAME H5REPACK_DMP-${testname}
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repack${tgt_file_ext}> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}
)
set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES
DEPENDS H5REPACK_DMP-${testname}-clear-objects
)
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
add_test (
NAME H5REPACK_DMP-h5dump-${testname}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5dump${tgt_file_ext}>"
-D "TEST_ARGS:STRING=out-${testname}.${resultfile}"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
-D "TEST_OUTPUT=${resultfile}-${testname}.out"
-D "TEST_EXPECT=${resultcode}"
-D "TEST_REFERENCE=${testname}.${resultfile}.ddl"
-P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
)
set_tests_properties (H5REPACK_DMP-h5dump-${testname} PROPERTIES
DEPENDS "H5REPACK_DMP-${testname}"
)
endif ()
endif ()
endmacro ()
macro (ADD_H5_DIFF_TEST testname testtype resultcode testfile)
if ("${testtype}" STREQUAL "SKIP")
if (NOT HDF5_ENABLE_USING_MEMCHECKER)
@@ -1665,6 +1716,10 @@ ADD_H5_DMP_TEST (textlink-mergeprune "TEST" 0 textlink.h5 --merge --prune --enab
### HDFFV-11128 needs fixed to enable the following test
#ADD_H5_DMP_TEST (textlinktar-mergeprune "TEST" 0 textlinktar.h5 --merge --prune --enable-error-stack)
ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_1d "TEST" 0 tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1)
ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_ext "TEST" 0 tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1)
ADD_H5_DMP_NO_OPT_TEST (tst_onion_objs "TEST" 0 tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1)
##############################################################################
### P L U G I N T E S T S
##############################################################################
tools/test/h5repack/h5repack.sh.in
+64
View File
@@ -177,6 +177,13 @@ $SRC_TOOLS_TESTFILES/vds/5_a.h5
$SRC_TOOLS_TESTFILES/vds/5_b.h5
$SRC_TOOLS_TESTFILES/vds/5_c.h5
$SRC_TOOLS_TESTFILES/vds/5_vds.h5
########tools/testfiles########
$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5
$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5.onion
$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5
$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5.onion
$SRC_TOOLS_TESTFILES/tst_onion_objs.h5
$SRC_TOOLS_TESTFILES/tst_onion_objs.h5.onion
"
LIST_OTHER_TEST_FILES="
@@ -251,6 +258,10 @@ $SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
########onion#files########
$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_1d.h5.ddl
$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_ext.h5.ddl
$SRC_H5REPACK_TESTFILES/onion.tst_onion_objs.h5.ddl
"
#
@@ -1130,6 +1141,56 @@ TOOLTEST_DUMP()
rm -f $outfile
}
# This is same as TOOLTEST_DUMP() with comparing h5dump output
# without any option
#
TOOLTEST_DUMP_NO_OPT()
{
infile=$2
outfile=out-$1.$2
expect="$TESTDIR/$1.$2.ddl"
actual="$TESTDIR/out-$1.$2.out"
actual_err="$TESTDIR/out-$1.$2.err"
shift
shift
# Run test.
TESTING $H5REPACK $@
(
cd $TESTDIR
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
) >$actual 2>$actual_err
RET=$?
if [ $RET != 0 ] ; then
echo "*FAILED*"
nerrors="`expr $nerrors + 1`"
else
echo " PASSED"
VERIFY h5dump output $@
(
cd $TESTDIR
$RUNSERIAL $H5DUMP_BIN $outfile
) >$actual 2>$actual_err
cat $actual_err >> $actual
RET=$?
fi
if cmp -s $expect $actual; then
echo " PASSED"
else
echo "*FAILED*"
echo " Expected result (*.ddl) differs from actual result (*.out)"
nerrors="`expr $nerrors + 1`"
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
fi
rm -f $actual $actual_err
rm -f $outfile
}
# This is similar to TOOLTEST_DUMP().
# Test h5repack with options added for paged aggregation.
# h5stat is used on the repacked file and the expected output
@@ -1830,6 +1891,9 @@ TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-sta
#TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack
### HDFFV-11128 needs fixed to enable the following test
#TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack
TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1
TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1
TOOLTEST_DUMP_NO_OPT onion tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR
tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl
+11
View File
@@ -0,0 +1,11 @@
HDF5 "out-onion.tst_onion_dset_1d.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) }
DATA {
(0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
}
}
}
}
tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl
+18
View File
@@ -0,0 +1,18 @@
HDF5 "out-onion.tst_onion_dset_ext.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6,
(4,0): 0, 1, 2, 3,
(5,0): 1, 2, 3, 4,
(6,0): 2, 3, 4, 5,
(7,0): 3, 4, 5, 6
}
}
}
}
tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl
+24
View File
@@ -0,0 +1,24 @@
HDF5 "out-onion.tst_onion_objs.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6
}
}
DATASET "DS2" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6
}
}
}
}
tools/testfiles/tst_onion_dset_1d.ddl
+12
View File
@@ -0,0 +1,12 @@
Using revision 1
HDF5 "tst_onion_dset_1d.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) }
DATA {
(0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
}
}
}
}
tools/testfiles/tst_onion_dset_1d.h5
BIN
View File
Binary file not shown.
tools/testfiles/tst_onion_dset_1d.h5.onion
BIN
View File
Binary file not shown.
tools/testfiles/tst_onion_dset_ext.ddl
+19
View File
@@ -0,0 +1,19 @@
Using revision 1
HDF5 "tst_onion_dset_ext.h5" {
GROUP "/" {
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6,
(4,0): 0, 1, 2, 3,
(5,0): 1, 2, 3, 4,
(6,0): 2, 3, 4, 5,
(7,0): 3, 4, 5, 6
}
}
}
}
tools/testfiles/tst_onion_dset_ext.h5
BIN
View File
Binary file not shown.
tools/testfiles/tst_onion_dset_ext.h5.onion
BIN
View File
Binary file not shown.
tools/testfiles/tst_onion_objs.ddl
+22
View File
@@ -0,0 +1,22 @@
Using revision 3
HDF5 "tst_onion_objs.h5" {
GROUP "/" {
ATTRIBUTE "file_attribute" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4 ) / ( 4 ) }
DATA {
(0): 0, 0, 0, 0
}
}
DATASET "DS1" {
DATATYPE H5T_STD_I32LE
DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
DATA {
(0,0): 0, 1, 2, 3,
(1,0): 1, 2, 3, 4,
(2,0): 2, 3, 4, 5,
(3,0): 3, 4, 5, 6
}
}
}
}
tools/testfiles/tst_onion_objs.h5
BIN
View File
Binary file not shown.
tools/testfiles/tst_onion_objs.h5.onion
BIN
View File
Binary file not shown.
tools/testfiles/tst_onion_revision_count.ddl
+1
View File
@@ -0,0 +1 @@
The number of revisions for the onion file is 6