diff --git a/.codespellrc b/.codespellrc index 9da7b27f2a9..5fddb495f09 100644 --- a/.codespellrc +++ b/.codespellrc @@ -3,4 +3,4 @@ skip = .git,*.svg,.codespellrc,./bin/trace,./hl/tools/h5watch/h5watch.c,./tools/test/h5jam/tellub.c,./config/sanitizer/LICENSE,./config/sanitizer/sanitizers.cmake,./tools/test/h5import/testfiles/*.conf,./tools/test/h5repack/testfiles/*.dat,./test/API/driver,./configure,./bin/ltmain.sh,./bin/depcomp,./bin/config.guess,./bin/config.sub,./autom4te.cache,./m4/libtool.m4,./c++/src/*.html,./HDF5Examples/depcomp check-hidden = true # ignore-regex = -ignore-words-list = ot,isnt,inout,nd,parms,parm,ba,offsetP,ser,ois,had,fiter,fo,clude,refere,minnum,offsetp,creat,ans:,eiter,lastr,ans,isn't,ifset,sur,trun,dne,tthe,hda,filname,te,htmp,ake,gord,numer,ro,oce,msdos,TEXTIN +ignore-words-list = ot,isnt,inout,nd,parms,parm,ba,offsetP,ser,ois,had,fiter,fo,clude,refere,minnum,offsetp,creat,ans:,eiter,lastr,ans,isn't,ifset,sur,trun,dne,tthe,hda,filname,te,htmp,ake,gord,numer,ro,oce,msdos,TEXTIN,FLE diff --git a/.github/workflows/linkchecker.yml b/.github/workflows/linkchecker.yml index 681ceaf4dbf..2eaf0530000 100644 --- a/.github/workflows/linkchecker.yml +++ b/.github/workflows/linkchecker.yml @@ -42,6 +42,6 @@ jobs: - name: check run: | cd build/hdf5lib_docs/html - linkchecker --check-extern ./index.html + linkchecker --ignore-url=/doxygen.css --check-extern ./index.html continue-on-error: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 48c25154dc7..b69da28f61d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -454,7 +454,7 @@ else () endif () set (HDF5_PACKAGE_STRING "${HDF5_PACKAGE_NAME} ${HDF5_PACKAGE_VERSION_STRING}") set (HDF5_PACKAGE_TARNAME "${HDF5_PACKAGE}${HDF_PACKAGE_EXT}") -set (HDF5_PACKAGE_URL "http://www.hdfgroup.org") +set (HDF5_PACKAGE_URL "https://www.hdfgroup.org") set (HDF5_PACKAGE_BUGREPORT "help@hdfgroup.org") #----------------------------------------------------------------------------- diff --git a/doc/branches-explained.md b/doc/branches-explained.md deleted file mode 100644 index f2aa6272a19..00000000000 --- a/doc/branches-explained.md +++ /dev/null @@ -1,40 +0,0 @@ -# HDF5 Git Branching Model Explained - -This document describes current HDF5 branches. - -Branches are tested nightly and testing results are available at https://cdash.hdfgroup.org/. -Commits that break daily testing should be fixed by 3:00 pm Central time or reverted. -We encourage code contributors to check the status of their commits. If you have any questions, please contact help@hdfgroup.org. - -## `develop` -Develop is the main branch whose source code always reflects a state with the latest delivered development changes for the next major release of HDF5. -This is also considered the integration branch, as **all** new features are integrated into this branch from respective feature branches. Although -develop is considered an integration branch, it is not an unstable branch. All code merged to develop is expected to pass all GitHub actions and daily tests. - -## `Maintenance branches` -Each currently supported release line of HDF5 (e.g. 1.8.x, 1.10.x, 1.12.x) has an associated branch with the name hdf5\_1\_10, etc.. -Maintenance branches are similar to the develop branch, except the source code in a maintenance branch always reflects a state -with the latest delivered development changes for the next **maintenance** release of that particular supported release-line of HDF5. -**Some** new features will be integrated into a release maintenance branch, depending on whether or not those features can be -introduced in minor releases. Maintenance branches are removed when a release-line is retired from support. - -## `Release branches` -Release branches are used to prepare a new production release. They are primarily used to allow for last minute dotting of i's and crossing of t's -(things like setting the release version, finalizing release notes, and generating Autotools files) and do not include new development. -They are created from the maintenance branch at the time of the maintenance release and have -names like hdf5\_1\_10\_N, where N is the minor release number. Once the release is done it is tagged, with a slightly different format: hdf5-1\_\10\_N. -Release branches are deleted after the tag has been created. If we have to create a patch version of a release (which is rare), we create a branch off of the tag. - -## `feature/*` -Feature branches are temporary branches used to develop new features in HDF5. -Feature branches branch off of develop and exist as long as the feature is under development. -When the feature is complete, the branch is merged back into develop, as well as into any support branches in which the change will be included, and then the feature branch is removed. - -Ideally, all feature branches should contain a BRANCH.md file in the root directory that explains the purpose of the branch, contact information for the person responsible, and, if possible, some clues about the branch's life cycle (so we have an idea about when it can be deleted, merged, or declared inactive). - -Minor bug fixes and refactoring work usually takes place on personal forks, not feature branches. - -## `inactive/*` -These branches are for experimental features that were developed in the past, have not been merged to develop, and are not under active development. The exception to this is that some feature branches are labeled inactive and preserved for a short time after merging to develop. Integration branches are usually not kept in sync with the develop branch. - -As for feature branches, inactive branches should have a BRANCH.md file as described above. diff --git a/doc/code-conventions.md b/doc/code-conventions.md deleted file mode 100644 index ff3b4cf4069..00000000000 --- a/doc/code-conventions.md +++ /dev/null @@ -1,57 +0,0 @@ -# HDF5 Library Code Conventions - -This document describes some practices that are new, or newly -documented, starting in 2020. - -## Function / Variable Attributes - -In H5private.h, the library provides platform-independent macros -for qualifying function and variable definitions. - -### Functions that accept `printf(3)` and `scanf(3)` format strings - -Label functions that accept a `printf(3)`-compliant format string with -`H5_ATTR_FORMAT(printf,format_argno,variadic_argno)`, where -the format string is the `format_argno`th argument (counting from 1) -and the variadic arguments start with the `variadic_argno`th. - -Functions that accept a `scanf(3)`-compliant format string should -be labeled `H5_ATTR_FORMAT(scanf,format_argno,variadic_argno)`. - -### Functions that do never return - -The definition of a function that always causes the program to abort and hang -should be labeled `H5_ATTR_NORETURN` to help the compiler see which flows of -control are infeasible. - -### Other attributes - -**TBD** - -### Unused variables and parameters - -Compilers will warn about unused parameters and variables—developers should pay -attention to those warnings and make an effort to prevent them. - -Some function parameters and variables are unused in *all* configurations of -the project. Ordinarily, such parameters and variables should be deleted. -However, sometimes it is possible to foresee a parameter being used, or -removing it would change an API, or a parameter has to be defined to conform a -function to some function pointer type. In those cases, it's permissible to -mark a symbol `H5_ATTR_UNUSED`. - -Other parameters and variables are unused in *some* configurations of the -project, but not all. A symbol may fall into disuse in some configuration in -the future—then the compiler should warn, and the symbol should not be -defined—so developers should try to label a sometimes-unused symbol with an -attribute that's specific to the configurations where the symbol is (or is not) -expected to be used. The library provides the following attributes for that -purpose: - -* `H5_ATTR_DEPRECATED_USED`: used only if deprecated symbols are enabled -* `H5_ATTR_NDEBUG_UNUSED`: used only if `NDEBUG` is *not* \#defined -* `H5_ATTR_DEBUG_API_USED`: used if the debug API is enabled -* `H5_ATTR_PARALLEL_UNUSED`: used only if Parallel HDF5 *is not* configured -* `H5_ATTR_PARALLEL_USED`: used only if Parallel HDF5 *is* configured - -Some attributes may be phased in or phased out in the future. diff --git a/doc/library-init-shutdown.md b/doc/library-init-shutdown.md deleted file mode 100644 index 917d213b094..00000000000 --- a/doc/library-init-shutdown.md +++ /dev/null @@ -1,56 +0,0 @@ -# HDF5 Library initialization and shutdown - -## Application perspective - -### Implicit initialization and shutdown - -When a developer exports a new symbol as part of the HDF5 library, -they should make sure that an application cannot enter the library in an -uninitialized state through a new API function, or read an uninitialized -value from a non-function HDF5 symbol. - -The HDF5 library initializes itself when an application either enters -the library through an API function call such as `H5Fopen`, or when -an application evaluates an HDF5 symbol that represents either a -property-list identifier such as `H5F_ACC_RDONLY` or `H5F_ACC_RDWR`, -a property-list class identifier such as `H5P_FILE_ACCESS`, a VFD -identifier such as `H5FD_FAMILY` or `H5FD_SEC2`, or a type identifier -such as `H5T_NATIVE_INT64`. - -The library sets a flag when initialization occurs and as long as the -flag is set, skips initialization. - -The library provides a couple of macros that initialize the library -as necessary. The library is initialized as a side-effect of the -`FUNC_ENTER_API*` macros used at the top of most API functions. HDF5 -library symbols other than functions are provided through `#define`s -that use `H5OPEN` to introduce a library-initialization call (`H5open`) -at each site where a non-function symbol is used. - -Ordinarily the library registers an `atexit(3)` handler to shut itself -down when the application exits. - -### Explicit initialization and shutdown - -An application may use an API call, `H5open`, to explicitly initialize -the library. `H5close` explicitly shuts down the library. - -## Library internals perspective - -No matter how library initializion begins, eventually the internal -function `H5_init_library` will be called. `H5_init_library` is -responsible for calling the initializers for every internal HDF5 -library module (aka "package") in the correct order so that no module is -initialized before its prerequisite modules. A table in `H5_init_library` -establishes the order of initialization. If a developer adds a -module to the library that it is appropriate to initialize with the rest -of the library, then they should insert its initializer into the right -place in the table. - -`H5_term_library` drives library shutdown. Library shutdown is -table-driven, too. If a developer adds a module that needs to release -resources during library shutdown, then they should add a call at the -right place to the shutdown table. Note that some entries in the shutdown -table are marked as "barriers," and if a new module should only be -shutdown *strictly after* the preceding modules, then it should be marked -as a barrier. See the comments in `H5_term_library` for more information. diff --git a/doc/threadsafety-warning.md b/doc/threadsafety-warning.md deleted file mode 100644 index 49abcb00af9..00000000000 --- a/doc/threadsafety-warning.md +++ /dev/null @@ -1,16 +0,0 @@ -## A Warning - -Any application that creates threads that use the HDF5 library must join those threads before either process exit or library close through H5close(). If all HDF5-using threads aren't joined, the threads may exhibit undefined behavior. - -## Discussion for Developers on Potential Improvements - -It would in principle be possible to make it safe to have threads continue using HDF5 resources after a call to H5close() by keeping a count of threads within the library. (There is probably no solution to an early process exit producing undefined behavior within threads.) This method would only be able to count (and presumably, only _need_ to count) threads that directly interact with the library. Because each thread would need to be counted exactly once, this would most likely be done by use of a thread-local key with e.g. a boolean value used to track whether the a global atomic thread counter has already counted this thread. Then, if H5close() is invoked while this thread counter is above one (because one thread must be doing the closing), the library would not close, and instead keep its resources valid to hopefully avoid bad behavior with the threads. - -The issues with this approach are as follows: - -1. The process of checking for the existence/value of the thread-local key is slow, or at least slow enough that it's probably not worth adding this to almost every single API call to prevent this particular edge case. -2. Even with this approach, bad behavior would still be possible if the application does something like expose HDF5 resources to threads indirectly via a global variable. -3. How to allow H5close() to fail is nonobvious. H5close() could be allowed to return an error indicating a failure to close, but the number of applications which could usefully respond to such an error by joining threads is small. If an application were able/willing to join its created threads, presumably it would have done so before calling H5close(). Alternatively, H5close() could succeed but silently leave the library open. This creates the potential for confusing, unexpected behavior when the user thinks they are closing and re-opening the library, e.g. if environment variables are modified between close and re-open, or if resources such as default property lists are modified. -4. Applications should join threads before closing libraries that those threads are using, so all of this work would constitute an above-and-beyond effort to maintain safe and defined behavior in the face of an unsafe application. - -Despite these issues, if a more performant method was found to perform threadcounting like this, it might still constitute a worthwhile change. \ No newline at end of file diff --git a/doxygen/aliases b/doxygen/aliases index cbc6e81f6cf..57326f99e06 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -5,8 +5,8 @@ ALIASES += THG="The HDF Group" ################################################################################ # Default URL for HDF Group Files ALIASES += HDFURL="support.hdfgroup.org" -# URL for archived files -ALIASES += ARCURL="\HDFURL/archive/support/HDF5/doc" +# URL for archived files - not currently used as all docs referenced are in doxygen +# ALIASES += ARCURL="\HDFURL/archive/support/HDF5/doc" # URL for release files ALIASES += RELURL="\HDFURL/releases/hdf5" # URL for documentation @@ -21,7 +21,7 @@ ALIASES += DOXURL="hdfgroup.github.io/hdf5/develop" #branch name (develop, hdf5_1_14) ALIASES += SRCURL="github.com/HDFGroup/hdf5/blob/develop" #Other projects that contribute to HDF5 -ALIASES += PRJURL="\HDFURL/projects" +ALIASES += PRJURL="\HDFURL/archive/support/projects" ALIASES += HVURL="github.com/HDFGroup/hdfview/blob/master" ALIASES += PLURL="github.com/HDFGroup/hdf5_plugins/blob/master" ################################################################################ @@ -257,6 +257,13 @@ ALIASES += callback_note="\attention \Bold{Leaving callback functions:}\n The ca ALIASES += par_compr_note="\attention If you are planning to use compression with parallel HDF5, ensure that calls to H5Dwrite() occur in collective mode. In other words, all MPI ranks (in the relevant communicator) call H5Dwrite() and pass a dataset transfer property list with the MPI-IO collective option property set to #H5FD_MPIO_COLLECTIVE_IO.\n Note that data transformations are currently \Bold{not} supported when writing to datasets in parallel and with compression enabled." ALIASES += sa_metadata_ops="\sa \li H5Pget_all_coll_metadata_ops() \li H5Pget_coll_metadata_write() \li H5Pset_all_coll_metadata_ops() \li H5Pset_coll_metadata_write() \li \ref maybe_metadata_reads" +################################################################################ +# Specifications +################################################################################ + +ALIASES += ref_spec_fileformat="\ref FMT3" +ALIASES += ref_spec_fileformat_btrees_v1="\ref subsubsec_fmt3_infra_btrees_v1" + ################################################################################ # References ################################################################################ @@ -264,7 +271,6 @@ ALIASES += sa_metadata_ops="\sa \li H5Pget_all_coll_metadata_ops() \li H5Pget_co ALIASES += ref_cons_semantics="Enabling a Strict Consistency Semantics Model in Parallel HDF5" ALIASES += ref_filter_pipe="Data Flow Pipeline for H5Dread()" ALIASES += ref_group_impls="Group implementations in HDF5" -ALIASES += ref_h5lib_relver="HDF5 Library Release Version Numbers" ALIASES += ref_mdc_in_hdf5="Metadata Caching in HDF5" ALIASES += ref_mdc_logging="Metadata Cache Logging" ALIASES += ref_news_112="New Features in HDF5 Release 1.12" diff --git a/doxygen/dox/CollectiveMetadataIO.dox b/doxygen/dox/CollectiveMetadataIO.dox new file mode 100644 index 00000000000..49332b2d978 --- /dev/null +++ b/doxygen/dox/CollectiveMetadataIO.dox @@ -0,0 +1,248 @@ +/** \page collective_metadata_io Collective Metadata I/O + * The purpose of this page is to briefly describe the new HDF5 Collective Metadata I/O feature and + * provide a gateway to available documentation. The page includes the following sections: + * + * + * \section sec_collective_metadata_io_overview Collective Metadata I/O Overview + * Calls for HDF5 metadata can result in many small reads and writes. On metadata reads, collective + * metadata I/O can improve performance by allowing the library to perform optimizations when reading + * the metadata, by having one rank read the data and broadcasting it to all other ranks. + * + * Collective metadata I/O improves metadata write performance through the construction of an MPI + * derived datatype that is then written collectively in a single call. + * + * \section sec_collective_metadata_io_resource Collective Metadata I/O User and Resource Documents + * HDF5 Collective Metadata I/O User Document (This document is not yet available.) + * + * Until an HDF5 Collective Metadata I/O User Document becomes available, users may find the + * following resources helpful: + * \li \ref_rfc20150424 + * \li \ref_rfc20150423 + * + * Taken together, these papers discuss the motivation, design, implementation, and API for HDF5’s + * Collective Metadata I/O feature. + * + * \section sec_collective_metadata_io_apis HDF5 Library APIs + * + * \subsection subsec_collective_metadata_io_apis_functions New Collective Metadata I/O Functions + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
APIDescription
#H5Pset_coll_metadata_writeEstablishes I/O mode property setting, collective or independent, for metadata writes
#H5Pget_coll_metadata_writeRetrieves I/O mode property setting for metadata writes
#H5Pset_all_coll_metadata_opsEstablishes I/O mode, collective or independent, for metadata read operations
#H5Pget_all_coll_metadata_opsRetrieves I/O mode for metadata read operations
+ * + * \subsection subsec_collective_metadata_io_apis_reference Additional API Reference + * + * \subsubsection subsubsec_collective_metadata_io_apis_reference_may Functions with No Access Property List Parameter that May Generate Metadata Reads + * + * Currently there are several operations in HDF5 that can issue metadata reads from the metadata + * cache, but that take no property list. It is therefore not possible set a collective requirement + * individually for those operations. The only solution with the HDF5 1.10.0 release is to set the + * collective requirement globally on #H5Fopen or #H5Fcreate for all metadata operations to be collective. + * + * The following is a list of those functions in the HDF5 library. This list is integral to the + * discussion in the #H5Pset_all_coll_metadata_ops entry: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
API
Attributes
#H5Awrite
#H5Aread
#H5Arename
#H5Aiterate2
#H5Adelete
#H5Aexists
Datasets
#H5Dget_space_status
#H5Dget_storage_size
#H5Dset_extent
H5Ddebug
#H5Dclose
#H5Dget_create_plist
#H5Dget_space (when dataset is a virtual dataset)
Groups
#H5Gget_create_plist
#H5Gget_info
#H5Gclose
Links
#H5Literate
#H5Lvisit
References
#H5Rcreate
#H5Rdereference2 (when reference is an object reference)
#H5Rget_region
#H5Rget_obj_type2
#H5Rget_name
Objects
#H5Ocopy
#H5Oopen_by_addr
#H5Oincr_refcount
#H5Odecr_refcount
#H5Oget_info
#H5Oset_comment
#H5Ovisit
Files
#H5Fis_hdf5
#H5Fflush
#H5Fclose
#H5Fget_file_image
#H5Freopen
#H5Fget_freespace
#H5Fget_info2
#H5Fget_free_sections
#H5Fmount
#H5Funmount
Identifiers
#H5Iget_name
Datatypes
#H5Tget_create_plist
#H5Tclose
Filters
#H5Zunregister
+ * + * In addition, most deprecated functions fall into this category. + * + * The HDF Group may address the above limitation in a future major release, but no decision has + * been made at this time. Such a change might, for example, include adding new versions of some + * or all the above functions with an extra property list parameter to allow an individual setting + * for the collective calling requirement. + * + */ diff --git a/doxygen/dox/CopyingCommited.dox b/doxygen/dox/CopyingCommited.dox new file mode 100644 index 00000000000..c005a2cd333 --- /dev/null +++ b/doxygen/dox/CopyingCommited.dox @@ -0,0 +1,143 @@ +/** \page copying_committed Copying Committed Datatypes with H5Ocopy + * HDF5 Release 1.8.9 May 2012 + * + * \section sec_copying_committed_datatypes Copying Committed Datatypes with H5Ocopy + * Committed datatypes can be a powerful feature in HDF5. They can be used to share a single datatype + * description among multiple datasets, to save space or ensure that the datatypes are truly identical, and to + * assign a name to that datatype within the HDF5 group structure. The object copy API, #H5Ocopy, can be used + * to copy HDF5 objects from one file to another, including committed datatypes and objects that use them. + * However, problems can occur when a dataset using a committed datatype or an object with an attribute that + * uses a committed datatype is copied to another file with #H5Ocopy. + * + * When copying a dataset that uses a committed datatype or an object with an attribute that uses a committed + * datatype between files, the library by default does not look for a matching committed datatype in the + * destination file. The library creates a new committed datatype in the destination file without any links to it (an + * anonymous committed datatype) and then links the dataset to the anonymous committed datatype. This + * means that, when copying multiple datasets in separate calls to #H5Ocopy, a new committed datatype is + * created for each #H5Ocopy call. While it is possible to have all of the copied datasets share the same + * committed datatype by copying them in a single call to #H5Ocopy, this is not always attainable. + * + * For example, imagine that a user has an application that automatically creates many data files, each with many + * datasets that all use a single committed datatype. At the end of a project, the user wants to merge all of these + * files into a single file. The HDF5 Library can have all of the datasets in the combined file use the same + * committed datatype, but the default behavior of the library is to create an anonymous committed datatype + * for each dataset. + * + * To make sure that shared committed datatypes in the source are shared in the copy, use the + * #H5Pset_copy_object property list API routine to set the #H5O_COPY_MERGE_COMMITTED_DTYPE_FLAG + * flag. When this flag is set and #H5Ocopy encounters an object or attribute that uses a committed datatype, + * H5Ocopy will search for a matching committed datatype in the destination file. If a matching committed + * datatype is found, then it will be used by the copied dataset or attribute. The next few paragraphs describe in + * more detail the process that #H5Ocopy goes through. + * + * When the #H5O_COPY_MERGE_COMMITTED_DTYPE_FLAG flag is set, H5Ocopy will search the destination file + * for committed datatypes and build a temporary list in memory of all the committed datatypes it finds. Then, + * whenever H5Ocopy encounters a dataset that uses a committed datatype or an object with an attribute that + * uses a committed datatype in the source, it will check that list to see if it contains a datatype equal to the + * source datatype. If #H5Ocopy finds an equal datatype, it will modify the copied object or attribute to use the + * found committed datatype as its datatype. #H5Ocopy will then update the list if a new committed datatype is + * created in the destination file as a result of the copy. When later datasets and attributes using committed + * datatypes are encountered, the library will again check to see if the list contains a matching datatype. + * + * To determine if two committed datatypes are equal, the library will compare their descriptions in a manner + * similar to #H5Tequal. In addition, if either committed datatype has one or more attributes, then all attributes + * must be present in both committed datatypes, and the attributes must all be identical. Each attribute’s + * datatype description, dataspace, and raw data must be identical. However, if an attribute uses a committed + * datatype, then the attributes of the attribute’s committed datatype will not be compared. + * + * When #H5Ocopy encounters a committed datatype object in the source file, it will similarly search for a + * matching committed datatype in the destination file. If a match is found, the library will create a hard link in + * the destination file to the found datatype. If a match is not found, the library will copy the committed datatype + * normally and add it to the temporary list of committed datatypes in the destination file. + * + * By default, #H5Ocopy will search the entire destination file for a matching committed datatype. It is possible to + * focus where #H5Ocopy will search. This focusing should result in a faster search. If there are locations in the + * destination file where a matching committed datatype might be found, then those locations can be specified + * with the #H5Padd_merge_committed_dtype_path property. + * + * The example below shows how to enable the feature described above for use with #H5Ocopy. + * + * Example1. Setting the object copy property list + * \code + * hid_t ocpypl_id; + * + * ocpypl_id = H5Pcreate(H5P_OBJECT_COPY); + * status = H5Pset_copy_object(ocpypl_id, H5O_COPY_MERGE_COMMITTED_DT_FLAG); + * status = H5Ocopy(file1_id, src_name, file2_id, dst_name, ocpypl_id, H5P_DEFAULT); + * \endcode + * + * \subsection subsec_copying_committed_datatypes_cb Callback Function + * If no matching datatype is found in the locations specified by the call to + * #H5Padd_merge_committed_dtype_path, then #H5Ocopy will by default search the entire destination file. In + * some cases, this may not be desirable. For instance, the user may expect the datatype to always have a match + * in the specified locations and may wish to return an error if a match is not found. The user may also have a + * very large file for which the full search incurs a substantial performance penalty. In this instance, the user may + * wish to log these events so that other datatypes can be added with + * #H5Padd_merge_committed_dtype_path, or the user may wish to abort the search and copy the datatype + * normally. + * + * To support these use cases, the functions #H5Pset_mcdt_search_cb and #H5Pget_mcdt_search_cb have + * been added. These functions allow the user to define a callback function that will be called every time the list + * of paths added by #H5Padd_merge_committed_dtype_path has been exhausted but before beginning the + * full search of the file. The prototype for the callback function is defined by #H5O_mcdt_search_cb_t. The + * only argument to the callback function is a user supplied user data pointer, and the return value is an enum, + * defined by #H5O_mcdt_search_ret_t, which tells the library to either continue with the full file search, abort + * the search and copy the datatype normally (create a new committed datatype in the destination file), or + * return an error. + * + * \subsection subsec_copying_committed_datatypes_sum Function Summary + * Functions used in committed datatype copying operations are listed below. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Committed Datatype Copying Related Functions
FunctionPurpose
#H5OcopyAllows an application to copy an object within an HDF5 file or to another HDF5 file.
#H5Pset_copy_objectAllows an application to set properties to be used + * when an object is copied.
#H5Padd_merge_committed_dtype_pathAllows an application to add a path to the list of + * paths that will be searched in the destination file for + * a matching committed datatype.
#H5Pfree_merge_committed_dtype_pathsAllows an application to clear the list of paths stored + * in the object copy property list \b ocpypl_id.
#H5Pset_mcdt_search_cbAllows an application to set the callback function + * that #H5Ocopy will invoke before searching the entire destination file for a matching committed + * datatype.
#H5Pget_mcdt_search_cbAllows an application to retrieve the callback + * function from the specified object copy property list.
#H5O_mcdt_search_cb_tDefinition of the callback function set by + * #H5Pset_mcdt_search_cb. Provides the mechanism by which a user application may set an + * action for #H5Ocopy to take after checking all suggested paths for a matching committed datatype + * but before starting the global search of the destination file.
+ * + * \subsection subsec_copying_committed_datatypes_res Resources + * See the following for more information. + * + * See the \ref sec_datatype chapter in the \ref UG. + * + * See these entries in the \ref RM + * \li #H5Ocopy + * \li #H5Pset_copy_object + * \li #H5Padd_merge_committed_dtype_path + * \li #H5Pfree_merge_committed_dtype_paths + * \li #H5Pset_mcdt_search_cb + * \li #H5Pget_mcdt_search_cb + * + * + */ + \ No newline at end of file diff --git a/doxygen/dox/FreeingMemoryAllocatedByTheHdf5Library.dox b/doxygen/dox/FreeingMemoryAllocatedByTheHdf5Library.dox new file mode 100644 index 00000000000..bec5abe5df5 --- /dev/null +++ b/doxygen/dox/FreeingMemoryAllocatedByTheHdf5Library.dox @@ -0,0 +1,101 @@ +/** \page freeing_memory Freeing Memory Allocated by the HDF5 Library + * Several functions in the HDF5 C API return buffers allocated by the HDF5 Library. When + * application code uses a different library for memory management than the HDF Library, + * a corrupt heap or a resource leak can occur when these allocated buffers are freed. This + * is most commonly a problem on Windows systems since Microsoft implements C library + * functions in Visual Studio-­­specific libraries which do not share heap state. + * + * Introduced with HDF5 Release 1.8.13 May 15, 2014 + * + * This document describes this problem and the steps users can take to mitigate the + * problem. This document also introduces the new #H5free_memory function. + * + * \section sec_freeing_memory_intro Introduction + * In the HDF5 Library, responsibility for the allocation and freeing of memory is usually the responsibility + * of the same component: either the library or the user's code. When data that would normally be stored + * in dynamically­allocated memory must be returned from the library, the user is usually asked to allocate + * a buffer which is passed to the function and then filled by the library. The complication is that the user + * must be able to determine the buffer's size. The mechanism for this is for the user to make a preliminary + * call, passing a NULL pointer in for the buffer. The function will then return the appropriate number of + * bytes for the user to allocate. See the example below. + * + * Example1. Determining the buffer size with a preliminary call + * \code + * ssize_t size; + * size_t bufsize; + * hid_t object_id; + * char *comment; + * … + * size = H5Oget_comment(object_id, NULL, bufsize); // determine size + * bufsize = size; + * comment = (char *)malloc(bufsize * sizeof(char)); + * size = H5Oget_comment(object_id, comment, bufsize); // fill buffer + * \endcode + * + * There are, however, several API calls in which the buffer is allocated by the HDF5 Library and returned to + * the user who is responsible for freeing it. This can be a problem when memory in the application and + * HDF5 Library are managed via different libraries as it can result in resource leaks or a corrupted heap. + * This heap corruption can result in subtle bugs that can be very difficult to reproduce and diagnose. In + * most cases, having the library allocate memory and the application free it is not a problem since + * memory operations will resolve down to the operating system's memory manager; however, there are + * cases where this is not true. For example, a debug memory manager may be in use by the application + * code but not the library. A complication that is unique to Windows is that the C standard library + * functions are implemented in Visual­Studio­specific C run-time (CRT) libraries. When different versions + * of Visual Studio are used to compile the library and application code, the allocate and free calls are + * made in different libraries, which do not share state, leading to the previously mentioned resource and + * corruption issues. + * + * \section sec_freeing_memory_crt The Windows C Run-time (CRT) + * Microsoft implements the standard C library functions in debug and release libraries that are specific to + * each version of Visual Studio1. Each library is a separate entity and maintains its own internal CRT object + * state, file handles, and heap information. Creating an object in one CRT and destroying it in another CRT + * may appear to work but can cause corruption of one CRT and resource leaks in the other. + * + * + * + * + * + *
+ * \image html FreeingMemory_fig1.png + *
+ * + * These problems are normally avoided on Windows by ensuring that all components that can return CRT + * resources are linked to the same CRT dynamic link library (DLL). Unfortunately, even debug and release + * CRTs are housed in separate DLLs, so this is not an easy solution to implement. Using static linkage does + * not avoid this problem since separate copies of the CRT are created in each statically linked component. + * + * \li 1 The names of these libraries are of the form MSVCR<#>.dll, where <#> is the Visual Studio version. For example, + * MSVCR110.dll corresponds to Visual Studio 11.0 (2012). + * + * \section sec_freeing_memory_api Affected API Calls + * This is a list of the API calls that are affected. + * \li #H5Eget_major + * \li #H5Eget_minor + * \li #H5Pget_class_name + * \li #H5Tget_member_name + * \li #H5Tget_tag + * + * \section sec_freeing_memory_mitigation Mitigation + * There are several potential solutions to the problem of freeing memory allocated by the HDF5 Library. + * + * \subsection subsec_freeing_memory_mitigation1 Use the Same Memory Manager/Correct C Run‐time Everywhere + * Both application code and the HDF5 Library must use the same memory allocator. When using Visual + * Studio, both the Visual Studio version and release/debug state must be identical. As of HDF5 1.8.12, this + * is the only available solution. + * + * \subsection subsec_freeing_memory_mitigation2 Use the H5free_memory Function + * A new function called #H5free_memory has been created and is essentially a thin wrapper for the run­ + * time's free() call. This function would be used to free any memory allocated by the library. This solution + * has the advantages of being extremely easy to implement and intuitive to use. It can also be used as a + * solution with legacy API calls, so it would be necessary even if we modify the HDF5 API. This function will + * also be extremely useful when HDF5 is wrapped for use with managed languages such as Java, .NET, and + * Python so that the wrappers can properly clean up resources. + * + * See the #H5free_memory entry in the \ref RM for more information. + * + * Note that the creation of this function does not imply that it will be acceptable for new API calls to be + * created that return library­allocated memory. The preferred mechanism will still be to use the + * "preliminary call" scheme described in the "Introduction" on page 4 where the user allocates the buffer. + * + */ + \ No newline at end of file diff --git a/doxygen/dox/H5.format.1.1.dox b/doxygen/dox/H5.format.1.1.dox index e120c35c3ab..136aab3dbd6 100644 --- a/doxygen/dox/H5.format.1.1.dox +++ b/doxygen/dox/H5.format.1.1.dox @@ -338,7 +338,7 @@ each group page, and a group entry for the root object in the file. Address of Global Free-space Index Free-space management is not yet defined in the HDF5 file format and is not handled by the library. - Currently this field always contains the undefined address.
+ Currently this field always contains the @ref FMT11UndefinedAddress "undefined address".
This field is present in version 0+ of the superblock. @@ -352,7 +352,7 @@ each group page, and a group entry for the root object in the file. Driver Information Block Address This is the relative file address of the file driver information block which contains driver-specific information needed to reopen the file. If there is no driver information block then this entry should - be the undefined address.
+ be the @ref FMT11UndefinedAddress "undefined address".
This field is present in version 0+ of the superblock. @@ -653,12 +653,12 @@ Aside from that difference, internal nodes and leaf nodes are identical. Address of Left Sibling This is the relative file address of the left sibling of the current node. If the current node is the - left-most node at this level then this field is the undefined address. + left-most node at this level then this field is the @ref FMT11UndefinedAddress "undefined address". Address of Right Sibling This is the relative file address of the right sibling of the current node. If the current node is the - right-most node at this level then this field is the undefined address. + right-most node at this level then this field is the @ref FMT11UndefinedAddress "undefined address". Keys and Child Pointers @@ -1044,7 +1044,7 @@ table; the names of those objects are stored in the local heap. Offset to Head of Free-list This is the offset within the heap data segment of the first free block (or the - undefined address if there is no no free block). The free block + @ref FMT11UndefinedAddress "undefined address" if there is no no free block). The free block contains "Size of Lengths" bytes that are the offset of the next free block (or the value '1' if this is the last free block) followed by "Size of Lengths" bytes that store the size of this free block. The size of the free block includes the space used to store the offset of the next free block and the of the current @@ -1226,7 +1226,7 @@ The free-space index is a collection of blocks of data, dispersed throughout the not used by any file objects. The super block contains a pointer to root of the free-space description; that pointer is currently required -to be the undefined address. +to be the @ref FMT11UndefinedAddress "undefined address". The format of the free-space index is not defined at this time. @@ -1487,7 +1487,7 @@ not described in this document.) Dimension \#n Maximum Size This value is the maximum size of the dimension of the data as stored in the file. This value may be - the special unlimited size which indicates that the data may expand along + the special \ref FMT11UnlimitedDim "unlimited" size which indicates that the data may expand along this dimension indefinitely. If these values are not stored, the maximum size of each dimension is assumed to be the dimension's current size. @@ -3041,7 +3041,7 @@ Class-specific information for chunked layout (Class 2): Address This is the address of the B-tree that is used to look up the addresses of the chunks. The - address may have the undefined address value, to indicate + address may have the @ref FMT11UndefinedAddress "undefined address" value, to indicate that storage has not yet been allocated for this array. @@ -3159,8 +3159,8 @@ applied to the data stream by providing filter identification numbers, flags, a the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group in the HDF5 library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found - at - https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md. + at + https://\PLURL/docs/RegisteredFilterPlugins.md. 1
To request a filter identifier, please contact The HDF Group’s Help Desk at HDF Help Desk. You will be asked to provide the following information: @@ -3246,7 +3246,7 @@ applied to the data stream by providing filter identification numbers, flags, a \anchor FMT11Footnote1Change 1 If you are reading an earlier version of this document, this link may have changed. If the link does not work, use the latest version of this document on The HDF Group’s github website, -HDF5 File Format Specification; the link there will always be correct. +\ref SPEC; the link there will always be correct. \subsubsection subsubsec_fmt11_dataobject_hdr_attribute Name: Attribute Header Message Type: 0x000C
diff --git a/doxygen/dox/H5.format.2.0.dox b/doxygen/dox/H5.format.2.0.dox index 53d84df8885..daf20e63c91 100644 --- a/doxygen/dox/H5.format.2.0.dox +++ b/doxygen/dox/H5.format.2.0.dox @@ -7502,7 +7502,7 @@ Class-specific information for chunked layout (Class 2): the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group in the HDF5 library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found - at The HDF Group’s + at The HDF Group’s Registered Filters page.
1
To request a filter identifier, please contact The HDF Group’s Help Desk at HDF Help Desk. @@ -7609,7 +7609,7 @@ Class-specific information for chunked layout (Class 2): \anchor FMT2Footnote1Change 1 If you are reading an earlier version of this document, this link may have changed. If the link does not work, use the latest version of this document on The HDF Group’s github website, -HDF5 File Format Specification; the link there will always be correct. +\ref SPEC; the link there will always be correct. @@ -7684,7 +7684,7 @@ link may have changed. If the link does not work, use the latest version of this the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group in the HDF5 library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found - at The HDF Group’s + at The HDF Group’s Registered Filters page.
1
To request a filter identifier, please contact The HDF Group’s Help Desk at HDF Help Desk. diff --git a/doxygen/dox/H5.format.3.0.dox b/doxygen/dox/H5.format.3.0.dox index e7ba0ef0e19..95df6095c95 100644 --- a/doxygen/dox/H5.format.3.0.dox +++ b/doxygen/dox/H5.format.3.0.dox @@ -8485,7 +8485,7 @@ The following information exists only when the chunk is filtered. In other words the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group in the HDF5 library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found - at The HDF Group’s + at The HDF Group’s Registered Filters page.
1
To request a filter identifier, please contact The HDF Group’s Help Desk at HDF Help Desk. @@ -8592,7 +8592,7 @@ The following information exists only when the chunk is filtered. In other words \anchor FMT3Footnote1Change 1 If you are reading an earlier version of this document, this link may have changed. If the link does not work, use the latest version of this document on The HDF Group’s github website, -HDF5 File Format Specification; the link there will always be correct. +\ref SPEC; the link there will always be correct.
Filter Pipeline Message - Version 2
@@ -8669,7 +8669,7 @@ link may have changed. If the link does not work, use the latest version of this the filter. Values from zero through 32,767 are reserved for filters supported by The HDF Group in the HDF5 library and for filters requested and supported by third parties. Filters supported by The HDF Group are documented immediately below. Information on 3rd-party filters can be found - at The HDF Group’s + at The HDF Group’s Registered Filters page.
1
To request a filter identifier, please contact The HDF Group’s Help Desk at HDF Help Desk. diff --git a/doxygen/dox/HDF5CompressionTroubleshooting.dox b/doxygen/dox/HDF5CompressionTroubleshooting.dox new file mode 100644 index 00000000000..5046a571fc6 --- /dev/null +++ b/doxygen/dox/HDF5CompressionTroubleshooting.dox @@ -0,0 +1,674 @@ +/** \page CompTS HDF5 Compression Troubleshooting +The purpose of this technical note is to help HDF5 users with troubleshooting problems +with \ref H5Z_UG, especially with compression filters. The document assumes that the +reader knows HDF5 basics and is aware of the compression feature in HDF5. + +\section sec_compts_intro Introduction +One of the most powerful features of HDF5 is the ability to modify, or “filter,” data during +I/O. Filters provided by the HDF5 Library, “predefined filters”, include several +types of data compression, data shuffling and checksum. Users can implement their +own “user-defined filters” and use them with the HDF5 Library. + +By far the most common user-defined filters are ones that perform data compression. While the +programming model and usage of the compression filters are straightforward, it is easy, especially for +novice users, to overlook important details when implementing compression filters and to end up with +data that is not modified as they would expect. + +The purpose of this document is to describe how to diagnose situations where the data in a file is not +compressed as expected. + +\section sec_compts_over An Overview of HDF5 Compression Troubleshooting +Sometimes users may find that HDF5 data was not compressed in a file or that the compression ratio is +very small. By themselves, these results do not mean that compression did not work or did not work +well. These results suggest that something might have gone wrong when a compression filter was +applied. How can users determine the true cause of the problem? + +There are two major reasons why a filter did not produce the desired result: it was not applied, or it was +not effective. + +

The filter was not applied

+If a filter was not applied at all, then it was not included at compile time when the library was built or +was not found at run time for dynamically loaded filters. + +The absence or presence of HDF5 predefined filters can be confirmed by examining the installed HDF5 +files or by using HDF5 API calls. The absence or presence of all filter types can be confirmed by running +HDF5 command-line tools on the produced HDF5 files. See \ref sec_compts_notapp for +more information. + +

The filter was applied but was not effective

+The effectiveness of compression filters is a complex matter and is only briefly covered this document. +See \ref sec_compts_ineff for more information. This section gives a +short overview of the problem and provides an example in which the advantages of different +compression filters and their combinations are shown. + +\section sec_compts_notapp If a Filter Was Not Applied +This section discusses how it may happen that a compression filter is not available to an application and +describes the behavior of the HDF5 Library in the absence of the filter. Then we walk through how to +troubleshoot the problem by checking the HDF5 installation, by examining what an application can do at +run time to see if a filter is available, and by using some HDF5 command line tools to see if a filter was +applied. + +Note that there are internal predefined filters: +\snippet{doc} H5Zmodule.h PreDefFilters + +These are enabled by default by both \b configure and +\b CMake builds. While these filters can be disabled intentionally with the \b configure flag +–disablefilters, disabling them is not recommended. The discussion and the examples in this document +focus on compression filters, but everything said can be applied to other missing internal filters as well. + +\subsection subsec_compts_notapp_miss How the HDF5 Library Configuration May Miss a Compression Filter +The HDF5 Library uses external libraries for data compression. The two predefined compression +methods are \b gzip3 and \b szip or \b libaec, and these can be requested at the HDF5 Library configuration time +(compile time). User-defined compression filters and the corresponding libraries are usually linked with +an application or provided as a dynamically loaded library. + +\b Note that the \b libaec library is a replacement for the original \b szip library. The \b libaec library is a +freely available, open-source library that provides compression and decompression functionality and is +compatible with the \b szip filter. The \b libaec library can be used as a drop-in replacement for the \b szip, +but requires two libraries to be present on the system: \b libaec.a(so,dylib,lib) and \b libsz.a(so,dylib,lib). +Everywhere in this document, the term \b szip refers to the \b szip filter and the \b libaec library. + +\b gzip and \b szip require the \b libz.a(so,dylib,lib) and \b libsz.a(so,dylib,lib)/libaec.a(so,dylib,lib) +libraries, respectively, to be present on the system and to be enabled during HDF5 configuration with +this autotools configure command: +\code + ./configure –with-zlib=/path… –with-szlib=/path… +\endcode +There is one important difference in the behavior of GNU Autotools configure between \b gzip and \b szip. +On Unix systems,\b gzip compression is enabled automatically if the \b zlib library is present on the system +in default locations without explicitly specifying –with-zlib=/path. For example, if +libz.so is installed under /usr/lib with the +header under /usr/include or under /usr/local/lib with the header under +/usr/local/include, the following HDF5 configure command will find the \b gzip library and will +configure the compression filter in: +\code + ./configure +\endcode +With GNU Autotools configure will not fail if libraries supporting the requested compression method are not found, for example, +because a specified path was not correct, or the library is missing. + +Or CMake configure command: +\code + cmake -G "Unix Makefiles" -DHDF5_ENABLE_ZLIB_SUPPORT=ON -DHDF5_ENABLE_SZIP_SUPPORT=ON -D +\endcode +With CMake both libraries have to be explicitly enabled. The source code distribution’s +config/cmake/cacheinit.cmake file will enable both filters along with setting other options. +Users can overwrite the defaults by using + -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF +with the “cmake –C” command. See the +INSTALL_CMake.txt file +under the release_docs directory in the HDF5 source distribution. + +If compression is not requested or found at configuration time, the compression method is not +registered with the library and cannot be applied when data is written or read. For example, the +\ref sec_cltools_h5repack tool will not be able to remove an \b szip compression filter from a dataset +if the \b szip library was not configured into the library against which the tool was built. The next +section discusses the behavior of the HDF5 Library in the absence of filters. + +\subsection subsec_compts_notapp_behave How Does the HDF5 Library Behave in the Absence of a Filter +By design, the HDF5 Library allows applications to create and write datasets using filters that are not +available at creation/write time. This feature makes it possible to create HDF5 files on one system and +to write data on another system where the HDF5 Library is configured with or without the requested +filter. + +Let’ s recall the HDF5 programming model for enabling filters. + +An HDF5 application uses one or more H5Pset_ calls to configure a dataset’s filter pipeline at +its creation time. The excerpt below shows how a \b gzip filter is added to a pipeline5 with +#H5Pset_deflate. +\code + // Create the dataset creation property list, add the gzip + // compression filter and set the chunk size. + dcpl = H5Pcreate (H5P_DATASET_CREATE); + status = H5Pset_deflate (dcpl, 9); + status = H5Pset_chunk (dcpl, 2, chunk); + dset = H5Dcreate (file, DATASET,…, dcpl,…); +\endcode +For all internal filters (\b shuffle, \b fletcher32, \b scaleoffset, and \b nbit) and the external \b gzip +filter, the HDF5 Library does not check to see if the filter is registered when the corresponding +H5Pset_ function is called. The only exception to this rule is #H5Pset_szip which will fail if +szip was not configured in or is configured with a decoder only. Hence, in the example above, #H5Pset_deflate will +succeed. The specified filter will be added to the dataset’s filter pipeline and will +be applied to any data written to this dataset. + +When H5Pset_ is called, a record for the filter is added to the dataset’s object header in the +file, and information about the filter can be queried with the HDF5 APIs and displayed by HDF5 tools +such as \ref sec_cltools_h5dump. The presence of filter information in a dataset’s header does not mean that the filter +was actually applied to the dataset’s data, as will be explained later in this document. See \ref +subsec_compts_notapp_tools for more information on how to use +\ref sec_cltools_h5ls and \ref sec_cltools_h5debug to determine if the filter was actually applied. + +The success of further write operations to a dataset when filters are missing depends on the filter type. + +By design, an HDF5 filter can be optional or required. This filter mode defines the behavior of the HDF5 +Library during write operations. In the absence of an optional filter, #H5Dwrite calls will succeed and +data will be written to the file, bypassing the filter. A missing required filter will cause #H5Dwrite calls to +fail. Clearly, #H5Dread calls will fail when filters that are needed to decode the data are missing. + +The HDF5 Library has only one required internal filter, \b Fletcher32 (checksum creation), and one +required external filter, \b szip. As mentioned earlier, only the \b szip compression (#H5Pset_szip) will +flag the absence of the filter. If, despite the missing filter, an application goes on to create a dataset via +#H5Dcreate, the call will succeed, but the \b szip filter will not be added to the filter pipeline. This +behavior is different from all other filters that may not be present, but will be added to the filter pipeline +and applied during I/O. See the \ref subsubsec_compts_notapp_need_api section for more information on how to +determine if a filter is available and to avoid writing data while the filter is missing. + +Developers who create their own filters should use the \b flags parameter in #H5Pset_filter to +declare if the filter is optional or required. The filter type can be determined by calling #H5Pget_filter +and checking the value of the \b flags parameter. + +For more information on filter behavior in HDF5, see \ref H5Z_UG. + +\subsection subsec_compts_notapp_need How to Determine if the HDF5 Library was Configured with a Needed Compression Filter +The previous section described how the HDF5 Library could be configured without certain compression +filters and the resulting expected library behavior. + +The following subsections explain how to determine if a compression method is configured in the HDF5 +Library and how to avoid accessing data if the filter is missing. + +\subsubsection subsubsec_compts_notapp_need_settins Examine the hdf5lib.settings File +To see how the library was configured and built, users should examine the hdf5lib.settings text file +found in the lib directory of the HDF5 installation point and search for the lines that contain the “I/O +filters” string. The hdf5lib.settings file is automatically generated at configuration time when +the HDF5 Library is built with \b configure on Unix or with \b CMake on Unix and Windows, and it should +contain the following lines: +\code + I/O filters (external): deflate(zlib),szip(encoder) + I/O filters (internal): shuffle,fletcher32,nbit,scaleoffset +\endcode + +The same lines in the file generated by \b CMake look slightly different: +\code + I/O filters (external): DEFLATE ENCODE DECODE + I/O filters (internal): SHUFFLE FLETCHER32 NBIT SCALEOFFSET +\endcode + +“ENCODE DECODE” indicates that both the \b szip compression encoder and decoder are present. This +inconsistency between configure and CMake generated files will be removed in a future release. +These lines show the compression libraries configured with HDF5. Here is an example of the same +output when external compression filters are absent: +\code + I/O filters (external): + I/O filters (internal): shuffle,fletcher32,nbit,scaleoffset +\endcode + +Depending on the values listed on the I/O filters (external) line, users will be able to tell if their HDF5 +files are compressed appropriately. If \b szip is not included in the build, data files will not be compressed +with \b szip. If \b gzip is not included in the build and is not installed on the system, then data files will not +be compressed with \b gzip. + +If the hdf5lib.settings file is not present on the system, then users can examine a public header +file or the library binary file to find out if a filter is present, as is discussed in the next two sections. + +\subsubsection subsubsec_compts_notapp_need_head Examine the H5pubconf.h Header File +To see if a filter is present, users can also inspect the HDF5 public header file installed under the +include directory of the HDF5 installation point. If the compression and internal filters are present, the +corresponding symbols will be defined as follows: +\code + // Define if support for deflate (zlib) filter is enabled + #define H5_HAVE_FILTER_DEFLATE 1 + // Define if support for Fletcher32 checksum is enabled + #define H5_HAVE_FILTER_FLETCHER32 1 + // Define if support for nbit filter is enabled + #define H5_HAVE_FILTER_NBIT 1 + // Define if support for scaleoffset filter is enabled + #define H5_HAVE_FILTER_SCALEOFFSET 1 + // Define if support for shuffle filter is enabled + #define H5_HAVE_FILTER_SHUFFLE 1 + // Define if support for Szip filter is enabled + #define H5_HAVE_FILTER_SZIP 1 +\endcode +If a compression or internal filter was not configured, the corresponding lines will be commented out as +follows: +\code + // Define if support for deflate (zlib) filter is enabled + // #undef H5_HAVE_FILTER_DEFLATE +\endcode + +\subsubsection subsubsec_compts_notapp_need_bin Check the HDF5 Library’s Binary +The HDF5 Library’s binary contains summary output similar to what is stored in the +hdf5lib.settings file. Users can run the Unix “strings” command to get information about the +configured filters: +\code + % strings libhdf5.a(so) | grep "I/O filters (" + I/O filters (external): deflate(zlib),szip(encoder) + I/O filters (internal): shuffle,fletcher32,nbit,scaleoffset +\endcode +When compression filters are not configured, the output of the command above will be: +\code + I/O filters (external): + I/O filters (internal): shuffle,fletcher32,nbit,scaleoffset +\endcode +On Windows one can use the dumpbin /all command, and then view and search the output for +strings like DEFLATE, FLETCHER32, DECODE, and ENCODE. +\code + ….. + 10201860: 4E 0A 20 20 20 20 20 20 20 20 20 49 2F 4F 20 66 N. I/O f + 10201870: 69 6C 74 65 72 73 20 28 65 78 74 65 72 6E 61 6C ilters (external + 10201880: 29 3A 20 20 44 45 46 4C 41 54 45 20 44 45 43 4F ): DEFLATE DECO + 10201890: 44 45 20 45 4E 43 4F 44 45 0A 20 20 20 20 20 20 DE ENCODE. + 102018A0: 20 20 20 49 2F 4F 20 66 69 6C 74 65 72 73 20 28 I/O filters ( + 102018B0: 69 6E 74 65 72 6E 61 6C 29 3A 20 20 53 48 55 46 internal): SHUF + 102018C0: 46 4C 45 20 46 4C 45 54 43 48 45 52 33 32 20 4E FLE FLETCHER32 N + 102018D0: 42 49 54 20 53 43 41 4C 45 4F 46 46 53 45 54 0A BIT SCALEOFFSET. +\endcode + +\subsubsection subsubsec_compts_notapp_need_script Check the Compiler Script +Developers can also use the compiler scripts such as h5cc to verify that a compression library is present +and configured in. Use the - show option with any of the compilers scripts found in the bin +subdirectory of the HDF5 installation directory. The presence of –lsz and –lz options among the linker +flags will confirm that \b szip or \b gzip were compiled with the HDF5 Library. See the sample below +\code + $ h5cc -show + gcc -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_BSD_SOURCE -L/mnt/hdf/packages/hdf5/v1812/Linux64_2.6/standard/lib + /mnt/hdf/packages/hdf5/v1812/Linux64_2.6/standard/lib/libhdf5_hl.a + /mnt/hdf/packages/hdf5/v1812/Linux64_2.6/standard/lib/libhdf5.a -lsz -lz -lrt + -ldl -lm -Wl,-rpath -Wl,/mnt/hdf/packages/hdf5/v1812/Linux64_2.6/standard/lib +\endcode + +\subsubsection subsubsec_compts_notapp_need_cmake Examine the hdf5-config.cmake File +\b CMake users can check the hdf5-config.cmake file in the \b CMake installation directory. The file will +indicate what options were used to configure the HDF5 Library. The variables in the "User Options" section +can be used by developers programmatically to determine if a filter was configured in. + +After using find-package(HDF5) \b CMake can test the setting of these variables as shown below: +\code + find_package (HDF5 NAMES hdf5 COMPONENTS C) + + if (HDF5_ENABLE_ZLIB_SUPPORT) + message(STATUS "gzip filter is available.") + else() + message(STATUS "gzip filter is not available.") + endif() +\endcode + +\subsubsection subsubsec_compts_notapp_need_api Using HDF5 APIs +Applications can check filter availability at run time. In order to check the filter’s availability with the +HDF5 Library, users should know the filter identifier (for example, #H5Z_FILTER_DEFLATE) and call the +#H5Zfilter_avail function as shown in the example below. Use #H5Zget_filter_info to determine +if the filter is configured to decode data, to encode data, neither, or both. +\code + // Check if gzip compression is available and can be used for both compression and decompression. + avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); + if (!avail) { + printf ("gzip filter not available.\n"); + return 1; + } + status = H5Zget_filter_info (H5Z_FILTER_DEFLATE, &filter_info); + if ( !(filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || + !(filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED) ) { + printf ("gzip filter not available for encoding and decoding.\n"); + return 1; + } +\endcode + +#H5Zfilter_avail can be used to find filters that are registered with the library or are available via +dynamically loaded libraries. For more information, see \ref subsubsec_dataset_filters_dyn. + +Currently there is no HDF5 API call to retrieve a list of all of the registered or dynamically loaded filters. +The default installation directories for HDF5 dynamically loaded filters are +/usr/local/hdf5/lib/plugin on Unix and %ALLUSERSPROFILE%\\hdf5\\lib\\plugin on +Windows. Users can also check to see if the environment variable HDF5_PLUGIN_PATH is set on the +system and refers to a directory with available plugins. + +\subsection subsec_compts_notapp_tools How to Use HDF5 Tools to Investigate Missing Compression Filters +In this section, we will use the \ref sec_cltools_h5dump, \ref sec_cltools_h5ls, and \ref sec_cltools_h5debug +command-line utilities to see if a file was +created with an HDF5 Library that did or did not have a compression filter configured in. For more +information on these tools, see the \ref sec_cltools page in the \ref UG. + +\subsubsection subsubsec_compts_notapp_tools_dump How to Use h5dump to Examine Files with Compressed Data +The \ref sec_cltools_h5dump command-line tool can be used to see if a file uses a compression filter. The tool has two +flags that will limit the output: the –p flag causes dataset properties including compression filters to be +displayed, and the –H flag is used to suppress the output of data. The program provided in the \ref sec_cltools +section creates a file called h5ex_d_gzip.h5. The output of \ref sec_cltools_h5dump shows +that the \b gzip compression filter set to level 9 was added to the DS1 dataset filter pipeline at creation +time. + +\code + $ hdf5/bin/h5dump -p -H *.h5 + HDF5 "h5ex_d_gzip.h5" { + GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 5, 9 ) + SIZE 5018 (1.633:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 9 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + } + } +\endcode + +The output also shows a compression ratio defined as (original size)/(storage size). The size of the stored +data is 5018 bytes vs. 8192 bytes of uncompressed data, a ratio of 1.663. This shows that the filter was +successfully applied. + +Now let’s look at what happens when the same program is linked against an HDF5 Library that was not +configured with the \b gzip library. + +Notice that some chunks are only partially filled. 56 chunks (7 along the first dimension and 8 along the +second dimension) are required to store the data. Since no compression was applied, each chunk has size +5x9x4 = 180 bytes, resulting in a total storage size of 10,080 bytes. With an original size of 8192 +bytes, the compression ratio is 0.813 (in other words, less than 1) and visible in the output below. +\code + $ hdf5/bin/h5dump -p -H *.h5 + HDF5 "h5ex_d_gzip.h5" { + GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 5, 9 ) + SIZE 10080 (0.813:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 9 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + } + } +\endcode +As discussed in the \ref subsec_compts_notapp_behave, the +presence of a filter in an object’s filter pipeline does not imply that it will be applied unconditionally +when data is written. + +If the compression ratio is less than 1, compression is not applied. If it is 1, and compression is shown by +\ref sec_cltools_h5dump, more investigation is needed; this will be discussed in the next section. + +\subsubsection subsubsec_compts_notapp_tools_debug How to Use h5ls and h5debug to Find a Missing Compression Filter +Filters operate on chunked datasets. A filter may be ineffective for one chunk (for example, the +compressed data is bigger than the original data), and succeed on another. How can users discern if a +filter is missing or just ineffective (and as a result non-compressed data was written)? The \ref sec_cltools_h5ls and +\ref sec_cltools_h5debug command-line tools can be used to investigate the issue. + +First, let’s take a look at what kind of information \ref sec_cltools_h5ls displays about the dataset DS1 in our example +file, which was written with an HDF5 library that has the \b deflate filter configured in: +\code + $ h5ls -vr h5ex_d_gzip.h5 + Opened "h5ex_d_gzip.h5" with sec2 driver. + / Group + Location: 1:96 + Links: 1 + /DS1 Dataset {32/32, 64/64} + Location: 1:800 + Links: 1 + Chunks: {5, 9} 180 bytes + Storage: 8192 logical bytes, 5018 allocated bytes, 163.25% utilization + Filter-0: deflate-1 OPT {9} + Type: native int +\endcode + +We see output similar to \ref sec_cltools_h5dump output with the compression ratio at 163%. + +Now let’s compare this output with another dataset \b DS1, but this time the dataset was written with a +program linked against an HDF5 library without the \b gzip filter present. +\code + $ h5ls -vr h5ex_d_gzip.h5 + Opened "h5ex_d_gzip.h5" with sec2 driver. + / Group + Location: 1:96 + Links: 1 + /DS1 Dataset {32/32, 64/64} + Location: 1:800 + Links: 1 + Chunks: {5, 9} 180 bytes + Storage: 8192 logical bytes, 10080 allocated bytes, 81.27% utilization + Filter-0: deflate-1 OPT {9} + Type: native int +\endcode + +The \ref sec_cltools_h5ls output above shows that the \b gzip filter was added to the filter pipeline of the dataset \b DS1. It +also shows that the compression ratio is less than 1. We can confirm by using \ref sec_cltools_h5debug that the filter +was not applied at all, and, as a result of the missing filter, the individual chunks were not compressed. + +From the \ref sec_cltools_h5ls output we know that the dataset object header is located at address 800. We retrieve +the dataset object header at address 800 and search the layout message for the address of the chunk +index B-tree as shown in the excerpt of the \ref sec_cltools_h5debug output below: +\code + $ h5debug h5ex_d_gzip.h5 800 + Reading signature at address 800 (rel) + Object Header... + ….. + Message 4... + Message ID (sequence number): 0x0008 `layout' (0) + Dirty: FALSE + Message flags: + Chunk number: 0 + Raw message data (offset, size) in chunk: (144, 24) bytes + Message Information: + Version: 3 + Type: Chunked + Number of dimensions: 3 + Size: {5, 9, 4} + Index Type: v1 B-tree + B-tree address: 1400 +\endcode + +Now we can retrieve the B-tree information: +\code + $ h5debug h5ex_d_gzip.h5 1400 3 + Reading signature at address 1400 (rel) + Tree type ID: H5B_CHUNK_ID + Size of node: 2616 + Size of raw (disk) key: 32 + Dirty flag: False + Level: 0 + Address of left sibling: UNDEF + Address of right sibling: UNDEF + Number of children (max): 56 (64) + Child 0... + Address: 4016 + Left Key: + Chunk size: 180 bytes + Filter mask: 0x00000001 + Logical offset: {0, 0, 0} + Right Key: + Chunk size: 180 bytes + Filter mask: 0x00000001 + Logical offset: {0, 9, 0} + Child 1... + Address: 4196 + Left Key: + Chunk size: 180 bytes +\endcode +\li Users have to supply the chunk rank. According to the HDF5 \ref sec_spec_ff Specification, this is the +dataset rank plus 1; in other words, 3. + +We see that the size of each chunk is 180 bytes: in other words, compression was not successful. The +filter mask value 0x00000001 indicates that filter was not applied. For more information on the filter +mask, see the \ref_spec_fileformat_btrees_v1 section in the HDF5 \ref sec_spec_ff Specification. + +\subsection subsec_compts_notapp_ex Example Program +The example program used to create the file discussed in this document is a modified version of the +program available at h5ex_d_gzip.c. It +was modified to have chunk dimensions not be factors of the +dataset dimensions. Chunk dimensions were chosen for demonstration purposes only and are not +recommended for real applications. +\code + #include + + #define FILE "h5ex_d_gzip.h5" + #define DATASET "DS1" + #define DIM0 32 + #define DIM1 64 + #define CHUNK0 5 + #define CHUNK1 9 + + int main (void) + { + hid_t file, space, dset, dcpl; // Handles + herr_t status; + htri_t avail; + H5Z_filter_t filter_type; + hsize_t dims[2] = {DIM0, DIM1}, chunk[2] = {CHUNK0, CHUNK1}; + size_t nelmts; + unsigned int flags, filter_info; + int wdata[DIM0][DIM1], // Write buffer + rdata[DIM0][DIM1], // Read buffer + max, i, j; + + // Initialize data. + for (i=0; iGCRIOREDRO_npp_d20030125_t0702533_e0711257_b00993_c20140501163427060570_XXXX_XXX.h5) +and used h5repack to apply three different compressions to the original file: +\li gzip with compression level 7 +\li szip compression using NN mode and 32-bit block size +\li Shuffle in combination with gzip compression level 7 + +Then we compared the sizes of the 32-bit floating dataset /All_Data/CrIMSS-EDR-GEOTC_All/Height when +different types of compression were used and compared for the sizes of the 32-bit integer dataset +/All_Data/CrIMSS-EDR_All/FORnum. The results are shown in the table below. +
Layout: Filter Pipeline Message - Version 2
+ + + + + + + + + + + + + + + + + + + + + + +
Table 1: Compression ratio for different types of compressions when using h5repack
DataOriginalgzip Level 7szip Using NN Mode and Blocksize 32Shuffle and gzip Level 7
32-bit Floats12.0871.6282.56
32-bit Integers13.64210.83238.20
+ +The combination of the \b shuffle filter and \b gzip compression level 7 worked well on both floating point +and integer datasets, as shown in the fifth column of the table above. \b gzip compression worked better +than \b szip on the floating point dataset, but not on the integer dataset as shown by the results in +columns three and four. Clearly, if the objective is to minimize the size of the file, datasets with different +types of data have to be compressed with different compression methods. + +For more information on the \b shuffle filter, see the \ref subsubsec_dataset_transfer_filter section in the +\ref sec_dataset chapter of the \ref UG. See also the \ref H5P in the \ref RM for the +#H5Pset_shuffle function call entry. + +\subsection subsec_compts_ineff_alt An Alternative to Compression +Sometimes HDF5 files contain unused space. The \ref sec_cltools_h5repack command-line tool can be used to reduce +the amount of unused space in a file without changing any storage parameters of the data. For example, +running \ref sec_cltools_h5stat on the file +GCRIOREDRO_npp_d20030125_t0702533_e0711257_b00993_c20140501163524579819_XXXX_XXX.h5 shows: +\code +Summary of file space information: + File metadata: 425632 bytes + Raw data: 328202 bytes + Unaccounted space: 449322 bytes +Total space: 1203156 bytes +\endcode + +After running \ref sec_cltools_h5repack, the file shows a 10-fold reduction in unaccounted space: +\code +Summary of file space information: + File metadata: 425176 bytes + Raw data: 328202 bytes + Unaccounted space: 45846 bytes +Total space: 799224 bytes +\endcode + +There is also a small reduction in file metadata space. +For more information on \ref sec_cltools_h5repack and \ref sec_cltools_h5stat, see the \ref sec_cltools page in the \ref UG. + +\section sec_compts_other Other Resources +See the following documents published by The HDF Group for more information. +\li See the \ref secLBComDsetCreate tutorial. +\li The “Filter Behavior in HDF5” note is part of the #H5Pset_filter function call entry. See + the \ref H5P in the \ref RM. + +*/ \ No newline at end of file diff --git a/doxygen/dox/HDF5ImprovingIOPerformanceCompressedDatasets.dox b/doxygen/dox/HDF5ImprovingIOPerformanceCompressedDatasets.dox new file mode 100644 index 00000000000..f47acc00a61 --- /dev/null +++ b/doxygen/dox/HDF5ImprovingIOPerformanceCompressedDatasets.dox @@ -0,0 +1,645 @@ +/** \page improve_compressed_perf Improving I/O Performance When Working with HDF5 Compressed Datasets + * Internal compression is one of several powerful HDF5 features that distinguish HDF5 + * from other binary formats and make it very attractive for storing and organizing data. + * Internal HDF5 compression saves storage space and I/O bandwidth and allows efficient + * partial access to data. Chunked storage has to be used when HDF5 compression is + * enabled. + * + * Certain combinations of compression, chunked storage, and access pattern may cause + * I/O performance degradation if used inappropriately, but the HDF5 Library provides + * tuning parameters to achieve I/O performance comparable with the I/O performance on + * raw data that uses contiguous storage. + * + * In this paper, we discuss the factors that should be considered when storing + * compressed data in HDF5 files and how to tune those parameters to optimize the I/O + * performance of an HDF5 application when working with compressed datasets. + * + * + * + * \section sec_improve_compressed_perf_intro Introduction + * One of the most powerful features of HDF5 is its ability to store and modify compressed data. The HDF5 + * Library comes with two pre-defined compression methods, GNU \b zip or \b gzip and \b szip or \b libaec, and has + * the capability to use third-party compression methods, \ref subsubsec_dataset_filters_dyn. The variety of available compression + * methods means users can choose the compression method that is best suited for achieving the desired + * balance between the CPU time needed to compress or un-compress data and storage performance. + * + * Compressed data is stored in a data array of an HDF5 dataset using a chunked storage mechanism. + * When chunked storage is used, the data array is split into equally sized chunks each of which is stored + * separately in the file. + * + * + * + *
Data array is logically split into equally sized chunks each of which is stored separately in the file
+ * \image html improve_perf-compress_fig_1.png "Figure 1" + *
+ * + * Compression is applied to each individual chunk. When an I/O operation is performed on a subset of the + * data array, only chunks that include data from the subset participate in I/O and need to be + * uncompressed or compressed. + * + * + * + *
Library will only read highlighted chunks when reading selected columns
+ * \image html improve_perf-compress_fig_2.png "Figure 2" + *
+ * + * Chunked storage also enables adding more data to a dataset without rewriting the whole dataset. Figure + * 3 below shows more rows and columns added to a data array stored in HDF5 by writing highlighted + * chunks that contain new data. + * + * + * + *
More rows and columns were added to the dataset
+ * \image html improve_perf-compress_fig_3.png "Figure 3" + *
+ * + * While HDF5 chunk storage and compression obviously provide great benefits in working with data, many + * HDF5 users have found that sometimes I/O performance is slower for compressed data than for + * uncompressed data. For example, as we show in this paper, there may be a huge performance + * difference between an application reading compressed data and reading the same data that was not + * compressed. For an application that writes compressed data, I/O performance may be excellent, but + * when data is moved to another system and read back, I/O performance drastically drops making data + * virtually unusable. + * + * Many of these cases of drastically slower reading performance can be ameliorated by more careful + * consideration of avoiding chunking arrangements that may cause poor reading performance when + * creating datasets or by a few simple changes to the application reading the data. In this paper, we will + * discuss the factors that should be considered when storing compressed data in HDF5 files and when + * tuning an HDF5 application that writes or reads compressed data. We assume that the reader knows + * HDF5 \ref LearnBasics and would like to learn a set of performance tuning techniques when working with + * compressed data. + * + * In our discussion, we use an HD5 file with Cross-track Infrared Sounder (CriS) data from the Suomi NPP + * satellite to illustrate several performance tuning techniques for HDF5 applications. The paper is + * organized as follows: + * \li The structure of the file and the properties of the datasets are discussed in the + \ref sec_improve_compressed_perf_case section. + * \li In the \ref sec_improve_compressed_perf_chunk section, we review HDF5 chunking and + * compression features in more detail. + * \li In the \ref sec_improve_compressed_perf_tune section, we discuss the performance tuning approach. + * \li The \ref sec_improve_compressed_perf_rec section summarizes our recommendations. + * + * In the near future, we intend to make available a new CCP (Chunking and Compression Performance) + * tool. This tool will allow users to vary access patterns, chunk sizes, compression method, and cache + * settings using the tool’s command options, reducing the need to create and compile test programs such + * as those used in the “Case Study” section on page 7. + * + * For more information on other things that can affect performance, see the “Things That Can Affect + * Performance” page in the FAQ on the website. + * + * \section sec_improve_compressed_perf_case Case Study + * We will use two HDF5 files to compare I/O performance and to illustrate the issues users may encounter + * when working with compressed data. These HDF5 files and the application programs used to read them + * can be downloaded [ 7 ] by readers wishing to reproduce the performance results discussed in this + * paper.1 + * \li 1:Performance results provided in the paper are intended to show the difference in + * performance when different HDF5 parameters are used. The reader should be aware that the numbers + * on his/her system would differ from those provided in the paper, but the effect of the HDF5 + * parameters should be the same. + * + * SCRIS_npp_d20140522_t0754579_e0802557_b13293_c20140522142425734814_noaa_pop.h5 is the + * first file we will use. It is an original data file with Cross-track Infrared Sounder (CriS) data from the + * Suomi NPP satellite. For brevity, we will refer to this file in this document as File.h5. + * + * The second file is gz6_SCRIS_npp_d20140522_t0754579_e0802557_b13293__noaa_pop.h5. We + * will refer to this file as File_with_compression.h5. The file was created from + * File.h5 by the \ref sec_cltools_h5repack tool that applied the \b gzip compression + * to all datasets using level 6 effort. Repacking File.h5 using \b gzip compression + * reduced the storage space by 1.3 times. We will use the file to demonstrate the most common issues + * HDF5 users encounter when working with compressed data in HDF5. + * + * We selected these files because they have characteristics that would be the first ones to look at when + * tuning I/O performance of both writing and reading HDF5 applications. First, this data file represents + * files generated on a big-endian system that is usually not available to general users of the data. The data + * provider used the HDF5 parameters to minimize storage space for data and to maximize write speed + * that were not necessarily the optimum parameters for the systems where the data would be read. + * Second, the users’ applications read data in a way that was optimized for scientific data analysis but not + * optimal for the HDF5 I/O performance. We will use the files to show what the users can do to improve + * performance of their applications, and which factors data providers should consider before creating + * data products. + * + * In our case study, we used a 4-dimensional array of 32-bit big-endian floating point numbers stored in + * the HDF5 dataset /All_Data/CrIS-SDR_All/ES_ImaginaryLW in both files. The data array is + * extensible and has the current dimension sizes 60x30x9x717. When compressed with \b gzip compression + * with level 6, the compression ratio is 1.0762. We used HDF5 command line tools + * \ref sec_cltools_h5dump and \ref sec_cltools_h5ls + * and the HDF Java-based browser HDFView to find various properties of the dataset that would help us to + * understand performance problems and propose solutions. If the reader decides to follow the discussion + * using a “hands on” approach, the examples below illustrate how to use h5dump and h5ls to get the + * characteristics of the /All_Data/CrIS-SDR_All/ES_ImaginaryLW dataset. + * \li 2:The ratio itself is not a subject of this paper, but the fact that the dataset was + * compressed is. It is one of the factors that affected the performance. While the total compression + * ratio for the file is 1.3, one should be careful about applying the same compression to all datasets + * in a file. For some datasets, compression will not significantly reduce storage space while requiring + * extra I/O time for decompression as this example shows. + * + * The \ref sec_cltools_h5dump command line below will yield the results shown in Figure 4 below: + * \code + * % h5dump -H -d /All_Data/CrIS-SDR_All/ES_ImaginaryLW File_with_compression.h5 + * + * HDF5 "gz6_SCRIS_npp_d20140522_t0754579_e0802557_b13293__noaa_pop.h5" { + * DATASET "/All_Data/CrIS-SDR_All/ES_ImaginaryLW" { + * DATATYPE H5T_IEEE_F32BE + * DATASPACE SIMPLE { ( 60, 30, 9, 717 ) / ( H5S_UNLIMITED, + * H5S_UNLIMITED, H5S_UNLIMITED, H5S_UNLIMITED ) } + * STORAGE_LAYOUT { + * CHUNKED ( 4, 30, 9, 717 ) + * SIZE 43162046 (1.076:1 COMPRESSION) + * } + * FILTERS { + * COMPRESSION DEFLATE { LEVEL 6 } + * } + * FILLVALUE { + * FILL_TIME H5D_FILL_TIME_IFSET + * VALUE -999.3 + * } + * ALLOCATION_TIME { + * H5D_ALLOC_TIME_INCR + * } + * } + * } + * \endcode + * Figure 4: Output of the h5dump command that shows properties of the dataset /All_Data/CrIS-SDR_All/ES_ImaginaryLW + * + * The \ref sec_cltools_h5ls command line below will yield the results shown in Figure 5 below: + * \code + * % h5ls -lrv gz6_SCRIS_npp_d20140522_t0754579_e0802557_b13293__noaa_pop.h5 + * + * /All_Data/CrIS-SDR_All/ES_ImaginaryLW Dataset {60/Inf, 30/Inf, 9/Inf, + * 717/Inf} + * Location: 1:60464 + * Links: 1 + * Chunks: {4, 30, 9, 717} 3097440 bytes + * Storage: 46461600 logical bytes, 43162046 allocated bytes, 107.64% + * utilization + * Filter-0: deflate-1 OPT {6} + * Type: IEEE 32-bit big-endian float + * \endcode + * Figure 5: Output of the h5ls command that shows properties of the dataset /All_Data/CrIS-SDR_All/ES_ImaginaryLW + * + * In HDFView, right click on the dataset to choose “Show Properties” option from the drop-down menu. + * The properties will appear in the new window as shown in Figure 6. + * + * + * + *
HDFView window with information about the dataset
+ * \image html improve_perf-compress_fig_6.png "Figure 6" + *
+ * + * Our application read the dataset along the fastest changing dimension, 717 elements at a time from the + * dataset in both files. In the 2-dimensional case, this would correspond to reading an array by “row”. + * There were 16,200 reads to get all of the data. What we found was a several orders of magnitude drop + * in the performance when data was read from the compressed dataset as shown in Table 1. + * + * + * + * + *
Table 1: Reading by 1x1x717 hyperslab (or “rows”) from original and compressed datasets. + * Performance drops more than 3000 times.
File NameFile.h5File_with_compression.h5 (gzip level 6)
Read Time0.1 seconds0.37 seconds
+ * + * We experimented with the HDF5 parameters such cache size and chunk size and modified our + * application to use different access patterns. The details of the experiments and achieved results will be + * discussed in the \ref sec_improve_compressed_perf_tune section. Here we provide the results just to show + * the difference in the read performance the change in the parameters made. + * + * The table below, Table 2, shows the result of reading data as in the example above with the difference + * that the application used a chunked cache size of 3MB instead of the default 1MB. Reading performance + * from the compressed dataset was only 4 times slower than for reading the uncompressed data. + * + * + * + * + *
Table 2: Reading by 1x1x717 hyperslab (or “rows”) from original and compressed datasets. Changing + * the chunk cache size from 1MB to 3MB improved application performance by a factor of 1000.
File NameFile.h5File_with_compression.h5 (gzip level 6)
Read Time0.1 seconds0.37 seconds
+ * + * We also experimented with a different access pattern to read data from both files. Instead of reading + * 717 elements at a time, we read a contiguous HDF5 hyperslab with dimensions 4x30x9x717. The reader + * who knows about HDF5 chunking will immediately recognize that we read one chunk at a time, a total + * 15 of them. With this change, reading from the non-compressed dataset was only 10 times better than + * reading from the compressed dataset; see Table 3 below and compare with the results in Table 1. + * + * + * + * + *
Table 3: Reading by 4x30x9x717 hyperslabs from original and compressed datasets. Performance for + * compressed dataset is several orders of magnitude better than the result in Table 1 and comparable + * to the result in Table 2.
File NameFile.h5File_with_compression.h5 (gzip level 6)
Read Time0.04 seconds0.36 seconds
+ * + * In our last experiment, we repacked both files with \ref sec_cltools_h5repack to use a chunk size of 1x30x9x717, 4 + * times smaller than the original chunks, and read the file by using the original access pattern of + * 1x1x1x717 hyperslab (by “row”). The result is shown below in Table 4. Once again, we got much better + * performance than shown in Table 1, even when considering the time to repack the file with h5repack. + * + * + * + * + * + *
Table 4: Reading by 1x1x1x717 hyperslab (by “row”) from non-compressed and compressed datasets; + * a smaller chunk size of 1x30x9x717 was used to store data in both files. Performance for the + * compressed dataset is comparable to the result in Table 2 and Table 3.
File NameFile.h5File_with_compression-small-chunk.h5 (gzip level 6)
Read Time0.08 seconds0.36 seconds
Repack Time3 seconds12 seconds
+ * \li Note that the read and repack times in the tables above are approximate values. + * + * \section sec_improve_compressed_perf_chunk Chunking and Compression in HDF5 + * In this section we will give a brief overview of the chunking and compression features needed to follow + * the approach presented later in the “Tuning for Performance” section on page 15. For more information + * on HDF5 chunking, see the \ref hdf5_chunking document. + * + * \subsection subsec_improve_compressed_perf_chunk_chunk Chunking in HDF5 + * Data of HDF5 dataset can be stored in several different ways in HDF5 file. See the + * \ref subsubsec_dataset_program_transfer + * section in the \ref sec_dataset chapter in the \ref UG for more information. + * + * The default storage layout of HDF5 files is contiguous storage: data of a multidimensional array is + * serialized (or flattened) along the fastest changing dimension and is stored as a contiguous block in the + * file. This storage mechanism is recommended if the size of a dataset is known and the storage size for + * the dataset is acceptable to the user: in other words, no data compression is desired. The contiguous + * storage is efficient for I/O if a whole HDF5 dataset is accessed or if a contiguous subset (as stored in the + * file) of an HDF5 dataset is accessed. The figure below shows an example with a row of a 2-dimensional + * array stored in an HDF5 dataset by a C application. In this case, the HDF5 Library seeks to the start + * position in the file and writes/reads the required number of bytes. + * + * + * + *
Elements of the rows of the 6x9 two-dimensional array are stored contiguously in the file while elements of the columns are not
+ * \image html improve_perf-compress_fig_7.png "Figure 7" + *
+ * + * If we change the access pattern to accessing the dataset by columns instead of by rows, the contiguous + * layout may not work well. The column’s elements are not stored contiguously in the file (see Figure 8). + * Accessing a column will require several seeks to find the data in the file and multiple reads/writes of one + * element at a time. Seeks and small size I/O operations may affect performance especially for large datasets. + * Obviously, contiguous storage is not as favorable for a column access pattern as it is for a row + * access pattern, and other storage options may be more beneficial. + * + * + * + *
Elements of the column are not stored contiguously in the file
+ * \image html improve_perf-compress_fig_8.png "Figure 8" + *
+ * + * An alternative is chunked storage (a chunked storage layout). When chunked storage is used, a + * multidimensional array is logically divided into equally sized chunks. For example, Figure 9 below shows + * the 6x9 array divided into 6 3x3 chunks. Chunked storage layout and chunk sizes (number of elements in + * a chunk along each dataset dimension) are specified at dataset creation time and cannot be changed + * without rewriting the dataset. Chunked storage is \b required if data will be added to an HDF5 dataset and + * the maximum size of the dataset is unknown at creation time (see Figure 3). Chunked storage is also + * \b required if data will be stored \b compressed. + * + * The logical chunk is stored as a contiguous block in the file (compare with the contiguous storage when + * the whole data array is stored contiguously in the file). When compression is used, it is applied to each + * chunk separately. During the I/O operation each chunk is accessed as a whole when the HDF5 Library + * reads or writes data elements stored in the chunk. For example, two chunks will be read (and + * uncompressed if needed) when accessing the 2nd column as shown in Figure 9. + * + * The chunk size is an important factor in achieving good I/O and storage performance. + * + * If the chunk size is too small, I/O performance degrades due to small reads/writes when a chunk is + * accessed. Storing a large number of small chunks increases the size of the internal HDF5 data structures + * needed to track the positions and sizes of chunks in the file, creating excessive storage overhead. + * + * On the other hand, if the chunk size is too big and compression is used, I/O performance may degrade + * with unsuitable combinations of access patterns and chunk cache sizes or on systems that do not have + * enough memory to compress or to uncompress chunks. For instance, an application that reads data by + * row from a chunk too large to fit in the configured cache will cause decompression of the entire chunk + * for each row that is read, resulting in a great deal of unnecessarily repeated disk reads and + * decompression processing. + * + * As was mentioned above, the storage layout cannot be changed after the dataset has been created. If + * desired, one can use the \ref sec_cltools_h5repack tool to modify the storage layout of a copy of a dataset; for + * example, the tool can be used to change the size of the chunk, to remove compression and store the + * dataset using contiguous storage, or to apply a different compression method. If data is read from the + * file many times, it may be much more efficient to rewrite the file using \ref sec_cltools_h5repack with the more + * appropriate storage parameters for reading, than to read data from the original file with an unfavorable + * compression and chunking arrangement. + * + * + * + *
Each chunk is stored separately in the HDF5 file. Two chunks will be read by HDF5 to access the 2nd column of the array
+ * \image html improve_perf-compress_fig_9.png "Figure 9" + *
+ * + * Another important aspect of HDF5 chunking is the chunk cache. + * + * HDF5 does not cache raw data unless chunked storage is used. When data is accessed for a chunked + * dataset, the chunks that contain the requested data are brought to the cache one by one and stay in + * cache until they are evicted. If a chunk is cached, then reading or writing data stored in the chunk does + * not require disk accesses. In other words, chunk caching helps when the same chunk is accessed + * multiple times during I/O operations. + * + * The HDF5 Library provides the #H5Pset_cache and #H5Pset_chunk_cache functions to control the + * size of the chunk cache and the chunk eviction policy to specify the appropriate cache parameters for a + * particular access pattern. + * + * As will be shown in the \ref sec_improve_compressed_perf_tune section, chunked storage and chunk cache + * parameters affect I/O performance and should be chosen with care depending on the I/O access + * pattern. + * + * \subsection subsec_improve_compressed_perf_chunk_comp Compression in HDF5 + * As it was mentioned in the previous sections, in HDF5 data can be stored compressed. The HDF5 Library + * comes with the built-in compression methods: + * \snippet{doc} H5Zmodule.h PreDefFilters + * One can also build in a custom filter, \ref subsec_dataset_filters, or use \ref subsubsec_dataset_filters_dyn. + * + * The compression method is chosen at a dataset creation time and cannot be changed later. As with the + * chunked layout, one can use \ref sec_cltools_h5repack to rewrite the dataset in a copy of the dataset using a different + * compression method or to remove compression completely. + * + * HDF5 tools such \ref sec_cltools_h5dump and \ref sec_cltools_h5ls can be used to check the efficiency of the compression. For + * example, both \ref sec_cltools_h5dump and \ref sec_cltools_h5ls show the compression ratio for a dataset. The compression ratio is + * defined as a ratio of the original data size to the size of compressed data. For example, the ratio for the + * dataset /All_Data/CrIS-SDR_All/ES_ImaginaryLW is 1.07 (see Figure 4) meaning that there was + * not much benefit in applying compression to save space in the file. For more information, see the \ref CompTS + * technical note for a discussion of compression efficiency. + * + * The HDF5 Library applies compression encoding or decoding when the chunk is moved between the + * chunk cache and the file. Since compression encoding and decoding takes CPU time, it affects HDF5 + * write and read performance. This is especially true when data is read or written many times from the + * same chunk and the chunk is not cached between the accesses; this means the chunk has to be brought + * from disk every time it is accessed. + * + * In the next section we will see the effect of compression on the I/O performance. + * + * \section sec_improve_compressed_perf_tune Tuning for Performance + * In this section we will discuss several strategies one can apply to get better I/O performance. We will + * explain in detail how a particular strategy works and when it should be applied. While the examples + * below focus on reading only, the same approach will work for writing too. + * + * The strategies for improving performance require modifications to the reading application or to the + * HDF5 file itself. The reader should choose the strategies that are appropriate for a particular use case. + * + * \subsection subsec_improve_compressed_perf_tune_cache Adjust Chunk Cache Size + * The HDF5 Library automatically creates a chunk cache for each opened chunked dataset. The first + * strategy is to check whether the current chunk cache settings work properly with the application access + * pattern and reset the chunk cache parameters as appropriate. + * + * The HDF5 Library provides two functions, #H5Pset_cache and #H5Pset_chunk_cache, to control + * chunk cache settings. #H5Pset_cache controls the chunk cache setting for ALL datasets in the file, and + * #H5Pset_chunk_cache controls the chunk cache settings per dataset. To find out the default or current + * settings, use the #H5Pget_cache or #H5Pget_chunk_cache functions and then reset appropriate + * parameters if necessary. See the \ref subsubsec_improve_compressed_perf_tune_cache_how section for more + * information. + * + * The default size of the cache is 1MB. The size can be modified by setting the \b nbytes parameter in + * #H5Pset_cache and #H5Pset_chunk_cache. Several chunks can be held in the cache if their total size + * in bytes is less or equal to 1MB. + * + * To look up a chunk in cache, the HDF5 Library uses an array of pointers to the chunks (hash table). The + * array has \b nslots elements (or slots in the hash table) with a default value of 511. One can use the + * \b nslots parameter in #H5Pset_cache and #H5Pset_chunk_cache to change the default size of the + * hash table. + * + * Each chunk has an associated hash value that is calculated as follows. All chunks of the dataset have an + * index (\b cindex) in a linear array of chunks. For example, chunks in Figure 9 will have indices from 0 to 5, + * with the upper left chunk having index 0, the middle one in the top row having index 1, and the lower + * right chunk having index 5. The hash value is calculated as the remainder of dividing \b cindex by \b nslots + * (known as a modulo operation cindex mod nslots). The hash table can contain only one chunk with + * the same hash value. This fact is important to remember to avoid situations when the needed chunks + * have the same hash value. For example, let’s assume \b nslots is 3. Then in Figure 9 the chunks with the + * indices 0 and 3 (in other words, the chunks that contain the first three columns) have the same hash + * values and cannot be in the chunk cache simultaneously even though their total sizes are less than 1MB. + * + * Now, we can analyze what happens when data is read by “rows” (contiguous 717 elements) from the + * /All_Data/CrIS-SDR_All/ES_ImaginaryLW dataset and the default chunk cache settings are + * used. The number of slots \b nslots in the hash table is not a concern since the default value is 511 and + * we have only 15 chunks. Now let’s analyze how the chunk cache size affects the performance. + * + * Each row is stored in one of the 15 chunks that comprise the dataset. Each chunk has 4x30x9 or 1,080 + * “rows”. To read the first row of the chunk, the whole chunk is read, uncompressed and the row is copied + * to the application buffer by the HDF5 Library. Since the size of the uncompressed chunk is 2.95 MB, the + * cache cannot hold the chunk. When the second row is read, the process repeats until all rows from the + * same chunk are read. Thus, the chunk will be read and uncompressed 1,080 times. When we increase + * the cache size to 3MB, the chunk stays in the cache and all rows can be copied to the application buffer + * without the HDF5 Library fetching data from disk and uncompressing the chunk every time the chunk is + * accessed. + * + * Since all 15 chunks have to be read, the HDF5 Library will be touching the disk 16,200 times when a 1MB + * size cache is used compared with 15 times when a 3MB cache is used. The first column in Table 5 below + * shows that it took 345 seconds to read a compressed dataset when using the default cache size of 1MB + * while it took only 0.37 seconds to read the dataset when using the chunk cache size of 3MB. We see + * several orders of magnitude performance improvements when we increase chunk cache size to 3MB. + * + * + * + * + * + *
Table 5: Performance improved when the chunk cache size was adjusted to 3MB by several orders of + * magnitude.
File NameFile_with_compression.h5File_with_compression.h5
Cache Size1MB (default)3MB
Read Time345 seconds0.37 seconds
+ * + * As shown in Table 6 below, the reading performance with the 3MB cache size is comparable to the + * reading performance of the data stored without compression applied. Please notice that the chunk + * cache size did not affect the reading performance for the uncompressed data. + * + * + * + * + * + *
Table 6: With the chunk cache size adjusted to 3MB, performance is comparable with the + * performance of reading data that was stored without compression.
File NameFile_with_compression.h5File.h5
Cache Size3MB1MB or 3MB
Read Time0.37 seconds0.1 seconds
+ * \li Note that the read times in the tables above are approximate values. + * + * \subsubsection subsubsec_improve_compressed_perf_tune_cache_how How to Adjust the Chunk Cache Size + * As was mentioned above, an application can adjust the chunk cache size by calling either + * #H5Pset_cache or #H5Pset_chunk_cache functions. #H5Pset_cache sets the chunk cache size for all + * chunked datasets in a file, and #H5Pset_chunk_cache sets the chunk cache size for a particular + * dataset. + * + * The programming model for using both functions is the following: + * \li Use #H5Pget_cache or #H5Pget_chunk_cache to retrieve the default parameters set by the + * library or by a previous call to the function. + * \li Use #H5Pset_cache or #H5Pset_chunk_cache to modify a subset of the parameters. + * + * Below are the code snippets that show the usage. + * + * The first example below shows how to change the cache size for all datasets in the file using + * #H5Pset_cache. Since the function sets a global setting for the file, it uses a file access property list + * identifier to modify the cache size. #H5Pget_cache is called first to retrieve default cache settings that + * will be modified by #H5Pset_cache. In the example below, every chunked dataset will have a cache size + * of 3MB. To overwrite this setting for a particular dataset one can use #H5Pset_chunk_cache as shown + * in the second example. + * + * + *
Code Example 1: Using H5Pset_cache to change the cache size for all datasets.
+ * \code + * hid_t fapl; // File access property identifier + * int nelemts; // Dummy parameter in API, no longer used + * size_t nslots; // Number of slots in the hash table + * size_t nbytes; // Size of chunk cache in bytes + * double w0; // Chunk preemption policy + * …… + * fapl = H5Pcreate (H5P_FILE_ACCESS); + * // Retrieve default cache parameters + * H5Pget_cache(fapl, &nelemts, &nslots, &nbytes, &w0) + * // Set cache size to 3MBs and instruct the cache to discard the fully read chunk + * nbytes = 3 * 1024 * 1024; + * w0 = 1. + * H5Pset_cache(fapl, nelemts, nslots, nbytes, w0); + * fid = H5Fopen (file, H5F_ACC_RDONLY, fapl); + * H5Dopen2 (fid, “/All_Data/CrIS-SDR_All/ES_ImaginaryLW”, H5P_DEAFULT); + * …… + * \endcode + * + * The second example, see below, shows how to set at dataset creation time the chunk cache size for the + * /All_Data/CrIS-SDR_All/ES_ImaginaryLW dataset. The cache sizes for other datasets will not be + * modified. + * + * + *
Code Example 2 : Using H5Pset_chunk_cache to change one dataset.
+ * \code + * hid_t dapl; // File access property identifier + * size_t nslots; // Number of slots in the hash table + * size_t nbytes; // Size of chunk cache in bytes + * double w0; // Chunk preemption policy + * …… + * dapl = H5Pcreate (H5P_DATASET_ACCESS); + * // Retrieve default cache parameters + * H5Pget_chunk_cache(dapl, &nslots, &nbytes, &w0) + * // Set cache size to 3MBs and instruct the cache to discard the fully read chunk + * nbytes = 3 * 1024 * 1024; + * w0 = 1. + * H5Pset_chunk_cache(dapl, nslots, nbytes, w0); + * H5Dopen2 (fid, “/All_Data/CrIS-SDR_All/ES_ImaginaryLW”, dapl); + * …… + * \endcode + * As we will see in the next section, care needs to be taken when working with chunked datasets and + * setting chunk cache sizes: an application’s memory footprint can be significantly affected. + * + * \subsubsection subsubsec_improve_compressed_perf_tune_cache_mem Chunk Cache Size and Application Memory + * A chunk cache is allocated for a dataset when the first I/O operation is performed. The chunk cache is + * discarded after the dataset is closed. If an application performs I/O on several datasets, memory + * consumed by an application increases by the total size of all chunk caches. One can also see an increase + * in the metadata cache size. + * + * If memory consumption is a concern, it is recommended that I/O be done on a few datasets at a time + * and to close the few datasets after I/O operation has been completed. As we will see in the next + * sections, there are access patterns that cannot take advantage of a chunk cache at all. If this is the case, + * the application can disable a chunk cache completely and thus reduce the memory footprint. To disable + * a chunk cache, use 0 for the value of the \b nbytes parameter in the calls to #H5Pset_cache or + * #H5Pset_chunk_cache. + * + * \subsection subsec_improve_compressed_perf_tune_access Change the Access Pattern + * When changing the chunk cache size is not an option (for example, there is no access to the program + * source code), one can consider a reading strategy that will minimize the effect of the chunk cache size. + * The strategy is to read as much data as possible in each read operation. + * + * As we mentioned before, the HDF5 Library performs I/O on the whole chunk. The chunk is read, + * uncompressed, and the requested data is copied to the application buffer. If in one read call the + * application requests all data in a chunk, then obviously chunk caching (and chunk cache size) is + * irrelevant since there is no need to access the same chunk again. + * + * In our case, suppose the application reads the selection that corresponds to the whole chunk. In other + * words, if a hyperslab with dimensions 4x30x9x717 is used instead of a hyperslab with dimensions + * 1x1x1x717, then the HDF5 Library would perform only 15 reading and decoding operations instead of + * 16,200. The significant improvement in performance is shown in Table 7 below. We see a similar I/O + * performance improvement as in the case when we increased the chunk cache size to 3MB (see Table 5). + * + * + * + * + * + * + *
Table 7: Leaving the chunk cache size unchanged and changing the access pattern to read more data + * improves performance by several orders of magnitude.
File NameFile_with_compression.h5File_with_compression.h5
Access Pattern1x1x1x7174x30x9x717
Read Time345 seconds0.36 seconds
+ * \li Note that the read times in the table above are approximate values. + * + * \subsection subsec_improve_compressed_perf_tune_size Change the Chunk Size + * Data producers should consider that users who cannot modify applications to increase the chunk cache + * size or to change the access pattern will not encounter the performance problem described in the + * \ref subsubsec_improve_compressed_perf_tune_cache_how section if chunks in the file are smaller than 1MB (1x30x9x717 by + * 4 bytes) because the whole chunk will fit into the chunk cache of the default size. Therefore if data in + * the HDF5 files is intended for reading by unknown user applications or on systems that might be + * different from the system where it was written, it is a good idea to consider a chunk size less than 1MB. + * In this case the applications that use default HDF5 settings will not be penalized. + * + * As shown in the \ref sec_improve_compressed_perf_case section, Table 4, the performance of reading by row (717 + * elements) when the chunk size is 1x30x9x717 (total size in bytes is approximately 0.74MB) is + * comparable to the performance of reading non-compressed data and is similar to the performance for + * reading compressed data when using a bigger cache size (Table 2) or bigger amount of data (Table 3). + * The above statement is summarized in the \ref sec_improve_compressed_perf_rec section. + * + * For users who encounter datasets with large chunk sizes and with applications that cannot be easily + * modified: since the chunk size is set at the dataset creation time and cannot be changed later, the only + * option is to recreate the dataset by using the \ref sec_cltools_h5repack tool to change the storage layout properties. + * The command below will change the chunk size of the /All_Data/CrIS-SDR_All/ES_ImaginaryLW + * dataset from 4x30x9x717 to 1x30x9x717 making chunk size in bytes 0.74MB instead of the original + * 2.96MBs size. + * \code + * % h5repack -l /All_Data/CrIS-SDR_All/ES_ImaginaryLW:CHUNK=1x30x9x717 + * gz6_SCRIS_npp_d20140522_t0754579_e0802557_b13293__noaa_pop.h5 new.h5 + * \endcode + * + * \section sec_improve_compressed_perf_rec Recommendations + * This section summarizes the discussion and recommendations for working with files that use the HDF5 + * chunking and compression feature. + * + * When compression is enabled for an HDF5 dataset, the library must always read an entire chunk for + * each call to #H5Dread unless the chunk is already in the cache. To avoid trashing the cache, make sure + * that the chunk cache size is big enough to hold the whole chunk or that the application reads the whole + * chunk in one read operation bypassing the chunk cache. + * + * When experiencing I/O performance problems with compressed data, find the size of the chunk and try + * the strategy that is most applicable to your use case: + * \li Increase the size of the chunk cache to hold the whole chunk. + * \li Increase the amount of the selected data to read (making selection to be the whole chunk will + * guarantee bypassing the chunk cache). + * \li Decrease the chunk size by using \ref sec_cltools_h5repack tool to fit into the default size chunk cache. + * + * The results of all three strategies provide similar performance and are summarized in Table 8 below. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table 8: By varying different parameters (highlighted) one can achieve good I/O performance for + * reading compressed data.
File NameFile_with_compression.h5File_with_compression.h5File_with_compression.h5File_with_compression-small-chunk.h5
Cache Size1MB3MB1MB1MB
Chunk Size4x30x9x7174x30x9x7174x30x9x7171x30x9x717
Access Pattern (Hyperslab Size)1x1x1x7171x1x1x7174x30x9x7171x1x1x717
Read Time345 seconds0.37 seconds0.36 seconds0.36 seconds
Repack TimeNANANA12 seconds
+ * \li Note that the read and repack times in the table above are approximate values. + * + * Please notice that when compression is disabled, the library’s behavior depends on the cache size + * relative to the chunk size. If the chunk fits the cache, the library reads entire chunk for each call to + * #H5Dread unless it is in cache already. If the chunk doesn’t fit the cache, the library reads only the data + * that is selected directly from the file. There will be more read operations, especially if the read plane + * does not include the fastest changing dimension. + * + * One can use \ref sec_cltools_h5repack tool to remove compression by using the following command: + * \code + * % h5repack -f /All_Data/CrIS-SDR_All/ES_ImaginaryLW:NONE + * gz6_SCRIS_npp_d20140522_t0754579_e0802557_b13293__noaa_pop.h5 new.h5 + * \endcode + * + * The CCP tool described in the introduction is intended to facilitate optimization of the parameters + * chosen when creating files and investigation of possible solutions when performance problems are + * encountered. + * + */ diff --git a/doxygen/dox/LearnBasics3.dox b/doxygen/dox/LearnBasics3.dox index c93c23971c0..fadd9fbe83a 100644 --- a/doxygen/dox/LearnBasics3.dox +++ b/doxygen/dox/LearnBasics3.dox @@ -107,7 +107,7 @@ which keeps track of the chunks associated with a dataset: \subsubsection susubsecLBDsetLayoutChunkWhy Why Chunking ? -Chunking is required for enabling compression and other filters, as well as for creating extendible +\ref hdf5_chunking is required for enabling compression and other filters, as well as for creating extendible or unlimited dimension datasets. It is also commonly used when subsetting very large datasets. Using the chunking layout can @@ -149,6 +149,8 @@ It is a good idea to: \li Test the data with different chunk sizes to determine the optimal chunk size to use. \li Consider the chunk size in terms of the most common access patterns that will be used once the dataset has been created. +@see hdf5_chunk_issues + \subsection subsecLBDsetLayoutCom Compact A compact dataset is one in which the raw data is stored in the object header of the dataset. This layout is for very small datasets that can easily fit in the object header. @@ -250,8 +252,7 @@ The following operations are required in order to create a compressed dataset: \li Create the dataset. \li Close the dataset creation property list and dataset. -For more information on troubleshooting compression issues, see the - HDF5 Compression Troubleshooting (PDF). +For more information on troubleshooting compression issues, see \ref CompTS. \section secLBComDsetProg Programming Example diff --git a/doxygen/dox/ModifiedRegionWrites.dox b/doxygen/dox/ModifiedRegionWrites.dox new file mode 100644 index 00000000000..fa9c745d054 --- /dev/null +++ b/doxygen/dox/ModifiedRegionWrites.dox @@ -0,0 +1,121 @@ +/** \page mod_region_writes Modified Region Writes + * + * The Core virtual file driver allows the manipulating of HDF5 files in memory instead of in + * physical storage. In previous versions, changing any part of a file in memory meant the + * entire file would be written to storage on file close or flush. To improve the + * performance of the writing to storage operation, a new feature, modified region writes, + * has been added. With modified region writes, only the changed regions of the file are + * written to storage. + * + * Introduced with HDF5-1.8.13 May 15, 2014 + * + * The intended audience for this feature is advanced users of the Core virtual file driver. + * + * \section sec_mod_region_writes_intro Introduction to Modified Region Writes + * In the 1.8.13 release of the HDF5 Library, a feature called modified region writes was added to improve + * the performance of writes to storage. The purpose of this document is to describe the feature and how + * to use it. The intended audience for this feature is advanced users of the Core virtual file driver (VFD). + * The Core (or Memory) VFD allows HDF5 files to be created or opened in memory instead of in physical + * storage. If an existing file is opened in memory, the entire contents of the file are copied into memory + * on open. All subsequent manipulations of created or opened files occur in memory. The advantage of + * working on files in memory is the file operations go much faster, but the disadvantage is significant + * memory resources may be required when working with large files. On file close or flush, the changes can + * optionally be propagated to physical storage. + + * The Core VFD is configured via the following API call: + * \code + * herr_t H5Pset_fapl_core(hid_t fapl_id, size_t increment, hbool_t backing_store) + * \endcode + * + * The backing_store parameter sets whether or not changes are propagated to physical storage on + * close. If this parameter is set to 0 (FALSE), then all changes will be lost when the file is closed. If set to 1 + * (TRUE), then the changes are written to storage on file close or flush. In previous versions of the library + * when a file was closed, the entire file would be written out if even a single byte has changed. This can be + * inefficient when very large files are written out after minimal changes have been made. + * + * If files being worked on in memory will be written to disk, the modified region writes feature can be + * enabled. + * + * \subsection subsec_mod_region_writes_intro_how How the Core VFD Tracks File Modifications + * When modified region writes are enabled, the Core VFD will track any changes made to the file. On file + * close or flush, the tracked changes will be written to storage. + * + * As write calls pass through the Core VFD, a list of “start address-end address” pairs representing the + * writes is updated. This list serves as a map of modified regions in the file. Overlapping or abutting + * regions are merged as they are inserted into the list. + * + * As a further optimization, a write page size can be set. This feature expands any dirty regions (regions + * with changed bytes) to the nearest page boundaries. Using write pages can minimize seeks and small, + * inefficient writes when a large number of small non-adjacent writes occur. See the figure below. + * + * Note that these marked regions are at the granularity of the write calls that the library makes. In other + * words, an entire metadata object or dataset chunk will be marked dirty if even a single byte is changed + * since the library uses a single write call when metadata objects or dataset chunks are evicted from their + * respective caches. The Core VFD will make no effort to determine the particular bytes that were + * modified with respect to the original data. + * + * + * + * + * + *
+ * \image html modregwrite.png + *
+ * + * \subsection subsec_mod_region_writes_intro_new Using the New Feature + * The modified region writes feature is turned off by default. Setting the \b backing_store flag to TRUE + * will not turn modified region writes on. + * + * The modified region writes feature is controlled via the + * #H5Pget_core_write_tracking/#H5Pset_core_write_tracking HDF5 + * API calls. The signatures of these function calls are the following: + * \code + * herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size) + * herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size) + * \endcode + * Setting the page size to a value greater than 1 turns write tracking on at that page size. Setting a page + * size of 1 byte disables paging. + * + * More information for these function calls can be found in the \ref RM. + * + * \subsection subsec_mod_region_writes_intro_perf Performance + * The performance benefits of the feature will depend heavily on the data access patterns of the + * application and will have to be evaluated on a case-by-case basis. In cases where the majority of the + * data would be written out (for example, creating and writing data to a new file), the new feature will + * likely not impart a significant performance benefit. In cases where a small amount of data will be added + * or changed (for example, opening an existing file and modifying a small amount of existing data), the + * performance benefits could be significant. + * + * When performance tuning, the following parameters are likely to have significant effects on I/O + * throughput: + * \li The size of the backing store pages (see #H5Pset_core_write_tracking) + * \li Dataset layout and chunk size (see #H5Pset_layout and H5Pset_chunk) + * \li Metadata aggregation size (see #H5Pset_meta_block_size) + * \li Using the latest file format (see #H5Pset_libver_bounds) + * \li Data layout considerations (arrangement of groups, datasets, and datatypes) + * + * In general, anything that promotes the aggregation of changes made to the file will enhance the + * performance of this feature. Unfortunately, empirical testing will typically be required to determine the + * “sweet spot” between reducing the number of seeks and minimizing the amount of data written out. + * + * More information for these function calls can be found in the \ref RM. + * + * \section sec_mod_region_writes_refs References + * For more information, see the entries for the #H5Pset_fapl_core, #H5Pget_core_write_tracking, + * and #H5Pset_core_write_tracking function calls in the \ref RM. + * + * \subsection subsec_mod_region_writes_refs_virt The Virtual File Layer and Virtual File Drivers + * The HDF5 Library uses a layered architecture. The lowest layer is the virtual file layer (VFL). The VFL + * handles low-level file I/O via virtual file drivers (VFDs). The VFL is an abstraction layer in the HDF5 Library + * that maps I/O operations such as “read” to concrete I/O calls like the POSIX read() call or the Win32 + * ReadFile() call. Each VFD implements a different I/O scheme: some examples are MPI-I/O, POSIX I/O, + * and in-memory I/O. This VFL/VFD scheme allows abstract HDF5 file manipulations to be separated from + * storage I/O operations. + * + * For more information, see \ref VFLTN + * + * For more information on virtual file drivers, see the \ref subsec_file_alternate_drivers section in the + * \ref sec_file chapter in the \ref UG. + * + * + */ \ No newline at end of file diff --git a/doxygen/dox/RelVersion.dox b/doxygen/dox/RelVersion.dox index aaace327b90..0bc7ab17409 100644 --- a/doxygen/dox/RelVersion.dox +++ b/doxygen/dox/RelVersion.dox @@ -260,4 +260,5 @@ Please note that this libtool version number for interface compatibility is unrelated to the HDF5 release version for a given release. +@see BRANCHEXPL */ diff --git a/doxygen/dox/Specifications.dox b/doxygen/dox/Specifications.dox index f59d0721cf7..d305b7f604b 100644 --- a/doxygen/dox/Specifications.dox +++ b/doxygen/dox/Specifications.dox @@ -1,20 +1,20 @@ /** \page SPEC Specifications -\section DDL +\section sec_spec_ddl DDL \li \ref DDLBNF110 \li \ref DDLBNF112 \li \ref DDLBNF114 \li \ref DDLBNF200 -\section File Format +\section sec_spec_ff File Format \li \ref FMT1 \li \ref FMT11 \li \ref FMT2 \li \ref FMT3 -\section Other +\section sec_spec_other Other \li \ref IMG \li \ref TBLSPEC diff --git a/doxygen/dox/TechnicalNotes.dox b/doxygen/dox/TechnicalNotes.dox index 3cb477561af..fa272193f68 100644 --- a/doxygen/dox/TechnicalNotes.dox +++ b/doxygen/dox/TechnicalNotes.dox @@ -3,18 +3,28 @@ \li \ref api-compat-macros \li \ref APPDBG \li \ref FMTDISC +\li \ref BRANCHEXPL +\li \ref CODECONV +\li \ref CompTS \li \ref sec_file_image \li \ref subsubsec_dataset_transfer_filter +\li \ref FileLock +\li \ref InitShut \li \ref IOFLOW +\li \ref improve_compressed_perf +\li \ref collective_metadata_io +\li \ref ParCompr \li \ref TNMDC \li \ref thread-safe-lib \li \ref SWMRTN +\li \ref ThrdSafe \li \ref VDSTN \li \ref RELVERSION \li \ref UNICODE \li \ref VFLTN \li HDF5 Library Architecture Overview \li \ref VOL_Connector +\li \ref CMakeVols */ diff --git a/doxygen/dox/UsersGuide.dox b/doxygen/dox/UsersGuide.dox index 14481a10f69..29eb8489af3 100644 --- a/doxygen/dox/UsersGuide.dox +++ b/doxygen/dox/UsersGuide.dox @@ -412,7 +412,7 @@ These documents provide additional information for the use and tuning of specifi -

Copying Committed Datatypes with H5Ocopy

+

@ref copying_committed

Describes how to copy to another file a dataset that uses a committed datatype or an object with an attribute that uses a committed datatype so that the committed datatype in the destination file can be used by multiple objects.

@@ -420,7 +420,7 @@ These documents provide additional information for the use and tuning of specifi -

Metadata Caching in HDF5

+

@ref TNMDC

Managing the HDF5 metadata cache and tuning it for performance.

@@ -428,7 +428,7 @@ These documents provide additional information for the use and tuning of specifi -

HDF5 Dynamically Loaded Filters

+

\ref_rfc20130316

Describes how an HDF5 application can apply a filter that is not registered with the HDF5 Library.

@@ -444,7 +444,7 @@ These documents provide additional information for the use and tuning of specifi -

Modified Region Writes

+

@ref mod_region_writes

Describes how to set write operations for in-memory files so that only modified regions are written to storage. Available when the Core (Memory) VFD is used.

@@ -452,7 +452,7 @@ These documents provide additional information for the use and tuning of specifi -

Using Identifiers

+

@ref UsingIdentifiers

Describes how identifiers behave and how they should be treated.

@@ -460,7 +460,7 @@ These documents provide additional information for the use and tuning of specifi -

Using UTF-8 Encoding in HDF5 Applications

+

@ref UNICODE

Describes the use of UTF-8 Unicode character encodings in HDF5 applications.

@@ -468,7 +468,7 @@ These documents provide additional information for the use and tuning of specifi -

Freeing Memory Allocated by the HDF5 Library

+

@ref freeing_memory

Describes how inconsistent memory management can cause heap corruption or resource leaks and possible solutions.

diff --git a/doxygen/dox/UsingIdentifiers.dox b/doxygen/dox/UsingIdentifiers.dox new file mode 100644 index 00000000000..7fe1923990c --- /dev/null +++ b/doxygen/dox/UsingIdentifiers.dox @@ -0,0 +1,97 @@ +/** \page UsingIdentifiers Using Identifiers + * + * \section sec_using_identifiers Using Identifiers + * + * The purpose of this topic is to describe how identifiers behave and how they should be treated by application programs. + * + * When an application program uses the HDF5 library to create or open an item, a unique identifier is + * returned. The items that return a unique identifier when they are created or opened include the following: + * \li dataset + * \li group + * \li datatype + * \li dataspace + * \li file + * \li attribute + * \li property list + * \li referenced object + * \li error stack + * \li error message + * + * An application may open one of the items listed above more than once at the same time. For example, an + * application might open a group twice, receiving two identifiers. Information from one dataset in the + * group could be handled through one identifier, and the information from another dataset in the group + * could be handled by a different identifier. + * + * An application program should track every identifier it receives as a result of creating or opening one of + * the items listed above. In order for an application to close properly, it must release every identifier it + * has opened. If an application opened a group twice for example, it would need to issue two #H5Gclose + * commands, one for each identifier. Not releasing identifiers causes resource leaks. Until an identifier + * is released, the item associated with the identifier is still open. + * + * The library considers a file open until all of the identifiers associated with the file and with the file’s + * various items have been released. The identifiers associated with these open items must be released separately. + * This means that an application can close a file and still work with one or more portions of the file. Suppose + * an application opened a file, a group within the file, and two datasets within the group. If the application + * closed the file with #H5Fclose, then the file would be considered closed to the application, but the group + * and two datasets would still be open. + * + * There are several exceptions to the above file closing rule. One is when the #H5close function is used + * instead of #H5Fclose. #H5close causes a general shut down of the library: all data is written to disk, + * all identifiers are closed, and all memory used by the library is cleaned up. Another exception occurs on + * parallel processing systems. Suppose on a parallel system an application has opened a file, a group in the + * file, and two datasets in the group. If the application uses the #H5Fclose function to close the file, the + * call will fail with an error. The open group and datasets must be closed before the file can be closed. + * A third exception is when the file access property list includes the property #H5F_CLOSE_STRONG. This + * property causes the closing of all of the file’s open items when the file is closed with #H5Fclose. For + * more information about #H5close, #H5Fclose, and #H5Pset_fclose_degree, see the \ref RM. + * + * The reference manual entries for functions that return identifiers describe what might be returned as + * follows: + * \b Returns: + * Returns an identifier if successful; otherwise returns a negative value. + * + * In other words, a successful operation will return a non-negative identifier which will never be 0 + * (zero) and will always be a positive value. + * + * \subsection subsec_using_identifiers_func Functions that Return Identifiers + * + * Some of the functions that return identifiers are listed below. + * + * \li #H5Acreate + * \li #H5Acreate_by_name + * \li #H5Aget_type + * \li #H5Aopen + * \li #H5Aopen_by_idx + * \li #H5Aopen_by_name + * \li #H5Dcreate + * \li #H5Dcreate_anon + * \li #H5Dget_access_plist + * \li #H5Dget_create_plist + * \li #H5Dget_space + * \li #H5Dget_type + * \li #H5Dopen + * \li #H5Ecreate_msg + * \li #H5Ecreate_stack + * \li #H5Fcreate + * \li #H5Fopen + * \li #H5Freopen + * \li #H5Gcreate + * \li #H5Gcreate_anon + * \li #H5Gopen + * \li #H5Oopen + * \li #H5Oopen_by_addr + * \li #H5Oopen_by_idx + * \li #H5Pcreate + * \li #H5Pget_virtual_srcspace + * \li #H5Pget_virtual_vspace + * \li #H5Rdereference + * \li #H5Rget_region + * \li #H5Screate + * \li #H5Screate_simple + * \li #H5Tcopy + * \li #H5Tcreate + * \li #H5Tdecode + * \li #H5Tget_member_type + * \li #H5Tget_super + * \li #H5Topen +*/ diff --git a/doxygen/dox/branches-explained.dox b/doxygen/dox/branches-explained.dox new file mode 100644 index 00000000000..46f9c00b70b --- /dev/null +++ b/doxygen/dox/branches-explained.dox @@ -0,0 +1,63 @@ +/** \page BRANCHEXPL HDF5 Git Branching Model Explained + +This document describes current HDF5 branches. + +Branches are tested nightly and testing results are available at +https://my.cdash.org/index.php?project=HDF5. +Commits that break daily testing should be fixed by 3:00 pm Central time or reverted. +We encourage code contributors to check the status of their commits. If you have any questions, +please contact help@hdfgroup.org. + +\section sec_branchexpl_develop develop branch +Develop is the main branch whose source code always reflects a state with the latest delivered +development changes for the next major release of HDF5. +This is also considered the integration branch, as \b all new features are integrated into this +branch from respective feature branches. Although +develop is considered an integration branch, it is not an unstable branch. All code merged to +develop is expected to pass all GitHub actions and daily tests. + +\section sec_branchexpl_maintenace Maintenance branches +Each currently supported release line of HDF5 (e.g. 1.8.x, 1.10.x, 1.12.x, 1.14.x) has an associated +branch with the name hdf5_1_10, etc.. +Maintenance branches are similar to the develop branch, except the source code in a maintenance +branch always reflects a state +with the latest delivered development changes for the next \b maintenance release of that particular +supported release-line of HDF5. +\b Some new features will be integrated into a release maintenance branch, depending on whether or +not those features can be +introduced in minor releases. Maintenance branches are removed when a release-line is retired from +support. + +\section sec_branchexpl_release Release branches +Release branches are used to prepare a new production release. They are primarily used to allow for +last minute dotting of i's and crossing of t's +(things like setting the release version, finalizing release notes, and generating Autotools files) +and do not include new development. +They are created from the maintenance branch at the time of the maintenance release and have +names like hdf5_1_10_N, where N is the minor release number. Once the release is done it is +tagged, with a slightly different format: hdf5-1_10_N. +Release branches are deleted after the tag has been created. If we have to create a patch version +of a release (which is rare), we create a branch off of the tag. + +\section sec_branchexpl_feature feature/\* +Feature branches are temporary branches used to develop new features in HDF5. +Feature branches branch off of develop and exist as long as the feature is under development. +When the feature is complete, the branch is merged back into develop, as well as into any support +branches in which the change will be included, and then the feature branch is removed. + +Ideally, all feature branches should contain a BRANCH.md file in the root directory that explains +the purpose of the branch, contact information for the person responsible, and, if possible, some +clues about the branch's life cycle (so we have an idea about when it can be deleted, merged, or +declared inactive). + +Minor bug fixes and refactoring work usually takes place on personal forks, not feature branches. + +\section sec_branchexpl_inactive inactive/\* +These branches are for experimental features that were developed in the past, have not been merged +to develop, and are not under active development. The exception to this is that some feature branches +are labeled inactive and preserved for a short time after merging to develop. Integration branches +are usually not kept in sync with the develop branch. + +As for feature branches, inactive branches should have a BRANCH.md file as described above. + +*/ diff --git a/doc/cmake-vols-fetchcontent.md b/doxygen/dox/cmake-vols-fetchcontent.dox similarity index 59% rename from doc/cmake-vols-fetchcontent.md rename to doxygen/dox/cmake-vols-fetchcontent.dox index f7b395dec7b..2d1d9420469 100644 --- a/doc/cmake-vols-fetchcontent.md +++ b/doxygen/dox/cmake-vols-fetchcontent.dox @@ -1,70 +1,64 @@ -# Building and testing HDF5 VOL connectors with CMake FetchContent +/** \page CMakeVols HDF5 Building and testing HDF5 VOL connectors with CMake FetchContent +\section sec_cmakevols_intro Introduction This document details the process of using CMake options to build and test an HDF5 VOL connector alongside the HDF5 library when building HDF5 from source. There are several benefits that this may provide, but among them are the following: - - * A VOL connector built this way can be tested at the same time that +\li A VOL connector built this way can be tested at the same time that HDF5 is, which eliminates the need to have a multi-step build process where one builds HDF5, uses it to build the VOL connector and then uses the external [HDF5 VOL tests](https://github.com/hdfGroup/vol-tests) repository to test their connector. - * Building VOL connectors in this manner will usually install the built +\li Building VOL connectors in this manner will usually install the built connector library alongside the HDF5 library, allowing future opportunities - for HDF5 to set a default plugin path such that the HDF5_PLUGIN_PATH + for HDF5 to set a default plugin path such that the #HDF5_PLUGIN_PATH environment variable doesn't need to be set. -## Building - +\section sec_cmakevols_build Building To enable building of an HDF5 VOL connector using HDF5's CMake functionality, a CMake variable must first be set: - - HDF5_VOL_ALLOW_EXTERNAL (Default: "NO") +\li HDF5_VOL_ALLOW_EXTERNAL (Default: "NO")
This variable is a string that specifies the manner in which the source code for an external VOL connector will be retrieved. This variable must be set - to "GIT" for building external VOL connectors from a Github repository, or - set to "LOCAL_DIR" to build from a local source directory. + to GIT for building external VOL connectors from a Github repository, or + set to LOCAL_DIR to build from a local source directory. - -### Building - -If the `HDF5_VOL_ALLOW_EXTERNAL` option is set to "GIT", the CMake cache will be populated with a predefined -(currently 10) amount of new variables, named: - - HDF5_VOL_URL01 - HDF5_VOL_URL02 - HDF5_VOL_URL03 - ... +\subsection subsec_cmakevols_build_git Building From GIT +If the HDF5_VOL_ALLOW_EXTERNAL option is set to GIT, the CMake cache +will be populated with a predefined (currently 10) amount of new variables, named: +\li HDF5_VOL_URL01 +\li HDF5_VOL_URL02 +\li HDF5_VOL_URL03 +\li ... For each of these variables, a URL that points to an HDF5 VOL connector Git repository can be specified. These URLs should currently be HTTPS URLs. For example, to specify the HDF5 Asynchronous I/O VOL Connector developed by the -ECP team, one can provide the following option to `cmake`: - - -DHDF5_VOL_URL01=https://github.com/hpc-io/vol-async.git +ECP team, one can provide the following option to CMake: +\li -DHDF5_VOL_URL01=https://github.com/hpc-io/vol-async.git For each URL specified, HDF5's CMake code will attempt to use CMake's [FetchContent](https://cmake.org/cmake/help/latest/module/FetchContent.html) functionality to retrieve the source code for a VOL connector pointed to by that URL and will try to build that VOL connector as part of the HDF5 library -build process. +build process. -If `HDF5_VOL_ALLOW_EXTERNAL` is instead set to "LOCAL_DIR", then the CMake cache -will instead be populated with the variables: +\subsection subsec_cmakevols_build_local Building From Local Folder +If HDF5_VOL_ALLOW_EXTERNAL is instead set to LOCAL_DIR, +then the CMake cache will instead be populated with the variables: - HDF5_VOL_PATH01 - HDF5_VOL_PATH02 - HDF5_VOL_PATH03 - ... +\li HDF5_VOL_PATH01 +\li HDF5_VOL_PATH02 +\li HDF5_VOL_PATH03 +\li ... -For each of these variables, an absolute path that points to a local +For each of these variables, an absolute path that points to a local directory containing source code for an HDF5 VOL connector -can be specified. For example, to specify a local clone of the -REST VOL connector stored under one's home directory, one can provide -the following option to `cmake`: - - -DHDF5_VOL_PATH01=/home/vol-rest +can be specified. For example, to specify a local clone of the +REST VOL connector stored under one's home directory, one can provide +the following option to CMake: +\li -DHDF5_VOL_PATH01=/home/vol-rest Regardless of the method used to obtain the VOL source code, the VOL connector must be able to be built by CMake and currently @@ -81,23 +75,21 @@ If the source was retrieved from a URL, then the name is generated by stripping off the last part of the Git repository URL given for the connector, removing the ".git" suffix and any whitespace and then upper-casing the result. For example, the name of the VOL connector located at the URL -https://github.com/hpc-io/vol-async.git would become "VOL-ASYNC". If the source was -retrieved from a local directory, then the source directory's name is trimmed of whitespace, -upper-cased, and has any trailing slashes removed. +https://github.com/hpc-io/vol-async.git would become VOL-ASYNC. +If the source was retrieved from a local directory, then the source directory's name is +trimmed of whitespace, upper-cased, and has any trailing slashes removed. After the VOL's internal name is generated, the following new variables get created: - - HDF5_VOL__NAME (Default: "") +\li HDF5_VOL__NAME (Default: "") This variable specifies the string that should be used when setting the - HDF5_VOL_CONNECTOR environment variable for testing the VOL connector - with the CMake-internal name ''. The value for this variable + #HDF5_VOL_CONNECTOR environment variable for testing the VOL connector + with the CMake-internal name \. The value for this variable can be determined according to the canonical name given to the connector by the connector's author(s), as well as any extra info that needs to be passed to the connector for its configuration (see example below). This variable must be set in order for the VOL connector to be testable with HDF5's tests. - - HDF5_VOL__CMAKE_PACKAGE_NAME (Default: ">") +\li HDF5_VOL__CMAKE_PACKAGE_NAME (Default: "\\>") This variable specifies the exact name that would be passed to CMake find_package(...) calls for the VOL connector in question. It is used as the dependency name when making CMake FetchContent calls to try to ensure @@ -105,43 +97,40 @@ After the VOL's internal name is generated, the following new variables get crea can make find_package(...) calls for this VOL connector at configure time. By default, this variable is set to a lowercased version of the internal name generated for the VOL connector (described above). - - HDF5_VOL__TEST_PARALLEL (Default: OFF) +\li HDF5_VOL__TEST_PARALLEL (Default: OFF) This variable determines whether the VOL connector with the CMake-internal - name '' should be tested against HDF5's parallel tests. + name \ should be tested against HDF5's parallel tests. If the source was retrieved from a Git URL, then the following variable will additionally be created: - - HDF5_VOL__BRANCH (Default: "main") +\li HDF5_VOL__BRANCH (Default: "main") This variable specifies the git branch name or tag to use when fetching the source code for the VOL connector with the CMake-internal name - ''. + \. As an example, this would create the following variables for the previously-mentioned VOL connector if it is retrieved from a URL: +\li HDF5_VOL_VOL-ASYNC_NAME "" +\li HDF5_VOL_VOL-ASYNC_CMAKE_PACKAGE_NAME "vol-async" +\li HDF5_VOL_VOL-ASYNC_BRANCH "main" +\li HDF5_VOL_VOL-ASYNC_TEST_PARALLEL OFF - HDF5_VOL_VOL-ASYNC_NAME "" - HDF5_VOL_VOL-ASYNC_CMAKE_PACKAGE_NAME "vol-async" - HDF5_VOL_VOL-ASYNC_BRANCH "main" - HDF5_VOL_VOL-ASYNC_TEST_PARALLEL OFF - -**NOTE** +NOTE
If a VOL connector requires extra information to be passed in its -HDF5_VOL__NAME variable and that information contains any semicolons, +HDF5_VOL__NAME variable and that information contains any semicolons, those semicolons should be escaped with a single backslash so that CMake -doesn't parse the string as a list. If `cmake` is run from a shell, extra care +doesn't parse the string as a list. If CMake is run from a shell, extra care may need to be taken when escaping the semicolons depending on how the shell interprets backslashes. -### Example - Build and test HDF5 Asynchronous I/O VOL connector from GIT - +\subsection subsec_cmakevols_build_ex Example - Build and test HDF5 Asynchronous I/O VOL connector from GIT Assuming that the HDF5 source code has been checked out and a build directory -has been created, running the following cmake command from that build directory +has been created, running the following CMake command from that build directory will retrieve, build and test the HDF5 Asynchronous I/O VOL connector while -building HDF5. Note that `[hdf5 options]` represents other build options that -would typically be passed when building HDF5, such as `CMAKE_INSTALL_PREFIX`, -`HDF5_BUILD_CPP_LIB`, etc. +building HDF5. Note that [hdf5 options] represents other build options that +would typically be passed when building HDF5, such as CMAKE_INSTALL_PREFIX, +HDF5_BUILD_CPP_LIB, etc. +\code cmake [hdf5 options] -DHDF5_ENABLE_THREADSAFE=ON -DHDF5_ENABLE_PARALLEL=ON @@ -153,78 +142,77 @@ would typically be passed when building HDF5, such as `CMAKE_INSTALL_PREFIX`, -DHDF5_VOL_VOL-ASYNC_NAME="async under_vol=0\;under_info={}" -DHDF5_VOL_VOL-ASYNC_TEST_PARALLEL=ON .. +\endcode Here, we are specifying that: - - * HDF5 should be built with thread-safety enabled (required by Async VOL connector) - * HDF5 should be built with parallel enabled (required by Async VOL connector) - * Allow unsupported HDF5 combinations (thread-safety and HL, which is on by default) - * Enable the API tests so that they can be tested with the Async VOL connector - * Build and use the HDF5 Asynchronous I/O VOL connector, located at +\li HDF5 should be built with thread-safety enabled (required by Async VOL connector) +\li HDF5 should be built with parallel enabled (required by Async VOL connector) +\li Allow unsupported HDF5 combinations (thread-safety and HL, which is on by default) +\li Enable the API tests so that they can be tested with the Async VOL connector +\li Build and use the HDF5 Asynchronous I/O VOL connector, located at https://github.com/hpc-io/vol-async.git - * Clone the Asynchronous I/O VOL connector from the repository's 'develop' branch - * When testing the Asynchronous I/O VOL connector, the `HDF5_VOL_CONNECTOR` environment - variable should be set to "async under_vol=0\;under_info={}", which - specifies that the VOL connector with the canonical name "async" should - be loaded and it should be passed the string "under_vol=0;under_info={}" +\li Clone the Asynchronous I/O VOL connector from the repository's develop branch +\li When testing the Asynchronous I/O VOL connector, the #HDF5_VOL_CONNECTOR environment + variable should be set to "async under_vol=0\;under_info={}", which + specifies that the VOL connector with the canonical name async should + be loaded and it should be passed the string "under_vol=0;under_info={}" for its configuration (note the backslash-escaping of semicolons in the string provided) - * The Asynchronous I/O VOL connector should be tested against HDF5's parallel API tests +\li The Asynchronous I/O VOL connector should be tested against HDF5's parallel API tests -Note that this also assumes that the Asynchronous I/O VOL connector's +Note that this also assumes that the Asynchronous I\/O VOL connector's [other dependencies](https://hdf5-vol-async.readthedocs.io/en/latest/gettingstarted.html#preparation) are installed on the system in a way that CMake can find them. If that is not the case, the locations for these dependencies may need to be provided to CMake by passing extra options, such as: - +\code -DABT_INCLUDE_DIR=/path/to/argobots/build/include -DABT_LIBRARY=/path/to/argbots/build/lib/libabt.so - +\endcode which would help CMake find an argobots installation in a non-standard location. -## Testing - +\section sec_cmakevols_test Testing To facilitate testing of HDF5 VOL connectors when building HDF5, tests from the [HDF5 VOL tests](https://github.com/hdfGroup/vol-tests) repository were integrated back into the library and the following new CMake options were added to HDF5 builds for the 1.14.1 release: - - HDF5_TEST_API (Default: OFF) +\li HDF5_TEST_API (Default: OFF) This variable determines whether the HDF5 API tests will be built and tested. - - HDF5_TEST_API_INSTALL (Default: OFF) +\li HDF5_TEST_API_INSTALL (Default: OFF) This variable determines whether the HDF5 API test executables will be installed on the system alongside the HDF5 library. - - HDF5_TEST_API_ENABLE_ASYNC (Default: OFF) +\li HDF5_TEST_API_ENABLE_ASYNC (Default: OFF) This variable determines whether the HDF5 Asynchronous I/O API tests will be built and tested. These tests will only run if a VOL connector reports that - it supports asynchronous I/O operations when queried via the H5Pget_vol_cap_flags + it supports asynchronous I/O operations when queried via the #H5Pget_vol_cap_flags API routine. - - HDF5_TEST_API_ENABLE_DRIVER (Default: OFF) +\li HDF5_TEST_API_ENABLE_DRIVER (Default: OFF) This variable determines whether the HDF5 API test driver program will be built and used for testing. This driver program is useful when a VOL connector - uses a client/server model where the server program needs to be up and running + uses a client\/server model where the server program needs to be up and running before the VOL connector can function. This option is currently not functional. -When the `HDF5_TEST_API` option is set to ON, HDF5's CMake code builds and tests +When the HDF5_TEST_API option is set to ON, HDF5's CMake code builds and tests the new API tests using the native VOL connector. When one or more external VOL connectors are built successfully with the process described in this document, the CMake code will duplicate some of these API tests by adding separate versions of the tests (for each VOL connector that was built) that set the -`HDF5_VOL_CONNECTOR` environment variable to the value specified for the -HDF5_VOL__NAME variable for each external VOL connector at build time. -Running the `ctest` command will then run these new tests which load and run with -each VOL connector that was built in turn. When run via the `ctest` command, the +#HDF5_VOL_CONNECTOR environment variable to the value specified for the +HDF5_VOL__NAME variable for each external VOL connector at build time. +Running the ctest command will then run these new tests which load and run with +each VOL connector that was built in turn. When run via the ctest command, the new tests typically follow the naming scheme: - +\code HDF5_VOL_-h5_api_test_ HDF5_VOL_-h5_api_test_parallel_ +\endcode -**NOTE** +\section sec_cmakevols_note NOTE If dependencies of a built VOL connector are installed on the system in -a non-standard location that would typically require one to set `LD_LIBRARY_PATH` +a non-standard location that would typically require one to set LD_LIBRARY_PATH or similar, one should ensure that those environment variables are set before running tests. Otherwise, the tests that run with that connector will likely fail due to being unable to load the necessary libraries for its dependencies. + +*/ + diff --git a/doxygen/dox/code-conventions.dox b/doxygen/dox/code-conventions.dox new file mode 100644 index 00000000000..54aa5155160 --- /dev/null +++ b/doxygen/dox/code-conventions.dox @@ -0,0 +1,58 @@ +/** \page CODECONV HDF5 Library Code Conventions + +This document describes some practices that are new, or newly +documented, starting in 2020. + +\section sec_codeconv_func Function / Variable Attributes + +In H5private.h, the library provides platform-independent macros +for qualifying function and variable definitions. + +\subsection subsec_codeconv_func_1 Functions that accept printf(3) and scanf(3) format strings + +Label functions that accept a printf(3)-compliant format string with +H5_ATTR_FORMAT(printf,format_argno,variadic_argno), where +the format string is the format_argnoth argument (counting from 1) +and the variadic arguments start with the variadic_argnoth. + +Functions that accept a scanf(3)-compliant format string should +be labeled H5_ATTR_FORMAT(scanf,format_argno,variadic_argno). + +\subsection subsec_codeconv_func_2 Functions that do never return + +The definition of a function that always causes the program to abort and hang +should be labeled H5_ATTR_NORETURN to help the compiler see which flows of +control are infeasible. + +\subsection subsec_codeconv_func_other Other attributes + +**TBD** + +\subsection subsec_codeconv_func_unused Unused variables and parameters + +Compilers will warn about unused parameters and variables—developers should pay +attention to those warnings and make an effort to prevent them. + +Some function parameters and variables are unused in \b all configurations of +the project. Ordinarily, such parameters and variables should be deleted. +However, sometimes it is possible to foresee a parameter being used, or +removing it would change an API, or a parameter has to be defined to conform a +function to some function pointer type. In those cases, it's permissible to +mark a symbol H5_ATTR_UNUSED. + +Other parameters and variables are unused in \b some configurations of the +project, but not all. A symbol may fall into disuse in some configuration in +the future—then the compiler should warn, and the symbol should not be +defined—so developers should try to label a sometimes-unused symbol with an +attribute that's specific to the configurations where the symbol is (or is not) +expected to be used. The library provides the following attributes for that +purpose: +\li H5_ATTR_DEPRECATED_USED: used only if deprecated symbols \b are enabled +\li H5_ATTR_NDEBUG_UNUSED: used only if NDEBUG is \b not \#defined +\li H5_ATTR_DEBUG_API_USED: used if the debug API \b is enabled +\li H5_ATTR_PARALLEL_UNUSED: used only if Parallel HDF5 is \b not configured +\li H5_ATTR_PARALLEL_USED: used only if Parallel HDF5 \b is configured + +Some attributes may be phased in or phased out in the future. + +*/ diff --git a/doc/file-locking.md b/doxygen/dox/file-locking.dox similarity index 57% rename from doc/file-locking.md rename to doxygen/dox/file-locking.dox index 067f7ab3993..aacf2135b04 100644 --- a/doc/file-locking.md +++ b/doxygen/dox/file-locking.dox @@ -1,5 +1,6 @@ -# File Locking in HDF5 +/** \page FileLock HDF5 File Locking in HDF5 +\section sec_filelock_intro Introduction This document describes the file locking scheme that was added to HDF5 in version 1.10.0 and how you can work around it, if you choose to do so. I'll try to keep it understandable for everyone, though diving into technical @@ -7,8 +8,7 @@ details is unavoidable, given the complexity of the material. We're in the process of converting the HDF5 user guide (UG) to Doxygen and this document will eventually be rolled up into those files as we update things. -**Parallel HDF5 Note** - +

Parallel HDF5 Note

Everything written here is from the perspective of serial HDF5. When we say that you can't access a file for write access from more than one process, we mean "from more than one independent, serial process". Parallel HDF5 can @@ -16,18 +16,15 @@ obviously write to a file from more than one process, but that involves IPC and multiple processes working together, not independent processes with no knowledge of each other, which is what the file locks are for. - -## Why file locks? - +\section sec_filelock_why Why file locks? The short answer is: "To prevent you from corrupting your HDF5 files and/or crashing your reader processes." The long answer is more complicated. An HDF5 file's state exists in two places when it is open for writing: - -1. The HDF5 file itself -2. The HDF5 library's various caches +\li The HDF5 file itself +\li The HDF5 library's various caches One of those caches is the metadata cache, which stores things like B-tree nodes that we use to locate data in the file. Problems arise when parent @@ -55,7 +52,7 @@ it wrong could result in corrupt files or crashed readers, we decided to add a file locking scheme to help users get it right. Since this would also help prevent harmful accesses when SWMR is not in use, we decided to switch the file locking scheme on by default. This scheme has been carried forward into -HDF5 1.12 and 1.13 (soon to be 1.14). +HDF5 1.12 and 1.14 (soon to be 2.0). Note that the current implementation of SWMR is only useful for appending to chunked datasets. Creating file objects like groups and datasets is not supported @@ -67,8 +64,7 @@ on parallel file systems, especially when file locks have been disabled, which often causes lock calls to fail. As a result of this, we've added work-arounds to disable the file locking scheme over the years. -## The existing scheme - +\section sec_filelock_scheme The existing scheme There are two parts to the file locking scheme. One is the file lock itself. The second is a mark we make in the HDF5 file's superblock. The superblock mark isn't really that important for understanding the file locking, but since @@ -82,11 +78,10 @@ SWMR and prevent dangerous file access. Here's how it all works: 1. The first thing we do is check if we're using file locks - - - We first check the file locking property in the file access property list + \li We first check the file locking property in the file access property list (fapl). The default value of this property is set at configure time when the library is built. - - Next we check the value of the `HDF5_USE_FILE_LOCKING` environment variable, + \li Next we check the value of the `HDF5_USE_FILE_LOCKING` environment variable, which was previously parsed at library startup. If this is set, we use the value to override the property list setting. @@ -97,16 +92,14 @@ Here's how it all works: take place. 2. We also check for ignoring file locks when they are disabled on the file system. - - - The environment variable setting for this is checked at VFD initialization + \li The environment variable setting for this is checked at VFD initialization time for all library VFDs. - - We check the value in the fapl in the `open` callback. The default value for + \li We check the value in the fapl in the `open` callback. The default value for this property was set at configure time when the library was built. 3. When we open a file, we lock it based on the file access flags: - - - If the `H5F_ACC_RDWR` flag is set, use an exclusive lock - - Otherwise use a shared lock + \li If the `H5F_ACC_RDWR` flag is set, use an exclusive lock + \li Otherwise use a shared lock If we are ignoring disabled file locks (see below), we will silently swallow lock API call failure when locks are not implemented on the file system. @@ -115,26 +108,23 @@ Here's how it all works: file consistency flags in the file's superblock to indicate this. **NOTE!** - - - The VFD has to have a lock callback for this to happen. It doesn't matter if + \li The VFD has to have a lock callback for this to happen. It doesn't matter if the locking was disabled - the check is simply for the callback. - - We mark the superblock in **ANY** write case - both SWMR and non-SWMR. - - Only the latest version of the superblock is marked in this way. If you + \li We mark the superblock in **ANY** write case - both SWMR and non-SWMR. + \li Only the latest version of the superblock is marked in this way. If you open up a file that wasn't created with the 1.10.0 or later file format, it won't get the superblock mark, even if it's been opened for writing. According to the file format document and H5Fpkg.h: - - - Bit 0 is set if the file is open for writing (`H5F_SUPER_WRITE_ACCESS`) - - Bit 2 is set if the file is open for SWMR writing (`H5F_SUPER_SWMR_WRITE_ACCESS`) + \li Bit 0 is set if the file is open for writing (`H5F_SUPER_WRITE_ACCESS`) + \li Bit 2 is set if the file is open for SWMR writing (`H5F_SUPER_SWMR_WRITE_ACCESS`) We check these superblock flags on file open and error out if they are unsuitable. - - - If the file is already opened for non-SWMR writing, no other process can open + \li If the file is already opened for non-SWMR writing, no other process can open it. - - If the file is open for SWMR writing, only SWMR readers can open the file. - - If you try to open a file for reading with `H5F_ACC_SWMR_READ` set and the + \li If the file is open for SWMR writing, only SWMR readers can open the file. + \li If you try to open a file for reading with `H5F_ACC_SWMR_READ` set and the file does not have the SWMR writer bits set in the superblock, the open call will fail. @@ -148,196 +138,178 @@ Here's how it all works: handle it when the file descriptors are closed since file locks don't normally surivive closing the underlying file descriptor. -**TL;DR** - When locks are available, HDF5 files will be exclusively locked while they are in use. The exception to this are files that are opened for SWMR writing, which are unlocked. Files that are open for any kind of writing get a "writing" superblock mark that HDF5 1.10.0+ will respect and refuse to open outside of SWMR. -## `H5Fstart_swmr_write()` - -This API call can be used to switch an open file to "SWMR writing" mode as -if it had been opened with the `H5F_ACC_SWMR_WRITE` flag set. This is used +\section sec_filelock_smrfunc H5Fstart_swmr_write +This #H5Fstart_swmr_write API call can be used to switch an open file to "SWMR writing" mode as +if it had been opened with the #H5F_ACC_SWMR_WRITE flag set. This is used when code needs to perform SWMR-forbidden operations like creating groups and datasets before appending data to datasets using SWMR. Most of the work of this API call involves flushing out the library caches in preparation for SWMR access, but there are a few locking operations that take place under the hood: +\li The file's superblock is marked as in the SWMR writer case, above. +\li For a brief period of time in the call, we convert the exclusive lock to + a shared lock. It's unclear why this was done and we'll look into removing + this. +\li At the end of the call, the lock is removed, as in the SWMR write open + case described above. -- The file's superblock is marked as in the SWMR writer case, above. -- For a brief period of time in the call, we convert the exclusive lock to - a shared lock. It's unclear why this was done and we'll look into removing - this. -- At the end of the call, the lock is removed, as in the SWMR write open - case described above. - -## Disabling the locks - +\section sec_filelock_disable Disabling the locks There are several ways to disable the locks, depending on which version of the HDF5 library you are working with. This section will describe the file lock disable schemes as they exist in late 2022. The current library versions at -this time are 1.10.9, 1.12.3, and 1.13.2. File locks are not present in HDF5 +this time were 1.10.9, 1.12.3, and 1.13.2. File locks are not present in HDF5 1.8. The lock feature matrix later in this document will describe the limitations of earlier versions. -### Configure option - +\subsection subsec_filelock_disable_config Configure option You can set the file locking defaults at configure time. This sets the defaults for the associated properties in the fapl. Users can override the configure -defaults using `H5Pset_file_locking()` or the `HDF5_USE_FILE_LOCKING` +defaults using #H5Pset_file_locking or the HDF5_USE_FILE_LOCKING environment variable. -- Autotools +

Autotools

+\li --enable-file-locking=(yesnobest-effort) sets the file locking behavior. + on and off should be self-explanatory. best-effort turns file locking + on but ignores file locks when they are disabled (default: best-effort). - `--enable-file-locking=(yes|no|best-effort)` sets the file locking behavior. - `on` and `off` should be self-explanatory. `best-effort` turns file locking - on but ignores file locks when they are disabled (default: `best-effort`). +

CMake

+\li set IGNORE_DISABLED_FILE_LOCK to ON to ignore file locks when they +\li are disabled on the file system (default: ON). +\li set HDF5_USE_FILE_LOCKING to OFF to disable file locks (default: ON) -- CMake - - - set `IGNORE_DISABLED_FILE_LOCK` to `ON` to ignore file locks when they - are disabled on the file system (default: `ON`). - - set `HDF5_USE_FILE_LOCKING` to `OFF` to disable file locks (default: `ON`) - -### `H5Pset_file_locking()` - -This API call can be used to override the configure defaults. It takes -`hbool_t` parameters for both the file locking and "ignore file locks when +\section sec_filelock_funcset H5Pset_file_locking +This #H5Pset_file_locking API call can be used to override the configure defaults. It takes +#hbool_t parameters for both the file locking and "ignore file locks when disabled on the file system" parameters. The values set here can be overridden by the file locking environment variable. -There is a corresponding `H5Pget_file_locking()` call that can be used to check -the currently set values of both properties in the fapl. **NOTE** that this -call just checks the property list values. It does **NOT** check the +There is a corresponding #H5Pget_file_locking call that can be used to check +the currently set values of both properties in the fapl. NOTE that this +call just checks the property list values. It does NOT check the environment variables! -### Environment variables - -The `HDF5_USE_FILE_LOCKING` environment variable overrides all other file +\section sec_filelock_env Environment variables +The HDF5_USE_FILE_LOCKING environment variable overrides all other file locking settings. HDF5 1.10.0 -- No file locking environment variable +\li No file locking environment variable HDF5 1.10.1 - 1.10.6, 1.12.0: -- `FALSE` turns file locking off -- Anything else turns file locking on -- Neither of these values ignores disabled file locks -- Environment variable parsed at file create/open time +\li FALSE turns file locking off +\li Anything else turns file locking on +\li Neither of these values ignores disabled file locks +\li Environment variable parsed at file create/open time -HDF5 1.10.7+, 1.12.1+, 1.13.x: -- `FALSE` or `0` disables file locking -- `TRUE` or `1` enables file locking -- `BEST_EFFORT` enables file locking and ignores disabled file locks -- Anything else gives you the defaults -- Environment variable parsed at library startup - -### Lock disable scheme interactions +HDF5 1.10.7+, 1.12.1+, 1.14.x: +\li FALSE or 0 disables file locking +\li TRUE or 1 enables file locking +\li BEST_EFFORT enables file locking and ignores disabled file locks +\li Anything else gives you the defaults +\li Environment variable parsed at library startup +\section sec_filelock_lockdisable Lock disable scheme interactions As mentioned above and reiterated here: -- Configure-time settings set fapl defaults -- `H5Pset_file_locking()` overrides configure-time defaults -- The environment variable setting overrides all +\li Configure-time settings set fapl defaults +\li #H5Pset_file_locking overrides configure-time defaults +\li The environment variable setting overrides all If you want to check that file locking is on, you'll need to check the fapl setting AND check the environment variable, which can override the fapl. -**!!! WARNING !!!** - +\subsection subsec_filelock_lockdisable_warn !!! WARNING !!! Disabling the file locks is at your own risk. If more than one writer process modifies an HDF5 file at the same time, the file could be corrupted. If a reader process reads a file that is being modified by a writer, the reader process might attempt to read garbage and encounter errors or even crash. In the case of: - -- A single process accessing a file with write access -- Any number of processes accessing a file read-only +\li A single process accessing a file with write access +\li Any number of processes accessing a file read-only You can safely disable the file locking scheme. If you are trying to set up SWMR without the benefit of the file locks, you'll just need to be extra careful that you hold to rules for SWMR access. -## Feature Matrix - +\section sec_filelock_feat Feature Matrix The following table indicates which versions of the library support which file lock features. 1.13.0 and 1.13.1 are experimental releases (basically glorified release candidates) so they are not included here. -**Locks** +\subsection subsec_filelock_feat_locks Locks +\li P = POSIX locks only, Windows was a no-op that always succeeded +\li WP = POSIX and Windows locks +\li (-) = POSIX no-op lock fails +\li (+) = POSIX no-op lock passes -- P = POSIX locks only, Windows was a no-op that always succeeded -- WP = POSIX and Windows locks -- (-) = POSIX no-op lock fails -- (+) = POSIX no-op lock passes +\subsection subsec_filelock_feat_var Configure Option and Environment Variable +\li on/off = sets file locks on/off +\li try = can also set "best effort", where locks are on but ignored if disabled -**Configure Option and Environment Variable** + + + + + + + + + + + + + + + + + +
VersionHas locksConfigure option#H5Pset_file_lockingHDF5_USE_FILE_LOCKING
1.8.xNo---
1.10.0P(-)---
1.10.1P(-)--on/off
1.10.2P(-)--on/off
1.10.3P(-)--on/off
1.10.4P(-)--on/off
1.10.5P(-)--on/off
1.10.6P(-)--on/off
1.10.7P(+)tryYtry
1.10.8WP(+)tryYtry
1.10.9WP(+)tryYtry
1.12.0P(-)--on/off
1.12.1WP(+)tryYtry
1.12.2WP(+)tryYtry
1.13.2WP(+)tryYtry
-- on/off = sets file locks on/off -- try = can also set "best effort", where locks are on but ignored if disabled - -|Version|Has locks|Configure option|`H5Pset_file_locking()`|`HDF5_USE_FILE_LOCKING`| -|-------|---------|----------------|-----------------------|-----------------------| -|1.8.x|No|-|-|-| -|1.10.0|P(-)|-|-|-| -|1.10.1|P(-)|-|-|on/off| -|1.10.2|P(-)|-|-|on/off| -|1.10.3|P(-)|-|-|on/off| -|1.10.4|P(-)|-|-|on/off| -|1.10.5|P(-)|-|-|on/off| -|1.10.6|P(-)|-|-|on/off| -|1.10.7|P(+)|try|Y|try| -|1.10.8|WP(+)|try|Y|try| -|1.10.9|WP(+)|try|Y|try| -|1.12.0|P(-)|-|-|on/off| -|1.12.1|WP(+)|try|Y|try| -|1.12.2|WP(+)|try|Y|try| -|1.13.2|WP(+)|try|Y|try| - - -## Appendix: File lock implementation - -The file lock system is implemented with `flock(2)` as the archetype since it +\section sec_filelock_appd Appendix: File lock implementation +The file lock system is implemented with flock(2) as the archetype since it has simple semantics and we don't need range locking. Locks are advisory on many systems, but this shouldn't be a problem for most users since the HDF5 library always respects them. If you have a program that parses or modifies HDF5 files independently of the HDF5 library, you'll want to be mindful of any potential for concurrent access across processes. -On Unix systems, we call `flock()` directly when it's available and pass -`LOCK_SH` (shared lock), `LOCK_EX` (exclusive lock), and `LOCK_UN` (unlock) as +On Unix systems, we call flock() directly when it's available and pass +LOCK_SH (shared lock), LOCK_EX (exclusive lock), and LOCK_UN (unlock) as described in the algorithm section. All locks are non-blocking, so we set the -`LOCK_NB` flag. Sadly, `flock(2)` is not POSIX and it doesn't lock files over +LOCK_NB flag. Sadly, flock(2) is not POSIX and it doesn't lock files over NFS. We didn't consider a lack of NFS support a problem since SWMR isn't supported on networked file systems like NFS (write order preservation isn't -guaranteed) and `flock(2)` usually doesn't fail when you attempt to lock NFS +guaranteed) and flock(2) usually doesn't fail when you attempt to lock NFS files. -On Unix systems without `flock(2)`, we implement a scheme based on `fcntl(2)` -(`Pflock()` in `H5system.c`). On these systems we use `F_SETLK` (non-blocking) -as the operation and set `l_type` in `struct flock` to be: +On Unix systems without flock(2), we implement a scheme based on fcntl(2) +(Pflock() in H5system.c). On these systems we use F_SETLK (non-blocking) +as the operation and set l_type in struct flock to be: +\li F_UNLOCK for LOCK_UNc +\li F_WRLOCK for LOCK_EXc +\li F_RDLOCK for LOCK_SHc -- `F_UNLOCK` for `LOCK_UN` -- `F_WRLOCK` for `LOCK_EX` -- `F_RDLOCK` for `LOCK_SH` - -We set the range to be the entire file. Most Unix-like systems have `flock()` +We set the range to be the entire file. Most Unix-like systems have flock() these days, so this system probably isn't very well tested. -We don't use `fcntl`-based open file locks or mandatory locking anywhere. The +We don't use fcntl-based open file locks or mandatory locking anywhere. The former scheme is non-POSIX and the latter is deprecated. -On Windows, we use `LockFileEx()` and `UnlockFileEx()` to lock the entire file -(`Wflock()` in `H5system.c`). We set `LOCKFILE_FAIL_IMMEDIATELY` to get -non-blocking locks and set `LOCKFILE_EXCLUSIVE_LOCK` when we want an exclusive +On Windows, we use LockFileEx() and UnlockFileEx() to lock the entire file +(Wflock() in H5system.c). We set LOCKFILE_FAIL_IMMEDIATELY to get +non-blocking locks and set LOCKFILE_EXCLUSIVE_LOCK when we want an exclusive lock. SWMR isn't well-tested on Windows, so this scheme hasn't been as -thoroughly vetted as the `flock`-based scheme. +thoroughly vetted as the flock-based scheme. -On non-Windows systems where neither `flock(2)` nor `fcntl(2)` is available, -we substitute a no-op stub that always succeeds (`Nflock()` in `H5system.c`). +On non-Windows systems where neither flock(2) nor fcntl(2) is available, +we substitute a no-op stub that always succeeds (Nflock() in H5system.c). In the past, the stub always failed (see the matrix for when we made the switch). We currently know of no non-Windows systems where neither call is available so this scheme is not well-tested. @@ -347,15 +319,15 @@ locking, is that all of these schemes have subtly different semantics. We're using file locking in a fairly crude manner, though, and lock use has always been optional, so we consider this a lower-order concern. -Locks are implemented at the VFD level via `lock` and `unlock` callbacks. The +Locks are implemented at the VFD level via lock and unlock callbacks. The VFDs that implement file locks are: core (w/ backing store), direct, log, sec2, -and stdio (`flock(2)` locks only). The family, multi, and splitter VFDs invoke +and stdio (flock(2) locks only). The family, multi, and splitter VFDs invoke the lock callback of their underlying sub-files. The onion and MPI-IO VFDs do NOT use locks, even though they create normal, on-disk native HDF5 files. The read-only S3 VFD and HDFS VFDs do not use file locking since they use alternative storage schemes. -Lock failures are detected by checking to see if `errno` is set to `ENOSYS`. +Lock failures are detected by checking to see if errno is set to ENOSYS. This is not particularly sophisticated and was implemented as a way of working around disabled locks on popular parallel file systems. @@ -363,4 +335,7 @@ One other thing to note here is that, in all of the locking schemes we use, the file locks do not survive process termination, so you don't have to worry about files being locked forever if a process exits abnormally. If a writer crashed and the library didn't clear the superblock mark, you can remove it with -the h5clear command-line tool, which is built with the library. +the \ref sec_cltools_h5clear command-line tool, which is built with the library. + +*/ + diff --git a/doxygen/dox/high_level/extension.dox b/doxygen/dox/high_level/extension.dox index fc0da48ee83..456692ed868 100644 --- a/doxygen/dox/high_level/extension.dox +++ b/doxygen/dox/high_level/extension.dox @@ -7,16 +7,14 @@ * for working with region references, hyperslab selections, and bit-fields. * These functions were created as part of a project supporting * NPP/NPOESS Data Production and Exploitation ( - * - * project, - * software ). + * project, + * software). * While they were written to facilitate access to NPP, NPOESS, and JPSS * data in the HDF5 format, these functions may be useful to anyone working * with region references, hyperslab selections, or bit-fields. * * Note that these functions are not part of the standard HDF5 distribution; - * the - * software + * the software * must be separately downloaded and installed. * * A comprehensive guide to this library, diff --git a/doxygen/dox/library-init-shutdown.dox b/doxygen/dox/library-init-shutdown.dox new file mode 100644 index 00000000000..0c48ee49d53 --- /dev/null +++ b/doxygen/dox/library-init-shutdown.dox @@ -0,0 +1,55 @@ +/** \page InitShut HDF5 Library initialization and shutdown + +\section sec_initshut_app Application perspective + +\subsection subsec_initshut_app_implicit Implicit initialization and shutdown +When a developer exports a new symbol as part of the HDF5 library, +they should make sure that an application cannot enter the library in an +uninitialized state through a new API function, or read an uninitialized +value from a non-function HDF5 symbol. + +The HDF5 library initializes itself when an application either enters +the library through an API function call such as #H5Fopen, or when +an application evaluates an HDF5 symbol that represents either a +property-list identifier such as #H5F_ACC_RDONLY or #H5F_ACC_RDWR, +a property-list class identifier such as #H5P_FILE_ACCESS, a VFD +identifier such as #H5FD_FAMILY or #H5FD_SEC2, or a type identifier +such as #H5T_NATIVE_INT64. + +The library sets a flag when initialization occurs and as long as the +flag is set, skips initialization. + +The library provides a couple of macros that initialize the library +as necessary. The library is initialized as a side-effect of the +FUNC_ENTER_API* macros used at the top of most API functions. HDF5 +library symbols other than functions are provided through \#defines +that use #H5OPEN to introduce a library-initialization call (#H5open) +at each site where a non-function symbol is used. + +Ordinarily the library registers an atexit(3) handler to shut itself +down when the application exits. + +\subsection subsec_initshut_app_explicit Explicit initialization and shutdown +An application may use an API call, #H5open, to explicitly initialize +the library. #H5close explicitly shuts down the library. + +\section sec_initshut_int Library internals perspective +No matter how library initialization begins, eventually the internal +function H5_init_library will be called. H5_init_library is +responsible for calling the initializers for every internal HDF5 +library module (aka "package") in the correct order so that no module is +initialized before its prerequisite modules. A table in H5_init_library +establishes the order of initialization. If a developer adds a +module to the library that it is appropriate to initialize with the rest +of the library, then they should insert its initializer into the right +place in the table. + +H5_term_library drives library shutdown. Library shutdown is +table-driven, too. If a developer adds a module that needs to release +resources during library shutdown, then they should add a call at the +right place to the shutdown table. Note that some entries in the shutdown +table are marked as "barriers," and if a new module should only be +shutdown strictly after the preceding modules, then it should be marked +as a barrier. See the comments in H5_term_library for more information. + +*/ diff --git a/doc/parallel-compression.md b/doxygen/dox/parallel-compression.dox similarity index 76% rename from doc/parallel-compression.md rename to doxygen/dox/parallel-compression.dox index 523aa758fec..962b9907857 100644 --- a/doc/parallel-compression.md +++ b/doxygen/dox/parallel-compression.dox @@ -1,7 +1,6 @@ -# HDF5 Parallel Compression - -## Introduction +/** \page ParCompr HDF5 Parallel Compression +\section sec_parcompr_intro Introduction When an HDF5 dataset is created, the application can specify optional data filters to be applied to the dataset (as long as the dataset uses a chunked data layout). These filters may @@ -45,28 +44,28 @@ their modifications to the owning MPI rank. The parallel compression feature is always enabled when HDF5 is built with parallel enabled, but the feature may be disabled if the necessary MPI-3 routines are not available. Therefore, -HDF5 conditionally defines the macro `H5_HAVE_PARALLEL_FILTERED_WRITES` +HDF5 conditionally defines the macro H5_HAVE_PARALLEL_FILTERED_WRITES which an application can check for to see if the feature is available. -## Examples +\section sec_parcompr_ex Examples Using the parallel compression feature is very similar to using compression in serial HDF5, except that dataset writes **must** be collective: -``` +\code hid_t dxpl_id = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); H5Dwrite(..., dxpl_id, ...); -``` +\endcode The following are two simple examples of using the parallel compression feature: -[ph5_filtered_writes.c][u1] +ph5_filtered_writes.c -[ph5_filtered_writes_no_sel.c][u2] +ph5_filtered_writes_no_sel.c The former contains simple examples of using the parallel compression feature to write to compressed datasets, while the @@ -76,10 +75,10 @@ Remember that the feature requires these writes to use collective I/O, so the MPI ranks which have nothing to contribute must still participate in the collective write call. -## Multi-dataset I/O support +\section sec_parcompr_multi Multi-dataset I/O support The parallel compression feature is supported when using the -multi-dataset I/O API routines ([H5Dwrite_multi][u3]/[H5Dread_multi][u4]), but the +multi-dataset I/O API routines (#H5Dwrite_multi/#H5Dread_multi), but the following should be kept in mind: - Parallel writes to filtered datasets **must** still be collective, @@ -97,17 +96,17 @@ following should be kept in mind: datasets if desired, while still performing collective writes to the filtered datasets. -## Incremental file space allocation support +\section sec_parcompr_incr Incremental file space allocation support -HDF5's [file space allocation time][u5] +HDF5's file space allocation time function. #H5Pset_alloc_time, is a dataset creation property that can have significant effects on application performance, especially if the application uses parallel HDF5. In a serial HDF5 application, the default file space -allocation time for chunked datasets is "incremental". This means +allocation time for chunked datasets is incremental. This means that allocation of space in the HDF5 file for data chunks is deferred until data is first written to those chunks. In parallel HDF5, the file space allocation time was previously always forced -to "early", which allocates space in the file for all of a dataset's +to early, which allocates space in the file for all of a dataset's data chunks at creation time (or during the first open of a dataset if it was created serially). This would ensure that all the necessary file space was allocated so MPI ranks could perform independent I/O @@ -118,7 +117,7 @@ While this strategy has worked in the past, it has some noticeable drawbacks. For one, the larger the chunked dataset being created, the more noticeable overhead there will be during dataset creation as all of the data chunks are being allocated in the HDF5 file. -Further, these data chunks will, by default, be [filled][u6] +Further, these data chunks will, by default, be filled, using #H5Pset_fill_value, with HDF5's default fill data value, leading to extraordinary dataset creation overhead and resulting in pre-filling large portions of a dataset that the application might have been planning @@ -126,16 +125,16 @@ to overwrite anyway. Even worse, there will be more initial overhead from compressing that fill data before writing it out, only to have it read back in, unfiltered and modified the first time a chunk is written to. In the past, it was typically suggested that parallel -HDF5 applications should use [H5Pset_fill_time][u7] -with a value of `H5D_FILL_TIME_NEVER` in order to disable writing of +HDF5 applications should use #H5Pset_fill_time +with a value of #H5D_FILL_TIME_NEVER in order to disable writing of the fill value to dataset chunks, but this isn't ideal if the application actually wishes to make use of fill values. -With [improvements made][u8] -to the parallel compression feature for the HDF5 1.13.1 release, -"incremental" file space allocation is now the default for datasets -created in parallel *only if they have filters applied to them*. -"Early" file space allocation is still supported for these datasets +With improvements made +to the parallel compression feature for the HDF5 1.14.0 release, +incremental file space allocation is now the default for datasets +created in parallel only if they have filters applied to them. +Early file space allocation is still supported for these datasets if desired and is still forced for datasets created in parallel that do *not* have filters applied to them. This change should significantly reduce the overhead of creating filtered datasets in parallel HDF5 @@ -144,7 +143,7 @@ use a fill value for these datasets. It should also help significantly reduce the size of the HDF5 file, as file space for the data chunks is allocated as needed rather than all at once. -## Performance Considerations +\section sec_parcompr_perf Performance Considerations Since getting good performance out of HDF5's parallel compression feature involves several factors, the following is a list of @@ -152,9 +151,9 @@ performance considerations (generally from most to least important) and best practices to take into account when trying to get the optimal performance out of the parallel compression feature. -### Begin with a good chunking strategy +\subsection subsec_parcompr_perf_begin Begin with a good chunking strategy -[Starting with a good chunking strategy][u9] +Starting with a good \ref hdf5_chunking strategy will generally have the largest impact on overall application performance. The different chunking parameters can be difficult to fine-tune, but it is essential to start with a well-performing @@ -166,11 +165,11 @@ chosen chunk size becomes a very important factor when compression is involved, as data chunks have to be completely read and re-written to perform partial writes to the chunk. -[Improving I/O performance with HDF5 compressed datasets][u10] +\ref improve_compressed_perf is a useful reference for more information on getting good performance when using a chunked dataset layout. -### Avoid chunk sharing +\subsection subsec_parcompr_perf_avoid Avoid chunk sharing Since the parallel compression feature has to assign ownership of data chunks to a single MPI rank in order to avoid the @@ -185,7 +184,7 @@ application will get the best performance out of parallel compression if it can avoid writing in a way that causes more than 1 MPI rank to write to any given data chunk in a dataset. -### Collective metadata operations +\subsection subsec_parcompr_perf_coll Collective metadata operations The parallel compression feature typically works with a significant amount of metadata related to the management of the data chunks @@ -203,7 +202,7 @@ performance and scalability and is generally always recommended unless application performance shows negative benefits by doing so. -``` +\code ... hid_t fapl_id = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); @@ -211,23 +210,23 @@ H5Pset_all_coll_metadata_ops(fapl_id, 1); H5Pset_coll_metadata_write(fapl_id, 1); hid_t file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); ... -``` +\endcode -### Align chunks in the file +\subsection subsec_parcompr_perf_align Align chunks in the file The natural layout of an HDF5 file may cause dataset data chunks to end up at addresses in the file that do not align well with the underlying file system, possibly leading to poor performance. As an example, Lustre performance is generally good when writes are aligned with the chosen stripe size. -The HDF5 application can use [H5Pset_alignment][u11] +The HDF5 application can use #H5Pset_alignment to have a bit more control over where objects in the HDF5 file end up. However, do note that setting the alignment of objects generally wastes space in the file and has the potential to dramatically increase its resulting size, so caution should be used when choosing the alignment parameters. -[H5Pset_alignment][u11] +#H5Pset_alignment has two parameters that control the alignment of objects in the HDF5 file, the "threshold" value and the alignment value. The threshold value specifies that any object greater @@ -246,16 +245,16 @@ the Lustre stripe size), this should cause dataset data chunks to be well-aligned and generally give good write performance. -``` +\code hid_t fapl_id = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); /* Assuming Lustre stripe size is 1MiB, align data chunks in the file to address multiples of 1MiB. */ H5Pset_alignment(fapl_id, dataset_chunk_size, 1048576); hid_t file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); -``` +\endcode -### File free space managers +\subsection subsec_parcompr_perf_space File free space managers As data chunks in a dataset get written to and compressed, they can change in size and be relocated in the HDF5 file. @@ -264,29 +263,29 @@ in a file, this can create significant amounts of free space in the file over its lifetime and eventually cause performance issues. -An HDF5 application can use [H5Pset_file_space_strategy][u12] -with a value of `H5F_FSPACE_STRATEGY_PAGE` to enable the paged +An HDF5 application can use #H5Pset_file_space_strategy +with a value of #H5F_FSPACE_STRATEGY_PAGE to enable the paged aggregation feature, which can accumulate metadata and raw data for dataset data chunks into well-aligned, configurably -sized "pages" for better performance. However, note that using +sized pages for better performance. However, note that using the paged aggregation feature will cause any setting from -[H5Pset_alignment][u11] +#H5Pset_alignment to be ignored. While an application should be able to get -comparable performance effects by [setting the size of these pages][u13] -to be equal to the value that would have been set for [H5Pset_alignment][u11], +comparable performance effects by setting the size of these pages, using #H5Pset_file_space_page_size, +to be equal to the value that would have been set for #H5Pset_alignment, this may not necessarily be the case and should be studied. -Note that [H5Pset_file_space_strategy][u12] -has a `persist` parameter. This determines whether or not the +Note that #H5Pset_file_space_strategy +has a persist parameter. This determines whether or not the file free space manager should include extra metadata in the HDF5 file about free space sections in the file. If this -parameter is `false`, any free space in the HDF5 file will +parameter is false, any free space in the HDF5 file will become unusable once the HDF5 file is closed. For parallel -compression, it's generally recommended that `persist` be set -to `true`, as this will keep better track of file free space +compression, it's generally recommended that persist be set +to true, as this will keep better track of file free space for data chunks between accesses to the HDF5 file. -``` +\code hid_t fcpl_id = H5Pcreate(H5P_FILE_CREATE); /* Use persistent free space manager with paged aggregation */ H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, 1, 1); @@ -294,58 +293,43 @@ H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, 1, 1); H5Pset_file_space_page_size(fcpl_id, 1048576); ... hid_t file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, fcpl_id, fapl_id); -``` +\endcode -### Low-level collective vs. independent I/O +\subsection subsec_parcompr_perf_low Low-level collective vs. independent I/O While the parallel compression feature requires that the HDF5 application set and maintain collective I/O at the application -interface level (via [H5Pset_dxpl_mpio][u14]), +interface level (via #H5Pset_dxpl_mpio), it does not require that the actual MPI I/O that occurs at the lowest layers of HDF5 be collective; independent I/O may perform better depending on the application I/O patterns and parallel file system performance, among other factors. The -application may use [H5Pset_dxpl_mpio_collective_opt][u15] +application may use #H5Pset_dxpl_mpio_collective_opt to control this setting and see which I/O method provides the best performance. -``` +\code hid_t dxpl_id = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); H5Pset_dxpl_mpio_collective_opt(dxpl_id, H5FD_MPIO_INDIVIDUAL_IO); /* Try independent I/O */ H5Dwrite(..., dxpl_id, ...); -``` +\endcode -### Runtime HDF5 Library version +\subsection subsec_parcompr_perf_libver Runtime HDF5 Library version -An HDF5 application can use the [H5Pset_libver_bounds][u16] +An HDF5 application can use the #H5Pset_libver_bounds routine to set the upper and lower bounds on library versions to use when creating HDF5 objects. For parallel compression specifically, setting the library version to the latest available version can allow access to better/more efficient chunk indexing types and data encoding methods. For example: -``` +\code ... hid_t fapl_id = H5Pcreate(H5P_FILE_ACCESS); H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); hid_t file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); ... -``` +\endcode -[u1]: https://github.com/HDFGroup/hdf5/blob/develop/HDF5Examples/C/H5PAR/ph5_filtered_writes.c -[u2]: https://github.com/HDFGroup/hdf5/blob/develop/HDF5Examples/C/H5PAR/ph5_filtered_writes_no_sel.c -[u3]: https://hdfgroup.github.io/hdf5/develop/group___h5_d.html#gaf6213bf3a876c1741810037ff2bb85d8 -[u4]: https://hdfgroup.github.io/hdf5/develop/group___h5_d.html#ga8eb1c838aff79a17de385d0707709915 -[u5]: https://hdfgroup.github.io/hdf5/develop/group___d_c_p_l.html#ga85faefca58387bba409b65c470d7d851 -[u6]: https://hdfgroup.github.io/hdf5/develop/group___d_c_p_l.html#ga4335bb45b35386daa837b4ff1b9cd4a4 -[u7]: https://hdfgroup.github.io/hdf5/develop/group___d_c_p_l.html#ga6bd822266b31f86551a9a1d79601b6a2 -[u8]: https://www.hdfgroup.org/2022/03/04/parallel-compression-improvements-in-hdf5-1-13-1/ -[u9]: https://hdfgroup.github.io/hdf5/develop/chunking__in__hdf5_8dox.html -[u10]: https://support.hdfgroup.org/releases/hdf5/documentation/hdf5_topics/HDF5ImprovingIOPerformanceCompressedDatasets.pdf -[u11]: https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a -[u12]: https://hdfgroup.github.io/hdf5/develop/group___f_c_p_l.html#ga167ff65f392ca3b7f1933b1cee1b9f70 -[u13]: https://hdfgroup.github.io/hdf5/develop/group___f_c_p_l.html#gad012d7f3c2f1e1999eb1770aae3a4963 -[u14]: https://hdfgroup.github.io/hdf5/develop/group___d_x_p_l.html#ga001a22b64f60b815abf5de8b4776f09e -[u15]: https://hdfgroup.github.io/hdf5/develop/group___d_x_p_l.html#gacb30d14d1791ec7ff9ee73aa148a51a3 -[u16]: https://hdfgroup.github.io/hdf5/develop/group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910 +*/ diff --git a/doxygen/dox/threadsafety-warning.dox b/doxygen/dox/threadsafety-warning.dox new file mode 100644 index 00000000000..16194f22a96 --- /dev/null +++ b/doxygen/dox/threadsafety-warning.dox @@ -0,0 +1,42 @@ +/** \page ThrdSafe HDF5 Threadsafety Warning +Any application that creates threads that use the HDF5 library must join those threads before +either process exit or library close through H5close(). If all HDF5-using threads aren't joined, +the threads may exhibit undefined behavior. + +\section sec_thrdsafe Discussion for Developers on Potential Improvements + +It would in principle be possible to make it safe to have threads continue using HDF5 resources +after a call to #H5close by keeping a count of threads within the library. (There is probably +no solution to an early process exit producing undefined behavior within threads.) This method +would only be able to count (and presumably, only _need_ to count) threads that directly interact +with the library. Because each thread would need to be counted exactly once, this would most +likely be done by use of a thread-local key with e.g. a boolean value used to track whether the +a global atomic thread counter has already counted this thread. Then, if #H5close is invoked +while this thread counter is above one (because one thread must be doing the closing), the library +would not close, and instead keep its resources valid to hopefully avoid bad behavior with the threads. + +The issues with this approach are as follows: +
    +
  1. The process of checking for the existence/value of the thread-local key is slow, or at least + slow enough that it's probably not worth adding this to almost every single API call to prevent + this particular edge case.
    2. +
  2. Even with this approach, bad behavior would still be possible if the application does something + like expose HDF5 resources to threads indirectly via a global variable.
    3. +
  3. How to allow #H5close to fail is nonobvious. #H5close could be allowed to return an error + indicating a failure to close, but the number of applications which could usefully respond to such + an error by joining threads is small. If an application were able/willing to join its created + threads, presumably it would have done so before calling #H5close. Alternatively, #H5close could + succeed but silently leave the library open. This creates the potential for confusing, unexpected + behavior when the user thinks they are closing and re-opening the library, e.g. if environment + variables are modified between close and re-open, or if resources such as default property lists + are modified.
    4. +
  4. Applications should join threads before closing libraries that those threads are using, so all + of this work would constitute an above-and-beyond effort to maintain safe and defined behavior in + the face of an unsafe application.
    5. +
+ +Despite these issues, if a more performant method was found to perform threadcounting like this, +it might still constitute a worthwhile change. + +*/ + \ No newline at end of file diff --git a/doxygen/examples/tables/fileDriverLists.dox b/doxygen/examples/tables/fileDriverLists.dox index 437d32a7b93..c321284944b 100644 --- a/doxygen/examples/tables/fileDriverLists.dox +++ b/doxygen/examples/tables/fileDriverLists.dox @@ -71,7 +71,7 @@ * //! [supported_file_driver_table] - + diff --git a/doxygen/examples/tables/propertyLists.dox b/doxygen/examples/tables/propertyLists.dox index 76727b58a59..4480cabc64f 100644 --- a/doxygen/examples/tables/propertyLists.dox +++ b/doxygen/examples/tables/propertyLists.dox @@ -2,7 +2,7 @@ * //! [plcr_table]
Supported file driversSupported file drivers
Driver Name Driver Identifier
- + @@ -32,7 +32,7 @@ * //! [plcra_table]
Property list class root functions (H5P)Property list class root functions (H5P)
Function Purpose
- + @@ -102,7 +102,7 @@ * //! [fcpl_table]
Property list class root (Advanced) functions (H5P)Property list class root (Advanced) functions (H5P)
Function Purpose
- + @@ -157,7 +157,7 @@ creation property list. * //! [fapl_table]
File creation property list functions (H5P)File creation property list functions (H5P)
Function Purpose
- + @@ -298,7 +298,7 @@ versions used when creating objects. * //! [fd_pl_table]
File access property list functions (H5P)File access property list functions (H5P)
Function Purpose
- + @@ -418,7 +418,7 @@ and one raw data file. * //! [dcpl_table]
File driver property list functions (H5P)File driver property list functions (H5P)
Function Purpose
- + @@ -579,7 +579,7 @@ encoding for object names. * //! [dapl_table]
Dataset creation property list functions (H5P)Dataset creation property list functions (H5P)
Function Purpose
- + @@ -621,7 +621,7 @@ encoding for object names. * //! [dxpl_table]
Dataset access property list functions (H5P)Dataset access property list functions (H5P)
Function Purpose
- + @@ -727,7 +727,7 @@ of the library for reading or writing the actual data. * //! [gcpl_table]
Data transfer property list functions (H5P)Data transfer property list functions (H5P)
C Function Purpose
- + @@ -820,7 +820,7 @@ encoding for object names. * //! [gapl_table]
Group creation property list functions (H5P)Group creation property list functions (H5P)
Function Purpose
- + @@ -834,7 +834,7 @@ encoding for object names. * //! [lapl_table]
Group access property list functions (H5P)Group access property list functions (H5P)
Function Purpose
- + @@ -864,7 +864,7 @@ encoding for object names. * //! [ocpl_table]
Link access property list functions (H5P)Link access property list functions (H5P)
Function Purpose
- + @@ -910,7 +910,7 @@ encoding for object names. * //! [ocpypl_table]
Object creation property list functions (H5P)Object creation property list functions (H5P)
Function Purpose
- + @@ -936,7 +936,7 @@ encoding for object names. * //! [strcpl_table]
Object copy property list functions (H5P)Object copy property list functions (H5P)
Function Purpose
- + @@ -950,7 +950,7 @@ encoding for object names. * //! [lcpl_table]
String creation property list functions (H5P)String creation property list functions (H5P)
Function Purpose
- + @@ -964,7 +964,7 @@ encoding for object names. * //! [acpl_table]
Link creation property list functions (H5P)Link creation property list functions (H5P)
Function Purpose
- + diff --git a/doxygen/img/FreeingMemory_fig1.png b/doxygen/img/FreeingMemory_fig1.png new file mode 100644 index 00000000000..811857869ca Binary files /dev/null and b/doxygen/img/FreeingMemory_fig1.png differ diff --git a/doxygen/img/improve_perf-compress_fig_1.png b/doxygen/img/improve_perf-compress_fig_1.png new file mode 100644 index 00000000000..ba7033ec903 Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_1.png differ diff --git a/doxygen/img/improve_perf-compress_fig_2.png b/doxygen/img/improve_perf-compress_fig_2.png new file mode 100644 index 00000000000..f817085b709 Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_2.png differ diff --git a/doxygen/img/improve_perf-compress_fig_3.png b/doxygen/img/improve_perf-compress_fig_3.png new file mode 100644 index 00000000000..64995bdbe71 Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_3.png differ diff --git a/doxygen/img/improve_perf-compress_fig_6.png b/doxygen/img/improve_perf-compress_fig_6.png new file mode 100644 index 00000000000..f6dfda7874b Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_6.png differ diff --git a/doxygen/img/improve_perf-compress_fig_7.png b/doxygen/img/improve_perf-compress_fig_7.png new file mode 100644 index 00000000000..367c519c122 Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_7.png differ diff --git a/doxygen/img/improve_perf-compress_fig_8.png b/doxygen/img/improve_perf-compress_fig_8.png new file mode 100644 index 00000000000..8d84d0530dc Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_8.png differ diff --git a/doxygen/img/improve_perf-compress_fig_9.png b/doxygen/img/improve_perf-compress_fig_9.png new file mode 100644 index 00000000000..2077ad89e42 Binary files /dev/null and b/doxygen/img/improve_perf-compress_fig_9.png differ diff --git a/doxygen/img/modregwrite.png b/doxygen/img/modregwrite.png new file mode 100644 index 00000000000..22e1a96f8af Binary files /dev/null and b/doxygen/img/modregwrite.png differ diff --git a/hl/src/H5DSpublic.h b/hl/src/H5DSpublic.h index f38f6fef0ba..28d70831bec 100644 --- a/hl/src/H5DSpublic.h +++ b/hl/src/H5DSpublic.h @@ -204,10 +204,10 @@ extern "C" { * extension of dimension scales when dataset dimensions are extended, so we suggest letting * applications be responsible for this operation. * \li Automatically deleting dimension scales. Should a dimension scale be deleted - * when all datasets that use it have been deleted? This is another case where different applications - * might have different requirements, so a general policy would be difficult to devise. Furthermore, + * when all datasets that use it have been deleted? This is another case where different applications + * might have different requirements, so a general policy would be difficult to devise. Furthermore, * enforcing a deletion policy, even a simple one, adds complexity to the library, and could also - * affect performance. Deletion policies seem best left to applications. + * affect performance. Deletion policies seem best left to applications. * * Section \ref sec_dim_scales_api presents an API and programming model that implements some of these * features. However, applications @@ -235,7 +235,7 @@ extern "C" { * \li A storage profile * \li An API and programming model * - * \subsection subsec_dim_scales_spec_store + * \subsection subsec_dim_scales_spec_store Storage Profile * This section specifies the storage profile for Dimension Scale objects and the association between * Dimensions and Dimension Scales. * @@ -246,7 +246,7 @@ extern "C" { * See Appendix 2 for a discussion of how to store converted HDF4 objects. See Appendix 3 for a * discussion of netCDF4 issues. See Appendix 4 for a discussion of HDF-EOS5. * - * \subsubsection subsubsec_dim_scales_spec_store_dset + * \subsubsection subsubsec_dim_scales_spec_store_dset Dimension Scale Dataset * A Dimension Scale dataset is stored as an HDF5 dataset. Table 1 summarizes the stored data, i.e., * the values of the scale. There are no restrictions on the dataspace or datatype, or storage properties of * the dataset. @@ -835,9 +835,8 @@ H5_HLDLL herr_t H5DSwith_new_ref(hid_t obj_id, hbool_t *with_new_ref); * dimension \p idx of dataset \p did. * * Entries are created in the #DIMENSION_LIST and - * #REFERENCE_LIST attributes, as defined in section 4.2 of - * - * HDF5 Dimension Scale Specification. + * #REFERENCE_LIST attributes, as defined in \ref subsec_dim_scales_spec_store section of + * the \ref H5DS_UG. * * Fails if: * - Bad arguments @@ -865,9 +864,8 @@ H5_HLDLL herr_t H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx); * \details If possible, deletes association of Dimension Scale \p dsid with * dimension \p idx of dataset \p did. This deletes the entries in the * #DIMENSION_LIST and #REFERENCE_LIST attributes, - * as defined in section 4.2 of - * - * HDF5 Dimension Scale Specification. + * as defined in \ref subsec_dim_scales_spec_store section of + * the \ref H5DS_UG. * * Fails if: * - Bad arguments @@ -898,10 +896,8 @@ H5_HLDLL herr_t H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx); * \details The dataset \p dsid is converted to a Dimension Scale dataset, * as defined above. Creates the CLASS attribute, set to the value * "DIMENSION_SCALE" and an empty #REFERENCE_LIST attribute, - * as described in - * - * HDF5 Dimension Scale Specification. - * (PDF, see section 4.2). + * as described in \ref subsec_dim_scales_spec_store section of + * the \ref H5DS_UG. * * If \p dimname is specified, then an attribute called NAME * is created, with the value \p dimname. diff --git a/java/src/hdf/hdf5lib/package-info.java b/java/src/hdf/hdf5lib/package-info.java index 77ef151621f..34efe584fde 100644 --- a/java/src/hdf/hdf5lib/package-info.java +++ b/java/src/hdf/hdf5lib/package-info.java @@ -19,7 +19,7 @@ * HDF5 entry point, generally with the arguments and return codes analogous to the C interface. *

* For details of the HDF5 library, see the HDF5 Documentation at: - * http://hdfgroup.org/HDF5/ + * https://support.hdfgroup.org/documentation/ *

*

* Mapping of arguments for Java @@ -196,7 +196,7 @@ * * @ref HDF5LIB * - * @see: HDF5" + * @see: HDF5" * */ package hdf.hdf5lib; diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index eed9e4a880a..e870c4321cf 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,8 @@ New Features Configuration: ------------- + - The documentation in the source folder, doc, has been converted to doxygen files. + - Added configuration option for API concurrency support: CMake: HDF5_ENABLE_CONCURRENCY (ON/OFF) (Default: OFF) diff --git a/src/H5Dmodule.h b/src/H5Dmodule.h index d7acb0d9f51..ab46f8a9260 100644 --- a/src/H5Dmodule.h +++ b/src/H5Dmodule.h @@ -852,7 +852,7 @@ *

* * + * filter. * *
Attribute creation property list functions (H5P)Attribute creation property list functions (H5P)
Function Purpose
szip compressionData compression using the szip library. The HDF Group now uses the libaec library for the szip -filter.
* diff --git a/src/H5Fmodule.h b/src/H5Fmodule.h index 2551e13aaeb..02e006cfbf9 100644 --- a/src/H5Fmodule.h +++ b/src/H5Fmodule.h @@ -679,6 +679,7 @@ * systems, environments, and applications. The drivers are listed in the table below. * *
+ * \subsubsection table_file_drivers Supported file drivers * \snippet{doc} tables/fileDriverLists.dox supported_file_driver_table *
* @@ -713,7 +714,7 @@ * If the application opens an HDF5 file without both determining the driver used to create the file * and setting up the use of that driver, the HDF5 Library will examine the superblock and the * driver definition block to identify the driver. - * See the HDF5 File Format Specification + * See the \ref_spec_fileformat * for detailed descriptions of the superblock and the driver definition block. * * \subsubsection subsubsec_file_alternate_drivers_sec2 The POSIX (aka SEC2) Driver @@ -2557,7 +2558,7 @@ * \subsection subsec_file_image_fort Fortran Signatures for File Image Operations API Calls * Fortran function call signatures for the file image operation APIs are described in this section. * - * \subsubsection subsubsec_file_image_fort_low + * \subsubsection subsubsec_file_image_fort_low Fortran Low-Level APIs * The Fortran low-level APIs make use of Fortran 2003’s ISO_C_BINDING module in order to achieve portable * and standard conforming interoperability with the C APIs. The C pointer (C_PTR) and function pointer * (C_FUN_PTR) types are returned from the intrinsic procedures C_LOC(X) and C_FUNLOC(X), respectively, @@ -2609,7 +2610,7 @@ * * * - * \subsubsection subsubsec_file_image_fort_high + * \subsubsection subsubsec_file_image_fort_high Fortran High-Level APIs * Fortran function call signatures for the file image operation APIs have not yet been implemented yet. * * Previous Chapter \ref sec_vol - Next Chapter \ref sec_async diff --git a/src/H5Gmodule.h b/src/H5Gmodule.h index 901f3b5b640..87df5e41322 100644 --- a/src/H5Gmodule.h +++ b/src/H5Gmodule.h @@ -342,7 +342,7 @@ * } * \endcode * - * h5dump is described on the “HDF5 Tools” page of the \ref RM. + * \ref sec_cltools_h5dump is described on the \ref sec_cltools page of the \ref UG. * * The HDF5 DDL grammar is described in the @ref DDLBNF200. * diff --git a/src/H5PLmodule.h b/src/H5PLmodule.h index 5548540c533..a76f8e199b4 100644 --- a/src/H5PLmodule.h +++ b/src/H5PLmodule.h @@ -48,7 +48,7 @@ * function, as a user would do in the past. The identification number and the filter parameters should be * available to the application. For example, if the application intends to apply the HDF5 bzip2 compression * filter that was registered with The HDF Group and has an identification number 307 - * (Registered + * (Registered * Filters) then the application would follow the steps as outlined below: * \code * dcpl = H5Pcreate (H5P_DATASET_CREATE); @@ -67,7 +67,7 @@ * H5Dread (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata[0]); * \endcode * - * The command-line utility h5dump, for example, will read and display the data as shown: + * The command-line utility \ref sec_cltools_h5dump, for example, will read and display the data as shown: * \code * HDF5 "h5ex_d_bzip2.h5" { * GROUP "/" { @@ -82,7 +82,7 @@ * USER_DEFINED_FILTER { * FILTER_ID 307 * COMMENT HDF5 bzip2 filter; see - * https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md + * https://\PLURL/docs/RegisteredFilterPlugins.md * PARAMS { 2 } * } * } @@ -101,7 +101,7 @@ * } * \endcode * - * If the filter can not be loaded then h5dump will show the following: + * If the filter can not be loaded then \ref sec_cltools_h5dump will show the following: * \code * ... * } @@ -162,7 +162,7 @@ * \endcode * * To avoid the problem make sure to close all objects to which the filter is applied and flush them using - * the H5Fflush call before unregistering the filter. + * the #H5Fflush call before unregistering the filter. * * \subsection subsec_filter_plugins_prog Programming Model for HDF5 Filter Plugins * This section describes how to create an HDF5 filter, an HDF5 filter plugin, and how to install the HDF5 @@ -179,50 +179,52 @@ * buffer if necessary, and appends the checksum to the end of the buffer. The input half calculates the * checksum on the first part of the buffer and compares it to the checksum already stored at the end of the * buffer. If the two differ then zero (failure) is returned, otherwise the buffer size is reduced to exclude - * the checksum. /code size_t md5_filter(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], - * size_t nbytes, size_t *buf_size, void **buf) - * { - * \c \#ifdef HAVE_MD5 - * unsigned char cksum[16]; + * the checksum. + * \code + * size_t md5_filter(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], + * size_t nbytes, size_t *buf_size, void **buf) + * { + * #ifdef HAVE_MD5 + * unsigned char cksum[16]; * - * if (flags & H5Z_REVERSE) { - * // Input - * assert(nbytes >= 16); - * md5(nbytes-16, *buf, cksum); - * // Compare - * if (memcmp(cksum, (char*)(*buf)+ nbytes- 16, 16)) { - * return 0; // fail - * } - * // Strip off checksum - * return nbytes - 16; - * } - * else { - * // Output - * md5(nbytes, *buf, cksum); - * // Increase buffer size if necessary - * if (nbytes + 16 > *buf_size) { - * *buf_size = nbytes + 16; - * *buf = realloc(*buf, *buf_size); - * } - * // Append checksum - * memcpy((char*)(*buf)+nbytes, cksum, 16); - * return nbytes+16; - * } - * \c \#else - * return 0; // fail - * \c \#endif - * } - * /endcode + * if (flags & H5Z_REVERSE) { + * // Input + * assert(nbytes >= 16); + * md5(nbytes-16, *buf, cksum); + * // Compare + * if (memcmp(cksum, (char*)(*buf)+ nbytes- 16, 16)) { + * return 0; // fail + * } + * // Strip off checksum + * return nbytes - 16; + * } + * else { + * // Output + * md5(nbytes, *buf, cksum); + * // Increase buffer size if necessary + * if (nbytes + 16 > *buf_size) { + * *buf_size = nbytes + 16; + * *buf = realloc(*buf, *buf_size); + * } + * // Append checksum + * memcpy((char*)(*buf)+nbytes, cksum, 16); + * return nbytes+16; + * } + * #else + * return 0; // fail + * #endif + * } + * \endcode * * Once the filter function is defined it must be registered so * the HDF5 library knows about it. Since we're testing this * filter we choose one of the #H5Z_filter_t numbers * from the reserved range. We'll randomly choose 305. * - * /code - * \c \#define FILTER_MD5 305 + * \code + * #define FILTER_MD5 305 * herr_t status = H5Zregister(FILTER_MD5, "md5 checksum", md5_filter); - * /endcode + * \endcode * * Now we can use the filter in a pipeline. We could have added * the filter to the pipeline before defining or registering the @@ -232,26 +234,28 @@ * have automatically removed it from the pipeline for each chunk * written before the filter was defined and registered). * - * /code + * \code * hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); * hsize_t chunk_size[3] = {10,10,10}; * H5Pset_chunk(dcpl, 3, chunk_size); * H5Pset_filter(dcpl, FILTER_MD5, 0, 0, NULL); * hid_t dset = H5Dcreate(file, "dset", H5T_NATIVE_DOUBLE, space, dcpl); - * /endcode + * \endcode * - * See the example of a more sophisticated HDF5 bzip2 filter function in the /ref subsec_filter_plugins_build - * section. The HDF5 bzip2 filter function is also available for download from Filter Plugin Repository. + * See the example of a more sophisticated HDF5 bzip2 filter function in the \ref subsec_filter_plugins_build + * section. The HDF5 bzip2 filter function is also available for download from + * Filter Plugin Repository. * * The user has to remember a few things when writing an HDF5 filter function. - *
  • 1. An HDF5 filter is bidirectional. - * The filter handles both input and output to the file; a flag is passed to the filter to indicate the - * direction.
    • - *
  • 2. An HDF5 filter operates on a buffer. - * The filter reads data from a buffer, performs some sort of transformation on the data, places - * the result in the same or new buffer, and returns the buffer pointer and size to the caller.
    • - *
  • 3. An HDF5 filter should return zero in the case of failure.
+ *
    + *
  1. An HDF5 filter is bidirectional.
    + * The filter handles both input and output to the file; a flag is passed to the filter to indicate the + * direction.
    2. + *
  2. An HDF5 filter operates on a buffer.
    + * The filter reads data from a buffer, performs some sort of transformation on the data, places + * he result in the same or new buffer, and returns the buffer pointer and size to the caller.
    3. + *
  3. An HDF5 filter should return zero in the case of failure.
    4. + *
* * The signature of the HDF5 filter function and the accompanying filter structure (see the section below) * are described in the \ref RM #H5Z_filter_t. @@ -259,7 +263,7 @@ * \subsubsection subsubsec_filter_plugins_prog_reg Registering a Filter with The HDF Group * If you are writing a filter that will be used by others, it would be a good idea to request a filter * identification number and register it with The HDF Group. Please follow the procedure described at - * Registered + * Registered * Filters. * * The HDF Group anticipates that developers of HDF5 filter plugins will not only register new filters, but @@ -274,7 +278,7 @@ * 1, // encoder_present flag (set to true) * 1, // decoder_present flag (set to true) * "HDF5 bzip2 filter; see - * https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md", + * https://\PLURL/docs/RegisteredFilterPlugins.md", * // Filter name for debugging * NULL, // The "can apply" callback * NULL, // The "set local" callback @@ -285,35 +289,42 @@ * The HDF5 Library and command-line tools have access to the “name” field. An application can * use the H5Pget_filter<*> functions to retrieve information about the filters. * - * Using the example of the structure above, the h5dump tool will print the string “HDF5 bzip2 - * filter found at …” pointing users to the applied filter (see the example in the \ref + * Using the example of the structure above, the \ref sec_cltools_h5dump tool will print the string “HDF5 + * bzip2 filter found at …” pointing users to the applied filter (see the example in the \ref * subsubsec_filter_plugins_model_read section) thus solving the problem of the filter’s origin. * * \subsubsection subsubsec_filter_plugins_prog_create Creating an HDF5 Filter Plugin * The HDF5 filter plugin source should include: - *
  • 1. The H5PLextern.h header file from the HDF5 distribution.
    • - *
  • 2. The definition of the filter structure (see the example shown in the section above).
    • - *
  • 3. The filter function (for example, H5Z_filter_bzip2).
    • - *
  • 4. The two functions necessary for the HDF5 Library to find the correct type of the plugin library - * while loading it at runtime and to get information about the filter function: - * - *
    H5PL_type_t H5PLget_plugin_type(void);
    const void* H5PLget_plugin_info(void);
    - * Here is an example of the functions above for the HDF5 bzip2 filter: - * - *
    H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
    const void* H5PLget_plugin_info(void) {return H5Z_BZIP2;}
    - *
    • - *
  • 5. Other functions such as the source of the compression library may also be included.
    • - *
+ *
    + *
  1. The H5PLextern.h header file from the HDF5 distribution.
    2. + *
  2. The definition of the filter structure (see the example shown in the section above).
    3. + *
  3. The filter function (for example, H5Z_filter_bzip2).
    4. + *
  4. The two functions necessary for the HDF5 Library to find the correct type of the plugin library + * while loading it at runtime and to get information about the filter function:
    + * + * + * + *
    H5PL_type_t H5PLget_plugin_type(void);
    const void* H5PLget_plugin_info(void);

    + * Here is an example of the functions above for the HDF5 bzip2 filter:
    + * + * + * + *
    H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
    const void* H5PLget_plugin_info(void) {return H5Z_BZIP2;}
    5. + *
  5. Other functions such as the source of the compression library may also be included.
    6. + *
+ * * Build the HDF5 filter plugin as a shared library. The following steps should be taken: - *
  • 1. When compiling, point to the HDF5 header files.
    • - *
  • 2. Use the appropriate linking flags.
    • - *
  • 3. Link with any required external libraries.
    • - *
  • 4. For example, if libbz2.so is installed on a Linux system, the HDF5 bzip2 plugin library - * libH5Zbzip2.so may be linked with libbz2.so instead of including bzip2 source into the - * plugin library. - * The complete example of the HDF5 bzip2 plugin library is provided at - * BZIP2 Filter Plugin - * and can be adopted for other plugins.
+ *
    + *
  1. When compiling, point to the HDF5 header files.
    2. + *
  2. Use the appropriate linking flags.
    3. + *
  3. Link with any required external libraries.
    4. + *
  4. For example, if libbz2.so is installed on a Linux system, the HDF5 bzip2 plugin library + * libH5Zbzip2.so may be linked with libbz2.so instead of including bzip2 source into the + * plugin library.
    + * The complete example of the HDF5 bzip2 plugin library is provided at + * BZIP2 Filter Plugin + * and can be adopted for other plugins.
    5. + *
* * \subsubsection subsubsec_filter_plugins_prog_install Installing an HDF5 Filter Plugin * The default directory for an HDF5 filter plugin library is defined on UNIX-like systems as @@ -355,9 +366,9 @@ * hdf5_plugins/docs folder. In * particular: * INSTALL_With_CMake + * href="https://\PLURL/docs/INSTALL_With_CMake.txt">INSTALL_With_CMake * USING_HDF5_AND_CMake + * href="https://\PLURL/docs/USING_HDF5_AND_CMake.txt">USING_HDF5_AND_CMake */ /** diff --git a/src/H5Zmodule.h b/src/H5Zmodule.h index 4458481d36f..360af0911d8 100644 --- a/src/H5Zmodule.h +++ b/src/H5Zmodule.h @@ -98,22 +98,18 @@ * * \snippet{doc} H5Zpublic.h FiltersIdTable * - * Filter identifiers for the filters distributed with the HDF5 - * Library are as follows: - * - * - * - * - * - * - * - *
#H5Z_FILTER_DEFLATEThe gzip compression, or - * deflation, filter
#H5Z_FILTER_SZIPThe SZIP compression - * filter
#H5Z_FILTER_NBITThe N-bit compression - * filter
#H5Z_FILTER_SCALEOFFSETThe scale-offset - * compression filter
#H5Z_FILTER_SHUFFLEThe shuffle algorithm - * filter
#H5Z_FILTER_FLETCHER32The Fletcher32 checksum, - * or error checking, filter
+ * Filter identifiers for the filters distributed with the HDF5 Library are as follows: +//! [PreDefFilters] + + + + + + + +
#H5Z_FILTER_DEFLATEThe gzip compression, or deflation, filter
#H5Z_FILTER_SZIPThe SZIP compressionfilter
#H5Z_FILTER_NBITThe N-bit compression filter
#H5Z_FILTER_SCALEOFFSETThe scale-offset compression filter
#H5Z_FILTER_SHUFFLEThe shuffle algorithm filter
#H5Z_FILTER_FLETCHER32The Fletcher32 checksum, or error checking, filter
+//! [PreDefFilters] + * * Custom filters that have been registered with the library will have * additional unique identifiers. * diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 06512f56435..f8d8448d84d 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -17,40 +17,40 @@ /** * \brief Filter identifiers * - * [FiltersIdTable] - - - - - - - - - - - - - - - - - - - - -
Values for #H5Z_filter_tDescription
0-255These values are reserved for filters predefined and - registered by the HDF5 library and of use to the general public.
256-511Filter values in this range are intended for testing only and can be - temporarily used by any organization. No attempts are made to resolve - numbering conflicts, as all definitions are temporary.
512-32,767Filter values within this range are designated for filters managed by - The HDF Group, but they are nominally requested, developed, and supported - by third parties. Please contact the - HDF5 development team - to reserve a value or range of values for use by your filters.
32,768-65,535Filter values in this range are designated for internal company use or - application testing when assessing a feature. The HDF Group does not - track or document the use of filters within this range. -
- * [FiltersIdTable] -*/ +//! [FiltersIdTable] + + + + + + + + + + + + + + + + + + + + +
Values for #H5Z_filter_tDescription
0-255These values are reserved for filters predefined and + registered by the HDF5 library and of use to the general public.
256-511Filter values in this range are intended for testing only and can be + temporarily used by any organization. No attempts are made to resolve + numbering conflicts, as all definitions are temporary.
512-32,767Filter values within this range are designated for filters managed by + The HDF Group, but they are nominally requested, developed, and supported + by third parties. Please contact the + HDF5 development team + to reserve a value or range of values for use by your filters.
32,768-65,535Filter values in this range are designated for internal company use or + application testing when assessing a feature. The HDF Group does not + track or document the use of filters within this range. +
+//! [FiltersIdTable] + */ typedef int H5Z_filter_t; diff --git a/src/H5module.h b/src/H5module.h index a5a448af6bf..cc63f6aec42 100644 --- a/src/H5module.h +++ b/src/H5module.h @@ -51,8 +51,7 @@ * The Abstract Data Model is a conceptual model of data, data types, and data organization. The * abstract data model is independent of storage medium or programming environment. The * Storage Model is a standard representation for the objects of the abstract data model. The - * HDF5 File Format Specification - * defines the storage model. + * \ref_spec_fileformat defines the storage model. * * The Programming Model is a model of the computing environment and includes platforms from * small single systems to large multiprocessors and clusters. The programming model manipulates @@ -104,7 +103,7 @@ * blocks. * * For more information on how these objects are organized; - * see HDF5 File Format Specification + * see \ref_spec_fileformat * * The HDF5 library can also use other * libraries and modules such as compression. @@ -129,8 +128,7 @@ * Specification. The organization of the data of application program, and how it is mapped to the * HDF5 abstract data model is up to the application developer. The application program only * needs to deal with the library and the abstract data model. Most applications need not consider - * any details of the - * HDF5 File Format Specification + * any details of the \ref_spec_fileformat * or the details of how objects of abstract data model are translated to and from storage. * * \subsection subsec_data_model_abstract The Abstract Data Model @@ -413,15 +411,14 @@ * * \subsection subsec_data_model_storage The HDF5 Storage Model * \subsubsection subsubsec_data_model_storage_spec The Abstract Storage Model: the HDF5 Format Specification - * The HDF5 File Format Specification - * defines how HDF5 objects and data are mapped to a linear + * The \ref_spec_fileformat defines how HDF5 objects and data are mapped to a linear * address space. The address space is assumed to be a contiguous array of bytes stored on some * random access medium. The format defines the standard for how the objects of the abstract data * model are mapped to linear addresses. The stored representation is self-describing in the sense * that the format defines all the information necessary to read and reconstruct the original objects * of the abstract data model. * - * The HDF5 File Format Specification is organized in three parts: + * The \ref_spec_fileformat is organized in three parts: *
  • Level 0: File signature and super block
    • *
  • Level 1: File infrastructure
    • *
    • Level 1A: B-link trees and B-tree nodes
      • @@ -445,7 +442,7 @@ * It is important to realize that the structures defined in the HDF5 file format are not the same as * the abstract data model: the object headers, heaps, and B-trees of the file specification are not * represented in the abstract data model. The format defines a number of objects for managing the - * storage including header blocks, B-trees, and heaps. The HDF5 File Format Specification defines + * storage including header blocks, B-trees, and heaps. The \ref_spec_fileformat defines * how the abstract objects (for example, groups and datasets) are represented as headers, B-tree * blocks, and other elements. * @@ -783,7 +780,7 @@ * item must be closed separately. * * For more information, - * @see Using Identifiers + * @see \ref UsingIdentifiers * in the HDF5 Application Developer's Guide under General Topics in HDF5. * *

      How Closing a File Effects Other Open Structural Elements

      @@ -1141,7 +1138,7 @@ * * * - * HDF5 requires the use of chunking when defining extendable datasets. Chunking makes it + * HDF5 requires the use of chunking when defining extendable datasets. \ref hdf5_chunking makes it * possible to extend datasets efficiently without having to reorganize contiguous storage * excessively. * diff --git a/src/H5public.h b/src/H5public.h index a49a4b34ad1..85e274ee422 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -127,7 +127,7 @@ * The #H5_VERSION_GE macro compares the version of the HDF5 library * being used against the version number specified in the parameters. * - * For more information about release versioning, see \ref_h5lib_relver. + * For more information about release versioning, see \ref RELVERSION. * * \since 1.8.7 * @@ -163,7 +163,7 @@ * The #H5_VERSION_LE macro compares the version of the HDF5 library * being used against the version number specified in the parameters. * - * For more information about release versioning, see \ref_h5lib_relver. + * For more information about release versioning, see \ref RELVERSION. * * \since 1.8.7 *