mirror of
https://github.com/HDFGroup/hdf5.git
synced 2026-09-25 04:09:44 +03:00
272 lines
39 KiB
HTML
272 lines
39 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
|
|
<meta http-equiv="X-UA-Compatible" content="IE=11"/>
|
|
<meta name="generator" content="Doxygen 1.16.1"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
<title>HDF5: HDF5 Parallel Compression</title>
|
|
<link href="tabs.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="jquery.js"></script>
|
|
<script type="text/javascript" src="dynsections.js"></script>
|
|
<link href="navtree.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="navtreedata.js"></script>
|
|
<script type="text/javascript" src="navtree.js"></script>
|
|
<script type="text/javascript" src="cookie.js"></script>
|
|
<link href="search/search.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="search/searchdata.js"></script>
|
|
<script type="text/javascript" src="search/search.js"></script>
|
|
<script type="text/javascript">
|
|
$(function() { init_search(); });
|
|
</script>
|
|
<link href="doxygen.css" rel="stylesheet" type="text/css" />
|
|
<link href="hdf5doxy.css" rel="stylesheet" type="text/css"/>
|
|
<link href="doxygen-awesome.css" rel="stylesheet" type="text/css"/>
|
|
<link href="hdf5doxy.css" rel="stylesheet" type="text/css">
|
|
<script type="text/javascript" src="hdf5_navtree_hacks.js"></script>
|
|
<div style="background:#FFDDDD;font-size:120%;text-align:center;margin:0;padding:5px">Help us improve by taking our short survey: <a href="https://www.hdfgroup.org/website-survey/">https://www.hdfgroup.org/website-survey/</a></div>
|
|
<!-- ... other metadata & script includes ... -->
|
|
<script type="text/javascript" src="doxygen-awesome-tabs.js"></script>
|
|
<script type="text/javascript">
|
|
DoxygenAwesomeTabs.init()
|
|
</script>
|
|
<!-- Google tag (gtag.js) -->
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-57FMZK4S7X"></script>
|
|
<script>
|
|
window.dataLayer = window.dataLayer || [];
|
|
function gtag(){dataLayer.push(arguments);}
|
|
gtag('js', new Date());
|
|
gtag('config', 'G-57FMZK4S7X');
|
|
</script>
|
|
</head>
|
|
<body>
|
|
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
|
|
<div id="titlearea">
|
|
<table cellspacing="0" cellpadding="0">
|
|
<tbody>
|
|
<tr id="projectrow">
|
|
<td id="projectlogo"><img alt="Logo" src="HDFG-logo.png"/></td>
|
|
<td id="projectalign">
|
|
<div id="projectname">HDF5<span id="projectnumber"> Last Updated on 2026-09-24</span>
|
|
</div>
|
|
<div id="projectbrief">The HDF5 Field Guide</div>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<!-- end header part -->
|
|
<!-- Generated by Doxygen 1.16.1 -->
|
|
<script type="text/javascript">
|
|
var searchBox = new SearchBox("searchBox", "search/",'.html');
|
|
</script>
|
|
<script type="text/javascript">
|
|
$(function() { codefold.init(); });
|
|
</script>
|
|
<div id="main-nav">
|
|
<div id="navrow1" class="tabs">
|
|
<ul class="tablist">
|
|
<li><a href="index.html"><span>Main Page</span></a></li>
|
|
<li><a href="_getting_started.html"><span>Getting started</span></a></li>
|
|
<li><a href="_u_g.html"><span>User Guide</span></a></li>
|
|
<li><a href="_r_m.html"><span>Reference Manual</span></a></li>
|
|
<li><a href="_cookbook.html"><span>Cookbook</span></a></li>
|
|
<li><a href="_t_n.html"><span>Technical Notes</span></a></li>
|
|
<li><a href="_r_f_c.html"><span>RFCs</span></a></li>
|
|
<li><a href="_s_p_e_c.html"><span>Specifications</span></a></li>
|
|
<li><a href="_g_l_s.html"><span>Glossary</span></a></li>
|
|
<li><a href="_f_t_s.html"><span>Full-Text Search</span></a></li>
|
|
<li><a href="_about.html"><span>About</span></a></li>
|
|
<li>
|
|
<div id="MSearchBox" class="MSearchBoxInactive">
|
|
<span class="left">
|
|
<span id="MSearchSelect" class="search-icon" onmouseover="return searchBox.OnSearchSelectShow()" onmouseout="return searchBox.OnSearchSelectHide()"><span class="search-icon-dropdown"></span></span>
|
|
<input type="text" id="MSearchField" value="" placeholder="Search" accesskey="S"
|
|
onfocus="searchBox.OnSearchFieldFocus(true)"
|
|
onblur="searchBox.OnSearchFieldFocus(false)"
|
|
onkeyup="searchBox.OnSearchFieldChange(event)"/>
|
|
</span><span class="right">
|
|
<a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><div id="MSearchCloseImg" class="close-icon"></div></a>
|
|
</span>
|
|
</div>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div><!-- main-nav -->
|
|
</div><!-- top -->
|
|
<div id="side-nav" class="ui-resizable side-nav-resizable">
|
|
<div id="nav-tree">
|
|
<div id="nav-tree-contents">
|
|
<div id="nav-sync" class="sync"></div>
|
|
</div>
|
|
</div>
|
|
<div id="splitbar" style="-moz-user-select:none;"
|
|
class="ui-resizable-handle">
|
|
</div>
|
|
</div>
|
|
<script type="text/javascript">
|
|
$(function(){initNavTree('_par_compr.html','',''); });
|
|
</script>
|
|
<div id="container">
|
|
<div id="doc-content">
|
|
<!-- window showing the filter options -->
|
|
<div id="MSearchSelectWindow"
|
|
onmouseover="return searchBox.OnSearchSelectShow()"
|
|
onmouseout="return searchBox.OnSearchSelectHide()"
|
|
onkeydown="return searchBox.OnSearchSelectKey(event)">
|
|
</div>
|
|
|
|
<!-- iframe showing the search results (closed by default) -->
|
|
<div id="MSearchResultsWindow">
|
|
<div id="MSearchResults">
|
|
<div class="SRPage">
|
|
<div id="SRIndex">
|
|
<div id="SRResults"></div>
|
|
<div class="SRStatus" id="Loading">Loading...</div>
|
|
<div class="SRStatus" id="Searching">Searching...</div>
|
|
<div class="SRStatus" id="NoMatches">No Matches</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div><div class="header">
|
|
<div class="headertitle"><div class="title">HDF5 Parallel Compression </div></div>
|
|
</div><!--header-->
|
|
<div class="contents">
|
|
<div class="textblock"><p>Navigate back: <a class="el" href="index.html" title="notitle">Main</a> / <a class="el" href="_t_n.html" title="Technical Notes">Technical Notes</a> </p><hr />
|
|
<h1 class="doxsection"><a class="anchor" id="sec_parcompr_intro"></a>
|
|
Introduction</h1>
|
|
<p>When an HDF5 dataset is created, the application can specify optional data filters to be applied to the dataset (as long as the dataset uses a chunked data layout). These filters may perform compression, shuffling, checksumming/error detection and more on the dataset data. The filters are added to a filter pipeline for the dataset and are automatically applied to the data during dataset writes and reads.</p>
|
|
<p>Prior to the HDF5 1.10.2 release, a parallel HDF5 application could read datasets with filters applied to them, but could not write to those datasets in parallel. The datasets would have to first be written in a serial HDF5 application or from a single MPI rank in a parallel HDF5 application. This restriction was in place because:</p>
|
|
<ul>
|
|
<li>Updating the data in filtered datasets requires management of file metadata, such as the dataset's chunk index and file space for data chunks, which must be done collectively in order for MPI ranks to have a consistent view of the file. At the time, HDF5 lacked the collective coordination of this metadata management.</li>
|
|
<li>When multiple MPI ranks are writing independently to the same chunk in a dataset (even if their selected portions of the chunk don't overlap), the whole chunk has to be read, unfiltered, modified, re-filtered and then written back to disk. This read-modify-write style of operation would cause conflicts among the MPI ranks and lead to an inconsistent view of the file.</li>
|
|
</ul>
|
|
<p>Introduced in the HDF5 1.10.2 release, the parallel compression feature allows an HDF5 application to write in parallel to datasets with filters applied to them, as long as collective I/O is used. The feature introduces new internal infrastructure that coordinates the collective management of the file metadata between MPI ranks during dataset writes. It also accounts for multiple MPI ranks writing to a chunk by assigning ownership to one of the MPI ranks, at which point the other MPI ranks send their modifications to the owning MPI rank.</p>
|
|
<p>The parallel compression feature is always enabled when HDF5 is built with parallel enabled, but the feature may be disabled if the necessary MPI-3 routines are not available. Therefore, HDF5 conditionally defines the macro <code>H5_HAVE_PARALLEL_FILTERED_WRITES</code> which an application can check for to see if the feature is available.</p>
|
|
<h1 class="doxsection"><a class="anchor" id="sec_parcompr_ex"></a>
|
|
Examples</h1>
|
|
<p>Using the parallel compression feature is very similar to using compression in serial HDF5, except that dataset writes <b>must</b> be collective:</p>
|
|
<div class="fragment"><div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> dxpl_id = <a class="code hl_function" href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a>(<a class="code hl_define" href="_h5_ppublic_8h.html#a6f9c8a5aba72c0445fff384bf418a80d">H5P_DATASET_XFER</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___d_x_p_l.html#ga22837d8504dc1f87f175b46b348ce0e5">H5Pset_dxpl_mpio</a>(dxpl_id, <a class="code hl_enumvalue" href="_h5_f_dmpi_8h.html#a99bc5a964089fea144e7056b004bcc16a75d4dc80546ad3c16d2d7647ab267fab">H5FD_MPIO_COLLECTIVE</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___h5_d.html#ga98f44998b67587662af8b0d8a0a75906">H5Dwrite</a>(..., dxpl_id, ...);</div>
|
|
<div class="ttc" id="a_h5_f_dmpi_8h_html_a99bc5a964089fea144e7056b004bcc16a75d4dc80546ad3c16d2d7647ab267fab"><div class="ttname"><a href="_h5_f_dmpi_8h.html#a99bc5a964089fea144e7056b004bcc16a75d4dc80546ad3c16d2d7647ab267fab">H5FD_MPIO_COLLECTIVE</a></div><div class="ttdeci">@ H5FD_MPIO_COLLECTIVE</div><div class="ttdef"><b>Definition</b> H5FDmpi.h:38</div></div>
|
|
<div class="ttc" id="a_h5_ipublic_8h_html_a0045db7ff9c22ad35db6ae91662e1943"><div class="ttname"><a href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a></div><div class="ttdeci">int64_t hid_t</div><div class="ttdef"><b>Definition</b> H5Ipublic.h:60</div></div>
|
|
<div class="ttc" id="a_h5_ppublic_8h_html_a6f9c8a5aba72c0445fff384bf418a80d"><div class="ttname"><a href="_h5_ppublic_8h.html#a6f9c8a5aba72c0445fff384bf418a80d">H5P_DATASET_XFER</a></div><div class="ttdeci">#define H5P_DATASET_XFER</div><div class="ttdef"><b>Definition</b> H5Ppublic.h:68</div></div>
|
|
<div class="ttc" id="agroup___d_x_p_l_html_ga22837d8504dc1f87f175b46b348ce0e5"><div class="ttname"><a href="group___d_x_p_l.html#ga22837d8504dc1f87f175b46b348ce0e5">H5Pset_dxpl_mpio</a></div><div class="ttdeci">herr_t H5Pset_dxpl_mpio(hid_t dxpl_id, H5FD_mpio_xfer_t xfer_mode)</div><div class="ttdoc">Sets data transfer mode.</div></div>
|
|
<div class="ttc" id="agroup___h5_d_html_ga98f44998b67587662af8b0d8a0a75906"><div class="ttname"><a href="group___h5_d.html#ga98f44998b67587662af8b0d8a0a75906">H5Dwrite</a></div><div class="ttdeci">herr_t H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, const void *buf)</div><div class="ttdoc">Writes raw data from a buffer to a dataset.</div></div>
|
|
<div class="ttc" id="agroup___p_l_c_r_html_gaf1b11da01d4d45d788c45f8bc5f0cbfa"><div class="ttname"><a href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a></div><div class="ttdeci">hid_t H5Pcreate(hid_t cls_id)</div><div class="ttdoc">Creates a new property list as an instance of a property list class.</div></div>
|
|
</div><!-- fragment --><p>The following are two simple examples of using the parallel compression feature:</p>
|
|
<p><a href="https://github.com/HDFGroup/hdf5/blob/develop/HDF5Examples/C/H5PAR/ph5_filtered_writes.c">ph5_filtered_writes.c</a></p>
|
|
<p><a href="https://github.com/HDFGroup/hdf5/blob/develop/HDF5Examples/C/H5PAR/ph5_filtered_writes_no_sel.c">ph5_filtered_writes_no_sel.c</a></p>
|
|
<p>The former contains simple examples of using the parallel compression feature to write to compressed datasets, while the latter contains an example of how to write to compressed datasets when one or MPI ranks don't have any data to write to a dataset. Remember that the feature requires these writes to use collective I/O, so the MPI ranks which have nothing to contribute must still participate in the collective write call.</p>
|
|
<h1 class="doxsection"><a class="anchor" id="sec_parcompr_multi"></a>
|
|
Multi-dataset I/O support</h1>
|
|
<p>The parallel compression feature is supported when using the multi-dataset I/O API routines (<a class="el" href="group___h5_d.html#gaf6213bf3a876c1741810037ff2bb85d8" title="Writes raw data from a set buffers to a set of datasets.">H5Dwrite_multi</a>/<a class="el" href="group___h5_d.html#ga8eb1c838aff79a17de385d0707709915" title="Reads raw data from a set of datasets into the provided buffers.">H5Dread_multi</a>), but the following should be kept in mind:</p>
|
|
<ul>
|
|
<li>Parallel writes to filtered datasets <b>must</b> still be collective, even when using the multi-dataset I/O API routines</li>
|
|
<li>When the multi-dataset I/O API routines are passed a mixture of filtered and unfiltered datasets, the library currently has to perform I/O on them separately in two phases. Since there is some slight complexity involved in this, it may be best (depending on the number of datasets, number of selected chunks, number of filtered vs. unfiltered datasets, etc.) to make two individual multi-dataset I/O calls, one for the filtered datasets and one for the unfiltered datasets. When performing writes to the datasets, this would also allow independent write access to the unfiltered datasets if desired, while still performing collective writes to the filtered datasets.</li>
|
|
</ul>
|
|
<h1 class="doxsection"><a class="anchor" id="sec_parcompr_incr"></a>
|
|
Incremental file space allocation support</h1>
|
|
<p>HDF5's file space allocation time function. <a class="el" href="group___d_c_p_l.html#ga85faefca58387bba409b65c470d7d851" title="Sets the timing for storage space allocation.">H5Pset_alloc_time</a>, is a dataset creation property that can have significant effects on application performance, especially if the application uses parallel HDF5. In a serial HDF5 application, the default file space allocation time for chunked datasets is <b>incremental</b>. This means that allocation of space in the HDF5 file for data chunks is deferred until data is first written to those chunks. In parallel HDF5, the file space allocation time was previously always forced to <b>early</b>, which allocates space in the file for all of a dataset's data chunks at creation time (or during the first open of a dataset if it was created serially). This would ensure that all the necessary file space was allocated so MPI ranks could perform independent I/O operations on a dataset without needing further coordination of file metadata as described previously.</p>
|
|
<p>While this strategy has worked in the past, it has some noticeable drawbacks. For one, the larger the chunked dataset being created, the more noticeable overhead there will be during dataset creation as all of the data chunks are being allocated in the HDF5 file. Further, these data chunks will, by default, be filled, using <a class="el" href="group___d_c_p_l.html#ga4335bb45b35386daa837b4ff1b9cd4a4" title="Sets the fill value for a dataset.">H5Pset_fill_value</a>, with HDF5's default fill data value, leading to extraordinary dataset creation overhead and resulting in pre-filling large portions of a dataset that the application might have been planning to overwrite anyway. Even worse, there will be more initial overhead from compressing that fill data before writing it out, only to have it read back in, unfiltered and modified the first time a chunk is written to. In the past, it was typically suggested that parallel HDF5 applications should use <a class="el" href="group___d_c_p_l.html#ga6bd822266b31f86551a9a1d79601b6a2" title="Sets the time when fill values are written to a dataset.">H5Pset_fill_time</a> with a value of <a class="el" href="_h5_dpublic_8h.html#aa39293626c4e68dd28b06c0dc84bde4aaa87fbf4f3ebf96f2f3effe7bf46c1528">H5D_FILL_TIME_NEVER</a> in order to disable writing of the fill value to dataset chunks, but this isn't ideal if the application actually wishes to make use of fill values.</p>
|
|
<p>With <a href="https://www.hdfgroup.org/2022/03/04/parallel-compression-improvements-in-hdf5-1-13-1/">improvements made</a> to the parallel compression feature for the HDF5 1.14.0 release, <b>incremental</b> file space allocation is now the default for datasets created in parallel <em>only if they have filters applied to them</em>. <b>Early</b> file space allocation is still supported for these datasets if desired and is still forced for datasets created in parallel that do <em>not</em> have filters applied to them. This change should significantly reduce the overhead of creating filtered datasets in parallel HDF5 applications and should be helpful to applications that wish to use a fill value for these datasets. It should also help significantly reduce the size of the HDF5 file, as file space for the data chunks is allocated as needed rather than all at once.</p>
|
|
<h1 class="doxsection"><a class="anchor" id="sec_parcompr_perf"></a>
|
|
Performance Considerations</h1>
|
|
<p>Since getting good performance out of HDF5's parallel compression feature involves several factors, the following is a list of performance considerations (generally from most to least important) and best practices to take into account when trying to get the optimal performance out of the parallel compression feature.</p>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_begin"></a>
|
|
Begin with a good chunking strategy</h2>
|
|
<p>Starting with a good <a class="el" href="hdf5_chunking.html" title="Chunking in HDF5">Chunking in HDF5</a> strategy will generally have the largest impact on overall application performance. The different chunking parameters can be difficult to fine-tune, but it is essential to start with a well-performing chunking layout before adding compression and parallel I/O into the mix. Compression itself adds overhead and may have side effects that necessitate further adjustment of the chunking parameters and HDF5 application settings. Consider that the chosen chunk size becomes a very important factor when compression is involved, as data chunks have to be completely read and re-written to perform partial writes to the chunk.</p>
|
|
<p><a class="el" href="improve_compressed_perf.html" title="Improving I/O Performance When Working with HDF5 Compressed Datasets">Improving I/O Performance When Working with HDF5 Compressed Datasets</a> is a useful reference for more information on getting good performance when using a chunked dataset layout.</p>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_avoid"></a>
|
|
Avoid chunk sharing</h2>
|
|
<p>Since the parallel compression feature has to assign ownership of data chunks to a single MPI rank in order to avoid the previously described read-modify-write issue, an HDF5 application may need to take care when determining how a dataset will be divided up among the MPI ranks writing to it. Each dataset data chunk that is written to by more than 1 MPI rank will incur extra MPI overhead as one of the ranks takes ownership and the other ranks send it their data and information about where in the chunk that data belongs. While not always possible to do, an HDF5 application will get the best performance out of parallel compression if it can avoid writing in a way that causes more than 1 MPI rank to write to any given data chunk in a dataset.</p>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_coll"></a>
|
|
Collective metadata operations</h2>
|
|
<p>The parallel compression feature typically works with a significant amount of metadata related to the management of the data chunks in datasets. In initial performance results gathered from various HPC machines, it was found that the parallel compression feature did not scale well at around 8k MPI ranks and beyond. On further investigation, it became obvious that the bottleneck was due to heavy filesystem pressure from the metadata management for dataset data chunks as they changed size (as a result of data compression) and moved around in the HDF5 file.</p>
|
|
<p>Enabling collective metadata operations in the HDF5 application (as in the below snippet) showed significant improvement in performance and scalability and is generally always recommended unless application performance shows negative benefits by doing so.</p>
|
|
<div class="fragment"><div class="line">...</div>
|
|
<div class="line">hid_t fapl_id = <a class="code hl_function" href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a>(<a class="code hl_define" href="_h5_ppublic_8h.html#a60ec2d4334addfc0eda89614598ee38e">H5P_FILE_ACCESS</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___f_a_p_l.html#ga7519d659a83ef5717d7a5d95baf0e9b1">H5Pset_fapl_mpio</a>(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);</div>
|
|
<div class="line"><a class="code hl_function" href="group___g_a_p_l.html#gac45976cb82e4d3f085f191e5872d0316">H5Pset_all_coll_metadata_ops</a>(fapl_id, 1);</div>
|
|
<div class="line"><a class="code hl_function" href="group___f_a_p_l.html#ga0163bef7ee102029286830c2c30162ca">H5Pset_coll_metadata_write</a>(fapl_id, 1);</div>
|
|
<div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> file_id = <a class="code hl_function" href="group___h5_f.html#gae64b51ee9ac0781bc4ccc599d98387f4">H5Fcreate</a>(<span class="stringliteral">"file.h5"</span>, <a class="code hl_define" href="_h5_fpublic_8h.html#a5a2d6726f9ad8d2bca8df2b817e5ad6a">H5F_ACC_TRUNC</a>, <a class="code hl_define" href="_h5_ppublic_8h.html#afa85e97bfbf9bf1c58e39263846c568f">H5P_DEFAULT</a>, fapl_id);</div>
|
|
<div class="line">...</div>
|
|
<div class="ttc" id="a_h5_fpublic_8h_html_a5a2d6726f9ad8d2bca8df2b817e5ad6a"><div class="ttname"><a href="_h5_fpublic_8h.html#a5a2d6726f9ad8d2bca8df2b817e5ad6a">H5F_ACC_TRUNC</a></div><div class="ttdeci">#define H5F_ACC_TRUNC</div><div class="ttdef"><b>Definition</b> H5Fpublic.h:30</div></div>
|
|
<div class="ttc" id="a_h5_ppublic_8h_html_a60ec2d4334addfc0eda89614598ee38e"><div class="ttname"><a href="_h5_ppublic_8h.html#a60ec2d4334addfc0eda89614598ee38e">H5P_FILE_ACCESS</a></div><div class="ttdeci">#define H5P_FILE_ACCESS</div><div class="ttdef"><b>Definition</b> H5Ppublic.h:56</div></div>
|
|
<div class="ttc" id="a_h5_ppublic_8h_html_afa85e97bfbf9bf1c58e39263846c568f"><div class="ttname"><a href="_h5_ppublic_8h.html#afa85e97bfbf9bf1c58e39263846c568f">H5P_DEFAULT</a></div><div class="ttdeci">#define H5P_DEFAULT</div><div class="ttdef"><b>Definition</b> H5Ppublic.h:220</div></div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_ga0163bef7ee102029286830c2c30162ca"><div class="ttname"><a href="group___f_a_p_l.html#ga0163bef7ee102029286830c2c30162ca">H5Pset_coll_metadata_write</a></div><div class="ttdeci">herr_t H5Pset_coll_metadata_write(hid_t plist_id, bool is_collective)</div><div class="ttdoc">Sets metadata write mode to be collective or independent (default).</div></div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_ga7519d659a83ef5717d7a5d95baf0e9b1"><div class="ttname"><a href="group___f_a_p_l.html#ga7519d659a83ef5717d7a5d95baf0e9b1">H5Pset_fapl_mpio</a></div><div class="ttdeci">herr_t H5Pset_fapl_mpio(hid_t fapl_id, MPI_Comm comm, MPI_Info info)</div><div class="ttdoc">Stores MPI IO communicator information to the file access property list.</div></div>
|
|
<div class="ttc" id="agroup___g_a_p_l_html_gac45976cb82e4d3f085f191e5872d0316"><div class="ttname"><a href="group___g_a_p_l.html#gac45976cb82e4d3f085f191e5872d0316">H5Pset_all_coll_metadata_ops</a></div><div class="ttdeci">herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, bool is_collective)</div><div class="ttdoc">Sets metadata I/O mode for read operations to be collective or independent (default).</div></div>
|
|
<div class="ttc" id="agroup___h5_f_html_gae64b51ee9ac0781bc4ccc599d98387f4"><div class="ttname"><a href="group___h5_f.html#gae64b51ee9ac0781bc4ccc599d98387f4">H5Fcreate</a></div><div class="ttdeci">hid_t H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)</div><div class="ttdoc">Creates an HDF5 file.</div></div>
|
|
</div><!-- fragment --><h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_align"></a>
|
|
Align chunks in the file</h2>
|
|
<p>The natural layout of an HDF5 file may cause dataset data chunks to end up at addresses in the file that do not align well with the underlying file system, possibly leading to poor performance. As an example, Lustre performance is generally good when writes are aligned with the chosen stripe size. The HDF5 application can use <a class="el" href="group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a" title="Sets alignment properties of a file access property list.">H5Pset_alignment</a> to have a bit more control over where objects in the HDF5 file end up. However, do note that setting the alignment of objects generally wastes space in the file and has the potential to dramatically increase its resulting size, so caution should be used when choosing the alignment parameters.</p>
|
|
<p><a class="el" href="group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a" title="Sets alignment properties of a file access property list.">H5Pset_alignment</a> has two parameters that control the alignment of objects in the HDF5 file, the "threshold" value and the alignment value. The threshold value specifies that any object greater than or equal in size to that value will be aligned in the file at addresses which are multiples of the chosen alignment value. While the value 0 can be specified for the threshold to make every object in the file be aligned according to the alignment value, this isn't generally recommended, as it will likely waste an excessive amount of space in the file.</p>
|
|
<p>In the example below, the chosen dataset chunk size is provided for the threshold value and 1MiB is specified for the alignment value. Assuming that 1MiB is an optimal alignment value (e.g., assuming that it matches well with the Lustre stripe size), this should cause dataset data chunks to be well-aligned and generally give good write performance.</p>
|
|
<div class="fragment"><div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> fapl_id = <a class="code hl_function" href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a>(<a class="code hl_define" href="_h5_ppublic_8h.html#a60ec2d4334addfc0eda89614598ee38e">H5P_FILE_ACCESS</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___f_a_p_l.html#ga7519d659a83ef5717d7a5d95baf0e9b1">H5Pset_fapl_mpio</a>(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);</div>
|
|
<div class="line"><span class="comment">/* Assuming Lustre stripe size is 1MiB, align data chunks</span></div>
|
|
<div class="line"><span class="comment"> in the file to address multiples of 1MiB. */</span></div>
|
|
<div class="line"><a class="code hl_function" href="group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a">H5Pset_alignment</a>(fapl_id, dataset_chunk_size, 1048576);</div>
|
|
<div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> file_id = <a class="code hl_function" href="group___h5_f.html#gae64b51ee9ac0781bc4ccc599d98387f4">H5Fcreate</a>(<span class="stringliteral">"file.h5"</span>, <a class="code hl_define" href="_h5_fpublic_8h.html#a5a2d6726f9ad8d2bca8df2b817e5ad6a">H5F_ACC_TRUNC</a>, <a class="code hl_define" href="_h5_ppublic_8h.html#afa85e97bfbf9bf1c58e39263846c568f">H5P_DEFAULT</a>, fapl_id);</div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_gab99d5af749aeb3896fd9e3ceb273677a"><div class="ttname"><a href="group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a">H5Pset_alignment</a></div><div class="ttdeci">herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold, hsize_t alignment)</div><div class="ttdoc">Sets alignment properties of a file access property list.</div></div>
|
|
</div><!-- fragment --><h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_space"></a>
|
|
File free space managers</h2>
|
|
<p>As data chunks in a dataset get written to and compressed, they can change in size and be relocated in the HDF5 file. Since parallel compression usually involves many data chunks in a file, this can create significant amounts of free space in the file over its lifetime and eventually cause performance issues.</p>
|
|
<p>An HDF5 application can use <a class="el" href="group___f_c_p_l.html#gae9ed9b56f290d6d24421242f1c04914e" title="Sets the file space handling strategy and persisting free-space values for a file creation property l...">H5Pset_file_space_strategy</a> with a value of <a class="el" href="_h5_fpublic_8h.html#a9cc492c4b5c936e48716a8dab3691bccacd625bd864903e71132c9098929f5a0a">H5F_FSPACE_STRATEGY_PAGE</a> to enable the paged aggregation feature, which can accumulate metadata and raw data for dataset data chunks into well-aligned, configurably sized <b>pages</b> for better performance. However, note that using the paged aggregation feature will cause any setting from <a class="el" href="group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a" title="Sets alignment properties of a file access property list.">H5Pset_alignment</a> to be ignored. While an application should be able to get comparable performance effects by setting the size of these pages, using <a class="el" href="group___f_c_p_l.html#gad012d7f3c2f1e1999eb1770aae3a4963" title="Sets the file space page size for a file creation property list.">H5Pset_file_space_page_size</a>, to be equal to the value that would have been set for <a class="el" href="group___f_a_p_l.html#gab99d5af749aeb3896fd9e3ceb273677a" title="Sets alignment properties of a file access property list.">H5Pset_alignment</a>, this may not necessarily be the case and should be studied.</p>
|
|
<p>Note that <a class="el" href="group___f_c_p_l.html#gae9ed9b56f290d6d24421242f1c04914e" title="Sets the file space handling strategy and persisting free-space values for a file creation property l...">H5Pset_file_space_strategy</a> has a <b>persist</b> parameter. This determines whether or not the file free space manager should include extra metadata in the HDF5 file about free space sections in the file. If this parameter is <b>false</b>, any free space in the HDF5 file will become unusable once the HDF5 file is closed. For parallel compression, it's generally recommended that <b>persist</b> be set to <b>true</b>, as this will keep better track of file free space for data chunks between accesses to the HDF5 file.</p>
|
|
<div class="fragment"><div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> fcpl_id = <a class="code hl_function" href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a>(<a class="code hl_define" href="_h5_ppublic_8h.html#a206f334f1e6c973e1215a3148b45b977">H5P_FILE_CREATE</a>);</div>
|
|
<div class="line"><span class="comment">/* Use persistent free space manager with paged aggregation */</span></div>
|
|
<div class="line"><a class="code hl_function" href="group___f_c_p_l.html#gae9ed9b56f290d6d24421242f1c04914e">H5Pset_file_space_strategy</a>(fcpl_id, <a class="code hl_enumvalue" href="_h5_fpublic_8h.html#a9cc492c4b5c936e48716a8dab3691bccacd625bd864903e71132c9098929f5a0a">H5F_FSPACE_STRATEGY_PAGE</a>, 1, 1);</div>
|
|
<div class="line"><span class="comment">/* Assuming Lustre stripe size is 1MiB, set page size to that */</span></div>
|
|
<div class="line"><a class="code hl_function" href="group___f_c_p_l.html#gad012d7f3c2f1e1999eb1770aae3a4963">H5Pset_file_space_page_size</a>(fcpl_id, 1048576);</div>
|
|
<div class="line">...</div>
|
|
<div class="line">hid_t file_id = <a class="code hl_function" href="group___h5_f.html#gae64b51ee9ac0781bc4ccc599d98387f4">H5Fcreate</a>(<span class="stringliteral">"file.h5"</span>, <a class="code hl_define" href="_h5_fpublic_8h.html#a5a2d6726f9ad8d2bca8df2b817e5ad6a">H5F_ACC_TRUNC</a>, fcpl_id, fapl_id);</div>
|
|
<div class="ttc" id="a_h5_fpublic_8h_html_a9cc492c4b5c936e48716a8dab3691bccacd625bd864903e71132c9098929f5a0a"><div class="ttname"><a href="_h5_fpublic_8h.html#a9cc492c4b5c936e48716a8dab3691bccacd625bd864903e71132c9098929f5a0a">H5F_FSPACE_STRATEGY_PAGE</a></div><div class="ttdeci">@ H5F_FSPACE_STRATEGY_PAGE</div><div class="ttdef"><b>Definition</b> H5Fpublic.h:198</div></div>
|
|
<div class="ttc" id="a_h5_ppublic_8h_html_a206f334f1e6c973e1215a3148b45b977"><div class="ttname"><a href="_h5_ppublic_8h.html#a206f334f1e6c973e1215a3148b45b977">H5P_FILE_CREATE</a></div><div class="ttdeci">#define H5P_FILE_CREATE</div><div class="ttdef"><b>Definition</b> H5Ppublic.h:52</div></div>
|
|
<div class="ttc" id="agroup___f_c_p_l_html_gad012d7f3c2f1e1999eb1770aae3a4963"><div class="ttname"><a href="group___f_c_p_l.html#gad012d7f3c2f1e1999eb1770aae3a4963">H5Pset_file_space_page_size</a></div><div class="ttdeci">herr_t H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size)</div><div class="ttdoc">Sets the file space page size for a file creation property list.</div></div>
|
|
<div class="ttc" id="agroup___f_c_p_l_html_gae9ed9b56f290d6d24421242f1c04914e"><div class="ttname"><a href="group___f_c_p_l.html#gae9ed9b56f290d6d24421242f1c04914e">H5Pset_file_space_strategy</a></div><div class="ttdeci">herr_t H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, bool persist, hsize_t threshold)</div><div class="ttdoc">Sets the file space handling strategy and persisting free-space values for a file creation property l...</div></div>
|
|
</div><!-- fragment --><h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_low"></a>
|
|
Low-level collective vs. independent I/O</h2>
|
|
<p>While the parallel compression feature requires that the HDF5 application set and maintain collective I/O at the application interface level (via <a class="el" href="group___d_x_p_l.html#ga22837d8504dc1f87f175b46b348ce0e5" title="Sets data transfer mode.">H5Pset_dxpl_mpio</a>), it does not require that the actual MPI I/O that occurs at the lowest layers of HDF5 be collective; independent I/O may perform better depending on the application I/O patterns and parallel file system performance, among other factors. The application may use <a class="el" href="group___d_x_p_l.html#gadd80f197d0e03841c5e7f0f4f02d4103" title="Sets low-level data transfer mode.">H5Pset_dxpl_mpio_collective_opt</a> to control this setting and see which I/O method provides the best performance.</p>
|
|
<div class="fragment"><div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> dxpl_id = <a class="code hl_function" href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a>(<a class="code hl_define" href="_h5_ppublic_8h.html#a6f9c8a5aba72c0445fff384bf418a80d">H5P_DATASET_XFER</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___d_x_p_l.html#ga22837d8504dc1f87f175b46b348ce0e5">H5Pset_dxpl_mpio</a>(dxpl_id, <a class="code hl_enumvalue" href="_h5_f_dmpi_8h.html#a99bc5a964089fea144e7056b004bcc16a75d4dc80546ad3c16d2d7647ab267fab">H5FD_MPIO_COLLECTIVE</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___d_x_p_l.html#gadd80f197d0e03841c5e7f0f4f02d4103">H5Pset_dxpl_mpio_collective_opt</a>(dxpl_id, <a class="code hl_enumvalue" href="_h5_f_dmpi_8h.html#afaf7d5667632176e8daca47549e29fb8aaacd91139f703159fde84fb5f7778886">H5FD_MPIO_INDIVIDUAL_IO</a>); <span class="comment">/* Try independent I/O */</span></div>
|
|
<div class="line"><a class="code hl_function" href="group___h5_d.html#ga98f44998b67587662af8b0d8a0a75906">H5Dwrite</a>(..., dxpl_id, ...);</div>
|
|
<div class="ttc" id="a_h5_f_dmpi_8h_html_afaf7d5667632176e8daca47549e29fb8aaacd91139f703159fde84fb5f7778886"><div class="ttname"><a href="_h5_f_dmpi_8h.html#afaf7d5667632176e8daca47549e29fb8aaacd91139f703159fde84fb5f7778886">H5FD_MPIO_INDIVIDUAL_IO</a></div><div class="ttdeci">@ H5FD_MPIO_INDIVIDUAL_IO</div><div class="ttdef"><b>Definition</b> H5FDmpi.h:51</div></div>
|
|
<div class="ttc" id="agroup___d_x_p_l_html_gadd80f197d0e03841c5e7f0f4f02d4103"><div class="ttname"><a href="group___d_x_p_l.html#gadd80f197d0e03841c5e7f0f4f02d4103">H5Pset_dxpl_mpio_collective_opt</a></div><div class="ttdeci">herr_t H5Pset_dxpl_mpio_collective_opt(hid_t dxpl_id, H5FD_mpio_collective_opt_t opt_mode)</div><div class="ttdoc">Sets low-level data transfer mode.</div></div>
|
|
</div><!-- fragment --><h2 class="doxsection"><a class="anchor" id="subsec_parcompr_perf_libver"></a>
|
|
Runtime HDF5 Library version</h2>
|
|
<p>An HDF5 application can use the <a class="el" href="group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910" title="Controls the range of library release versions used when creating objects in a file.">H5Pset_libver_bounds</a> routine to set the upper and lower bounds on library versions to use when creating HDF5 objects. For parallel compression specifically, setting the library version to the latest available version can allow access to better/more efficient chunk indexing types and data encoding methods. For example:</p>
|
|
<div class="fragment"><div class="line">...</div>
|
|
<div class="line">hid_t fapl_id = <a class="code hl_function" href="group___p_l_c_r.html#gaf1b11da01d4d45d788c45f8bc5f0cbfa">H5Pcreate</a>(<a class="code hl_define" href="_h5_ppublic_8h.html#a60ec2d4334addfc0eda89614598ee38e">H5P_FILE_ACCESS</a>);</div>
|
|
<div class="line"><a class="code hl_function" href="group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910">H5Pset_libver_bounds</a>(fapl_id, <a class="code hl_enumvalue" href="_h5_fpublic_8h.html#a2d963b599894f684571fbd4d5e8a96a2aa1212669916e7389d0a687a3673153b0">H5F_LIBVER_LATEST</a>, <a class="code hl_enumvalue" href="_h5_fpublic_8h.html#a2d963b599894f684571fbd4d5e8a96a2aa1212669916e7389d0a687a3673153b0">H5F_LIBVER_LATEST</a>);</div>
|
|
<div class="line"><a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> file_id = <a class="code hl_function" href="group___h5_f.html#gae64b51ee9ac0781bc4ccc599d98387f4">H5Fcreate</a>(<span class="stringliteral">"file.h5"</span>, <a class="code hl_define" href="_h5_fpublic_8h.html#a5a2d6726f9ad8d2bca8df2b817e5ad6a">H5F_ACC_TRUNC</a>, <a class="code hl_define" href="_h5_ppublic_8h.html#afa85e97bfbf9bf1c58e39263846c568f">H5P_DEFAULT</a>, fapl_id);</div>
|
|
<div class="line">...</div>
|
|
<div class="ttc" id="a_h5_fpublic_8h_html_a2d963b599894f684571fbd4d5e8a96a2aa1212669916e7389d0a687a3673153b0"><div class="ttname"><a href="_h5_fpublic_8h.html#a2d963b599894f684571fbd4d5e8a96a2aa1212669916e7389d0a687a3673153b0">H5F_LIBVER_LATEST</a></div><div class="ttdeci">@ H5F_LIBVER_LATEST</div><div class="ttdef"><b>Definition</b> H5Fpublic.h:187</div></div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_gacbe1724e7f70cd17ed687417a1d2a910"><div class="ttname"><a href="group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910">H5Pset_libver_bounds</a></div><div class="ttdeci">herr_t H5Pset_libver_bounds(hid_t plist_id, H5F_libver_t low, H5F_libver_t high)</div><div class="ttdoc">Controls the range of library release versions used when creating objects in a file.</div></div>
|
|
</div><!-- fragment --><hr />
|
|
<p> Navigate back: <a class="el" href="index.html" title="notitle">Main</a> / <a class="el" href="_t_n.html" title="Technical Notes">Technical Notes</a> </p>
|
|
</div></div><!-- contents -->
|
|
</div><!-- PageDoc -->
|
|
</div><!-- doc-content -->
|
|
<div id="page-nav" class="page-nav-panel">
|
|
<div id="page-nav-resize-handle"></div>
|
|
<div id="page-nav-tree">
|
|
<div id="page-nav-contents">
|
|
</div><!-- page-nav-contents -->
|
|
</div><!-- page-nav-tree -->
|
|
</div><!-- page-nav -->
|
|
</div><!-- container -->
|
|
<!-- start footer part -->
|
|
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
|
|
<ul>
|
|
<li class="footer">Generated by <a href="https://www.doxygen.org/index.html"><img class="footer" src="doxygen.svg" width="104" height="31" alt="doxygen"/></a> 1.16.1 </li>
|
|
</ul>
|
|
</div>
|
|
</body>
|
|
</html>
|