mirror of
https://github.com/HDFGroup/hdf5.git
synced 2026-09-25 04:09:44 +03:00
213 lines
19 KiB
HTML
213 lines
19 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
|
|
<meta http-equiv="X-UA-Compatible" content="IE=11"/>
|
|
<meta name="generator" content="Doxygen 1.16.1"/>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
<title>HDF5: Modified Region Writes</title>
|
|
<link href="tabs.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="jquery.js"></script>
|
|
<script type="text/javascript" src="dynsections.js"></script>
|
|
<link href="navtree.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="navtreedata.js"></script>
|
|
<script type="text/javascript" src="navtree.js"></script>
|
|
<script type="text/javascript" src="cookie.js"></script>
|
|
<link href="search/search.css" rel="stylesheet" type="text/css"/>
|
|
<script type="text/javascript" src="search/searchdata.js"></script>
|
|
<script type="text/javascript" src="search/search.js"></script>
|
|
<script type="text/javascript">
|
|
$(function() { init_search(); });
|
|
</script>
|
|
<link href="doxygen.css" rel="stylesheet" type="text/css" />
|
|
<link href="hdf5doxy.css" rel="stylesheet" type="text/css"/>
|
|
<link href="doxygen-awesome.css" rel="stylesheet" type="text/css"/>
|
|
<link href="hdf5doxy.css" rel="stylesheet" type="text/css">
|
|
<script type="text/javascript" src="hdf5_navtree_hacks.js"></script>
|
|
<div style="background:#FFDDDD;font-size:120%;text-align:center;margin:0;padding:5px">Help us improve by taking our short survey: <a href="https://www.hdfgroup.org/website-survey/">https://www.hdfgroup.org/website-survey/</a></div>
|
|
<!-- ... other metadata & script includes ... -->
|
|
<script type="text/javascript" src="doxygen-awesome-tabs.js"></script>
|
|
<script type="text/javascript">
|
|
DoxygenAwesomeTabs.init()
|
|
</script>
|
|
<!-- Google tag (gtag.js) -->
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-57FMZK4S7X"></script>
|
|
<script>
|
|
window.dataLayer = window.dataLayer || [];
|
|
function gtag(){dataLayer.push(arguments);}
|
|
gtag('js', new Date());
|
|
gtag('config', 'G-57FMZK4S7X');
|
|
</script>
|
|
</head>
|
|
<body>
|
|
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
|
|
<div id="titlearea">
|
|
<table cellspacing="0" cellpadding="0">
|
|
<tbody>
|
|
<tr id="projectrow">
|
|
<td id="projectlogo"><img alt="Logo" src="HDFG-logo.png"/></td>
|
|
<td id="projectalign">
|
|
<div id="projectname">HDF5<span id="projectnumber"> Last Updated on 2026-09-02</span>
|
|
</div>
|
|
<div id="projectbrief">The HDF5 Field Guide</div>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<!-- end header part -->
|
|
<!-- Generated by Doxygen 1.16.1 -->
|
|
<script type="text/javascript">
|
|
var searchBox = new SearchBox("searchBox", "search/",'.html');
|
|
</script>
|
|
<script type="text/javascript">
|
|
$(function() { codefold.init(); });
|
|
</script>
|
|
<div id="main-nav">
|
|
<div id="navrow1" class="tabs">
|
|
<ul class="tablist">
|
|
<li><a href="index.html"><span>Main Page</span></a></li>
|
|
<li><a href="_getting_started.html"><span>Getting started</span></a></li>
|
|
<li><a href="_u_g.html"><span>User Guide</span></a></li>
|
|
<li><a href="_r_m.html"><span>Reference Manual</span></a></li>
|
|
<li><a href="_cookbook.html"><span>Cookbook</span></a></li>
|
|
<li><a href="_t_n.html"><span>Technical Notes</span></a></li>
|
|
<li><a href="_r_f_c.html"><span>RFCs</span></a></li>
|
|
<li><a href="_s_p_e_c.html"><span>Specifications</span></a></li>
|
|
<li><a href="_g_l_s.html"><span>Glossary</span></a></li>
|
|
<li><a href="_f_t_s.html"><span>Full-Text Search</span></a></li>
|
|
<li><a href="_about.html"><span>About</span></a></li>
|
|
<li>
|
|
<div id="MSearchBox" class="MSearchBoxInactive">
|
|
<span class="left">
|
|
<span id="MSearchSelect" class="search-icon" onmouseover="return searchBox.OnSearchSelectShow()" onmouseout="return searchBox.OnSearchSelectHide()"><span class="search-icon-dropdown"></span></span>
|
|
<input type="text" id="MSearchField" value="" placeholder="Search" accesskey="S"
|
|
onfocus="searchBox.OnSearchFieldFocus(true)"
|
|
onblur="searchBox.OnSearchFieldFocus(false)"
|
|
onkeyup="searchBox.OnSearchFieldChange(event)"/>
|
|
</span><span class="right">
|
|
<a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><div id="MSearchCloseImg" class="close-icon"></div></a>
|
|
</span>
|
|
</div>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div><!-- main-nav -->
|
|
</div><!-- top -->
|
|
<div id="side-nav" class="ui-resizable side-nav-resizable">
|
|
<div id="nav-tree">
|
|
<div id="nav-tree-contents">
|
|
<div id="nav-sync" class="sync"></div>
|
|
</div>
|
|
</div>
|
|
<div id="splitbar" style="-moz-user-select:none;"
|
|
class="ui-resizable-handle">
|
|
</div>
|
|
</div>
|
|
<script type="text/javascript">
|
|
$(function(){initNavTree('mod_region_writes.html','',''); });
|
|
</script>
|
|
<div id="container">
|
|
<div id="doc-content">
|
|
<!-- window showing the filter options -->
|
|
<div id="MSearchSelectWindow"
|
|
onmouseover="return searchBox.OnSearchSelectShow()"
|
|
onmouseout="return searchBox.OnSearchSelectHide()"
|
|
onkeydown="return searchBox.OnSearchSelectKey(event)">
|
|
</div>
|
|
|
|
<!-- iframe showing the search results (closed by default) -->
|
|
<div id="MSearchResultsWindow">
|
|
<div id="MSearchResults">
|
|
<div class="SRPage">
|
|
<div id="SRIndex">
|
|
<div id="SRResults"></div>
|
|
<div class="SRStatus" id="Loading">Loading...</div>
|
|
<div class="SRStatus" id="Searching">Searching...</div>
|
|
<div class="SRStatus" id="NoMatches">No Matches</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div><div class="header">
|
|
<div class="headertitle"><div class="title">Modified Region Writes </div></div>
|
|
</div><!--header-->
|
|
<div class="contents">
|
|
<div class="textblock"><p>Navigate back: <a class="el" href="index.html" title="notitle">Main</a> / <a class="el" href="_u_g.html" title="HDF5 User Guide">HDF5 User Guide</a> / <a class="el" href="_a_r__u_g.html" title="Additional Resources">Additional Resources</a> </p><hr />
|
|
<p>The Core virtual file driver allows the manipulating of HDF5 files in memory instead of in physical storage. In previous versions, changing any part of a file in memory meant the entire file would be written to storage on file close or flush. To improve the performance of the writing to storage operation, a new feature, modified region writes, has been added. With modified region writes, only the changed regions of the file are written to storage.</p>
|
|
<p>Introduced with HDF5-1.8.13 May 15, 2014</p>
|
|
<p>The intended audience for this feature is advanced users of the Core virtual file driver.</p>
|
|
<h1 class="doxsection"><a class="anchor" id="sec_mod_region_writes_intro"></a>
|
|
Introduction to Modified Region Writes</h1>
|
|
<p>In the 1.8.13 release of the HDF5 Library, a feature called modified region writes was added to improve the performance of writes to storage. The purpose of this document is to describe the feature and how to use it. The intended audience for this feature is advanced users of the Core virtual file driver (VFD). The Core (or Memory) VFD allows HDF5 files to be created or opened in memory instead of in physical storage. If an existing file is opened in memory, the entire contents of the file are copied into memory on open. All subsequent manipulations of created or opened files occur in memory. The advantage of working on files in memory is the file operations go much faster, but the disadvantage is significant memory resources may be required when working with large files. On file close or flush, the changes can optionally be propagated to physical storage.</p>
|
|
<p>The Core VFD is configured via the following API call: </p><div class="fragment"><div class="line"><a class="code hl_typedef" href="_h5public_8h.html#a3b079ecf932a5c599499cf7e298af160">herr_t</a> <a class="code hl_function" href="group___f_a_p_l.html#gac9f8c916dbe7b5e4ef23e9a7b15f370f">H5Pset_fapl_core</a>(<a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> fapl_id, <span class="keywordtype">size_t</span> increment, <span class="keywordtype">bool</span> backing_store)</div>
|
|
<div class="ttc" id="a_h5_ipublic_8h_html_a0045db7ff9c22ad35db6ae91662e1943"><div class="ttname"><a href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a></div><div class="ttdeci">int64_t hid_t</div><div class="ttdef"><b>Definition</b> H5Ipublic.h:60</div></div>
|
|
<div class="ttc" id="a_h5public_8h_html_a3b079ecf932a5c599499cf7e298af160"><div class="ttname"><a href="_h5public_8h.html#a3b079ecf932a5c599499cf7e298af160">herr_t</a></div><div class="ttdeci">int herr_t</div><div class="ttdef"><b>Definition</b> H5public.h:269</div></div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_gac9f8c916dbe7b5e4ef23e9a7b15f370f"><div class="ttname"><a href="group___f_a_p_l.html#gac9f8c916dbe7b5e4ef23e9a7b15f370f">H5Pset_fapl_core</a></div><div class="ttdeci">herr_t H5Pset_fapl_core(hid_t fapl_id, size_t increment, bool backing_store)</div><div class="ttdoc">Modifies the file access property list to use the H5FD_CORE driver.</div></div>
|
|
</div><!-- fragment --><p>The backing_store parameter sets whether or not changes are propagated to physical storage on close. If this parameter is set to 0 (FALSE), then all changes will be lost when the file is closed. If set to 1 (TRUE), then the changes are written to storage on file close or flush. In previous versions of the library when a file was closed, the entire file would be written out if even a single byte has changed. This can be inefficient when very large files are written out after minimal changes have been made.</p>
|
|
<p>If files being worked on in memory will be written to disk, the modified region writes feature can be enabled.</p>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_mod_region_writes_intro_how"></a>
|
|
How the Core VFD Tracks File Modifications</h2>
|
|
<p>When modified region writes are enabled, the Core VFD will track any changes made to the file. On file close or flush, the tracked changes will be written to storage.</p>
|
|
<p>As write calls pass through the Core VFD, a list of “start address-end address” pairs representing the writes is updated. This list serves as a map of modified regions in the file. Overlapping or abutting regions are merged as they are inserted into the list.</p>
|
|
<p>As a further optimization, a write page size can be set. This feature expands any dirty regions (regions with changed bytes) to the nearest page boundaries. Using write pages can minimize seeks and small, inefficient writes when a large number of small non-adjacent writes occur. See the figure below.</p>
|
|
<p>Note that these marked regions are at the granularity of the write calls that the library makes. In other words, an entire metadata object or dataset chunk will be marked dirty if even a single byte is changed since the library uses a single write call when metadata objects or dataset chunks are evicted from their respective caches. The Core VFD will make no effort to determine the particular bytes that were modified with respect to the original data.</p>
|
|
<table class="doxtable">
|
|
<tr>
|
|
<td><div class="image">
|
|
<img src="modregwrite.png" alt=""/>
|
|
</div>
|
|
</td></tr>
|
|
</table>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_mod_region_writes_intro_new"></a>
|
|
Using the New Feature</h2>
|
|
<p>The modified region writes feature is turned off by default. Setting the <b>backing_store</b> flag to TRUE will not turn modified region writes on.</p>
|
|
<p>The modified region writes feature is controlled via the <a class="el" href="group___f_a_p_l.html#ga767be5f6f42a14bb1550c47ac08ad469" title="Gets information about the write tracking feature used by the core VFD.">H5Pget_core_write_tracking</a>/<a class="el" href="group___f_a_p_l.html#gada5811359151b195aa5e1172becd0250" title="Sets write tracking information for core driver, H5FD_CORE.">H5Pset_core_write_tracking</a> HDF5 API calls. The signatures of these function calls are the following: </p><div class="fragment"><div class="line"><a class="code hl_typedef" href="_h5public_8h.html#a3b079ecf932a5c599499cf7e298af160">herr_t</a> <a class="code hl_function" href="group___f_a_p_l.html#gada5811359151b195aa5e1172becd0250">H5Pset_core_write_tracking</a>(<a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> fapl_id, <span class="keywordtype">bool</span> is_enabled, <span class="keywordtype">size_t</span> page_size)</div>
|
|
<div class="line"><a class="code hl_typedef" href="_h5public_8h.html#a3b079ecf932a5c599499cf7e298af160">herr_t</a> <a class="code hl_function" href="group___f_a_p_l.html#ga767be5f6f42a14bb1550c47ac08ad469">H5Pget_core_write_tracking</a>(<a class="code hl_typedef" href="_h5_ipublic_8h.html#a0045db7ff9c22ad35db6ae91662e1943">hid_t</a> fapl_id, <span class="keywordtype">bool</span> *is_enabled, <span class="keywordtype">size_t</span> *page_size)</div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_ga767be5f6f42a14bb1550c47ac08ad469"><div class="ttname"><a href="group___f_a_p_l.html#ga767be5f6f42a14bb1550c47ac08ad469">H5Pget_core_write_tracking</a></div><div class="ttdeci">herr_t H5Pget_core_write_tracking(hid_t fapl_id, bool *is_enabled, size_t *page_size)</div><div class="ttdoc">Gets information about the write tracking feature used by the core VFD.</div></div>
|
|
<div class="ttc" id="agroup___f_a_p_l_html_gada5811359151b195aa5e1172becd0250"><div class="ttname"><a href="group___f_a_p_l.html#gada5811359151b195aa5e1172becd0250">H5Pset_core_write_tracking</a></div><div class="ttdeci">herr_t H5Pset_core_write_tracking(hid_t fapl_id, bool is_enabled, size_t page_size)</div><div class="ttdoc">Sets write tracking information for core driver, H5FD_CORE.</div></div>
|
|
</div><!-- fragment --><p> Setting the page size to a value greater than 1 turns write tracking on at that page size. Setting a page size of 1 byte disables paging.</p>
|
|
<p>More information for these function calls can be found in the <a class="el" href="_r_m.html" title="HDF5 Reference Manual">HDF5 Reference Manual</a>.</p>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_mod_region_writes_intro_perf"></a>
|
|
Performance</h2>
|
|
<p>The performance benefits of the feature will depend heavily on the data access patterns of the application and will have to be evaluated on a case-by-case basis. In cases where the majority of the data would be written out (for example, creating and writing data to a new file), the new feature will likely not impart a significant performance benefit. In cases where a small amount of data will be added or changed (for example, opening an existing file and modifying a small amount of existing data), the performance benefits could be significant.</p>
|
|
<p>When performance tuning, the following parameters are likely to have significant effects on I/O throughput: </p><ul>
|
|
<li>The size of the backing store pages (see <a class="el" href="group___f_a_p_l.html#gada5811359151b195aa5e1172becd0250" title="Sets write tracking information for core driver, H5FD_CORE.">H5Pset_core_write_tracking</a>) </li>
|
|
<li>Dataset layout and chunk size (see <a class="el" href="group___d_c_p_l.html#ga75d80991a8f467e0d454c53a383ae7f9" title="Sets the type of storage used to store the raw data for a dataset.">H5Pset_layout</a> and H5Pset_chunk) </li>
|
|
<li>Metadata aggregation size (see <a class="el" href="group___f_a_p_l.html#ga8822e3dedc8e1414f20871a87d533cb1" title="Sets the minimum metadata block size.">H5Pset_meta_block_size</a>) </li>
|
|
<li>Using the latest file format (see <a class="el" href="group___f_a_p_l.html#gacbe1724e7f70cd17ed687417a1d2a910" title="Controls the range of library release versions used when creating objects in a file.">H5Pset_libver_bounds</a>) </li>
|
|
<li>Data layout considerations (arrangement of groups, datasets, and datatypes)</li>
|
|
</ul>
|
|
<p>In general, anything that promotes the aggregation of changes made to the file will enhance the performance of this feature. Unfortunately, empirical testing will typically be required to determine the “sweet spot” between reducing the number of seeks and minimizing the amount of data written out.</p>
|
|
<p>More information for these function calls can be found in the <a class="el" href="_r_m.html" title="HDF5 Reference Manual">HDF5 Reference Manual</a>.</p>
|
|
<h1 class="doxsection"><a class="anchor" id="sec_mod_region_writes_refs"></a>
|
|
References</h1>
|
|
<p>For more information, see the entries for the <a class="el" href="group___f_a_p_l.html#gac9f8c916dbe7b5e4ef23e9a7b15f370f" title="Modifies the file access property list to use the H5FD_CORE driver.">H5Pset_fapl_core</a>, <a class="el" href="group___f_a_p_l.html#ga767be5f6f42a14bb1550c47ac08ad469" title="Gets information about the write tracking feature used by the core VFD.">H5Pget_core_write_tracking</a>, and <a class="el" href="group___f_a_p_l.html#gada5811359151b195aa5e1172becd0250" title="Sets write tracking information for core driver, H5FD_CORE.">H5Pset_core_write_tracking</a> function calls in the <a class="el" href="_r_m.html" title="HDF5 Reference Manual">HDF5 Reference Manual</a>.</p>
|
|
<h2 class="doxsection"><a class="anchor" id="subsec_mod_region_writes_refs_virt"></a>
|
|
The Virtual File Layer and Virtual File Drivers</h2>
|
|
<p>The HDF5 Library uses a layered architecture. The lowest layer is the virtual file layer (VFL). The VFL handles low-level file I/O via virtual file drivers (VFDs). The VFL is an abstraction layer in the HDF5 Library that maps I/O operations such as “read” to concrete I/O calls like the POSIX read() call or the Win32 ReadFile() call. Each VFD implements a different I/O scheme: some examples are MPI-I/O, POSIX I/O, and in-memory I/O. This VFL/VFD scheme allows abstract HDF5 file manipulations to be separated from storage I/O operations.</p>
|
|
<p>For more information, see <a class="el" href="_v_f_l_t_n.html" title="HDF5 Virtual File Layer">HDF5 Virtual File Layer</a></p>
|
|
<p>For more information on virtual file drivers, see the <a class="el" href="_h5_f__u_g.html#subsec_file_alternate_drivers" title="Alternate File Storage Layouts and Low-level File Drivers">Alternate File Storage Layouts and Low-level File Drivers</a> section in the <a class="el" href="_h5_f__u_g.html#sec_file" title="The HDF5 File">The HDF5 File</a> chapter in the <a class="el" href="_u_g.html" title="HDF5 User Guide">HDF5 User Guide</a>.</p>
|
|
<hr />
|
|
<p> Navigate back: <a class="el" href="index.html" title="notitle">Main</a> / <a class="el" href="_u_g.html" title="HDF5 User Guide">HDF5 User Guide</a> / <a class="el" href="_a_r__u_g.html" title="Additional Resources">Additional Resources</a> </p>
|
|
</div></div><!-- contents -->
|
|
</div><!-- PageDoc -->
|
|
</div><!-- doc-content -->
|
|
<div id="page-nav" class="page-nav-panel">
|
|
<div id="page-nav-resize-handle"></div>
|
|
<div id="page-nav-tree">
|
|
<div id="page-nav-contents">
|
|
</div><!-- page-nav-contents -->
|
|
</div><!-- page-nav-tree -->
|
|
</div><!-- page-nav -->
|
|
</div><!-- container -->
|
|
<!-- start footer part -->
|
|
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
|
|
<ul>
|
|
<li class="footer">Generated by <a href="https://www.doxygen.org/index.html"><img class="footer" src="doxygen.svg" width="104" height="31" alt="doxygen"/></a> 1.16.1 </li>
|
|
</ul>
|
|
</div>
|
|
</body>
|
|
</html>
|