mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Adds new subfiling API H5FDsubfiling_get_file_mapping() (#5828)

Browse Source 
Adds H5FDsubfiling_get_file_mapping() API to map logical HDF5 files to physical subfiles, with Fortran support and comprehensive testing.

Adds H5FDsubfiling_get_file_mapping() in H5FDsubfiling.c to retrieve subfile paths for a logical HDF5 file.
Updates Fortran interface in H5VFDff.F90 to include h5fdsubfiling_get_file_mapping_f().

Updates h5fuse.in to support a list of subfiles for processing with a new -l option.
This commit is contained in:
Scot Breitenfeld
2025-09-26 11:30:16 -05:00
committed by GitHub
parent 2e88b9da1a
commit 877cbf79a7
13 changed files with 1287 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doxygen/dox/ReferenceManual.dox
+1 -1
View File
@@ -78,7 +78,7 @@ The functions provided by the HDF5 API are grouped into the following
</td>
</tr>
<tr>
<th>Virtual File Driver (H5VFD)</th><td>@ref H5VFD "C"</td><td style="white-space: nowrap;">C++</td><td>Fortran</td><td>Java</td><td>Manage HDF5 Virtual File Driver.
<th>Virtual File Driver (H5VFD)</th><td>@ref H5VFD "C"</td><td style="white-space: nowrap;">C++</td><td>@ref FH5VFD "Fortran"</td><td>Java</td><td>Manage HDF5 Virtual File Driver.
</td>
<tr>
<th>VOL Connector (H5VL)</th><td>@ref H5VL "C"</td><td style="white-space: nowrap;">C++</td><td>@ref FH5VL "Fortran"</td><td>@ref JH5VL "Java"</td><td>Manage HDF5 VOL connector plugins.
fortran/src/CMakeLists.txt
+1
View File
@@ -341,6 +341,7 @@ set (f90_F_BASE_SOURCES
${HDF5_F90_SRC_SOURCE_DIR}/H5Rff.F90
${HDF5_F90_SRC_SOURCE_DIR}/H5Sff.F90
${HDF5_F90_SRC_SOURCE_DIR}/H5Tff.F90
${HDF5_F90_SRC_SOURCE_DIR}/H5VFDff.F90
${HDF5_F90_SRC_SOURCE_DIR}/H5VLff.F90
${HDF5_F90_SRC_SOURCE_DIR}/H5Zff.F90
)
fortran/src/H5VFDff.F90
+283
View File
@@ -0,0 +1,283 @@
!> @defgroup FH5VFD Fortran VFD (H5VFD) Interface
!!
!! @see H5FD, C-API
!!
!> @ingroup FH5VFD
!!
!! @brief This module contains Fortran interfaces for H5VFD (Virtual File Driver) functions.
!!
!! The H5VFD module provides Fortran bindings for HDF5 Virtual File Driver operations,
!! including subfiling functionality for parallel I/O optimization.
!
! COPYRIGHT
! * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
! Copyright by The HDF Group. *
! All rights reserved. *
! *
! This file is part of HDF5. The full HDF5 copyright notice, including *
! terms governing use, modification, and redistribution, is contained in *
! the LICENSE file, which can be found at the root of the source code *
! distribution tree, or in https://www.hdfgroup.org/licenses. *
! If you do not have access to either file, you may request a copy from *
! help@hdfgroup.org. *
! * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
!
! NOTES
! _____ __ __ _____ ____ _____ _______ _ _ _______
! |_ _| \/ | __ \ / __ \| __ \__ __|/\ | \ | |__ __|
! **** | | | \ / | |__) | | | | |__) | | | / \ | \| | | | ****
! **** | | | |\/| | ___/| | | | _ / | | / /\ \ | . ` | | | ****
! **** _| |_| | | | | | |__| | | \ \ | |/ ____ \| |\ | | | ****
! |_____|_| |_|_| \____/|_| \_\ |_/_/ \_\_| \_| |_|
!
! If you add a new H5VFD function you must add the function name to the
! Windows dll file 'hdf5_fortrandll.def.in' in the fortran/src directory.
! This is needed for Windows based operating systems.
!
#include <H5config_f.inc>
MODULE H5VFD
USE H5GLOBAL
USE H5fortkit
IMPLICIT NONE
#ifndef H5_DOXYGEN
INTERFACE
! Helper function to free the C memory allocated by H5FDsubfiling_get_file_mapping
INTEGER(C_INT) FUNCTION h5free_string_array_memory_c(filenames_ptr, num_files) &
BIND(C, NAME='h5free_string_array_memory_c')
IMPORT :: C_INT, C_PTR, C_SIZE_T
IMPLICIT NONE
TYPE(C_PTR) :: filenames_ptr
INTEGER(C_SIZE_T) :: num_files
END FUNCTION h5free_string_array_memory_c
END INTERFACE
#endif
CONTAINS
#ifdef H5_HAVE_SUBFILING_VFD
!>
!! \ingroup FH5VFD
!!
!! \brief Retrieve the list of subfile names for a HDF5 file for the subfiling VFD
!!
!! \details This function retrieves the names of all subfiles associated with an HDF5 file
!! that uses the subfiling Virtual File Driver (VFD). The subfiling VFD distributes
!! file data across multiple subfiles to improve parallel I/O performance on shared
!! file systems.
!!
!! The returned filenames correspond to the physical subfiles stored on the file system
!! that collectively make up the logical HDF5 file. This information is useful for:
!! - File management and backup operations
!! - Understanding the physical storage layout
!! - Tools like h5fuse for recombining subfiles
!! - Debugging subfiling configurations
!!
!! \param file_id [in] HDF5 file identifier for a file using the subfiling VFD
!! \param filenames [out] Allocatable array of subfile names. Memory is automatically
!! allocated by the function and must be deallocated by the caller.
!! See **Compiler Compatibility** note below.
!! \param num_files [out] Number of subfiles in the filenames array
!! \param hdferr [out] Error code:
!! \li 0 on success
!! \li -1 on failure
!!
!! \since 2.0.0
!!
!! \note **Memory Management**: The filenames array is automatically allocated. The caller
!! is responsible for deallocating when it is no longer needed:
!! \code{.f90}
!! DEALLOCATE(filenames)
!! \endcode
!!
!! \note
!! \parblock
!! **Compiler Compatibility**:
!! - With H5_FORTRAN_HAVE_CHAR_ALLOC: Variable-length character strings (Fortran 2003+)
!! - Without H5_FORTRAN_HAVE_CHAR_ALLOC: Fixed-length 8192 character strings (older compilers),
!! filenames longer than 8192 characters will cause the function to fail with hdferr = -1.
!! \endparblock
!!
!! \note
!! \parblock
!! This function will not be accessible if support for the subfiling VFD is unavailable or disabled.
!! \endparblock
!!
!! \note
!! \parblock
!! **Optimized Allocation**: The function uses knowledge of the subfiling filename template
!! to estimate optimal string lengths, typically reducing memory usage compared to the
!! maximum 8192 character limit. Subfile names follow the pattern:
!! `basename.subfile_<inode>_<index>_of_<total>`
!! \endparblock
!!
!! \par Example Usage:
!! \code{.f90}
!! USE HDF5
!! IMPLICIT NONE
!!
!! INTEGER(HID_T) :: file_id
!! CHARACTER(LEN=:), ALLOCATABLE, DIMENSION(:) :: subfile_names
!! INTEGER(SIZE_T) :: num_subfiles
!! INTEGER :: hdferr
!! INTEGER :: i
!!
!! ! Open file with subfiling VFD (file_id setup not shown)
!!
!! ! Get subfile mapping
!! CALL h5fdsubfiling_get_file_mapping_f(file_id, subfile_names, num_subfiles, hdferr)
!!
!! IF (hdferr == 0) THEN
!! PRINT *, 'Found', num_subfiles, 'subfiles:'
!! DO i = 1, num_subfiles
!! PRINT *, ' ', TRIM(subfile_names(i))
!! END DO
!!
!! ! Clean up
!! DEALLOCATE(subfile_names)
!! ELSE
!! PRINT *, 'Error getting file mapping:', hdferr
!! END IF
!! \endcode
!!
!! \see H5FDsubfiling_get_file_mapping() (C API)
!! \see H5Pset_fapl_subfiling_f() for setting up subfiling VFD
!!
SUBROUTINE h5fdsubfiling_get_file_mapping_f(file_id, filenames, num_files, hdferr)
IMPLICIT NONE
INTEGER(HID_T), INTENT(IN) :: file_id
#ifdef H5_FORTRAN_HAVE_CHAR_ALLOC
CHARACTER(LEN=:), ALLOCATABLE, DIMENSION(:), INTENT(OUT) :: filenames
#else
INTEGER, PARAMETER :: DEFAULT_MAX_LEN = 8192
CHARACTER(LEN=DEFAULT_MAX_LEN), ALLOCATABLE, DIMENSION(:), INTENT(OUT) :: filenames
#endif
INTEGER(SIZE_T), INTENT(OUT) :: num_files
INTEGER, INTENT(OUT) :: hdferr
TYPE(C_PTR) :: filenames_ptr
INTEGER(C_SIZE_T) :: c_num_files
INTEGER(C_INT) :: ret_val
INTEGER(SIZE_T) :: i, str_len
TYPE(C_PTR), POINTER, DIMENSION(:) :: c_filename_ptrs
INTEGER :: max_len
#ifdef H5_FORTRAN_HAVE_CHAR_ALLOC
CHARACTER(LEN=:), ALLOCATABLE :: temp_filenames(:)
INTEGER(SIZE_T) :: k
#endif
INTERFACE
INTEGER(C_INT) FUNCTION h5fdsubfiling_get_file_mapping(file_id, filenames_ptr, len) &
BIND(C, NAME='H5FDsubfiling_get_file_mapping')
IMPORT :: C_INT, HID_T, C_PTR, C_SIZE_T
IMPLICIT NONE
INTEGER(HID_T), VALUE :: file_id
TYPE(C_PTR) :: filenames_ptr
INTEGER(C_SIZE_T) :: len
END FUNCTION h5fdsubfiling_get_file_mapping
END INTERFACE
hdferr = INT(h5fdsubfiling_get_file_mapping(file_id, filenames_ptr, c_num_files))
num_files = INT(c_num_files, SIZE_T)
IF (hdferr .NE. 0 .OR. num_files .EQ. 0) THEN
! Allocate empty array on error or no files
#ifdef H5_FORTRAN_HAVE_CHAR_ALLOC
ALLOCATE(CHARACTER(LEN=0) :: filenames(0))
#else
ALLOCATE(filenames(0))
#endif
num_files = 0_SIZE_T
RETURN
END IF
! Convert the C char** array to Fortran character array
CALL C_F_POINTER(filenames_ptr, c_filename_ptrs, [num_files])
! Allocate with a reasonable default - will expand if needed
#ifdef H5_FORTRAN_HAVE_CHAR_ALLOC
ALLOCATE(CHARACTER(LEN=1024) :: filenames(num_files))
#else
ALLOCATE(filenames(num_files))
#endif
max_len = 0
! determine lengths and copy strings
DO i = 1, num_files
IF (C_ASSOCIATED(c_filename_ptrs(i))) THEN
BLOCK
CHARACTER(KIND=C_CHAR), POINTER :: c_string(:)
INTEGER :: current_size, j
INTEGER, PARAMETER :: INITIAL_SIZE = 1024
current_size = INITIAL_SIZE
DO
CALL C_F_POINTER(c_filename_ptrs(i), c_string, [current_size])
! Find string length by searching for null terminator
str_len = 0
DO WHILE (str_len < current_size .AND. c_string(str_len + 1) /= C_NULL_CHAR)
str_len = str_len + 1
END DO
! If we found the null terminator, we're done
IF (str_len < current_size) EXIT
! Otherwise, double the size and try again
current_size = current_size * 2
IF (current_size > 65536) THEN
! Sanity check - if path is longer than 64K, something is wrong
hdferr = -1
RETURN
END IF
END DO
max_len = MAX(max_len, INT(str_len))
#ifdef H5_FORTRAN_HAVE_CHAR_ALLOC
! Reallocate if this string is longer than current allocation
IF (INT(str_len) > LEN(filenames(1))) THEN
ALLOCATE(CHARACTER(LEN=INT(str_len)) :: temp_filenames(num_files))
DO k = 1, i-1
temp_filenames(k) = filenames(k)
END DO
DEALLOCATE(filenames)
ALLOCATE(CHARACTER(LEN=INT(str_len)) :: filenames(num_files))
DO k = 1, i-1
filenames(k) = temp_filenames(k)
END DO
DEALLOCATE(temp_filenames)
END IF
#else
! Check if string exceeds our fixed buffer
IF (INT(str_len) > DEFAULT_MAX_LEN) THEN
hdferr = -1
RETURN
END IF
#endif
! Copy the string
filenames(i) = ""
DO j = 1, INT(str_len)
filenames(i)(j:j) = c_string(j)
END DO
END BLOCK
ELSE
filenames(i) = ""
END IF
END DO
! Free the C memory allocated by H5FDsubfiling_get_file_mapping
ret_val = h5free_string_array_memory_c(filenames_ptr, c_num_files)
! Note: We ignore the return value of the free function since the main operation succeeded
END SUBROUTINE h5fdsubfiling_get_file_mapping_f
#endif
END MODULE H5VFD
fortran/src/H5_f.c
+46
View File
@@ -1125,3 +1125,49 @@ h5dont_atexit_c(void)
ret_value = 0;
return ret_value;
}
/****if* H5_f/h5free_string_array_memory_c
* NAME
* h5free_string_array_memory_c
* PURPOSE
* Frees internal memory allocated for a char** string array
* INPUTS
* array_ptr - pointer to char** array
* num_files - number of strings in the array
* RETURNS
* 0 on success, -1 on failure
* SOURCE
*/
int_f
h5free_string_array_memory_c(void **array_ptr, size_t_f *num_files)
/******/
{
int ret_value = 0;
char **array;
size_t len;
if (array_ptr == NULL || num_files == NULL) {
return ret_value; /* Nothing to free */
}
array = (char **)(*array_ptr);
len = (size_t)(*num_files);
if (array == NULL || len == 0) {
return ret_value; /* Nothing to free */
}
/* Free each individual string */
for (size_t i = 0; i < len; i++) {
if (array[i] != NULL) {
H5free_memory(array[i]);
array[i] = NULL;
}
}
/* Free the array of pointers */
H5free_memory(array);
*array_ptr = NULL;
return ret_value;
}
fortran/src/H5f90proto.h
+3
View File
@@ -632,4 +632,7 @@ H5FC_DLL int_f h5literate_by_name_c(hid_t_f *loc_id, _fcd name, size_t_f *namele
int_f *order, hsize_t_f *idx, H5L_iterate2_t op, void *op_data,
hid_t_f *lapl_id);
/* Subfiling VFD support functions */
H5FC_DLL int_f h5free_string_array_memory_c(void **array_ptr, size_t_f *num_files);
#endif /* H5f90proto_H */
fortran/src/HDF5.F90
+1
View File
@@ -39,6 +39,7 @@ MODULE HDF5
USE H5P
USE H5R
USE H5VL
USE H5VFD
USE H5Z
USE H5_gen
USE H5LIB
fortran/src/hdf5_fortrandll.def.in
+2
View File
@@ -598,6 +598,8 @@ H5VL_mp_H5VLCMP_CONNECTOR_CLS_F
H5VL_mp_H5VLUNREGISTER_CONNECTOR_F
H5VL_mp_H5VLNATIVE_ADDR_TO_TOKEN_F
H5VL_mp_H5VLNATIVE_TOKEN_TO_ADDR_F
; H5VFD
@H5_NOSUBFILING@H5VFD_mp_H5FDSUBFILING_GET_FILE_MAPPING_F
; H5Z
H5Z_mp_H5ZUNREGISTER_F
H5Z_mp_H5ZFILTER_AVAIL_F
fortran/testpar/subfiling.F90
+87
View File
@@ -380,6 +380,17 @@ PROGRAM subfiling_test
IF(mpi_rank==0) CALL write_test_status(nerrors, &
'Testing H5Fcreate with subfiling with custom settings', total_error)
! *********************************************************
! Testing H5FDsubfiling_get_file_mapping_f Fortran wrapper
! *********************************************************
nerrors = 0
CALL test_subfiling_get_file_mapping_f(filename, nerrors, mpi_rank)
CALL MPI_Barrier(MPI_COMM_WORLD, mpierror)
IF(mpi_rank==0) CALL write_test_status(nerrors, &
'Testing H5FDsubfiling_get_file_mapping_f wrapper', total_error)
!
! close HDF5 interface
!
@@ -411,6 +422,82 @@ PROGRAM subfiling_test
IF(mpi_rank==0) CALL write_test_footer()
CONTAINS
! **********************************************************************
! Function: test_subfiling_get_file_mapping_f
!
! Purpose: Test the Fortran wrapper for H5FDsubfiling_get_file_mapping
! This is a focused test for the Fortran wrapper functionality
!
! Return: none (errors incremented in nerrors)
! **********************************************************************
SUBROUTINE test_subfiling_get_file_mapping_f(filename, nerrors, mpi_rank)
IMPLICIT NONE
CHARACTER(LEN=*), INTENT(IN) :: filename
INTEGER, INTENT(INOUT) :: nerrors
INTEGER(KIND=MPI_INTEGER_KIND), INTENT(IN) :: mpi_rank
INTEGER(HID_T) :: file_id, fapl_id
INTEGER :: hdferror
INTEGER(SIZE_T) :: num_files
! Variable declarations matching the exact interface signature
#ifdef H5_FORTRAN_HAVE_CHAR_ALLOC
CHARACTER(LEN=:), ALLOCATABLE, DIMENSION(:) :: filenames
#else
CHARACTER(LEN=4096), ALLOCATABLE, DIMENSION(:) :: filenames
#endif
! All ranks will test the API to ensure parallel consistency
! Set up file access property list with subfiling
CALL h5pcreate_f(H5P_FILE_ACCESS_F, fapl_id, hdferror)
CALL check("h5pcreate_f", hdferror, nerrors)
CALL H5Pset_mpi_params_f(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL, hdferror)
CALL check("H5Pset_mpi_params_f", hdferror, nerrors)
CALL h5pset_fapl_subfiling_f(fapl_id, hdferror)
CALL check("h5pset_fapl_subfiling_f", hdferror, nerrors)
! Open existing subfiling file (created earlier in the test)
CALL h5fopen_f(filename, H5F_ACC_RDONLY_F, file_id, hdferror, access_prp = fapl_id)
CALL check("h5fopen_f", hdferror, nerrors)
! Test the Fortran wrapper for getting file mapping
CALL h5fdsubfiling_get_file_mapping_f(file_id, filenames, num_files, hdferror)
CALL check("h5fdsubfiling_get_file_mapping_f", hdferror, nerrors)
! Basic validation of the Fortran wrapper functionality
IF (hdferror .EQ. 0) THEN
IF (num_files .GT. 0) THEN
! Just verify we got some results and they have the expected pattern
IF (INDEX(filenames(1), ".subfile_") .LE. 0) THEN
IF (mpi_rank .EQ. 0) WRITE(*,*) "ERROR: Subfile names don't contain expected pattern"
nerrors = nerrors + 1
END IF
ELSE
IF (mpi_rank .EQ. 0) WRITE(*,*) "ERROR: No subfiles returned from Fortran wrapper"
nerrors = nerrors + 1
END IF
! Clean up memory allocated by the Fortran wrapper
IF (ALLOCATED(filenames)) DEALLOCATE(filenames)
ELSE
IF (mpi_rank .EQ. 0) WRITE(*,*) "ERROR: h5fdsubfiling_get_file_mapping_f failed with hdferr =", hdferror
nerrors = nerrors + 1
END IF
! Clean up
CALL h5fclose_f(file_id, hdferror)
CALL check("h5fclose_f", hdferror, nerrors)
CALL h5pclose_f(fapl_id, hdferror)
CALL check("h5pclose_f", hdferror, nerrors)
END SUBROUTINE test_subfiling_get_file_mapping_f
!
! end main program
!
release_docs/CHANGELOG.md
+9
View File
@@ -441,8 +441,17 @@ Simple example programs showing how to use complex number datatypes have been ad
## Parallel Library
### Added H5FDsubfiling_get_file_mapping() API function for subfiling VFD
Added H5FDsubfiling_get_file_mapping() API function to retrieve the names of all physical subfiles that collectively make up a logical HDF5 file when using the subfiling Virtual File Driver.
## Fortran Library
### Added Fortran wrapper h5fdsubfiling_get_file_mapping_f() for subfiling VFD
Added Fortran wrapper h5fdsubfiling_get_file_mapping_f() for the subfiling file mapping functionality, ensuring complete language binding support.
## C++ Library
## Java Library
src/H5FDsubfiling/H5FDsubfiling.c
+110
View File
@@ -564,6 +564,116 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_subfiling() */
herr_t
H5FDsubfiling_get_file_mapping(hid_t file_id, char ***filenames, size_t *len)
{
subfiling_context_t *sf_context = NULL;
H5FD_t *driver = NULL;
H5F_t *file_ptr = NULL;
char **filenames_arr = NULL;
char *filepath = NULL;
char *subfile_dir = NULL;
char *base = NULL;
int num_subfiles = 0;
int num_digits = 0;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
if (file_id < 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file ID");
if (!filenames)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`filenames` was NULL");
if (!len)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`len` was NULL");
*filenames = NULL;
*len = 0;
if (NULL == (file_ptr = H5VL_object(file_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file ID");
if (H5FD_SUBFILING != H5F_get_driver_id(file_ptr))
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file is not using Subfiling VFD");
if (H5F_shared_get_file_driver(H5F_SHARED(file_ptr), &driver) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get driver structure from file ID");
if (NULL == (sf_context = H5FD__subfiling_get_object(((H5FD_subfiling_t *)driver)->context_id)))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get subfiling context from ID");
if (!sf_context->topology)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"application topology hasn't been initialized yet for this file");
assert(sf_context->h5_file_id != UINT64_MAX);
assert(sf_context->h5_filename);
assert(sf_context->sf_num_subfiles > 0);
assert(sf_context->topology);
if (sf_context->topology->rank_is_ioc) {
assert(sf_context->sf_fids);
assert(sf_context->sf_num_fids > 0);
/* Get the basename of the full HDF5 filename */
if (H5_basename(sf_context->h5_filename, &base) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't get HDF5 file basename");
/*
* Get the directory prefix where subfiles will be placed.
* Under normal circumstances, the subfiles are co-located
* with the HDF5 file, but users may specify a different
* directory name.
*/
if (sf_context->subfile_prefix) {
if (NULL == (subfile_dir = H5MM_strdup(sf_context->subfile_prefix)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't copy subfile prefix");
}
else {
if (H5_dirname(sf_context->h5_filename, &subfile_dir) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't get HDF5 file dirname");
}
num_subfiles = sf_context->sf_num_subfiles;
num_digits = (int)(log10(num_subfiles) + 1);
if (NULL == (filenames_arr = calloc((size_t)sf_context->sf_num_fids, sizeof(char *))))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate filenames array");
for (int i = 0; i < sf_context->sf_num_fids; i++) {
int subfile_idx;
if (NULL == (filepath = malloc(PATH_MAX)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate space for subfile filename");
subfile_idx = (i * sf_context->topology->n_io_concentrators) + sf_context->topology->ioc_idx + 1;
snprintf(filepath, PATH_MAX, "%s/" H5FD_SUBFILING_FILENAME_TEMPLATE, subfile_dir, base,
sf_context->h5_file_id, num_digits, subfile_idx, num_subfiles);
filenames_arr[i] = filepath;
filepath = NULL;
}
*filenames = filenames_arr;
*len = (size_t)sf_context->sf_num_fids;
}
done:
if (ret_value < 0) {
if (filenames_arr) {
for (int i = 0; i < sf_context->sf_num_fids; i++) {
free(filenames_arr[i]);
}
}
free(filenames_arr);
}
H5MM_free(base);
H5MM_free(subfile_dir);
FUNC_LEAVE_API(ret_value);
}
static herr_t
H5FD__subfiling_get_default_config(hid_t fapl_id, H5FD_subfiling_config_t *config_out)
{
src/H5FDsubfiling/H5FDsubfiling.h
+97
View File
@@ -417,6 +417,103 @@ H5_DLL herr_t H5Pset_fapl_subfiling(hid_t fapl_id, const H5FD_subfiling_config_t
*/
H5_DLL herr_t H5Pget_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *config_out);
/**
* \ingroup H5VFD
*
* \brief Retrieve the subfile names for an HDF5 file using the subfiling VFD
*
* \file_id{file_id}
*
* \param[out] filenames Pointer to an array of C strings containing the subfile names.
* Memory is allocated by the function and must be freed by the caller.
* \param[out] len Pointer to the number of subfiles in the \p filenames array.
*
* \returns \herr_t
*
* \details H5FDsubfiling_get_file_mapping() retrieves the names of all physical subfiles
* that collectively make up a logical HDF5 file using the subfiling Virtual File
* Driver (VFD). The subfiling VFD distributes file data across multiple subfiles
* to improve parallel I/O performance, particularly systems where metadata operations
* can become a bottleneck.
*
* The function returns an array of subfile names corresponding to the physical files
* stored on the file system. Each MPI rank may be responsible for different subfiles,
* and this function provides visibility into which subfiles are associated with the
* calling rank.
*
* **Typical use cases include:**
* - **File management**: Understanding the physical storage layout for backup and archival
* - **Performance analysis**: Identifying subfile distribution patterns
* - **File fusion**: Providing subfile lists to tools like h5fuse for recombining files
* - **Debugging**: Troubleshooting subfiling configuration and I/O patterns
* - **Storage optimization**: Analyzing subfile sizes and distribution
*
* \note
* \parblock
* **Memory management**: The caller must call H5free_memory() on each string in the
* \p filenames array, and then call H5free_memory() on the \p filenames array itself:
* \code{.c}
* for (size_t i = 0; i < len; i++) {
* H5free_memory(filenames[i]);
* }
* H5free_memory(filenames);
* \endcode
* \endparblock
*
* \note
* \parblock
* **VFD requirement**: This function only works with files that use the subfiling VFD.
* Calling it on files using other VFDs will result in an error. This function will not
* be accessible if support for the subfiling VFD is unavailable or disabled.
* \endparblock
*
* \note
* \parblock
* **MPI context**: The function returns subfiles associated with the calling MPI rank.
* Different ranks may receive different subfile lists depending on the subfiling
* configuration and I/O concentrator (IOC) assignment.
* \endparblock
*
* \note
* \parblock
* \warning
* \endparblock
*
* \par Example
* \code{.c}
* #include "hdf5.h"
*
* hid_t file_id;
* char **subfile_names = NULL;
* size_t num_subfiles;
* herr_t ret;
*
* // Open file with subfiling VFD (setup not shown)
* // ...
*
* // Get subfile mapping
* ret = H5FDsubfiling_get_file_mapping(file_id, &subfile_names, &num_subfiles);
* if (ret >= 0) {
* printf("Found %zu subfiles:\n", num_subfiles);
* for (size_t i = 0; i < num_subfiles; i++) {
* printf(" %s\n", subfile_names[i]);
* H5free_memory(subfile_names[i]); // Free each string
* }
* H5free_memory(subfile_names); // Free the array
* } else {
* printf("Error getting file mapping\n");
* }
* \endcode
*
* \see H5Pset_fapl_subfiling()
* \see H5Pget_fapl_subfiling()
* \see H5free_memory()
*
* \since 2.0.0
*
*/
H5_DLL herr_t H5FDsubfiling_get_file_mapping(hid_t file_id, char ***filenames, size_t *len);
#ifdef __cplusplus
}
#endif
testpar/t_subfiling_vfd.c
+603
View File
@@ -99,6 +99,7 @@ static hid_t create_dcpl_id(int rank, const hsize_t dims[], hid_t dxpl_id);
/* Test functions */
static void test_create_and_close(void);
static void test_subfiling_file_mapping_api(void);
static void test_ioc_only_fail(void);
static void test_config_file(void);
static void test_stripe_sizes(void);
@@ -124,6 +125,7 @@ static test_func tests[] = {
test_subfiling_write_many_read_few,
test_subfiling_vector_io_extension,
test_subfiling_h5fuse,
test_subfiling_file_mapping_api,
};
/* ---------------------------------------------------------------------------
@@ -275,6 +277,601 @@ test_create_and_close(void)
}
#undef SUBF_FILENAME
/* Helper function to cleanup file mapping memory */
static void
cleanup_file_mapping_memory(char **filenames, size_t len)
{
if (filenames && len > 0) {
for (size_t i = 0; i < len; i++) {
if (filenames[i]) {
H5free_memory(filenames[i]);
}
}
H5free_memory(filenames);
}
}
/* Helper function to validate HDF5 file using h5fuse */
static herr_t
validate_file_with_h5fuse(const char *config_filename, char **subfile_names, size_t num_subfiles,
const char *main_filename)
{
char *h5fuse_cmd = NULL;
char subfile_list[2048];
int system_ret;
hid_t fused_file_id = H5I_INVALID_HID;
herr_t ret_value = SUCCEED;
/* Only validate on main process */
if (!MAINPROCESS)
return SUCCEED;
/* Allocate command buffer */
h5fuse_cmd = malloc(4096);
if (!h5fuse_cmd) {
ret_value = FAIL;
goto done;
}
/* Check if h5fuse script exists - similar to test_subfiling_h5fuse() */
int skip_validation = 0;
FILE *h5fuse_script;
h5fuse_script = fopen("./h5fuse", "r");
if (h5fuse_script)
fclose(h5fuse_script);
else
skip_validation = 1;
if (skip_validation) {
return SUCCEED; /* Skip validation if h5fuse script not found */
}
/* Build comma-separated list of subfiles for h5fuse */
subfile_list[0] = '\0';
for (size_t i = 0; i < num_subfiles; i++) {
if (i > 0) {
strcat(subfile_list, ",");
}
strcat(subfile_list, subfile_names[i]);
/* Verify each subfile exists before attempting to fuse */
if (access(subfile_names[i], F_OK) != 0) {
printf("ERROR: Subfile %s does not exist\n", subfile_names[i]);
ret_value = FAIL;
goto done;
}
}
/* Construct h5fuse command with proper arguments */
if (config_filename && access(config_filename, F_OK) == 0) {
/* Use configuration file if available */
snprintf(h5fuse_cmd, 4096, "./h5fuse -r -q -f %s -l %s", config_filename, subfile_list);
}
else {
/* Use subfile list only */
snprintf(h5fuse_cmd, 4096, "./h5fuse -r -q -l %s", subfile_list);
}
system_ret = system(h5fuse_cmd);
if (system_ret != 0) {
printf("ERROR: h5fuse command failed with return code: %d\n", system_ret);
ret_value = FAIL;
goto done;
}
/* Check if the fused file is accessible as an HDF5 file */
H5E_BEGIN_TRY
{
htri_t is_accessible = H5Fis_accessible(main_filename, H5P_DEFAULT);
if (is_accessible <= 0) {
printf("ERROR: Fused file %s is not accessible as an HDF5 file\n", main_filename);
ret_value = FAIL;
goto done;
}
}
H5E_END_TRY;
/* Open file with sec2 driver and verify the data */
{
hid_t sec2_file_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t fspace_id = H5I_INVALID_HID;
void *buf = NULL;
hsize_t dset_dims[1];
size_t dset_size;
H5E_BEGIN_TRY
{
sec2_file_id = H5Fopen(main_filename, H5F_ACC_RDONLY, H5P_DEFAULT);
}
H5E_END_TRY;
if (sec2_file_id >= 0) {
H5E_BEGIN_TRY
{
dset_id = H5Dopen2(sec2_file_id, "DSET", H5P_DEFAULT);
}
H5E_END_TRY;
if (dset_id >= 0) {
/* Get dataset dimensions */
fspace_id = H5Dget_space(dset_id);
if (fspace_id >= 0 && H5Sget_simple_extent_dims(fspace_id, dset_dims, NULL) >= 0) {
dset_size = dset_dims[0] * sizeof(int);
buf = malloc(dset_size);
if (buf) {
/* Read the entire dataset */
if (H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) >= 0) {
printf("Successfully read dataset data from fused file\n");
/* Note: Full data verification would require knowledge of the original data
* pattern */
/* For now, just verify we can read the data without errors */
}
free(buf);
}
if (fspace_id >= 0)
H5Sclose(fspace_id);
}
H5Dclose(dset_id);
}
H5Fclose(sec2_file_id);
}
else {
printf("ERROR: Could not re-open fused file with sec2 driver for data validation\n");
ret_value = FAIL;
goto done;
}
}
done:
if (h5fuse_cmd)
free(h5fuse_cmd);
H5E_BEGIN_TRY
{
if (fused_file_id >= 0)
H5Fclose(fused_file_id);
}
H5E_END_TRY;
return ret_value;
}
/* Helper function to find configuration file */
static char *
find_config_file(const char *main_filename)
{
h5_stat_t file_info;
char *config_filename = NULL;
/* Get file inode for config filename construction */
if (HDstat(main_filename, &file_info) < 0) {
return NULL;
}
config_filename = malloc(PATH_MAX);
if (!config_filename) {
return NULL;
}
/* Construct expected config filename using the same pattern as the main code */
snprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE,
config_dir ? config_dir : ".", main_filename, (uint64_t)file_info.st_ino);
/* Check if config file exists */
if (access(config_filename, F_OK) == 0) {
return config_filename;
}
/* Try alternative locations */
snprintf(config_filename, PATH_MAX, "./%s.config", main_filename);
if (access(config_filename, F_OK) == 0) {
return config_filename;
}
/* Search for any .config file in current directory */
snprintf(config_filename, PATH_MAX, "./subfiling.config");
if (access(config_filename, F_OK) == 0) {
return config_filename;
}
free(config_filename);
return NULL;
}
/* Helper function to create FAPL with specific IOC selection and count */
static hid_t
create_subfiling_fapl_with_ioc_selection_and_count(H5FD_subfiling_ioc_select_t selection_type, int ioc_count)
{
hid_t fapl_id = H5I_INVALID_HID;
H5FD_subfiling_config_t subf_config;
herr_t ret;
/* Create base FAPL with MPI settings */
fapl_id = create_subfiling_ioc_fapl(comm_g, info_g, false, NULL, 0);
VRFY((fapl_id >= 0), "Base FAPL creation succeeded");
/* Get current subfiling configuration */
ret = H5Pget_fapl_subfiling(fapl_id, &subf_config);
VRFY((ret >= 0), "H5Pget_fapl_subfiling succeeded");
/* Modify IOC selection method and stripe count */
subf_config.shared_cfg.ioc_selection = selection_type;
subf_config.shared_cfg.stripe_count = ioc_count;
/* Apply modified configuration */
ret = H5Pset_fapl_subfiling(fapl_id, &subf_config);
VRFY((ret >= 0), "H5Pset_fapl_subfiling succeeded");
VRFY((H5Pclose(subf_config.ioc_fapl_id) >= 0), "FAPL close succeeded");
return fapl_id;
}
/*
* Test H5FDsubfiling_get_file_mapping with different IOC selection methods
*/
#define SUBF_FILENAME_IOC "test_subfiling_ioc_selection.h5"
static void
test_subfiling_get_file_mapping_ioc_selection(void)
{
const H5FD_subfiling_ioc_select_t selection_types[] = {SELECT_IOC_ONE_PER_NODE, SELECT_IOC_EVERY_NTH_RANK,
/* SELECT_IOC_WITH_CONFIG, */
SELECT_IOC_TOTAL};
const char *selection_names[] = {"One IOC Per Node", "Every Nth Rank",
/* "With Config", */
"Total IOCs"};
const size_t num_selections = sizeof(selection_types) / sizeof(selection_types[0]);
curr_nerrors = nerrors;
if (MAINPROCESS)
TESTING_MULTIPART("H5FDsubfiling_get_file_mapping with different IOC selections");
/* Test each IOC selection with both 1 and 3 IOCs */
const int test_ioc_counts[] = {1, 3};
const size_t num_ioc_counts = sizeof(test_ioc_counts) / sizeof(test_ioc_counts[0]);
for (size_t i = 0; i < num_selections; i++) {
for (size_t j = 0; j < num_ioc_counts; j++) {
hid_t file_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
char **filenames = NULL;
size_t len = 0;
size_t len_total = 0;
herr_t ret;
char filename[256];
int expected_ioc_count = test_ioc_counts[j];
snprintf(filename, sizeof(filename), "test_subfiling_ioc_selection_%zu_%d.h5", i,
expected_ioc_count);
if (MAINPROCESS) {
char test_desc[256];
snprintf(test_desc, sizeof(test_desc), "IOC Selection: %s with %d IOCs", selection_names[i],
expected_ioc_count);
TESTING_2(test_desc);
}
/* Create FAPL with specific IOC selection and count */
fapl_id =
create_subfiling_fapl_with_ioc_selection_and_count(selection_types[i], expected_ioc_count);
VRFY((fapl_id >= 0), "IOC-specific FAPL creation succeeded");
/* Create file */
file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
VRFY((file_id >= 0), "H5Fcreate succeeded");
/* Test H5FDsubfiling_get_file_mapping */
ret = H5FDsubfiling_get_file_mapping(file_id, &filenames, &len);
VRFY((ret >= 0), "H5FDsubfiling_get_file_mapping succeeded");
MPI_Allreduce(&len, &len_total, 1, H5_SIZE_T_AS_MPI_TYPE, MPI_SUM, comm_g);
/* Validate that the number of subfiles matches expected IOC count */
VRFY((len_total == (size_t)expected_ioc_count), "Number of subfiles matches expected IOC count");
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
/* All ranks participate in h5fuse validation */
int validation_result = 1; /* Default to success */
char *config_filename = NULL;
if (len > 0) {
config_filename = find_config_file(filename);
ret = validate_file_with_h5fuse(config_filename, filenames, len, filename);
validation_result = (ret >= 0) ? 1 : 0;
}
/* Synchronize validation result across all ranks */
MPI_Allreduce(MPI_IN_PLACE, &validation_result, 1, MPI_INT, MPI_LAND, comm_g);
VRFY(validation_result, "h5fuse validation succeeded for IOC selection");
if (config_filename) {
free(config_filename);
}
cleanup_file_mapping_memory(filenames, len);
H5E_BEGIN_TRY
{
H5Fdelete(filename, fapl_id);
}
H5E_END_TRY
VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded");
CHECK_PASSED();
}
}
}
#undef SUBF_FILENAME_IOC
/*
* Test H5FDsubfiling_get_file_mapping consistency
*/
#define SUBF_FILENAME_CONSISTENCY "test_subfiling_mapping_consistency.h5"
static void
test_subfiling_get_file_mapping_consistency(void)
{
hid_t file_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
char **filenames1 = NULL, **filenames2 = NULL;
size_t len1 = 0, len2 = 0;
herr_t ret;
curr_nerrors = nerrors;
if (MAINPROCESS)
TESTING("H5FDsubfiling_get_file_mapping consistency across calls");
/* Create FAPL */
fapl_id = create_subfiling_ioc_fapl(comm_g, info_g, false, NULL, 0);
VRFY((fapl_id >= 0), "FAPL creation succeeded");
/* Create file */
file_id = H5Fcreate(SUBF_FILENAME_CONSISTENCY, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
VRFY((file_id >= 0), "H5Fcreate succeeded");
/* First call to H5FDsubfiling_get_file_mapping */
ret = H5FDsubfiling_get_file_mapping(file_id, &filenames1, &len1);
VRFY((ret >= 0), "First H5FDsubfiling_get_file_mapping call succeeded");
/* Second call to H5FDsubfiling_get_file_mapping */
ret = H5FDsubfiling_get_file_mapping(file_id, &filenames2, &len2);
VRFY((ret >= 0), "Second H5FDsubfiling_get_file_mapping call succeeded");
/* Verify consistency */
VRFY((len1 == len2), "Both calls returned same number of subfiles");
if (len1 > 0 && len2 > 0) {
for (size_t i = 0; i < len1; i++) {
VRFY((strcmp(filenames1[i], filenames2[i]) == 0), "Filenames are identical between calls");
}
cleanup_file_mapping_memory(filenames1, len1);
cleanup_file_mapping_memory(filenames2, len2);
}
/* Cleanup */
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
H5E_BEGIN_TRY
{
H5Fdelete(SUBF_FILENAME_CONSISTENCY, fapl_id);
}
H5E_END_TRY
VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded");
CHECK_PASSED();
}
#undef SUBF_FILENAME_CONSISTENCY
/*
* Test H5FDsubfiling_get_file_mapping error conditions
*/
static void
test_subfiling_get_file_mapping_errors(void)
{
hid_t file_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
char **filenames = NULL;
size_t len = 0;
herr_t ret;
curr_nerrors = nerrors;
if (MAINPROCESS)
TESTING("H5FDsubfiling_get_file_mapping error conditions");
/* Test with invalid file ID */
H5E_BEGIN_TRY
{
ret = H5FDsubfiling_get_file_mapping(H5I_INVALID_HID, &filenames, &len);
}
H5E_END_TRY
VRFY((ret < 0), "H5FDsubfiling_get_file_mapping failed with invalid file ID as expected");
/* Create valid file for parameter testing */
fapl_id = create_subfiling_ioc_fapl(comm_g, info_g, false, NULL, 0);
VRFY((fapl_id >= 0), "FAPL creation succeeded");
file_id = H5Fcreate("test_error_conditions.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
VRFY((file_id >= 0), "H5Fcreate succeeded");
/* Test with NULL filenames parameter */
H5E_BEGIN_TRY
{
ret = H5FDsubfiling_get_file_mapping(file_id, NULL, &len);
}
H5E_END_TRY
VRFY((ret < 0), "H5FDsubfiling_get_file_mapping failed with NULL filenames as expected");
/* Test with NULL len parameter */
H5E_BEGIN_TRY
{
ret = H5FDsubfiling_get_file_mapping(file_id, &filenames, NULL);
}
H5E_END_TRY
VRFY((ret < 0), "H5FDsubfiling_get_file_mapping failed with NULL len as expected");
/* Test with both parameters NULL */
H5E_BEGIN_TRY
{
ret = H5FDsubfiling_get_file_mapping(file_id, NULL, NULL);
}
H5E_END_TRY
VRFY((ret < 0), "H5FDsubfiling_get_file_mapping failed with both NULL parameters as expected");
/* Cleanup */
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
H5E_BEGIN_TRY
{
H5Fdelete("test_error_conditions.h5", fapl_id);
}
H5E_END_TRY
VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded");
CHECK_PASSED();
}
/*
* Test H5FDsubfiling_get_file_mapping with data I/O operations
*/
#define SUBF_FILENAME_IO "test_subfiling_mapping_with_io.h5"
static void
test_subfiling_get_file_mapping_with_io(void)
{
hid_t file_id = H5I_INVALID_HID;
hid_t fapl_id = H5I_INVALID_HID;
hid_t dset_id = H5I_INVALID_HID;
hid_t dspace_id = H5I_INVALID_HID;
char **filenames_before = NULL, **filenames_after = NULL;
size_t len_before = 0, len_after = 0;
herr_t ret;
hsize_t dims[2] = {100, 50};
int *write_buf = NULL;
curr_nerrors = nerrors;
if (MAINPROCESS)
TESTING("H5FDsubfiling_get_file_mapping before/after I/O operations");
/* Allocate write buffer */
write_buf = malloc(dims[0] * dims[1] * sizeof(int));
VRFY((write_buf != NULL), "Write buffer allocation succeeded");
/* Initialize data */
for (hsize_t i = 0; i < dims[0] * dims[1]; i++) {
write_buf[i] = (int)(mpi_rank * 1000 + (int)i);
}
/* Create FAPL */
fapl_id = create_subfiling_ioc_fapl(comm_g, info_g, false, NULL, 0);
VRFY((fapl_id >= 0), "FAPL creation succeeded");
/* Create file */
file_id = H5Fcreate(SUBF_FILENAME_IO, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
VRFY((file_id >= 0), "H5Fcreate succeeded");
/* Get file mapping before I/O */
ret = H5FDsubfiling_get_file_mapping(file_id, &filenames_before, &len_before);
VRFY((ret >= 0), "H5FDsubfiling_get_file_mapping before I/O succeeded");
/* Create dataset and write data */
dspace_id = H5Screate_simple(2, dims, NULL);
VRFY((dspace_id >= 0), "H5Screate_simple succeeded");
dset_id =
H5Dcreate2(file_id, "/dataset", H5T_NATIVE_INT, dspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
VRFY((dset_id >= 0), "H5Dcreate2 succeeded");
ret = H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, write_buf);
VRFY((ret >= 0), "H5Dwrite succeeded");
/* Get file mapping after I/O */
ret = H5FDsubfiling_get_file_mapping(file_id, &filenames_after, &len_after);
VRFY((ret >= 0), "H5FDsubfiling_get_file_mapping after I/O succeeded");
/* Verify mapping consistency */
VRFY((len_before == len_after), "File mapping length unchanged after I/O");
for (size_t i = 0; i < len_before; i++) {
VRFY((strcmp(filenames_before[i], filenames_after[i]) == 0), "File mapping unchanged after I/O");
}
VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded");
VRFY((H5Sclose(dspace_id) >= 0), "Dataspace close succeeded");
VRFY((H5Fclose(file_id) >= 0), "File close succeeded");
/* All ranks participate in h5fuse validation with actual data */
int validation_result = 1; /* Default to success */
char *config_filename = NULL;
if (len_after > 0) {
config_filename = find_config_file(SUBF_FILENAME_IO);
ret = validate_file_with_h5fuse(config_filename, filenames_after, len_after, SUBF_FILENAME_IO);
validation_result = (ret >= 0) ? 1 : 0;
}
/* Synchronize validation result across all ranks */
MPI_Allreduce(MPI_IN_PLACE, &validation_result, 1, MPI_INT, MPI_LAND, comm_g);
VRFY(validation_result, "h5fuse validation with I/O data succeeded");
if (config_filename) {
free(config_filename);
}
/* All ranks participate in cleanup */
cleanup_file_mapping_memory(filenames_before, len_before);
cleanup_file_mapping_memory(filenames_after, len_after);
/* Cleanup */
free(write_buf);
H5E_BEGIN_TRY
{
H5Fdelete(SUBF_FILENAME_IO, fapl_id);
}
H5E_END_TRY
VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded");
CHECK_PASSED();
}
#undef SUBF_FILENAME_IO
static void
test_subfiling_file_mapping_api(void)
{
int saved_num_iocs_g = num_iocs_g; /* Save original value */
if (MAINPROCESS) {
printf("\n");
printf("=======================================================\n");
printf("Testing H5FDsubfiling_get_file_mapping API Functions\n");
printf("=======================================================\n");
}
test_subfiling_get_file_mapping_ioc_selection(); /* Different IOC selections */
test_subfiling_get_file_mapping_consistency(); /* Consistency testing */
test_subfiling_get_file_mapping_errors(); /* Error conditions */
test_subfiling_get_file_mapping_with_io(); /* With actual I/O */
if (MAINPROCESS) {
printf("H5FDsubfiling_get_file_mapping API tests completed.\n");
printf("=======================================================\n\n");
}
MPI_Barrier(comm_g);
/* Restore original num_iocs_g value to prevent affecting subsequent tests */
num_iocs_g = saved_num_iocs_g;
}
/*
* A simple test that ensures file creation fails when
* attempting to use the IOC VFD by itself, without it
@@ -3442,6 +4039,12 @@ exit:
HDunsetenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX);
if (MAINPROCESS) {
/* Remove any .config files in the directory before removing the directory */
char cleanup_cmd[512];
snprintf(cleanup_cmd, sizeof(cleanup_cmd), "rm -f %s/*.config 2>/dev/null",
SUBFILING_CONFIG_FILE_DIR);
system(cleanup_cmd);
if (HDrmdir(SUBFILING_CONFIG_FILE_DIR) < 0 && (errno != ENOENT)) {
printf("couldn't remove temporary testing directory\n");
nerrors++;
utils/subfiling_vfd/h5fuse.in
+44 -13
View File
@@ -26,10 +26,11 @@ function usage {
configuration file either as a command-line argument or the script will
search for the *.config file in the current directory."
echo ""
echo "usage: h5fuse [-f filename] [-h] [-p] [-q] [-r] [-v] "
echo "usage: h5fuse [-f filename] [-h] [-p] [-q] [-r] [-v] [-l subfile1,subfile2,...]"
echo "-f filename Subfile configuration file."
echo "-h Print this help."
echo "-q Quiet all output. [no]"
echo "-l Comma separated list of subfiles to process"
echo "-p h5fuse is being run in parallel, with more than one rank. [no]"
echo "-r Remove subfiles after being processed. [no]"
echo "-v Verbose output. [no]"
@@ -131,8 +132,7 @@ for i in "${subfiles[@]}"; do
if [ "$verbose" == "true" ]; then
echo -e "$GRN $EXEC $NC"
fi
err=$( $EXEC 2>&1 1>/dev/null )
if [ $? -ne 0 ]; then
if ! err=$( $EXEC 2>&1 1>/dev/null ); then
echo -e "$CYN ERR: dd Utility Failed $NC"
echo -e "$CYN MSG: $err $NC"
exit $FAILED
@@ -172,7 +172,8 @@ verbose="false"
quiet="false"
rm_subf="false"
parallel="false"
while getopts "hpqrvf:" option; do
subf_list=()
while getopts "hpqrvf:l:" option; do
case $option in
f) # subfiling configuration file
file_config=$OPTARG;;
@@ -187,6 +188,9 @@ while getopts "hpqrvf:" option; do
rm_subf="true";;
v) # verbose output
verbose="true";;
l) # process a list of subfiles
# Split the comma-separated list into an array
IFS=',' read -r -a subf_list <<< "${OPTARG}";;
\?) # Invalid option
echo -e "$RED ERROR: Invalid option ${BLD}-${OPTARG}${RED} $NC"
usage
@@ -259,17 +263,44 @@ fi
nsubfiles=${#subfiles[@]}
# Get the number of local subfiles
subfiles_loc=()
subfiles_size=()
for i in "${subfiles[@]}"; do
subfile="${subfile_dir}/${i}"
if [ -f "${subfile}" ]; then
subfiles_loc+=("$subfile")
subfiles_size+=($(wc -c "${subfile}" | awk '{print $1}'))
else
subfiles_size+=(0)
fi
done
if [[ "${#subf_list[@]}" == "0" ]]; then
for i in "${subfiles[@]}"; do
subfile="${subfile_dir}/${i}"
if [ -f "${subfile}" ]; then
subfiles_loc+=("$subfile")
subfiles_size+=("$(wc -c "${subfile}" | awk '{print $1}')")
else
subfiles_size+=(0)
fi
done
else
for i in "${subfiles[@]}"; do
subfile="${subfile_dir}/${i}"
match=0
for ((j=0; j<${#subf_list[@]}; ++j)); do
if [[ "${subf_list[j]}" == "$subfile" ]]; then
unset "subf_list[$j]" # Use unset within loop for proper removal
match=1
break
fi
done
if [[ "$match" == "1" ]]; then
subfiles_loc+=("$subfile")
subfiles_size+=("$(wc -c "${subfile}" | awk '{print $1}')")
else
subfiles_size+=(0)
fi
done
fi
if [ "$quiet" == "false" ]; then
TIMEFORMAT="COMPLETION TIME = %R s"