diff --git a/config/apple b/config/apple index a8a219b6798..39ed454a11f 100644 --- a/config/apple +++ b/config/apple @@ -55,30 +55,19 @@ fi # Figure out C compiler flags . $srcdir/config/gnu-flags . $srcdir/config/clang-flags +. $srcdir/config/oneapi-flags . $srcdir/config/intel-flags -# temp patch: if GCC 4.2.1 is used in Lion or Mountain Lion systems, do not -# use -O option as it causes failures in test/dt_arith. -case "$host_os" in - darwin1[12].*) # lion & mountain lion - #echo cc_vendor=$cc_vendor'-'cc_version=$cc_version - case "$cc_vendor-$cc_version" in - gcc-4.2.1) - # Remove any -O flags - #echo PROD_CFLAGS=$PROD_CFLAGS - PROD_CFLAGS="`echo $PROD_CFLAGS | sed -e 's/-O[0-3]*//'`" - #echo new PROD_CFLAGS=$PROD_CFLAGS - ;; - esac - ;; -esac - if test "X-" = "X-$FC"; then case $CC_BASENAME in gcc*) FC=gfortran FC_BASENAME=gfortran ;; + icx*) + FC=ifx + FC_BASENAME=ifx + ;; icc*) FC=ifort FC_BASENAME=ifort @@ -97,6 +86,7 @@ fi # Figure out FORTRAN compiler flags . $srcdir/config/gnu-fflags +. $srcdir/config/oneapi-fflags . $srcdir/config/intel-fflags @@ -107,6 +97,10 @@ if test "X-" = "X-$CXX"; then CXX=g++ CXX_BASENAME=g++ ;; + icx) + CXX=icpx + CXX_BASENAME=icpx + ;; icc) CXX=icpc CXX_BASENAME=icpc @@ -123,6 +117,7 @@ if test "X-" = "X-$CXX"; then fi # Figure out C++ compiler flags +. $srcdir/config/oneapi-cxxflags . $srcdir/config/intel-cxxflags # Do this ahead of GNU to avoid icpc being detected as g++ . $srcdir/config/gnu-cxxflags . $srcdir/config/clang-cxxflags @@ -139,6 +134,11 @@ case $CC in grep 'GCC' | sed 's/.*\((GCC) [-a-z0-9\. ]*.*\)/\1/'` ;; + *icx*) + cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *icc*) cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` @@ -156,6 +156,11 @@ case $FC in grep 'GCC' | sed 's/\(.*(GCC) [-a-z0-9\. ]*\).*/\1/'` ;; + *ifx*) + fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *ifc*|*ifort*) fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` @@ -179,6 +184,11 @@ case $CXX in grep 'GCC' | sed 's/.*\((GCC) [-a-z0-9\. ]*.*\)/\1/'` ;; + *icpx*) + cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *icpc*) cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` diff --git a/config/freebsd b/config/freebsd index 2fb962fe308..b0e825a9e26 100644 --- a/config/freebsd +++ b/config/freebsd @@ -29,7 +29,10 @@ fi # Figure out GNU C compiler flags . $srcdir/config/gnu-flags -# Figure out Intel C compiler flags +# Figure out Intel oneAPI C compiler flags +. $srcdir/config/oneapi-flags + +# Figure out Intel classic C compiler flags . $srcdir/config/intel-flags # The default Fortran 90 compiler @@ -43,6 +46,10 @@ if test "X-" = "X-$FC"; then FC=gfortran FC_BASENAME=gfortran ;; + icx*) + FC=ifx + FC_BASENAME=ifx + ;; icc*) FC=ifort FC_BASENAME=ifort @@ -57,8 +64,11 @@ fi # Figure out FORTRAN compiler flags . $srcdir/config/gnu-fflags -# Figure out Intel F90 compiler flags -. $srcdir/config/intel-fflags +# Figure out Intel oneAPI FC compiler flags +. $srcdir/config/oneapi-fflags + +# Figure out Intel classic FC compiler flags +. $srcdir/config/classic-fflags # The default C++ compiler diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1 index 7614b07852f..328f8d3cec3 100644 --- a/config/linux-gnulibc1 +++ b/config/linux-gnulibc1 @@ -38,7 +38,10 @@ fi # Figure out CCE C compiler flags . $srcdir/config/cce-flags -# Figure out Intel C compiler flags +# Figure out Intel oneAPI C compiler flags +. $srcdir/config/oneapi-flags + +# Figure out Intel classic C compiler flags . $srcdir/config/intel-flags # Figure out Clang C compiler flags @@ -55,6 +58,10 @@ if test "X-" = "X-$FC"; then FC=pgf90 FC_BASENAME=pgf90 ;; + icx*) + FC=ifx + FC_BASENAME=ifx + ;; icc*) FC=ifort FC_BASENAME=ifort @@ -119,7 +126,10 @@ fi # Figure out CCE FC compiler flags . $srcdir/config/cce-fflags -# Figure out Intel FC compiler flags +# Figure out Intel oneAPI FC compiler flags +. $srcdir/config/oneapi-fflags + +# Figure out Intel classic FC compiler flags . $srcdir/config/intel-fflags # Figure out Clang FC compiler flags @@ -200,7 +210,10 @@ if test -z "$CXX"; then CXX_BASENAME=g++ fi -# Figure out Intel CXX compiler flags +# Figure out Intel oneAPI CXX compiler flags +. $srcdir/config/oneapi-cxxflags + +# Figure out Intel classic CXX compiler flags # Do this ahead of GNU to avoid icpc being detected as g++ . $srcdir/config/intel-cxxflags @@ -237,6 +250,11 @@ case $CC in cc_version_info=`echo $cc_version_info` ;; + *icx*) + cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *icc*) cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` @@ -271,6 +289,11 @@ case $FC in fc_version_info=`echo $fc_version_info` ;; + *ifx*) + fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *ifc*|*ifort*) fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` @@ -333,6 +356,10 @@ case $CXX in cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS --version 2>&1 |\ grep 'GCC' | sed 's/\(.*(GCC) [-a-z0-9\. ]*\).*/\1/'` ;; + *icpx*) + cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; *icpc*) cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\ sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` diff --git a/config/netbsd b/config/netbsd index 04761f294a8..0ed84f7b3d2 100644 --- a/config/netbsd +++ b/config/netbsd @@ -26,7 +26,10 @@ fi # Figure out C compiler flags . $srcdir/config/gnu-flags -# Figure out Intel C compiler flags +# Figure out Intel oneAPI C compiler flags +. $srcdir/config/oneapi-flags + +# Figure out Intel classic C compiler flags . $srcdir/config/intel-flags # The default Fortran 90 compiler @@ -36,6 +39,10 @@ if test "X-" = "X-$FC"; then FC=gfortran FC_BASENAME=gfortran ;; + icx*) + FC=ifx + FC_BASENAME=ifx + ;; icc*) FC=ifort FC_BASENAME=ifort @@ -50,6 +57,8 @@ fi # Figure out FORTRAN compiler flags . $srcdir/config/gnu-fflags -# Figure out Intel F90 compiler flags +# Figure out Intel oneAPI FC compiler flags . $srcdir/config/intel-fflags +# Figure out Intel classic FC compiler flags +. $srcdir/config/oneapi-fflags diff --git a/config/oneapi-cxxflags b/config/oneapi-cxxflags new file mode 100644 index 00000000000..d9819b94c44 --- /dev/null +++ b/config/oneapi-cxxflags @@ -0,0 +1,155 @@ +# -*- shell-script -*- +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + + +# This file should be sourced into configure if the compiler is the +# Intel icpx compiler or a derivative. It is careful not to do anything +# if the compiler is not Intel; otherwise `cxx_flags_set' is set to `yes' +# + +# +# Prepend `$srcdir/config/intel-warnings/` to the filename suffix(es) given as +# subroutine argument(s), remove comments starting with # and ending +# at EOL, replace spans of whitespace (including newlines) with spaces, +# and re-emit the file(s) thus filtered on the standard output stream. +# +load_intel_arguments() +{ + set -- $(for arg; do + sed 's,#.*$,,' $srcdir/config/intel-warnings/${arg} + done) + IFS=' ' echo "$*" +} + +# Get the compiler version in a way that works for icpx +# icpx unless a compiler version is already known +# +# cxx_vendor: The compiler name: icpx +# cxx_version: Version number: 2023.2.0 +# +if test X = "X$cxx_flags_set"; then + cxx_version="`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 |grep 'oneAPI'`" + if test X != "X$cxx_version"; then + cxx_vendor=icpx + cxx_version=`echo $cxx_version |sed 's/.*Version \([-a-z0-9\.\-]*\).*/\1/'` + echo "compiler '$CXX' is Intel oneAPI $cxx_vendor-$cxx_version" + + # Some version numbers + # Intel oneAPI version numbers are of the form: "major.minor.patch" + cxx_vers_major=`echo $cxx_version | cut -f1 -d.` + cxx_vers_minor=`echo $cxx_version | cut -f2 -d.` + cxx_vers_patch=`echo $cxx_version | cut -f2 -d.` + test -n "$cxx_vers_major" || cxx_vers_major=0 + test -n "$cxx_vers_minor" || cxx_vers_minor=0 + test -n "$cxx_vers_patch" || cxx_vers_patch=0 + cxx_vers_all=`expr $cxx_vers_major '*' 1000000 + $cxx_vers_minor '*' 1000 + $cxx_vers_patch` + fi +fi + +# Common Intel flags for various situations +if test "X-icpx" = "X-$cxx_vendor"; then + # Insert section about version specific problems from compiler flags here, + # if necessary. + + arch= + # Architecture-specific flags + # Nothing currently. (Uncomment code below and modify to add any) + #case "$host_os-$host_cpu" in + # *-i686) + # arch="-march=i686" + # ;; + #esac + + # Host-specific flags + # Nothing currently. (Uncomment code below and modify to add any) + #case "`hostname`" in + # sleipnir.ncsa.uiuc.edu) + # arch="$arch -pipe" + # ;; + #esac + + ########### + # General # + ########### + + # Default to C++11 standard + H5_CXXFLAGS="$H5_CXXFLAGS $arch -std=c++11" + + ############## + # Production # + ############## + + PROD_CXXFLAGS= + + ######### + # Debug # + ######### + + # NDEBUG is handled explicitly in configure + # -g is handled by the symbols flags + DEBUG_CXXFLAGS= + + ########### + # Symbols # + ########### + + NO_SYMBOLS_CXXFLAGS="-Wl,-s" + SYMBOLS_CXXFLAGS="-g" + + ############# + # Profiling # + ############# + + PROFILE_CXXFLAGS="-p" + + ################ + # Optimization # + ################ + + HIGH_OPT_CXXFLAGS="-O3" + DEBUG_OPT_CXXFLAGS="-O0" + NO_OPT_CXXFLAGS="-O0" + + ############ + # Warnings # + ############ + + ########### + # General # + ########### + + # Add various general warning flags in intel-warnings. + # Use the C warnings as CXX warnings are the same + H5_CXXFLAGS="$H5_CXXFLAGS $(load_intel_arguments oneapi/general)" + + ###################### + # Developer warnings # + ###################### + + # Use the C warnings as CXX warnings are the same + DEVELOPER_WARNING_CXXFLAGS=$(load_intel_arguments oneapi/developer-general) + + ############################# + # Version-specific warnings # + ############################# + + ################# + # Flags are set # + ################# + cxx_flags_set=yes +fi + +# Clear cxx info if no flags set +if test "X-$cxx_flags_set" = "X-"; then + cxx_vendor= + cxx_version= +fi diff --git a/config/oneapi-fflags b/config/oneapi-fflags new file mode 100644 index 00000000000..a63108d0b99 --- /dev/null +++ b/config/oneapi-fflags @@ -0,0 +1,145 @@ +# -*- shell-script -*- +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + + +# This file should be sourced into configure if the compiler is the +# Intel oneAPI ifx compiler or a derivative. It is careful not to do anything +# if the compiler is not Intel; otherwise `f9x_flags_set' is set to `yes' +# + +# +# Prepend `$srcdir/config/intel-warnings/` to the filename suffix(es) given as +# subroutine argument(s), remove comments starting with # and ending +# at EOL, replace spans of whitespace (including newlines) with spaces, +# and re-emit the file(s) thus filtered on the standard output stream. +# +load_intel_arguments() +{ + set -- $(for arg; do + sed 's,#.*$,,' $srcdir/config/intel-warnings/${arg} + done) + IFS=' ' echo "$*" +} + +# Get the compiler version in a way that works for ifx +# ifx unless a compiler version is already known +# +# f9x_vendor: The compiler name: ifx +# f9x_version: Version number: 2023.2.0 +# +if test X = "X$f9x_flags_set"; then + f9x_version="`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 |grep '^Intel'`" + if test X != "X$f9x_version"; then + f9x_vendor=ifx + f9x_version="`echo $f9x_version |sed 's/.*Version \([-a-z0-9\.\-]*\).*/\1/'`" + echo "compiler '$FC' is Intel oneAPI $f9x_vendor-$f9x_version" + + # Some version numbers + # Intel oneAPI version numbers are of the form: "major.minor.patch" + f9x_vers_major=`echo $f9x_version | cut -f1 -d.` + f9x_vers_minor=`echo $f9x_version | cut -f2 -d.` + f9x_vers_patch=`echo $f9x_version | cut -f2 -d.` + test -n "$f9x_vers_major" || f9x_vers_major=0 + test -n "$f9x_vers_minor" || f9x_vers_minor=0 + test -n "$f9x_vers_patch" || f9x_vers_patch=0 + f9x_vers_all=`expr $f9x_vers_major '*' 1000000 + $f9x_vers_minor '*' 1000 + $f9x_vers_patch` + fi +fi + +if test "X-ifx" = "X-$f9x_vendor"; then + + FC_BASENAME=ifx + F9XSUFFIXFLAG="" + FSEARCH_DIRS="" + + ############################### + # Architecture-specific flags # + ############################### + + arch= + # Nothing currently. (Uncomment code below and modify to add any) + #case "$host_os-$host_cpu" in + # *-i686) + # arch="-march=i686" + # ;; + #esac + + # Host-specific flags + # Nothing currently. (Uncomment code below and modify to add any) + #case "`hostname`" in + # sleipnir.ncsa.uiuc.edu) + # arch="$arch -pipe" + # ;; + #esac + + ############## + # Production # + ############## + + PROD_FCFLAGS= + + ######### + # Debug # + ######### + + # Don't use -check uninit or you'll get false positives from H5_buildiface + DEBUG_FCFLAGS="-check all,nouninit" + + ########### + # Symbols # + ########### + + NO_SYMBOLS_FCFLAGS= + SYMBOLS_FCFLAGS="-g" + + ############# + # Profiling # + ############# + + PROFILE_FCFLAGS="-p" + + ################ + # Optimization # + ################ + + HIGH_OPT_FCFLAGS="-O3" + DEBUG_OPT_FCFLAGS="-O0" + NO_OPT_FCFLAGS="-O0" + + ############ + # Warnings # + ############ + + ########### + # General # + ########### + + H5_FCFLAGS="$H5_FCFLAGS -free" + H5_FCFLAGS="$H5_FCFLAGS $(load_intel_arguments oneapi/ifort-general)" + + ############################# + # Version-specific warnings # + ############################# + + + ################# + # Flags are set # + ################# + f9x_flags_set=yes +fi + +# Clear f9x info if no flags set +if test "X-$f9x_flags_set" = "X-"; then + f9x_vendor= + f9x_version= +fi + diff --git a/config/oneapi-flags b/config/oneapi-flags new file mode 100644 index 00000000000..629e93f02f1 --- /dev/null +++ b/config/oneapi-flags @@ -0,0 +1,151 @@ +# -*- shell-script -*- +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + + +# This file should be sourced into configure if the compiler is the +# Intel icx compiler or a derivative. It is careful not to do anything +# if the compiler is not Intel; otherwise `cc_flags_set' is set to `yes' +# + +# +# Prepend `$srcdir/config/intel-warnings/` to the filename suffix(es) given as +# subroutine argument(s), remove comments starting with # and ending +# at EOL, replace spans of whitespace (including newlines) with spaces, +# and re-emit the file(s) thus filtered on the standard output stream. +# +load_intel_arguments() +{ + set -- $(for arg; do + sed 's,#.*$,,' $srcdir/config/intel-warnings/${arg} + done) + IFS=' ' echo "$*" +} + +# Get the compiler version in a way that works for icx +# icx unless a compiler version is already known +# cc_vendor: The compiler name: icx +# cc_version: Version number: 2023.2.0 +# +if test X = "X$cc_flags_set"; then + cc_version="`$CC $CFLAGS $H5_CFLAGS -V 2>&1 |grep 'oneAPI'`" + if test X != "X$cc_version"; then + cc_vendor=icx + cc_version=`echo $cc_version |sed 's/.*Version \([-a-z0-9\.\-]*\).*/\1/'` + echo "compiler '$CC' is Intel oneAPI $cc_vendor-$cc_version" + + # Some version numbers + # Intel oneAPI version numbers are of the form: "major.minor.patch" + cc_vers_major=`echo $cc_version | cut -f1 -d.` + cc_vers_minor=`echo $cc_version | cut -f2 -d.` + cc_vers_patch=`echo $cc_version | cut -f2 -d.` + test -n "$cc_vers_major" || cc_vers_major=0 + test -n "$cc_vers_minor" || cc_vers_minor=0 + test -n "$cc_vers_patch" || cc_vers_patch=0 + cc_vers_all=`expr $cc_vers_major '*' 1000000 + $cc_vers_minor '*' 1000 + $cc_vers_patch` + fi +fi + +# Common Intel flags for various situations +if test "X-icx" = "X-$cc_vendor"; then + # Insert section about version specific problems from compiler flags here, + # if necessary. + + arch= + # Architecture-specific flags + # Nothing currently. (Uncomment code below and modify to add any) + #case "$host_os-$host_cpu" in + # *-i686) + # arch="-march=i686" + # ;; + #esac + + # Host-specific flags + # Nothing currently. (Uncomment code below and modify to add any) + #case "`hostname`" in + # sleipnir.ncsa.uiuc.edu) + # arch="$arch -pipe" + # ;; + #esac + + ########### + # General # + ########### + + # Default to C99 standard. + H5_CFLAGS="$H5_CFLAGS $arch -std=c99" + + ############## + # Production # + ############## + + PROD_CFLAGS= + + ######### + # Debug # + ######### + + # NDEBUG is handled explicitly in configure + DEBUG_CFLAGS= + + ########### + # Symbols # + ########### + + NO_SYMBOLS_CFLAGS="-Wl,-s" + SYMBOLS_CFLAGS="-g" + + ############# + # Profiling # + ############# + + PROFILE_CFLAGS="-p" + + ################ + # Optimization # + ################ + + HIGH_OPT_CFLAGS="-O3" + DEBUG_OPT_CFLAGS="-O0" + NO_OPT_CFLAGS="-O0" + + ############ + # Warnings # + ############ + + ########### + # General # + ########### + + # Add various general warning flags in intel-warnings. + H5_CFLAGS="$H5_CFLAGS $(load_intel_arguments oneapi/general)" + + ###################### + # Developer warnings # + ###################### + + DEVELOPER_WARNING_CFLAGS=$(load_intel_arguments oneapi/developer-general) + + ############################# + # Version-specific warnings # + ############################# + + ################# + # Flags are set # + ################# + cc_flags_set=yes +fi + +# Clear cc info if no flags set +if test "X-$cc_flags_set" = "X-"; then + cc_vendor= + cc_version= +fi diff --git a/fortran/src/H5Fff.F90 b/fortran/src/H5Fff.F90 index fee4d3c7cf8..d31117784e5 100644 --- a/fortran/src/H5Fff.F90 +++ b/fortran/src/H5Fff.F90 @@ -72,12 +72,6 @@ MODULE H5F INTEGER(HSIZE_T) :: tot_space !< Amount of free space in the file END TYPE H5F_info_free_t -!> @brief H5_ih_info_t derived type. - TYPE, BIND(C) :: H5_ih_info_t - INTEGER(HSIZE_T) :: index_size !< btree and/or list - INTEGER(HSIZE_T) :: heap_size !< Heap size - END TYPE H5_ih_info_t - !> @brief H5F_info_t_sohm derived type. TYPE, BIND(C) :: H5F_info_sohm_t INTEGER(C_INT) :: version !< Version # of shared object header info diff --git a/fortran/src/H5Off.F90 b/fortran/src/H5Off.F90 index 4a0a1632e78..b705ba324d7 100644 --- a/fortran/src/H5Off.F90 +++ b/fortran/src/H5Off.F90 @@ -110,12 +110,6 @@ MODULE H5O TYPE(mesg_t) :: mesg END TYPE c_hdr_t -!> @brief Extra metadata storage for obj & attributes - TYPE, BIND(C) :: H5_ih_info_t - INTEGER(hsize_t) :: index_size !< btree and/or list - INTEGER(hsize_t) :: heap_size !< heap - END TYPE H5_ih_info_t - !> @brief meta_size_t derived type TYPE, BIND(C) :: meta_size_t TYPE(H5_ih_info_t) :: obj !< v1/v2 B-tree & local/fractal heap for groups, B-tree for chunked datasets diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index bbc7a9dc066..5821889c3e9 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -6405,7 +6405,7 @@ END SUBROUTINE h5pget_virtual_dsetname_f !! \brief Gets the file space handling strategy and persisting free-space values for a file creation property list. !! !! \param plist_id File creation property list identifier -!! \param strategy The file space handling strategy to be used. +!! \param strategy The file space handling strategy to be used !! \param persist Indicate whether free space should be persistent or not !! \param threshold The free-space section size threshold value !! \param hdferr \fortran_error @@ -6507,6 +6507,42 @@ END SUBROUTINE h5pget_virtual_dsetname_f hdferr = INT(h5pget_file_space_page_size(prp_id, fsp_size)) END SUBROUTINE h5pget_file_space_page_size_f +!> +!! \ingroup FH5P +!! +!! \brief Retrieves the type(s) of I/O that HDF5 actually performed on raw data +!! during the last I/O call. +!! +!! \param plist_id File creation property list identifier +!! \param actual_selection_io_mode A bitwise set value indicating the type(s) of I/O performed +!! \param hdferr \fortran_error +!! +!! See C API: @ref H5Pget_actual_selection_io_mode() +!! + SUBROUTINE h5pget_actual_selection_io_mode_f(plist_id, actual_selection_io_mode, hdferr) + + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN) :: plist_id + INTEGER , INTENT(OUT) :: actual_selection_io_mode + INTEGER , INTENT(OUT) :: hdferr + + INTEGER(C_INT32_T) :: c_actual_selection_io_mode + + INTERFACE + INTEGER(C_INT) FUNCTION H5Pget_actual_selection_io_mode(plist_id, actual_selection_io_mode) & + BIND(C, NAME='H5Pget_actual_selection_io_mode') + IMPORT :: HID_T, C_INT32_T, C_INT + IMPLICIT NONE + INTEGER(HID_T), VALUE :: plist_id + INTEGER(C_INT32_T) :: actual_selection_io_mode + END FUNCTION H5Pget_actual_selection_io_mode + END INTERFACE + + hdferr = INT(H5Pget_actual_selection_io_mode(plist_id, c_actual_selection_io_mode)) + + actual_selection_io_mode = INT(c_actual_selection_io_mode) + + END SUBROUTINE h5pget_actual_selection_io_mode_f END MODULE H5P diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c index 181047b5454..0392c2bdfa6 100644 --- a/fortran/src/H5_f.c +++ b/fortran/src/H5_f.c @@ -477,6 +477,10 @@ h5init_flags_c(int_f *h5d_flags, size_t_f *h5d_size_flags, int_f *h5e_flags, hid h5d_flags[55] = (int_f)H5D_MPIO_LINK_CHUNK; h5d_flags[56] = (int_f)H5D_MPIO_MULTI_CHUNK; + h5d_flags[57] = (int_f)H5D_SCALAR_IO; + h5d_flags[58] = (int_f)H5D_VECTOR_IO; + h5d_flags[59] = (int_f)H5D_SELECTION_IO; + /* * H5E flags */ diff --git a/fortran/src/H5_ff.F90 b/fortran/src/H5_ff.F90 index 68b3dd874bb..53156731946 100644 --- a/fortran/src/H5_ff.F90 +++ b/fortran/src/H5_ff.F90 @@ -74,7 +74,7 @@ MODULE H5LIB ! ! H5D flags declaration ! - INTEGER, PARAMETER :: H5D_FLAGS_LEN = 57 + INTEGER, PARAMETER :: H5D_FLAGS_LEN = 60 INTEGER, DIMENSION(1:H5D_FLAGS_LEN) :: H5D_flags INTEGER, PARAMETER :: H5D_SIZE_FLAGS_LEN = 2 INTEGER(SIZE_T), DIMENSION(1:H5D_SIZE_FLAGS_LEN) :: H5D_size_flags @@ -467,6 +467,9 @@ CONTAINS H5D_MPIO_NO_CHUNK_OPTIMIZATION_F = H5D_flags(55) H5D_MPIO_LINK_CHUNK_F = H5D_flags(56) H5D_MPIO_MULTI_CHUNK_F = H5D_flags(57) + H5D_SCALAR_IO_F = H5D_flags(58) + H5D_VECTOR_IO_F = H5D_flags(59) + H5D_SELECTION_IO_F = H5D_flags(60) H5D_CHUNK_CACHE_NSLOTS_DFLT_F = H5D_size_flags(1) H5D_CHUNK_CACHE_NBYTES_DFLT_F = H5D_size_flags(2) diff --git a/fortran/src/H5config_f.inc.cmake b/fortran/src/H5config_f.inc.cmake index 34fb091c787..71bce0e18c2 100644 --- a/fortran/src/H5config_f.inc.cmake +++ b/fortran/src/H5config_f.inc.cmake @@ -23,6 +23,12 @@ #undef H5_HAVE_SUBFILING_VFD #endif +! Define if on APPLE +#cmakedefine01 H5_HAVE_DARWIN +#if H5_HAVE_DARWIN == 0 +#undef H5_HAVE_DARWIN +#endif + ! Define if the intrinsic function STORAGE_SIZE exists #define H5_FORTRAN_HAVE_STORAGE_SIZE @H5_FORTRAN_HAVE_STORAGE_SIZE@ @@ -81,4 +87,4 @@ #cmakedefine01 H5_NO_DEPRECATED_SYMBOLS #if H5_NO_DEPRECATED_SYMBOLS == 0 #undef H5_NO_DEPRECATED_SYMBOLS -#endif \ No newline at end of file +#endif diff --git a/fortran/src/H5config_f.inc.in b/fortran/src/H5config_f.inc.in index 7fb76e12449..991e4b0750b 100644 --- a/fortran/src/H5config_f.inc.in +++ b/fortran/src/H5config_f.inc.in @@ -20,6 +20,9 @@ ! Define if we have subfiling support #undef HAVE_SUBFILING_VFD +! Define if on APPLE +#undef HAVE_DARWIN + ! Define if the intrinsic function STORAGE_SIZE exists #undef FORTRAN_HAVE_STORAGE_SIZE diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90 index e60f1e83320..aa046235eb3 100644 --- a/fortran/src/H5f90global.F90 +++ b/fortran/src/H5f90global.F90 @@ -25,6 +25,12 @@ MODULE H5GLOBAL IMPLICIT NONE +!> @brief H5_ih_info_t derived type. + TYPE, BIND(C) :: H5_ih_info_t + INTEGER(HSIZE_T) :: index_size !< btree and/or list + INTEGER(HSIZE_T) :: heap_size !< Heap size + END TYPE H5_ih_info_t + !> \addtogroup FH5 !> @{ ! Parameters used in the function 'h5kind_to_type' located in H5_ff.F90. @@ -368,6 +374,12 @@ MODULE H5GLOBAL !DEC$ATTRIBUTES DLLEXPORT :: H5D_MPIO_NO_CHUNK_OPTIMIZATION_F !DEC$ATTRIBUTES DLLEXPORT :: H5D_MPIO_LINK_CHUNK_F !DEC$ATTRIBUTES DLLEXPORT :: H5D_MPIO_MULTI_CHUNK_F + + !DEC$ATTRIBUTES DLLEXPORT :: H5D_SCALAR_IO_F + !DEC$ATTRIBUTES DLLEXPORT :: H5D_VECTOR_IO_F + !DEC$ATTRIBUTES DLLEXPORT :: H5D_SELECTION_IO_F + + !DEC$endif !> \addtogroup FH5D !> @{ @@ -444,6 +456,10 @@ MODULE H5GLOBAL INTEGER :: H5D_MPIO_NO_CHUNK_OPTIMIZATION_F !< H5D_MPIO_NO_CHUNK_OPTIMIZATION INTEGER :: H5D_MPIO_LINK_CHUNK_F !< H5D_MPIO_LINK_CHUNK INTEGER :: H5D_MPIO_MULTI_CHUNK_F !< H5D_MPIO_MULTI_CHUNK + + INTEGER :: H5D_SCALAR_IO_F !< Scalar (or legacy MPIO) I/O was performed + INTEGER :: H5D_VECTOR_IO_F !< Vector I/O was performed + INTEGER :: H5D_SELECTION_IO_F !< Selection I/O was performed ! ! H5E flags declaration ! diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 3b6600c061e..2ded00222ba 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -417,6 +417,7 @@ H5P_mp_H5PSET_FILE_SPACE_STRATEGY_F H5P_mp_H5PGET_FILE_SPACE_STRATEGY_F H5P_mp_H5PSET_FILE_SPACE_PAGE_SIZE_F H5P_mp_H5PGET_FILE_SPACE_PAGE_SIZE_F +H5P_mp_H5PGET_ACTUAL_SELECTION_IO_MODE_F ; Parallel @H5_NOPAREXP@H5P_mp_H5PSET_FAPL_MPIO_F @H5_NOPAREXP@H5P_mp_H5PGET_FAPL_MPIO_F diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index c73016bc7cf..78d665f0aa1 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -869,6 +869,7 @@ SUBROUTINE test_in_place_conversion(cleanup, total_error) REAL(KIND=C_DOUBLE), DIMENSION(1:array_len) :: wbuf_d_org REAL(KIND=C_FLOAT), DIMENSION(1:array_len), TARGET :: rbuf INTEGER :: i + INTEGER :: actual_selection_io_mode TYPE(C_PTR) :: f_ptr ! create the data @@ -919,6 +920,10 @@ SUBROUTINE test_in_place_conversion(cleanup, total_error) ! Should not be equal for in-place buffer use CALL VERIFY("h5dwrite_f -- in-place", wbuf_d(1), wbuf_d_org(1), total_error, .FALSE.) + CALL h5pget_actual_selection_io_mode_f(plist_id, actual_selection_io_mode, error) + CALL check("h5pget_actual_selection_io_mode_f", error, total_error) + CALL VERIFY("h5pget_actual_selection_io_mode_f", actual_selection_io_mode, H5D_SCALAR_IO_F, total_error) + f_ptr = C_LOC(rbuf) CALL h5dread_f(dset_id, h5kind_to_type(KIND(rbuf(1)), H5_REAL_KIND), f_ptr, error) CALL check("h5dread_f", error, total_error) diff --git a/fortran/testpar/CMakeTests.cmake b/fortran/testpar/CMakeTests.cmake index 8c157241500..473049fb976 100644 --- a/fortran/testpar/CMakeTests.cmake +++ b/fortran/testpar/CMakeTests.cmake @@ -17,3 +17,4 @@ ############################################################################## add_test (NAME MPI_TEST_FORT_parallel_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) add_test (NAME MPI_TEST_FORT_subfiling_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) +add_test (NAME MPI_TEST_FORT_async_test COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) diff --git a/fortran/testpar/Makefile.am b/fortran/testpar/Makefile.am index 7f9f2846928..afdda980c5c 100644 --- a/fortran/testpar/Makefile.am +++ b/fortran/testpar/Makefile.am @@ -36,7 +36,7 @@ TEST_PROG_PARA=parallel_test subfiling_test async_test check_PROGRAMS=$(TEST_PROG_PARA) # Temporary files -CHECK_CLEANFILES+=parf[12].h5 subf.h5* +CHECK_CLEANFILES+=parf[12].h5 h5*_tests.h5 subf.h5* test_async_apis.mod # Test source files parallel_test_SOURCES=ptest.F90 hyper.F90 mdset.F90 multidsetrw.F90 diff --git a/fortran/testpar/hyper.F90 b/fortran/testpar/hyper.F90 index edd93cf9b8f..ec3a657afbb 100644 --- a/fortran/testpar/hyper.F90 +++ b/fortran/testpar/hyper.F90 @@ -55,6 +55,7 @@ SUBROUTINE hyper(length,do_collective,do_chunk, mpi_size, mpi_rank, nerrors) INTEGER :: local_no_collective_cause INTEGER :: global_no_collective_cause INTEGER :: no_selection_io_cause + INTEGER :: actual_selection_io_mode ! ! initialize the array data between the processes (3) @@ -236,6 +237,20 @@ SUBROUTINE hyper(length,do_collective,do_chunk, mpi_size, mpi_rank, nerrors) CALL h5dwrite_f(dset_id,H5T_NATIVE_INTEGER,wbuf,dims,hdferror,file_space_id=fspace_id,mem_space_id=mspace_id,xfer_prp=dxpl_id) CALL check("h5dwrite_f", hdferror, nerrors) + CALL h5pget_actual_selection_io_mode_f(dxpl_id, actual_selection_io_mode, hdferror) + CALL check("h5pget_actual_selection_io_mode_f", hdferror, nerrors) + IF(do_collective)THEN + IF(actual_selection_io_mode .NE. H5D_SELECTION_IO_F)THEN + PRINT*, "Incorrect actual selection io mode" + nerrors = nerrors + 1 + ENDIF + ELSE + IF(actual_selection_io_mode .NE. IOR(H5D_SELECTION_IO_F, H5D_SCALAR_IO_F))THEN + PRINT*, "Incorrect actual selection io mode" + nerrors = nerrors + 1 + ENDIF + ENDIF + ! Check h5pget_mpio_actual_io_mode_f function CALL h5pget_mpio_actual_io_mode_f(dxpl_id, actual_io_mode, hdferror) CALL check("h5pget_mpio_actual_io_mode_f", hdferror, nerrors) diff --git a/fortran/testpar/subfiling.F90 b/fortran/testpar/subfiling.F90 index 043ac6cb771..a677bea6121 100644 --- a/fortran/testpar/subfiling.F90 +++ b/fortran/testpar/subfiling.F90 @@ -54,6 +54,7 @@ PROGRAM subfiling_test INTEGER(HID_T) :: driver_id CHARACTER(len=8) :: hex1, hex2 + CHARACTER(len=1) :: arg ! ! initialize MPI @@ -336,10 +337,14 @@ PROGRAM subfiling_test WRITE(*,"(A,A)") "Failed to find the stub subfile ",TRIM(filename) nerrors = nerrors + 1 ENDIF - - CALL EXECUTE_COMMAND_LINE("stat --format='%i' "//filename//" >> tmp_inode", EXITSTAT=i) +#ifdef H5_HAVE_DARWIN + arg(1:1)="f" +#else + arg(1:1)="c" +#endif + CALL EXECUTE_COMMAND_LINE("stat -"//arg(1:1)//" %i "//filename//" >> tmp_inode", EXITSTAT=i) IF(i.ne.0)THEN - WRITE(*,"(A,A)") "Failed to stat the stub subfile ",TRIM(filename) + WRITE(*,"(A,A)") "Failed to stat the stub subfile ",TRIM(filename) nerrors = nerrors + 1 ENDIF diff --git a/hl/test/Makefile.am b/hl/test/Makefile.am index 1d1cb0f92fb..6f66291b25d 100644 --- a/hl/test/Makefile.am +++ b/hl/test/Makefile.am @@ -20,7 +20,7 @@ include $(top_srcdir)/config/commence.am # Add include directories to C preprocessor flags AM_CPPFLAGS+=-I. -I$(srcdir) -I$(top_builddir)/src -I$(top_srcdir)/src -I$(top_builddir)/test -I$(top_srcdir)/test -I$(top_srcdir)/hl/src -# The tests depend on the hdf5, hdf5 test, and hdf5_hl libraries +# The tests depend on the hdf5, hdf5 test, and hdf5_hl libraries LDADD=$(LIBH5_HL) $(LIBH5TEST) $(LIBHDF5) # Test programs. These are our main targets. They should be listed in the diff --git a/java/test/Makefile.am b/java/test/Makefile.am index 9f39be9ca1c..7f6ab0169a8 100644 --- a/java/test/Makefile.am +++ b/java/test/Makefile.am @@ -90,7 +90,8 @@ noinst_DATA = $(jarfile) check_SCRIPTS = junit.sh TEST_SCRIPT = $(check_SCRIPTS) -CLEANFILES = classnoinst.stamp $(jarfile) $(JAVAROOT)/$(pkgpath)/*.class junit.sh +CLEANFILES = classnoinst.stamp $(jarfile) $(JAVAROOT)/$(pkgpath)/*.class junit.sh \ + *.h5 testExport*.txt clean: rm -rf $(JAVAROOT)/* diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 1734c01b9fd..f228e3984d8 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,14 @@ New Features Configuration: ------------- + - Improved support for Intel oneAPI + + * Separates the old 'classic' Intel compiler settings and warnings + from the oneAPI settings + * Uses `-check nouninit` in debug builds to avoid false positives + when building H5_buildiface with `-check all` + * Both Autotools and CMake + - Added new options for CMake and Autotools to control the Doxygen warnings as errors setting. @@ -127,6 +135,16 @@ New Features Library: -------- + - Added a simple cache to the read-only S3 (ros3) VFD + + The read-only S3 VFD now caches the first N bytes of a file stored + in S3 to avoid a lot of small I/O operations when opening files. + This cache is per-file and created when the file is opened. + + N is currently 16 MiB or the size of the file, whichever is smaller. + + Addresses GitHub issue #3381 + - Added new API function H5Pget_actual_selection_io_mode() This function allows the user to determine if the library performed @@ -159,7 +177,8 @@ New Features - Fortran async APIs H5A, H5D, H5ES, H5G, H5F, H5L and H5O were added. - Added Fortran APIs: - h5pset_selection_io_f, h5pget_selection_io_f + h5pset_selection_io_f, h5pget_selection_io_f, + h5pget_actual_selection_io_mode_f, h5pset_modify_write_buf_f, h5pget_modify_write_buf_f - Added Fortran APIs: @@ -219,6 +238,66 @@ Bug Fixes since HDF5-1.14.2 release =================================== Library ------- + - Fixed some issues with chunk index metadata not getting read + collectively when collective metadata reads are enabled + + When looking up dataset chunks during I/O, the parallel library + temporarily disables collective metadata reads since it's generally + unlikely that the application will read the same chunks from all + MPI ranks. Leaving collective metadata reads enabled during + chunk lookups can lead to hangs or other bad behavior depending + on the chunk indexing structure used for the dataset in question. + However, due to the way that dataset chunk index metadata was + previously loaded in a deferred manner, this could mean that + the metadata for the main chunk index structure or its + accompanying pieces of metadata (e.g., fixed array data blocks) + could end up being read independently if these chunk lookup + operations are the first chunk index-related operation that + occurs on a dataset. This behavior is generally observed when + opening a dataset for which the metadata isn't in the metadata + cache yet and then immediately performing I/O on that dataset. + This behavior is not generally observed when creating a dataset + and then performing I/O on it, as the relevant metadata will + usually be in the metadata cache as a side effect of creating + the chunk index structures during dataset creation. + + This issue has been fixed by adding callbacks to the different + chunk indexing structure classes that allow more explicit control + over when chunk index metadata gets loaded. When collective + metadata reads are enabled, the necessary index metadata will now + get loaded collectively by all MPI ranks at the start of dataset + I/O to ensure that the ranks don't unintentionally read this + metadata independently further on. These changes fix collective + loading of the main chunk index structure, as well as v2 B-tree + root nodes, extensible array index blocks and fixed array data + blocks. There are still pieces of metadata that cannot currently + be loaded collectively, however, such as extensible array data + blocks, data block pages and super blocks, as well as fixed array + data block pages. These pieces of metadata are not necessarily + read in by all MPI ranks since this depends on which chunks the + ranks have selected in the dataset. Therefore, reading of these + pieces of metadata remains an independent operation. + + - Fixed potential hangs in parallel library during collective I/O with + independent metadata writes + + When performing collective parallel writes to a dataset where metadata + writes are requested as (or left as the default setting of) independent, + hangs could potentially occur during metadata cache sync points. This + was due to incorrect management of the internal state tracking whether + an I/O operation should be collective or not, causing the library to + attempt collective writes of metadata when they were meant to be + independent writes. During the metadata cache sync points, if the number + of cache entries being flushed was a multiple of the number of MPI ranks + in the MPI communicator used to access the HDF5 file, an equal amount of + collective MPI I/O calls were made and the dataset write call would be + successful. However, when the number of cache entries being flushed was + NOT a multiple of the number of MPI ranks, the ranks with more entries + than others would get stuck in an MPI_File_set_view call, while other + ranks would get stuck in a post-write MPI_Barrier call. This issue has + been fixed by correctly switching to independent I/O temporarily when + writing metadata independently during collective dataset I/O. + - Fixed a bug with the way the Subfiling VFD assigns I/O concentrators During a file open operation, the Subfiling VFD determines the topology diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index d7bf5b1dbda..c8db5352ff6 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -154,8 +154,9 @@ herr_t H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, haddr_t *candidates_list_ptr, int mpi_rank, int mpi_size) { - unsigned first_entry_to_flush; - unsigned last_entry_to_flush; + H5FD_mpio_xfer_t orig_xfer_mode; + unsigned first_entry_to_flush; + unsigned last_entry_to_flush; #ifndef NDEBUG unsigned total_entries_to_clear = 0; unsigned total_entries_to_flush = 0; @@ -169,11 +170,12 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha haddr_t last_addr; #endif /* H5C_DO_SANITY_CHECKS */ #if H5C_APPLY_CANDIDATE_LIST__DEBUG - char tbl_buf[1024]; + char *tbl_buf = NULL; #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ unsigned m, n; - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + unsigned u; /* Local index variable */ + bool restore_io_mode = false; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -185,21 +187,57 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha assert(0 <= mpi_rank); assert(mpi_rank < mpi_size); + /* Get I/O transfer mode */ + if (H5CX_get_io_xfer_mode(&orig_xfer_mode) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode"); + /* Initialize the entries_to_flush and entries_to_clear arrays */ memset(entries_to_flush, 0, sizeof(entries_to_flush)); memset(entries_to_clear, 0, sizeof(entries_to_clear)); #if H5C_APPLY_CANDIDATE_LIST__DEBUG - fprintf(stdout, "%s:%d: setting up candidate assignment table.\n", __func__, mpi_rank); + { + const char *const table_header = "candidate list = "; + size_t tbl_buf_size; + size_t tbl_buf_left; + size_t entry_nchars; + int bytes_printed; - memset(tbl_buf, 0, sizeof(tbl_buf)); + fprintf(stdout, "%s:%d: setting up candidate assignment table.\n", __func__, mpi_rank); - snprintf(tbl_buf, sizeof(tbl_buf), "candidate list = "); - for (u = 0; u < num_candidates; u++) - sprintf(&(tbl_buf[strlen(tbl_buf)]), " 0x%llx", (long long)(*(candidates_list_ptr + u))); - sprintf(&(tbl_buf[strlen(tbl_buf)]), "\n"); + /* Calculate maximum number of characters printed for each + * candidate entry, including the leading space and "0x" + */ + entry_nchars = (sizeof(long long) * CHAR_BIT / 4) + 3; - fprintf(stdout, "%s", tbl_buf); + tbl_buf_size = strlen(table_header) + (num_candidates * entry_nchars) + 1; + if (NULL == (tbl_buf = H5MM_malloc(tbl_buf_size))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "can't allocate debug buffer"); + tbl_buf_left = tbl_buf_size; + + if ((bytes_printed = snprintf(tbl_buf, tbl_buf_left, table_header)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list"); + assert((size_t)bytes_printed < tbl_buf_left); + tbl_buf_left -= (size_t)bytes_printed; + + for (u = 0; u < num_candidates; u++) { + if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, " 0x%llx", + (long long)(*(candidates_list_ptr + u)))) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list"); + assert((size_t)bytes_printed < tbl_buf_left); + tbl_buf_left -= (size_t)bytes_printed; + } + + if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, "\n")) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list"); + assert((size_t)bytes_printed < tbl_buf_left); + tbl_buf_left -= (size_t)bytes_printed + 1; /* NUL terminator */ + + fprintf(stdout, "%s", tbl_buf); + + H5MM_free(tbl_buf); + tbl_buf = NULL; + } #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ if (f->shared->coll_md_write) { @@ -258,18 +296,50 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha last_entry_to_flush = candidate_assignment_table[mpi_rank + 1] - 1; #if H5C_APPLY_CANDIDATE_LIST__DEBUG - for (u = 0; u < 1024; u++) - tbl_buf[u] = '\0'; - snprintf(tbl_buf, sizeof(tbl_buf), "candidate assignment table = "); - for (u = 0; u <= (unsigned)mpi_size; u++) - sprintf(&(tbl_buf[strlen(tbl_buf)]), " %u", candidate_assignment_table[u]); - sprintf(&(tbl_buf[strlen(tbl_buf)]), "\n"); - fprintf(stdout, "%s", tbl_buf); + { + const char *const table_header = "candidate assignment table = "; + unsigned umax = UINT_MAX; + size_t tbl_buf_size; + size_t tbl_buf_left; + size_t entry_nchars; + int bytes_printed; - fprintf(stdout, "%s:%d: flush entries [%u, %u].\n", __func__, mpi_rank, first_entry_to_flush, - last_entry_to_flush); + /* Calculate the maximum number of characters printed for each entry */ + entry_nchars = (size_t)(log10(umax) + 1) + 1; - fprintf(stdout, "%s:%d: marking entries.\n", __func__, mpi_rank); + tbl_buf_size = strlen(table_header) + ((size_t)mpi_size * entry_nchars) + 1; + if (NULL == (tbl_buf = H5MM_malloc(tbl_buf_size))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "can't allocate debug buffer"); + tbl_buf_left = tbl_buf_size; + + if ((bytes_printed = snprintf(tbl_buf, tbl_buf_left, table_header)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list"); + assert((size_t)bytes_printed < tbl_buf_left); + tbl_buf_left -= (size_t)bytes_printed; + + for (u = 0; u <= (unsigned)mpi_size; u++) { + if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, " %u", + candidate_assignment_table[u])) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list"); + assert((size_t)bytes_printed < tbl_buf_left); + tbl_buf_left -= (size_t)bytes_printed; + } + + if ((bytes_printed = snprintf(&(tbl_buf[tbl_buf_size - tbl_buf_left]), tbl_buf_left, "\n")) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSERRSTR, FAIL, "can't add to candidate list"); + assert((size_t)bytes_printed < tbl_buf_left); + tbl_buf_left -= (size_t)bytes_printed + 1; /* NUL terminator */ + + fprintf(stdout, "%s", tbl_buf); + + H5MM_free(tbl_buf); + tbl_buf = NULL; + + fprintf(stdout, "%s:%d: flush entries [%u, %u].\n", __func__, mpi_rank, first_entry_to_flush, + last_entry_to_flush); + + fprintf(stdout, "%s:%d: marking entries.\n", __func__, mpi_rank); + } #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ for (u = 0; u < num_candidates; u++) { @@ -354,6 +424,19 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha num_candidates, total_entries_to_clear, total_entries_to_flush); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + /* + * If collective I/O was requested, but collective metadata + * writes were not requested, temporarily disable collective + * I/O while flushing candidate entries so that we don't cause + * a hang in the case where the number of candidate entries + * to flush isn't a multiple of mpi_size. + */ + if ((orig_xfer_mode == H5FD_MPIO_COLLECTIVE) && !f->shared->coll_md_write) { + if (H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode"); + restore_io_mode = true; + } + /* We have now marked all the entries on the candidate list for * either flush or clear -- now scan the LRU and the pinned list * for these entries and do the deed. Do this via a call to @@ -367,6 +450,13 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha if (H5C__flush_candidate_entries(f, entries_to_flush, entries_to_clear) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates failed"); + /* Restore collective I/O if we temporarily disabled it */ + if (restore_io_mode) { + if (H5CX_set_io_xfer_mode(orig_xfer_mode) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode"); + restore_io_mode = false; + } + /* If we've deferred writing to do it collectively, take care of that now */ if (f->shared->coll_md_write) { /* Sanity check */ @@ -378,6 +468,10 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha } /* end if */ done: + /* Restore collective I/O if we temporarily disabled it */ + if (restore_io_mode && (H5CX_set_io_xfer_mode(orig_xfer_mode) < 0)) + HDONE_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode"); + if (candidate_assignment_table != NULL) candidate_assignment_table = (unsigned *)H5MM_xfree((void *)candidate_assignment_table); if (cache_ptr->coll_write_list) { diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index d79f7d0b031..4f8a867974e 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -24,30 +24,32 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Bprivate.h" /* B-link trees */ -#include "H5Dpkg.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* Files */ -#include "H5FDprivate.h" /* File drivers */ +#include "H5private.h" /* Generic Functions */ +#include "H5Bprivate.h" /* B-link trees */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5Iprivate.h" /* IDs */ -#include "H5MFprivate.h" /* File space management */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Oprivate.h" /* Object headers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MFprivate.h" /* File space management */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Oprivate.h" /* Object headers */ #include "H5Sprivate.h" /* Dataspaces */ -#include "H5VMprivate.h" /* Vector and array functions */ +#include "H5VMprivate.h" /* Vector and array functions */ /****************/ /* Local Macros */ /****************/ +#define H5D_BTREE_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree.shared) + /******************/ /* Local Typedefs */ /******************/ /* - * B-tree key. A key contains the minimum logical N-dimensional coordinates and + * B-tree key. A key contains the minimum logical N-dimensional coordinates and * the logical size of the chunk to which this key refers. The * fastest-varying dimension is assumed to reference individual bytes of the * array, so a 100-element 1-d array of 4-byte integers would really be a 2-d @@ -61,9 +63,9 @@ * The chunk's file address is part of the B-tree and not part of the key. */ typedef struct H5D_btree_key_t { - hsize_t scaled[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ - uint32_t nbytes; /*size of stored data */ - unsigned filter_mask; /*excluded filters */ + hsize_t scaled[H5O_LAYOUT_NDIMS]; /*logical offset to start */ + uint32_t nbytes; /*size of stored data */ + unsigned filter_mask; /*excluded filters */ } H5D_btree_key_t; /* B-tree callback info for iteration over chunks */ @@ -111,10 +113,14 @@ static herr_t H5D__btree_debug_key(FILE *stream, int indent, int fwidth, const v static herr_t H5D__btree_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space, haddr_t dset_ohdr_addr); static herr_t H5D__btree_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__btree_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__btree_idx_close(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__btree_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open); static bool H5D__btree_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__btree_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, const H5D_t *dset); static herr_t H5D__btree_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__btree_idx_load_metadata(const H5D_chk_idx_info_t *idx_info); static int H5D__btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); static herr_t H5D__btree_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata); @@ -137,9 +143,13 @@ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{ false, /* v1 B-tree indices does not support SWMR access */ H5D__btree_idx_init, /* insert */ H5D__btree_idx_create, /* create */ + H5D__btree_idx_open, /* open */ + H5D__btree_idx_close, /* close */ + H5D__btree_idx_is_open, /* is_open */ H5D__btree_idx_is_space_alloc, /* is_space_alloc */ H5D__btree_idx_insert, /* insert */ H5D__btree_idx_get_addr, /* get_addr */ + H5D__btree_idx_load_metadata, /* load_metadata */ NULL, /* resize */ H5D__btree_idx_iterate, /* iterate */ H5D__btree_idx_remove, /* remove */ @@ -158,21 +168,21 @@ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{ /* inherits B-tree like properties from H5B */ static H5B_class_t H5B_BTREE[1] = {{ - H5B_CHUNK_ID, /*id */ - sizeof(H5D_btree_key_t), /*sizeof_nkey */ - H5D__btree_get_shared, /*get_shared */ - H5D__btree_new_node, /*new */ - H5D__btree_cmp2, /*cmp2 */ - H5D__btree_cmp3, /*cmp3 */ - H5D__btree_found, /*found */ - H5D__btree_insert, /*insert */ - false, /*follow min branch? */ - false, /*follow max branch? */ - H5B_LEFT, /*critical key */ - H5D__btree_remove, /*remove */ - H5D__btree_decode_key, /*decode */ - H5D__btree_encode_key, /*encode */ - H5D__btree_debug_key /*debug */ + H5B_CHUNK_ID, /* id */ + sizeof(H5D_btree_key_t), /* sizeof_nkey */ + H5D__btree_get_shared, /* get_shared */ + H5D__btree_new_node, /* new */ + H5D__btree_cmp2, /* cmp2 */ + H5D__btree_cmp3, /* cmp3 */ + H5D__btree_found, /* found */ + H5D__btree_insert, /* insert */ + false, /* follow min branch? */ + false, /* follow max branch? */ + H5B_LEFT, /* critical key */ + H5D__btree_remove, /* remove */ + H5D__btree_decode_key, /* decode */ + H5D__btree_encode_key, /* encode */ + H5D__btree_debug_key /* debug */ }}; /*******************/ @@ -183,13 +193,13 @@ static H5B_class_t H5B_BTREE[1] = {{ H5FL_DEFINE_STATIC(H5O_layout_chunk_t); /*------------------------------------------------------------------------- - * Function: H5D__btree_get_shared + * Function: H5D__btree_get_shared * - * Purpose: Returns the shared B-tree info for the specified UDATA. + * Purpose: Returns the shared B-tree info for the specified UDATA. * - * Return: Success: Pointer to the raw B-tree page for this dataset + * Return: Success: Pointer to the raw B-tree page for this dataset * - * Failure: Can't fail + * Failure: Can't fail * *------------------------------------------------------------------------- */ @@ -210,17 +220,17 @@ H5D__btree_get_shared(const H5F_t H5_ATTR_UNUSED *f, const void *_udata) } /* end H5D__btree_get_shared() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_new_node + * Function: H5D__btree_new_node * - * Purpose: Adds a new entry to an i-storage B-tree. We can assume that - * the domain represented by UDATA doesn't intersect the domain - * already represented by the B-tree. + * Purpose: Adds a new entry to an i-storage B-tree. We can assume + * that the domain represented by UDATA doesn't intersect the + * domain already represented by the B-tree. * - * Return: Success: Non-negative. The address of leaf is returned - * through the ADDR argument. It is also added - * to the UDATA. + * Return: Success: Non-negative. The address of leaf is returned + * through the ADDR argument. It is also added + * to the UDATA. * - * Failure: Negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -275,18 +285,18 @@ H5D__btree_new_node(H5F_t H5_ATTR_NDEBUG_UNUSED *f, H5B_ins_t op, void *_lt_key, } /* end H5D__btree_new_node() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_cmp2 + * Function: H5D__btree_cmp2 * - * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer - * is only to supply extra information not carried in the keys - * (in this case, the dimensionality) and is not compared - * against the keys. + * Purpose: Compares two keys sort of like strcmp(). The UDATA pointer + * is only to supply extra information not carried in the keys + * (in this case, the dimensionality) and is not compared + * against the keys. * - * Return: Success: -1 if LT_KEY is less than RT_KEY; - * 1 if LT_KEY is greater than RT_KEY; - * 0 if LT_KEY and RT_KEY are equal. + * Return: Success: -1 if LT_KEY is less than RT_KEY; + * 1 if LT_KEY is greater than RT_KEY; + * 0 if LT_KEY and RT_KEY are equal. * - * Failure: FAIL (same as LT_KEYstorage); + assert(H5D_CHUNK_IDX_BTREE == idx_info->storage->idx_type); + assert(is_open); + + *is_open = H5D_BTREE_IDX_IS_OPEN(idx_info); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__btree_idx_is_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__btree_idx_is_space_alloc + * + * Purpose: Query if space is allocated for index method + * + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -886,11 +960,11 @@ H5D__btree_idx_is_space_alloc(const H5O_storage_chunk_t *storage) } /* end H5D__btree_idx_is_space_alloc() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_insert + * Function: H5D__btree_idx_insert * - * Purpose: Insert chunk entry into the indexing structure. + * Purpose: Insert chunk entry into the indexing structure. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -922,13 +996,13 @@ done: } /* H5D__btree_idx_insert() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_get_addr + * Function: H5D__btree_idx_get_addr * - * Purpose: Get the file address of a chunk if file space has been - * assigned. Save the retrieved information in the udata - * supplied. + * Purpose: Get the file address of a chunk if file space has been + * assigned. Save the retrieved information in the udata + * supplied. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -959,14 +1033,34 @@ done: } /* H5D__btree_idx_get_addr() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_iterate_cb + * Function: H5D__btree_idx_load_metadata * - * Purpose: Translate the B-tree specific chunk record into a generic + * Purpose: Load additional chunk index metadata beyond the chunk index + * itself. Currently a no-op. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__btree_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__btree_idx_load_metadata() */ + +/*------------------------------------------------------------------------- + * Function: H5D__btree_idx_iterate_cb + * + * Purpose: Translate the B-tree specific chunk record into a generic * form and make the callback to the generic chunk callback * routine. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -1001,12 +1095,12 @@ H5D__btree_idx_iterate_cb(H5F_t H5_ATTR_UNUSED *f, const void *_lt_key, haddr_t } /* H5D__btree_idx_iterate_cb() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_iterate + * Function: H5D__btree_idx_iterate * - * Purpose: Iterate over the chunks in an index, making a callback + * Purpose: Iterate over the chunks in an index, making a callback * for each one. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1043,11 +1137,11 @@ H5D__btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t c } /* end H5D__btree_idx_iterate() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_remove + * Function: H5D__btree_idx_remove * - * Purpose: Remove chunk from index. + * Purpose: Remove chunk from index. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1077,13 +1171,13 @@ done: } /* H5D__btree_idx_remove() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_delete + * Function: H5D__btree_idx_delete * - * Purpose: Delete index and raw data storage for entire dataset + * Purpose: Delete index and raw data storage for entire dataset * (i.e. all chunks) * - * Return: Success: Non-negative - * Failure: negative + * Return: Success: Non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -1134,11 +1228,11 @@ done: } /* end H5D__btree_idx_delete() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_copy_setup + * Function: H5D__btree_idx_copy_setup * - * Purpose: Set up any necessary information for copying chunks + * Purpose: Set up any necessary information for copying chunks * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1178,11 +1272,11 @@ done: } /* end H5D__btree_idx_copy_setup() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_copy_shutdown + * Function: H5D__btree_idx_copy_shutdown * - * Purpose: Shutdown any information from copying chunks + * Purpose: Shutdown any information from copying chunks * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1250,11 +1344,11 @@ done: } /* end H5D__btree_idx_size() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_reset + * Function: H5D__btree_idx_reset * - * Purpose: Reset indexing information. + * Purpose: Reset indexing information. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1274,11 +1368,11 @@ H5D__btree_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr) } /* end H5D__btree_idx_reset() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_dump + * Function: H5D__btree_idx_dump * - * Purpose: Dump indexing information to a stream. + * Purpose: Dump indexing information to a stream. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1296,11 +1390,11 @@ H5D__btree_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream) } /* end H5D__btree_idx_dump() */ /*------------------------------------------------------------------------- - * Function: H5D__btree_idx_dest + * Function: H5D__btree_idx_dest * - * Purpose: Release indexing information in memory. + * Purpose: Release indexing information in memory. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1328,11 +1422,11 @@ done: } /* end H5D__btree_idx_dest() */ /*------------------------------------------------------------------------- - * Function: H5D_btree_debug + * Function: H5D_btree_debug * - * Purpose: Debugs a B-tree node for indexed raw data storage. + * Purpose: Debugs a B-tree node for indexed raw data storage. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c index 4da9555d0c0..7a26b6d016c 100644 --- a/src/H5Dbtree2.c +++ b/src/H5Dbtree2.c @@ -27,16 +27,18 @@ /* Headers */ /***********/ #include "H5private.h" /* Generic Functions */ -#include "H5Dpkg.h" /* Datasets */ +#include "H5Dpkg.h" /* Datasets */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File space management */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5VMprivate.h" /* Vector and array functions */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5VMprivate.h" /* Vector and array functions */ /****************/ /* Local Macros */ /****************/ +#define H5D_BT2_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree2.bt2) + /******************/ /* Local Typedefs */ /******************/ @@ -92,7 +94,6 @@ static herr_t H5D__bt2_filt_debug(FILE *stream, int indent, int fwidth, const vo const void *u_ctx); /* Helper routine */ -static herr_t H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info); static herr_t H5D__btree2_idx_depend(const H5D_chk_idx_info_t *idx_info); /* Callback for H5B2_iterate() which is called in H5D__bt2_idx_iterate() */ @@ -114,10 +115,14 @@ static herr_t H5D__bt2_mod_cb(void *_record, void *_op_data, bool *changed); static herr_t H5D__bt2_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space, haddr_t dset_ohdr_addr); static herr_t H5D__bt2_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__bt2_idx_close(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__bt2_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open); static bool H5D__bt2_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__bt2_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, const H5D_t *dset); static herr_t H5D__bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__bt2_idx_load_metadata(const H5D_chk_idx_info_t *idx_info); static int H5D__bt2_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); static herr_t H5D__bt2_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata); @@ -139,9 +144,13 @@ const H5D_chunk_ops_t H5D_COPS_BT2[1] = {{ true, /* Fixed array indices support SWMR access */ H5D__bt2_idx_init, /* init */ H5D__bt2_idx_create, /* create */ + H5D__bt2_idx_open, /* open */ + H5D__bt2_idx_close, /* close */ + H5D__bt2_idx_is_open, /* is_open */ H5D__bt2_idx_is_space_alloc, /* is_space_alloc */ H5D__bt2_idx_insert, /* insert */ H5D__bt2_idx_get_addr, /* get_addr */ + H5D__bt2_idx_load_metadata, /* load_metadata */ NULL, /* resize */ H5D__bt2_idx_iterate, /* iterate */ H5D__bt2_idx_remove, /* remove */ @@ -203,8 +212,8 @@ H5FL_ARR_DEFINE_STATIC(uint32_t, H5O_LAYOUT_NDIMS); * * Purpose: Create client callback context * - * Return: Success: non-NULL - * Failure: NULL + * Return: Success: non-NULL + * Failure: NULL * *------------------------------------------------------------------------- */ @@ -258,8 +267,8 @@ done: * * Purpose: Destroy client callback context * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -286,10 +295,10 @@ H5D__bt2_dst_context(void *_ctx) * Function: H5D__bt2_store * * Purpose: Store native information into record for v2 B-tree - * (non-filtered) + * (non-filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -308,8 +317,8 @@ H5D__bt2_store(void *record, const void *_udata) /*------------------------------------------------------------------------- * Function: H5D__bt2_compare * - * Purpose: Compare two native information records, according to some key - * (non-filtered) + * Purpose: Compare two native information records, according to some + * key (non-filtered) * * Return: <0 if rec1 < rec2 * =0 if rec1 == rec2 @@ -341,10 +350,10 @@ H5D__bt2_compare(const void *_udata, const void *_rec2, int *result) * Function: H5D__bt2_unfilt_encode * * Purpose: Encode native information into raw form for storing on disk - * (non-filtered) + * (non-filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -373,10 +382,10 @@ H5D__bt2_unfilt_encode(uint8_t *raw, const void *_record, void *_ctx) * Function: H5D__bt2_unfilt_decode * * Purpose: Decode raw disk form of record into native form - * (non-filtered) + * (non-filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -403,12 +412,12 @@ H5D__bt2_unfilt_decode(const uint8_t *raw, void *_record, void *_ctx) } /* H5D__bt2_unfilt_decode() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_unfilt_debug + * Function: H5D__bt2_unfilt_debug * - * Purpose: Debug native form of record (non-filtered) + * Purpose: Debug native form of record (non-filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -440,10 +449,10 @@ H5D__bt2_unfilt_debug(FILE *stream, int indent, int fwidth, const void *_record, * Function: H5D__bt2_filt_encode * * Purpose: Encode native information into raw form for storing on disk - * (filtered) + * (filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -473,13 +482,13 @@ H5D__bt2_filt_encode(uint8_t *raw, const void *_record, void *_ctx) } /* H5D__bt2_filt_encode() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_filt_decode + * Function: H5D__bt2_filt_decode * - * Purpose: Decode raw disk form of record into native form - * (filtered) + * Purpose: Decode raw disk form of record into native form + * (filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -511,12 +520,12 @@ H5D__bt2_filt_decode(const uint8_t *raw, void *_record, void *_ctx) } /* H5D__bt2_filt_decode() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_filt_debug + * Function: H5D__bt2_filt_debug * - * Purpose: Debug native form of record (filtered) + * Purpose: Debug native form of record (filtered) * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -570,13 +579,13 @@ H5D__bt2_idx_init(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, const H5S_t } /* end H5D__bt2_idx_init() */ /*------------------------------------------------------------------------- - * Function: H5D__btree2_idx_depend + * Function: H5D__btree2_idx_depend * - * Purpose: Create flush dependency between v2 B-tree and dataset's + * Purpose: Create flush dependency between v2 B-tree and dataset's * object header. * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -629,63 +638,9 @@ done: } /* end H5D__btree2_idx_depend() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_open() + * Function: H5D__bt2_idx_create * - * Purpose: Opens an existing v2 B-tree. - * - * Note: This information is passively initialized from each index - * operation callback because those abstract chunk index operations - * are designed to work with the v2 B-tree chunk indices also, - * which don't require an 'open' for the data structure. - * - * Return: Success: non-negative - * Failure: negative - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info) -{ - H5D_bt2_ctx_ud_t u_ctx; /* user data for creating context */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check args */ - assert(idx_info); - assert(idx_info->f); - assert(idx_info->pline); - assert(idx_info->layout); - assert(H5D_CHUNK_IDX_BT2 == idx_info->layout->idx_type); - assert(idx_info->storage); - assert(H5_addr_defined(idx_info->storage->idx_addr)); - assert(NULL == idx_info->storage->u.btree2.bt2); - - /* Set up the user data */ - u_ctx.f = idx_info->f; - u_ctx.ndims = idx_info->layout->ndims - 1; - u_ctx.chunk_size = idx_info->layout->size; - u_ctx.dim = idx_info->layout->dim; - - /* Open v2 B-tree for the chunk index */ - if (NULL == - (idx_info->storage->u.btree2.bt2 = H5B2_open(idx_info->f, idx_info->storage->idx_addr, &u_ctx))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open v2 B-tree for tracking chunked dataset"); - - /* Check for SWMR writes to the file */ - if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) - if (H5D__btree2_idx_depend(idx_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, - "unable to create flush dependency on object header"); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__bt2_idx_open() */ - -/*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_create - * - * Purpose: Create the v2 B-tree for tracking dataset chunks + * Purpose: Create the v2 B-tree for tracking dataset chunks * * Return: SUCCEED/FAIL * @@ -758,11 +713,120 @@ done: } /* end H5D__bt2_idx_create() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_is_space_alloc + * Function: H5D__bt2_idx_open() * - * Purpose: Query if space is allocated for index method + * Purpose: Opens an existing v2 B-tree. * - * Return: Non-negative on success/Negative on failure + * Note: This information is passively initialized from each index + * operation callback because those abstract chunk index + * operations are designed to work with the v2 B-tree chunk + * indices also, which don't require an 'open' for the data + * structure. + * + * Return: Success: non-negative + * Failure: negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info) +{ + H5D_bt2_ctx_ud_t u_ctx; /* user data for creating context */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args */ + assert(idx_info); + assert(idx_info->f); + assert(idx_info->pline); + assert(idx_info->layout); + assert(H5D_CHUNK_IDX_BT2 == idx_info->layout->idx_type); + assert(idx_info->storage); + assert(H5_addr_defined(idx_info->storage->idx_addr)); + assert(NULL == idx_info->storage->u.btree2.bt2); + + /* Set up the user data */ + u_ctx.f = idx_info->f; + u_ctx.ndims = idx_info->layout->ndims - 1; + u_ctx.chunk_size = idx_info->layout->size; + u_ctx.dim = idx_info->layout->dim; + + /* Open v2 B-tree for the chunk index */ + if (NULL == + (idx_info->storage->u.btree2.bt2 = H5B2_open(idx_info->f, idx_info->storage->idx_addr, &u_ctx))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open v2 B-tree for tracking chunked dataset"); + + /* Check for SWMR writes to the file */ + if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if (H5D__btree2_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, + "unable to create flush dependency on object header"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__bt2_idx_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__bt2_idx_close() + * + * Purpose: Closes an existing v2 B-tree. + * + * Return: Success: non-negative + * Failure: negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__bt2_idx_close(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_BT2 == idx_info->storage->idx_type); + assert(idx_info->storage->u.btree2.bt2); + + if (H5B2_close(idx_info->storage->u.btree2.bt2) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close v2 B-tree"); + idx_info->storage->u.btree2.bt2 = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__bt2_idx_close() */ + +/*------------------------------------------------------------------------- + * Function: H5D__bt2_idx_is_open + * + * Purpose: Query if the index is opened or not + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__bt2_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open) +{ + FUNC_ENTER_PACKAGE_NOERR + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_BT2 == idx_info->storage->idx_type); + assert(is_open); + + *is_open = H5D_BT2_IDX_IS_OPEN(idx_info); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__bt2_idx_is_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__bt2_idx_is_space_alloc + * + * Purpose: Query if space is allocated for index method + * + * Return: true/false * *------------------------------------------------------------------------- */ @@ -778,14 +842,14 @@ H5D__bt2_idx_is_space_alloc(const H5O_storage_chunk_t *storage) } /* end H5D__bt2_idx_is_space_alloc() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_mod_cb + * Function: H5D__bt2_mod_cb * - * Purpose: Modify record for dataset chunk when it is found in a v2 B-tree. - * This is the callback for H5B2_update() which is called in - * H5D__bt2_idx_insert(). + * Purpose: Modify record for dataset chunk when it is found in a v2 + * B-tree. This is the callback for H5B2_update() which is + * called in H5D__bt2_idx_insert(). * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -817,18 +881,21 @@ H5D__bt2_mod_cb(void *_record, void *_op_data, bool *changed) } /* end H5D__bt2_mod_cb() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_insert + * Function: H5D__bt2_idx_insert * - * Purpose: Insert chunk address into the indexing structure. - * A non-filtered chunk: - * Should not exist - * Allocate the chunk and pass chunk address back up - * A filtered chunk: - * If it was not found, create the chunk and pass chunk address back up - * If it was found but its size changed, reallocate the chunk and pass chunk address back up - * If it was found but its size was the same, pass chunk address back up + * Purpose: Insert chunk address into the indexing structure. + * A non-filtered chunk: + * Should not exist + * Allocate the chunk and pass chunk address back up + * A filtered chunk: + * If it was not found, create the chunk and pass chunk + * address back up + * If it was found but its size changed, reallocate the chunk + * and pass chunk address back up + * If it was found but its size was the same, pass chunk + * address back up * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -854,7 +921,7 @@ H5D__bt2_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, assert(H5_addr_defined(udata->chunk_block.offset)); /* Check if the v2 B-tree is open yet */ - if (NULL == idx_info->storage->u.btree2.bt2) { + if (!H5D_BT2_IDX_IS_OPEN(idx_info)) { /* Open existing v2 B-tree */ if (H5D__bt2_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree"); @@ -889,14 +956,14 @@ done: } /* H5D__bt2_idx_insert() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_found_cb + * Function: H5D__bt2_found_cb * - * Purpose: Retrieve record for dataset chunk when it is found in a v2 B-tree. - * This is the callback for H5B2_find() which is called in - * H5D__bt2_idx_get_addr() and H5D__bt2_idx_insert(). + * Purpose: Retrieve record for dataset chunk when it is found in a v2 + * B-tree. This is the callback for H5B2_find() which is called + * in H5D__bt2_idx_get_addr() and H5D__bt2_idx_insert(). * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -911,13 +978,13 @@ H5D__bt2_found_cb(const void *nrecord, void *op_data) } /* H5D__bt2_found_cb() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_get_addr + * Function: H5D__bt2_idx_get_addr * - * Purpose: Get the file address of a chunk if file space has been - * assigned. Save the retrieved information in the udata - * supplied. + * Purpose: Get the file address of a chunk if file space has been + * assigned. Save the retrieved information in the udata + * supplied. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -944,7 +1011,7 @@ H5D__bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) assert(udata); /* Check if the v2 B-tree is open yet */ - if (NULL == idx_info->storage->u.btree2.bt2) { + if (!H5D_BT2_IDX_IS_OPEN(idx_info)) { /* Open existing v2 B-tree */ if (H5D__bt2_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree"); @@ -1003,16 +1070,59 @@ done: } /* H5D__bt2_idx_get_addr() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_iterate_cb + * Function: H5D__bt2_idx_load_metadata * - * Purpose: Translate the B-tree specific chunk record into a generic + * Purpose: Load additional chunk index metadata beyond the chunk index + * itself. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__bt2_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + H5D_chunk_ud_t chunk_ud; + hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* + * After opening a dataset that uses a v2 Btree, the root + * node will generally not be read in until an element is + * looked up for the first time. Since there isn't currently + * a good way of controlling that explicitly, perform a fake + * lookup of a chunk to cause it to be read in. + */ + chunk_ud.common.layout = idx_info->layout; + chunk_ud.common.storage = idx_info->storage; + chunk_ud.common.scaled = scaled; + + chunk_ud.chunk_block.offset = HADDR_UNDEF; + chunk_ud.chunk_block.length = 0; + chunk_ud.filter_mask = 0; + chunk_ud.new_unfilt_chunk = false; + chunk_ud.idx_hint = UINT_MAX; + + if (H5D__bt2_idx_get_addr(idx_info, &chunk_ud) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't load v2 B-tree root node"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__bt2_idx_load_metadata() */ + +/*------------------------------------------------------------------------- + * Function: H5D__bt2_idx_iterate_cb + * + * Purpose: Translate the B-tree specific chunk record into a generic * form and make the callback to the generic chunk callback * routine. - * This is the callback for H5B2_iterate() which is called in - * H5D__bt2_idx_iterate(). + * This is the callback for H5B2_iterate() which is called in + * H5D__bt2_idx_iterate(). * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -1033,12 +1143,12 @@ H5D__bt2_idx_iterate_cb(const void *_record, void *_udata) } /* H5D__bt2_idx_iterate_cb() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_iterate + * Function: H5D__bt2_idx_iterate * - * Purpose: Iterate over the chunks in an index, making a callback + * Purpose: Iterate over the chunks in an index, making a callback * for each one. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1062,7 +1172,7 @@ H5D__bt2_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chu assert(chunk_udata); /* Check if the v2 B-tree is open yet */ - if (NULL == idx_info->storage->u.btree2.bt2) { + if (!H5D_BT2_IDX_IS_OPEN(idx_info)) { /* Open existing v2 B-tree */ if (H5D__bt2_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree"); @@ -1087,15 +1197,16 @@ done: } /* end H5D__bt2_idx_iterate() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_remove_cb() + * Function: H5D__bt2_remove_cb() * - * Purpose: Free space for 'dataset chunk' object as v2 B-tree - * is being deleted or v2 B-tree node is removed. - * This is the callback for H5B2_remove() and H5B2_delete() which - * which are called in H5D__bt2_idx_remove() and H5D__bt2_idx_delete(). + * Purpose: Free space for 'dataset chunk' object as v2 B-tree + * is being deleted or v2 B-tree node is removed. + * This is the callback for H5B2_remove() and H5B2_delete() + * which are called in H5D__bt2_idx_remove() and + * H5D__bt2_idx_delete(). * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -1121,11 +1232,11 @@ done: } /* H5D__bt2_remove_cb() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_remove + * Function: H5D__bt2_idx_remove * - * Purpose: Remove chunk from index. + * Purpose: Remove chunk from index. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1149,7 +1260,7 @@ H5D__bt2_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *u assert(udata); /* Check if the v2 B-tree is open yet */ - if (NULL == idx_info->storage->u.btree2.bt2) { + if (!H5D_BT2_IDX_IS_OPEN(idx_info)) { /* Open existing v2 B-tree */ if (H5D__bt2_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree"); @@ -1180,13 +1291,13 @@ done: } /* H5D__bt2_idx_remove() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_delete + * Function: H5D__bt2_idx_delete * - * Purpose: Delete index and raw data storage for entire dataset + * Purpose: Delete index and raw data storage for entire dataset * (i.e. all chunks) * - * Return: Success: Non-negative - * Failure: negative + * Return: Success: Non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -1233,11 +1344,11 @@ done: } /* end H5D__bt2_idx_delete() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_copy_setup + * Function: H5D__bt2_idx_copy_setup * - * Purpose: Set up any necessary information for copying chunks + * Purpose: Set up any necessary information for copying chunks * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1264,7 +1375,7 @@ H5D__bt2_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_id assert(!H5_addr_defined(idx_info_dst->storage->idx_addr)); /* Check if the source v2 B-tree is open yet */ - if (NULL == idx_info_src->storage->u.btree2.bt2) + if (!H5D_BT2_IDX_IS_OPEN(idx_info_src)) if (H5D__bt2_idx_open(idx_info_src) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open v2 B-tree"); @@ -1284,11 +1395,11 @@ done: } /* end H5D__bt2_idx_copy_setup() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_copy_shutdown + * Function: H5D__bt2_idx_copy_shutdown * - * Purpose: Shutdown any information from copying chunks + * Purpose: Shutdown any information from copying chunks * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1324,8 +1435,8 @@ done: * * Purpose: Retrieve the amount of index storage for chunked dataset * - * Return: Success: Non-negative - * Failure: negative + * Return: Success: Non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -1355,23 +1466,23 @@ H5D__bt2_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) /* Get v2 B-tree size for indexing chunked dataset */ if (H5B2_size(bt2_cdset, index_size) < 0) - HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't retrieve v2 B-tree storage info for chunked dataset"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't retrieve v2 B-tree storage info for chunked dataset"); done: /* Close v2 B-tree index */ - if (bt2_cdset && H5B2_close(bt2_cdset) < 0) - HDONE_ERROR(H5E_SYM, H5E_CLOSEERROR, FAIL, "can't close v2 B-tree for tracking chunked dataset"); - idx_info->storage->u.btree2.bt2 = NULL; + if (H5D__bt2_idx_close(idx_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close v2 B-tree for tracking chunked dataset"); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__bt2_idx_size() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_reset + * Function: H5D__bt2_idx_reset * - * Purpose: Reset indexing information. + * Purpose: Reset indexing information. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1392,11 +1503,11 @@ H5D__bt2_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr) } /* end H5D__bt2_idx_reset() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_dump + * Function: H5D__bt2_idx_dump * - * Purpose: Dump indexing information to a stream. + * Purpose: Dump indexing information to a stream. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1415,11 +1526,11 @@ H5D__bt2_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream) } /* end H5D__bt2_idx_dump() */ /*------------------------------------------------------------------------- - * Function: H5D__bt2_idx_dest + * Function: H5D__bt2_idx_dest * - * Purpose: Release indexing information in memory. + * Purpose: Release indexing information in memory. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1436,16 +1547,14 @@ H5D__bt2_idx_dest(const H5D_chk_idx_info_t *idx_info) assert(idx_info->storage); /* Check if the v2-btree is open */ - if (idx_info->storage->u.btree2.bt2) { - + if (H5D_BT2_IDX_IS_OPEN(idx_info)) { /* Patch the top level file pointer contained in bt2 if needed */ if (H5B2_patch_file(idx_info->storage->u.btree2.bt2, idx_info->f) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't patch v2 B-tree file pointer"); /* Close v2 B-tree */ - if (H5B2_close(idx_info->storage->u.btree2.bt2) < 0) + if (H5D__bt2_idx_close(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "can't close v2 B-tree"); - idx_info->storage->u.btree2.bt2 = NULL; } /* end if */ done: diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 9f4bd90b68a..41d774d0d3e 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -1124,18 +1124,33 @@ H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo) if (H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_HAS_MPI) && H5F_shared_get_coll_metadata_reads(io_info->f_sh) && H5D__chunk_is_space_alloc(&dataset->shared->layout.storage)) { - H5D_chunk_ud_t udata; - hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; + H5O_storage_chunk_t *sc = &(dataset->shared->layout.storage.u.chunk); + H5D_chk_idx_info_t idx_info; + bool index_is_open; + + idx_info.f = dataset->oloc.file; + idx_info.pline = &dataset->shared->dcpl_cache.pline; + idx_info.layout = &dataset->shared->layout.u.chunk; + idx_info.storage = sc; + + assert(sc && sc->ops && sc->ops->is_open); + if (sc->ops->is_open(&idx_info, &index_is_open) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to check if dataset chunk index is open"); + + if (!index_is_open) { + assert(sc->ops->open); + if (sc->ops->open(&idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to open dataset chunk index"); + } /* - * TODO: Until the dataset chunk index callback structure has - * callbacks for checking if an index is opened and also for - * directly opening the index, the following fake chunk lookup - * serves the purpose of forcing a chunk index open operation - * on all ranks + * Load any other chunk index metadata that we can, + * such as fixed array data blocks, while we know all + * MPI ranks will do so with collective metadata reads + * enabled */ - if (H5D__chunk_lookup(dataset, scaled, &udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to collectively open dataset chunk index"); + if (sc->ops->load_metadata && sc->ops->load_metadata(&idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to load additional chunk index metadata"); } #endif @@ -3827,15 +3842,29 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat idx_info.storage = sc; #ifdef H5_HAVE_PARALLEL - /* Disable collective metadata read for chunk indexes as it is - * highly unlikely that users would read the same chunks from all - * processes. - */ if (H5F_HAS_FEATURE(idx_info.f, H5FD_FEAT_HAS_MPI)) { - md_reads_file_flag = H5P_FORCE_FALSE; - md_reads_context_flag = false; - H5F_set_coll_metadata_reads(idx_info.f, &md_reads_file_flag, &md_reads_context_flag); - restore_md_reads_state = true; + /* Disable collective metadata read for chunk indexes as it is + * highly unlikely that users would read the same chunks from all + * processes. + */ + if (H5F_get_coll_metadata_reads(idx_info.f)) { +#ifndef NDEBUG + bool index_is_open; + + /* + * The dataset's chunk index should be open at this point. + * Otherwise, we will end up reading it in independently, + * which may not be desired. + */ + sc->ops->is_open(&idx_info, &index_is_open); + assert(index_is_open); +#endif + + md_reads_file_flag = H5P_FORCE_FALSE; + md_reads_context_flag = false; + H5F_set_coll_metadata_reads(idx_info.f, &md_reads_file_flag, &md_reads_context_flag); + restore_md_reads_state = true; + } } #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dearray.c b/src/H5Dearray.c index c713b6f18bb..965eaacaca3 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -26,19 +26,21 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Dpkg.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5EAprivate.h" /* Extensible arrays */ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5EAprivate.h" /* Extensible arrays */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5MFprivate.h" /* File space management */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5MFprivate.h" /* File space management */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ /* Local Macros */ /****************/ +#define H5D_EARRAY_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.earray.ea) + /* Value to fill unset array elements with */ #define H5D_EARRAY_FILL HADDR_UNDEF #define H5D_EARRAY_FILT_FILL \ @@ -106,10 +108,14 @@ static herr_t H5D__earray_filt_debug(FILE *stream, int indent, int fwidth, hsize static herr_t H5D__earray_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space, haddr_t dset_ohdr_addr); static herr_t H5D__earray_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__earray_idx_close(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__earray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open); static bool H5D__earray_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, const H5D_t *dset); static herr_t H5D__earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__earray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info); static herr_t H5D__earray_idx_resize(H5O_layout_chunk_t *layout); static int H5D__earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); @@ -125,7 +131,6 @@ static herr_t H5D__earray_idx_dump(const H5O_storage_chunk_t *storage, FILE *str static herr_t H5D__earray_idx_dest(const H5D_chk_idx_info_t *idx_info); /* Generic extensible array routines */ -static herr_t H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info); static herr_t H5D__earray_idx_depend(const H5D_chk_idx_info_t *idx_info); /*********************/ @@ -137,9 +142,13 @@ const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{ true, /* Extensible array indices support SWMR access */ H5D__earray_idx_init, /* init */ H5D__earray_idx_create, /* create */ + H5D__earray_idx_open, /* open */ + H5D__earray_idx_close, /* close */ + H5D__earray_idx_is_open, /* is_open */ H5D__earray_idx_is_space_alloc, /* is_space_alloc */ H5D__earray_idx_insert, /* insert */ H5D__earray_idx_get_addr, /* get_addr */ + H5D__earray_idx_load_metadata, /* load_metadata */ H5D__earray_idx_resize, /* resize */ H5D__earray_idx_iterate, /* iterate */ H5D__earray_idx_remove, /* remove */ @@ -270,10 +279,10 @@ H5D__earray_dst_context(void *_ctx) /*------------------------------------------------------------------------- * Function: H5D__earray_fill * - * Purpose: Fill "missing elements" in block of elements + * Purpose: Fill "missing elements" in block of elements * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -705,59 +714,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__earray_idx_depend() */ -/*------------------------------------------------------------------------- - * Function: H5D__earray_idx_open - * - * Purpose: Opens an existing extensible array. - * - * Note: This information is passively initialized from each index - * operation callback because those abstract chunk index operations - * are designed to work with the v1 B-tree chunk indices also, - * which don't require an 'open' for the data structure. - * - * Return: Success: non-negative - * Failure: negative - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info) -{ - H5D_earray_ctx_ud_t udata; /* User data for extensible array open call */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check args */ - assert(idx_info); - assert(idx_info->f); - assert(idx_info->pline); - assert(idx_info->layout); - assert(H5D_CHUNK_IDX_EARRAY == idx_info->layout->idx_type); - assert(idx_info->storage); - assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type); - assert(H5_addr_defined(idx_info->storage->idx_addr)); - assert(NULL == idx_info->storage->u.earray.ea); - - /* Set up the user data */ - udata.f = idx_info->f; - udata.chunk_size = idx_info->layout->size; - - /* Open the extensible array for the chunk index */ - if (NULL == - (idx_info->storage->u.earray.ea = H5EA_open(idx_info->f, idx_info->storage->idx_addr, &udata))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array"); - - /* Check for SWMR writes to the file */ - if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) - if (H5D__earray_idx_depend(idx_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, - "unable to create flush dependency on object header"); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__earray_idx_open() */ - /*------------------------------------------------------------------------- * Function: H5D__earray_idx_init * @@ -905,12 +861,120 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__earray_idx_create() */ +/*------------------------------------------------------------------------- + * Function: H5D__earray_idx_open + * + * Purpose: Opens an existing extensible array. + * + * Note: This information is passively initialized from each index + * operation callback because those abstract chunk index + * operations are designed to work with the v1 B-tree chunk + * indices also, which don't require an 'open' for the data + * structure. + * + * Return: Success: non-negative + * Failure: negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info) +{ + H5D_earray_ctx_ud_t udata; /* User data for extensible array open call */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args */ + assert(idx_info); + assert(idx_info->f); + assert(idx_info->pline); + assert(idx_info->layout); + assert(H5D_CHUNK_IDX_EARRAY == idx_info->layout->idx_type); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type); + assert(H5_addr_defined(idx_info->storage->idx_addr)); + assert(NULL == idx_info->storage->u.earray.ea); + + /* Set up the user data */ + udata.f = idx_info->f; + udata.chunk_size = idx_info->layout->size; + + /* Open the extensible array for the chunk index */ + if (NULL == + (idx_info->storage->u.earray.ea = H5EA_open(idx_info->f, idx_info->storage->idx_addr, &udata))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array"); + + /* Check for SWMR writes to the file */ + if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if (H5D__earray_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, + "unable to create flush dependency on object header"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__earray_idx_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__earray_idx_close + * + * Purpose: Closes an existing extensible array. + * + * Return: Success: non-negative + * Failure: negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__earray_idx_close(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type); + assert(idx_info->storage->u.earray.ea); + + if (H5EA_close(idx_info->storage->u.earray.ea) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array"); + idx_info->storage->u.earray.ea = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__earray_idx_close() */ + +/*------------------------------------------------------------------------- + * Function: H5D__earray_idx_is_open + * + * Purpose: Query if the index is opened or not + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__earray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open) +{ + FUNC_ENTER_PACKAGE_NOERR + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type); + assert(is_open); + + *is_open = H5D_EARRAY_IDX_IS_OPEN(idx_info); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__earray_idx_is_open() */ + /*------------------------------------------------------------------------- * Function: H5D__earray_idx_is_space_alloc * * Purpose: Query if space is allocated for index method * - * Return: Non-negative on success/Negative on failure + * Return: true/false * *------------------------------------------------------------------------- */ @@ -953,7 +1017,7 @@ H5D__earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata assert(udata); /* Check if the extensible array is open yet */ - if (NULL == idx_info->storage->u.earray.ea) { + if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) { /* Open the extensible array in file */ if (H5D__earray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array"); @@ -1021,7 +1085,7 @@ H5D__earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda assert(udata); /* Check if the extensible array is open yet */ - if (NULL == idx_info->storage->u.earray.ea) { + if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) { /* Open the extensible array in file */ if (H5D__earray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array"); @@ -1086,6 +1150,51 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D__earray_idx_get_addr() */ +/*------------------------------------------------------------------------- + * Function: H5D__earray_idx_load_metadata + * + * Purpose: Load additional chunk index metadata beyond the chunk index + * itself. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__earray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info) +{ + H5D_chunk_ud_t chunk_ud; + hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* + * After opening a dataset that uses an extensible array, + * the extensible array header index block will generally + * not be read in until an element is looked up for the + * first time. Since there isn't currently a good way of + * controlling that explicitly, perform a fake lookup of + * a chunk to cause it to be read in or created if it + * doesn't exist yet. + */ + chunk_ud.common.layout = idx_info->layout; + chunk_ud.common.storage = idx_info->storage; + chunk_ud.common.scaled = scaled; + + chunk_ud.chunk_block.offset = HADDR_UNDEF; + chunk_ud.chunk_block.length = 0; + chunk_ud.filter_mask = 0; + chunk_ud.new_unfilt_chunk = false; + chunk_ud.idx_hint = UINT_MAX; + + if (H5D__earray_idx_get_addr(idx_info, &chunk_ud) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't load extensible array header index block"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__earray_idx_load_metadata() */ + /*------------------------------------------------------------------------- * Function: H5D__earray_idx_resize * @@ -1195,10 +1304,6 @@ H5D__earray_idx_iterate_cb(hsize_t H5_ATTR_UNUSED idx, const void *_elmt, void * * Purpose: Iterate over the chunks in an index, making a callback * for each one. * - * Note: This implementation is slow, particularly for sparse - * extensible arrays, replace it with call to H5EA_iterate() - * when that's available. - * * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- @@ -1223,10 +1328,10 @@ H5D__earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t assert(chunk_udata); /* Check if the extensible array is open yet */ - if (NULL == idx_info->storage->u.earray.ea) { + if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) { /* Open the extensible array in file */ if (H5D__earray_idx_open(idx_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, H5_ITER_ERROR, "can't open extensible array"); } else /* Patch the top level file pointer contained in ea if needed */ H5EA_patch_file(idx_info->storage->u.earray.ea, idx_info->f); @@ -1236,7 +1341,7 @@ H5D__earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t /* Get the extensible array statistics */ if (H5EA_get_stats(ea, &ea_stat) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query extensible array statistics"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, H5_ITER_ERROR, "can't query extensible array statistics"); if (ea_stat.stored.max_idx_set > 0) { H5D_earray_it_ud_t udata; /* User data for iteration callback */ @@ -1291,7 +1396,7 @@ H5D__earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t assert(udata); /* Check if the extensible array is open yet */ - if (NULL == idx_info->storage->u.earray.ea) { + if (!H5D_EARRAY_IDX_IS_OPEN(idx_info)) { /* Open the extensible array in file */ if (H5D__earray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array"); @@ -1444,9 +1549,8 @@ H5D__earray_idx_delete(const H5D_chk_idx_info_t *idx_info) HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk addresses"); /* Close extensible array */ - if (H5EA_close(idx_info->storage->u.earray.ea) < 0) + if (H5D__earray_idx_close(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array"); - idx_info->storage->u.earray.ea = NULL; /* Set up the context user data */ ctx_udata.f = idx_info->f; @@ -1494,7 +1598,7 @@ H5D__earray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk assert(!H5_addr_defined(idx_info_dst->storage->idx_addr)); /* Check if the source extensible array is open yet */ - if (NULL == idx_info_src->storage->u.earray.ea) + if (!H5D_EARRAY_IDX_IS_OPEN(idx_info_src)) /* Open the extensible array in file */ if (H5D__earray_idx_open(idx_info_src) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array"); @@ -1593,9 +1697,8 @@ H5D__earray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) done: if (idx_info->storage->u.earray.ea) { - if (H5EA_close(idx_info->storage->u.earray.ea) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array"); - idx_info->storage->u.earray.ea = NULL; + if (H5D__earray_idx_close(idx_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array"); } /* end if */ FUNC_LEAVE_NOAPI(ret_value) @@ -1673,16 +1776,14 @@ H5D__earray_idx_dest(const H5D_chk_idx_info_t *idx_info) assert(idx_info->storage); /* Check if the extensible array is open */ - if (idx_info->storage->u.earray.ea) { - + if (H5D_EARRAY_IDX_IS_OPEN(idx_info)) { /* Patch the top level file pointer contained in ea if needed */ if (H5EA_patch_file(idx_info->storage->u.earray.ea, idx_info->f) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't patch earray file pointer"); /* Close extensible array */ - if (H5EA_close(idx_info->storage->u.earray.ea) < 0) + if (H5D__earray_idx_close(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array"); - idx_info->storage->u.earray.ea = NULL; } /* end if */ done: diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index 450d466755c..8d06de47b02 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -37,6 +37,8 @@ /* Local Macros */ /****************/ +#define H5D_FARRAY_IDX_IS_OPEN(idx_info) (NULL != (idx_info)->storage->u.btree2.bt2) + /* Value to fill unset array elements with */ #define H5D_FARRAY_FILL HADDR_UNDEF #define H5D_FARRAY_FILT_FILL \ @@ -105,10 +107,14 @@ static herr_t H5D__farray_filt_debug(FILE *stream, int indent, int fwidth, hsize static herr_t H5D__farray_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space, haddr_t dset_ohdr_addr); static herr_t H5D__farray_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__farray_idx_close(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__farray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open); static bool H5D__farray_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__farray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, const H5D_t *dset); static herr_t H5D__farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__farray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info); static int H5D__farray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); static herr_t H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata); @@ -123,7 +129,6 @@ static herr_t H5D__farray_idx_dump(const H5O_storage_chunk_t *storage, FILE *str static herr_t H5D__farray_idx_dest(const H5D_chk_idx_info_t *idx_info); /* Generic fixed array routines */ -static herr_t H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info); static herr_t H5D__farray_idx_depend(const H5D_chk_idx_info_t *idx_info); /*********************/ @@ -135,9 +140,13 @@ const H5D_chunk_ops_t H5D_COPS_FARRAY[1] = {{ true, /* Fixed array indices support SWMR access */ H5D__farray_idx_init, /* init */ H5D__farray_idx_create, /* create */ + H5D__farray_idx_open, /* open */ + H5D__farray_idx_close, /* close */ + H5D__farray_idx_is_open, /* is_open */ H5D__farray_idx_is_space_alloc, /* is_space_alloc */ H5D__farray_idx_insert, /* insert */ H5D__farray_idx_get_addr, /* get_addr */ + H5D__farray_idx_load_metadata, /* load_metadata */ NULL, /* resize */ H5D__farray_idx_iterate, /* iterate */ H5D__farray_idx_remove, /* remove */ @@ -726,55 +735,6 @@ H5D__farray_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR_UNU FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__farray_idx_init() */ -/*------------------------------------------------------------------------- - * Function: H5D__farray_idx_open - * - * Purpose: Opens an existing fixed array and initializes - * the layout struct with information about the storage. - * - * Return: Success: non-negative - * Failure: negative - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info) -{ - H5D_farray_ctx_ud_t udata; /* User data for fixed array open call */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check args */ - assert(idx_info); - assert(idx_info->f); - assert(idx_info->pline); - assert(idx_info->layout); - assert(H5D_CHUNK_IDX_FARRAY == idx_info->layout->idx_type); - assert(idx_info->storage); - assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type); - assert(H5_addr_defined(idx_info->storage->idx_addr)); - assert(NULL == idx_info->storage->u.farray.fa); - - /* Set up the user data */ - udata.f = idx_info->f; - udata.chunk_size = idx_info->layout->size; - - /* Open the fixed array for the chunk index */ - if (NULL == - (idx_info->storage->u.farray.fa = H5FA_open(idx_info->f, idx_info->storage->idx_addr, &udata))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open fixed array"); - - /* Check for SWMR writes to the file */ - if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) - if (H5D__farray_idx_depend(idx_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, - "unable to create flush dependency on object header"); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__farray_idx_open() */ - /*------------------------------------------------------------------------- * Function: H5D__farray_idx_create * @@ -853,12 +813,115 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__farray_idx_create() */ +/*------------------------------------------------------------------------- + * Function: H5D__farray_idx_open + * + * Purpose: Opens an existing fixed array and initializes + * the layout struct with information about the storage. + * + * Return: Success: non-negative + * Failure: negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info) +{ + H5D_farray_ctx_ud_t udata; /* User data for fixed array open call */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args */ + assert(idx_info); + assert(idx_info->f); + assert(idx_info->pline); + assert(idx_info->layout); + assert(H5D_CHUNK_IDX_FARRAY == idx_info->layout->idx_type); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type); + assert(H5_addr_defined(idx_info->storage->idx_addr)); + assert(NULL == idx_info->storage->u.farray.fa); + + /* Set up the user data */ + udata.f = idx_info->f; + udata.chunk_size = idx_info->layout->size; + + /* Open the fixed array for the chunk index */ + if (NULL == + (idx_info->storage->u.farray.fa = H5FA_open(idx_info->f, idx_info->storage->idx_addr, &udata))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open fixed array"); + + /* Check for SWMR writes to the file */ + if (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if (H5D__farray_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, + "unable to create flush dependency on object header"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__farray_idx_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__farray_idx_close + * + * Purpose: Closes an existing fixed array. + * + * Return: Success: non-negative + * Failure: negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__farray_idx_close(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type); + assert(idx_info->storage->u.farray.fa); + + if (H5FA_close(idx_info->storage->u.farray.fa) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array"); + idx_info->storage->u.farray.fa = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__farray_idx_close() */ + +/*------------------------------------------------------------------------- + * Function: H5D__farray_idx_is_open + * + * Purpose: Query if the index is opened or not + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__farray_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open) +{ + FUNC_ENTER_PACKAGE_NOERR + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type); + assert(is_open); + + *is_open = H5D_FARRAY_IDX_IS_OPEN(idx_info); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__farray_idx_is_open() */ + /*------------------------------------------------------------------------- * Function: H5D__farray_idx_is_space_alloc * * Purpose: Query if space is allocated for index method * - * Return: Non-negative on success/Negative on failure + * Return: true/false * *------------------------------------------------------------------------- */ @@ -901,7 +964,7 @@ H5D__farray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata assert(udata); /* Check if the fixed array is open yet */ - if (NULL == idx_info->storage->u.farray.fa) { + if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) { /* Open the fixed array in file */ if (H5D__farray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array"); @@ -969,7 +1032,7 @@ H5D__farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda assert(udata); /* Check if the fixed array is open yet */ - if (NULL == idx_info->storage->u.farray.fa) { + if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) { /* Open the fixed array in file */ if (H5D__farray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array"); @@ -1016,6 +1079,50 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D__farray_idx_get_addr() */ +/*------------------------------------------------------------------------- + * Function: H5D__farray_idx_load_metadata + * + * Purpose: Load additional chunk index metadata beyond the chunk index + * itself. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__farray_idx_load_metadata(const H5D_chk_idx_info_t *idx_info) +{ + H5D_chunk_ud_t chunk_ud; + hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* + * After opening a dataset that uses a fixed array, the + * fixed array data block will generally not be read in + * until an element is looked up for the first time. Since + * there isn't currently a good way of controlling that + * explicitly, perform a fake lookup of a chunk to cause + * it to be read in. + */ + chunk_ud.common.layout = idx_info->layout; + chunk_ud.common.storage = idx_info->storage; + chunk_ud.common.scaled = scaled; + + chunk_ud.chunk_block.offset = HADDR_UNDEF; + chunk_ud.chunk_block.length = 0; + chunk_ud.filter_mask = 0; + chunk_ud.new_unfilt_chunk = false; + chunk_ud.idx_hint = UINT_MAX; + + if (H5D__farray_idx_get_addr(idx_info, &chunk_ud) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't load fixed array data block"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__farray_idx_load_metadata() */ + /*------------------------------------------------------------------------- * Function: H5D__farray_idx_iterate_cb * @@ -1102,7 +1209,7 @@ H5D__farray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t assert(chunk_udata); /* Check if the fixed array is open yet */ - if (NULL == idx_info->storage->u.farray.fa) { + if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) { /* Open the fixed array in file */ if (H5D__farray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array"); @@ -1171,7 +1278,7 @@ H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t assert(udata); /* Check if the fixed array is open yet */ - if (NULL == idx_info->storage->u.farray.fa) { + if (!H5D_FARRAY_IDX_IS_OPEN(idx_info)) { /* Open the fixed array in file */ if (H5D__farray_idx_open(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array"); @@ -1302,9 +1409,8 @@ H5D__farray_idx_delete(const H5D_chk_idx_info_t *idx_info) HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk addresses"); /* Close fixed array */ - if (H5FA_close(idx_info->storage->u.farray.fa) < 0) + if (H5D__farray_idx_close(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array"); - idx_info->storage->u.farray.fa = NULL; /* Set up the user data */ ctx_udata.f = idx_info->f; @@ -1352,10 +1458,11 @@ H5D__farray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk assert(!H5_addr_defined(idx_info_dst->storage->idx_addr)); /* Check if the source fixed array is open yet */ - if (NULL == idx_info_src->storage->u.farray.fa) + if (!H5D_FARRAY_IDX_IS_OPEN(idx_info_src)) { /* Open the fixed array in file */ if (H5D__farray_idx_open(idx_info_src) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open fixed array"); + } /* Set copied metadata tag */ H5_BEGIN_TAG(H5AC__COPIED_TAG) @@ -1450,9 +1557,8 @@ H5D__farray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) done: if (idx_info->storage->u.farray.fa) { - if (H5FA_close(idx_info->storage->u.farray.fa) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array"); - idx_info->storage->u.farray.fa = NULL; + if (H5D__farray_idx_close(idx_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array"); } /* end if */ FUNC_LEAVE_NOAPI(ret_value) @@ -1528,16 +1634,14 @@ H5D__farray_idx_dest(const H5D_chk_idx_info_t *idx_info) assert(idx_info->storage); /* Check if the fixed array is open */ - if (idx_info->storage->u.farray.fa) { - + if (H5D_FARRAY_IDX_IS_OPEN(idx_info)) { /* Patch the top level file pointer contained in fa if needed */ if (H5FA_patch_file(idx_info->storage->u.farray.fa, idx_info->f) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't patch fixed array file pointer"); /* Close fixed array */ - if (H5FA_close(idx_info->storage->u.farray.fa) < 0) + if (H5D__farray_idx_close(idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close fixed array"); - idx_info->storage->u.farray.fa = NULL; } /* end if */ done: diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 8ba9a14fc11..5c2ee87a4ae 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -3024,6 +3024,26 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_dset_io_info_t *di, uint8_t as * metadata reads are enabled. */ if (H5F_get_coll_metadata_reads(di->dset->oloc.file)) { +#ifndef NDEBUG + { + H5D_chk_idx_info_t idx_info; + bool index_is_open; + + idx_info.f = di->dset->oloc.file; + idx_info.pline = &di->dset->shared->dcpl_cache.pline; + idx_info.layout = &di->dset->shared->layout.u.chunk; + idx_info.storage = &di->dset->shared->layout.storage.u.chunk; + + /* + * The dataset's chunk index should be open at this point. + * Otherwise, we will end up reading it in independently, + * which may not be desired. + */ + idx_info.storage->ops->is_open(&idx_info, &index_is_open); + assert(index_is_open); + } +#endif + md_reads_file_flag = H5P_FORCE_FALSE; md_reads_context_flag = false; H5F_set_coll_metadata_reads(di->dset->oloc.file, &md_reads_file_flag, &md_reads_context_flag); @@ -3446,26 +3466,6 @@ H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t *io_info, const chunk_node = H5SL_next(chunk_node); } } - else if (H5F_get_coll_metadata_reads(di[dset_idx].dset->oloc.file)) { - hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; - - /* - * If this rank has no selection in the dataset and collective - * metadata reads are enabled, do a fake lookup of a chunk to - * ensure that this rank has the chunk index opened. Otherwise, - * only the ranks that had a selection will have opened the - * chunk index and they will have done so independently. Therefore, - * when ranks with no selection participate in later collective - * metadata reads, they will try to open the chunk index collectively - * and issues will occur since other ranks won't participate. - * - * In the future, we should consider having a chunk index "open" - * callback that can be used to ensure collectivity between ranks - * in a more natural way, but this hack should suffice for now. - */ - if (H5D__chunk_lookup(di[dset_idx].dset, scaled, &udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address"); - } /* Reset metadata tagging */ H5AC_tag(prev_tag, NULL); diff --git a/src/H5Dnone.c b/src/H5Dnone.c index 472a2214dc7..d4eb9188840 100644 --- a/src/H5Dnone.c +++ b/src/H5Dnone.c @@ -14,9 +14,9 @@ * Purpose: Implicit (Non Index) chunked I/O functions. * * This is used when the dataset is: - * - extendible but with fixed max. dims - * - with early allocation - * - without filter + * - extendible but with fixed max. dims + * - with early allocation + * - without filter * * The chunk coordinate is mapped into the actual disk addresses * for the chunk without indexing. @@ -31,12 +31,12 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Dpkg.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5MFprivate.h" /* File space management */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5MFprivate.h" /* File space management */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ /* Local Macros */ @@ -52,8 +52,12 @@ /* Non Index chunking I/O ops */ static herr_t H5D__none_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__none_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__none_idx_close(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__none_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open); static bool H5D__none_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__none_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__none_idx_load_metadata(const H5D_chk_idx_info_t *idx_info); static int H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); static herr_t H5D__none_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata); @@ -73,9 +77,13 @@ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{ false, /* Non-indexed chunking don't current support SWMR access */ NULL, /* init */ H5D__none_idx_create, /* create */ + H5D__none_idx_open, /* open */ + H5D__none_idx_close, /* close */ + H5D__none_idx_is_open, /* is_open */ H5D__none_idx_is_space_alloc, /* is_space_alloc */ NULL, /* insert */ H5D__none_idx_get_addr, /* get_addr */ + H5D__none_idx_load_metadata, /* load_metadata */ NULL, /* resize */ H5D__none_idx_iterate, /* iterate */ H5D__none_idx_remove, /* remove */ @@ -97,12 +105,12 @@ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{ /*******************/ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_create + * Function: H5D__none_idx_create * - * Purpose: Allocate memory for the maximum # of chunks in the dataset. + * Purpose: Allocate memory for the maximum # of chunks in the dataset. * - * Return: Non-negative on success - * Negative on failure. + * Return: Non-negative on success + * Negative on failure. * *------------------------------------------------------------------------- */ @@ -141,11 +149,73 @@ done: } /* end H5D__none_idx_create() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_is_space_alloc + * Function: H5D__none_idx_open * - * Purpose: Query if space for the dataset chunks is allocated + * Purpose: Opens an existing "none" index. Currently a no-op. * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__none_idx_open(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__none_idx_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__none_idx_close + * + * Purpose: Closes an existing "none" index. Currently a no-op. + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__none_idx_close(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__none_idx_close() */ + +/*------------------------------------------------------------------------- + * Function: H5D__none_idx_is_open + * + * Purpose: Query if the index is opened or not + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__none_idx_is_open(const H5D_chk_idx_info_t H5_ATTR_NDEBUG_UNUSED *idx_info, bool *is_open) +{ + FUNC_ENTER_PACKAGE_NOERR + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_NONE == idx_info->storage->idx_type); + assert(is_open); + + *is_open = true; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__none_idx_is_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__none_idx_is_space_alloc + * + * Purpose: Query if space for the dataset chunks is allocated + * + * Return: true/false * *------------------------------------------------------------------------- */ @@ -161,12 +231,12 @@ H5D__none_idx_is_space_alloc(const H5O_storage_chunk_t *storage) } /* end H5D__none_idx_is_space_alloc() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_get_addr + * Function: H5D__none_idx_get_addr * - * Purpose: Get the file address of a chunk. - * Save the retrieved information in the udata supplied. + * Purpose: Get the file address of a chunk. + * Save the retrieved information in the udata supplied. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -200,12 +270,32 @@ H5D__none_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata } /* H5D__none_idx_get_addr() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_iterate + * Function: H5D__none_idx_load_metadata * - * Purpose: Iterate over the chunks in an index, making a callback + * Purpose: Load additional chunk index metadata beyond the chunk index + * itself. Currently a no-op. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__none_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5D__none_idx_load_metadata() */ + +/*------------------------------------------------------------------------- + * Function: H5D__none_idx_iterate + * + * Purpose: Iterate over the chunks in an index, making a callback * for each one. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -275,13 +365,13 @@ done: } /* end H5D__none_idx_iterate() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_remove + * Function: H5D__none_idx_remove * - * Purpose: Remove chunk from index. + * Purpose: Remove chunk from index. * - * Note: Chunks can't be removed (or added) to datasets with this - * form of index - all the space for all the chunks is always - * allocated in the file. + * Note: Chunks can't be removed (or added) to datasets with this + * form of index - all the space for all the chunks is always + * allocated in the file. * * Return: Non-negative on success/Negative on failure * @@ -299,12 +389,12 @@ H5D__none_idx_remove(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, } /* H5D__none_idx_remove() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_delete + * Function: H5D__none_idx_delete * - * Purpose: Delete raw data storage for entire dataset (i.e. all chunks) + * Purpose: Delete raw data storage for entire dataset (i.e. all chunks) * - * Return: Success: Non-negative - * Failure: negative + * Return: Success: Non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -337,11 +427,11 @@ done: } /* end H5D__none_idx_delete() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_copy_setup + * Function: H5D__none_idx_copy_setup * - * Purpose: Set up any necessary information for copying chunks + * Purpose: Set up any necessary information for copying chunks * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -407,11 +497,11 @@ H5D__none_idx_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *i } /* end H5D__none_idx_size() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_reset + * Function: H5D__none_idx_reset * - * Purpose: Reset indexing information. + * Purpose: Reset indexing information. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -431,11 +521,11 @@ H5D__none_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr) } /* end H5D__none_idx_reset() */ /*------------------------------------------------------------------------- - * Function: H5D__none_idx_dump + * Function: H5D__none_idx_dump * - * Purpose: Dump + * Purpose: Dump * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 82fec0ea1ff..a3695ae8544 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -393,10 +393,14 @@ typedef int (*H5D_chunk_cb_func_t)(const H5D_chunk_rec_t *chunk_rec, void *udata typedef herr_t (*H5D_chunk_init_func_t)(const H5D_chk_idx_info_t *idx_info, const H5S_t *space, haddr_t dset_ohdr_addr); typedef herr_t (*H5D_chunk_create_func_t)(const H5D_chk_idx_info_t *idx_info); +typedef herr_t (*H5D_chunk_open_func_t)(const H5D_chk_idx_info_t *idx_info); +typedef herr_t (*H5D_chunk_close_func_t)(const H5D_chk_idx_info_t *idx_info); +typedef herr_t (*H5D_chunk_is_open_func_t)(const H5D_chk_idx_info_t *idx_info, bool *is_open); typedef bool (*H5D_chunk_is_space_alloc_func_t)(const H5O_storage_chunk_t *storage); typedef herr_t (*H5D_chunk_insert_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, const H5D_t *dset); typedef herr_t (*H5D_chunk_get_addr_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +typedef herr_t (*H5D_chunk_load_metadata_func_t)(const H5D_chk_idx_info_t *idx_info); typedef herr_t (*H5D_chunk_resize_func_t)(H5O_layout_chunk_t *layout); typedef int (*H5D_chunk_iterate_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); @@ -413,13 +417,18 @@ typedef herr_t (*H5D_chunk_dest_func_t)(const H5D_chk_idx_info_t *idx_info); /* Typedef for grouping chunk I/O routines */ typedef struct H5D_chunk_ops_t { - bool can_swim; /* Flag to indicate that the index supports SWMR access */ - H5D_chunk_init_func_t init; /* Routine to initialize indexing information in memory */ - H5D_chunk_create_func_t create; /* Routine to create chunk index */ + bool can_swim; /* Flag to indicate that the index supports SWMR access */ + H5D_chunk_init_func_t init; /* Routine to initialize indexing information in memory */ + H5D_chunk_create_func_t create; /* Routine to create chunk index */ + H5D_chunk_open_func_t open; /* Routine to open chunk index */ + H5D_chunk_close_func_t close; /* Routine to close chunk index */ + H5D_chunk_is_open_func_t is_open; /* Query routine to determine if index is open or not */ H5D_chunk_is_space_alloc_func_t - is_space_alloc; /* Query routine to determine if storage/index is allocated */ - H5D_chunk_insert_func_t insert; /* Routine to insert a chunk into an index */ - H5D_chunk_get_addr_func_t get_addr; /* Routine to retrieve address of chunk in file */ + is_space_alloc; /* Query routine to determine if storage/index is allocated */ + H5D_chunk_insert_func_t insert; /* Routine to insert a chunk into an index */ + H5D_chunk_get_addr_func_t get_addr; /* Routine to retrieve address of chunk in file */ + H5D_chunk_load_metadata_func_t + load_metadata; /* Routine to load additional chunk index metadata, such as fixed array data blocks */ H5D_chunk_resize_func_t resize; /* Routine to update chunk index info after resizing dataset */ H5D_chunk_iterate_func_t iterate; /* Routine to iterate over chunks */ H5D_chunk_remove_func_t remove; /* Routine to remove a chunk from an index */ diff --git a/src/H5Dsingle.c b/src/H5Dsingle.c index 9cb18d35278..dd9f2353d7b 100644 --- a/src/H5Dsingle.c +++ b/src/H5Dsingle.c @@ -27,12 +27,12 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Dpkg.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5MFprivate.h" /* File space management */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5MFprivate.h" /* File space management */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ /* Local Macros */ @@ -50,10 +50,14 @@ static herr_t H5D__single_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t *space, haddr_t dset_ohdr_addr); static herr_t H5D__single_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__single_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__single_idx_close(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__single_idx_is_open(const H5D_chk_idx_info_t *idx_info, bool *is_open); static bool H5D__single_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__single_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, const H5D_t *dset); static herr_t H5D__single_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__single_idx_load_metadata(const H5D_chk_idx_info_t *idx_info); static int H5D__single_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); static herr_t H5D__single_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata); @@ -73,9 +77,13 @@ const H5D_chunk_ops_t H5D_COPS_SINGLE[1] = {{ false, /* Single Chunk indexing doesn't current support SWMR access */ H5D__single_idx_init, /* init */ H5D__single_idx_create, /* create */ + H5D__single_idx_open, /* open */ + H5D__single_idx_close, /* close */ + H5D__single_idx_is_open, /* is_open */ H5D__single_idx_is_space_alloc, /* is_space_alloc */ H5D__single_idx_insert, /* insert */ H5D__single_idx_get_addr, /* get_addr */ + H5D__single_idx_load_metadata, /* load_metadata */ NULL, /* resize */ H5D__single_idx_iterate, /* iterate */ H5D__single_idx_remove, /* remove */ @@ -133,12 +141,12 @@ H5D__single_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR_UNU } /* end H5D__single_idx_init() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_create + * Function: H5D__single_idx_create * - * Purpose: Set up Single Chunk Index: filtered or non-filtered + * Purpose: Set up Single Chunk Index: filtered or non-filtered * - * Return: Non-negative on success - * Negative on failure. + * Return: Non-negative on success + * Negative on failure. * *------------------------------------------------------------------------- */ @@ -166,11 +174,73 @@ H5D__single_idx_create(const H5D_chk_idx_info_t *idx_info) } /* end H5D__single_idx_create() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_is_space_alloc + * Function: H5D__single_idx_open * - * Purpose: Query if space is allocated for the single chunk + * Purpose: Opens an existing "single" index. Currently a no-op. * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__single_idx_open(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__single_idx_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__single_idx_close + * + * Purpose: Closes an existing "single" index. Currently a no-op. + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__single_idx_close(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__single_idx_close() */ + +/*------------------------------------------------------------------------- + * Function: H5D__single_idx_is_open + * + * Purpose: Query if the index is opened or not + * + * Return: SUCCEED (can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__single_idx_is_open(const H5D_chk_idx_info_t H5_ATTR_NDEBUG_UNUSED *idx_info, bool *is_open) +{ + FUNC_ENTER_PACKAGE_NOERR + + assert(idx_info); + assert(idx_info->storage); + assert(H5D_CHUNK_IDX_SINGLE == idx_info->storage->idx_type); + assert(is_open); + + *is_open = true; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D__single_idx_is_open() */ + +/*------------------------------------------------------------------------- + * Function: H5D__single_idx_is_space_alloc + * + * Purpose: Query if space is allocated for the single chunk + * + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -186,11 +256,11 @@ H5D__single_idx_is_space_alloc(const H5O_storage_chunk_t *storage) } /* end H5D__single_idx_is_space_alloc() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_insert + * Function: H5D__single_idx_insert * - * Purpose: Allocate space for the single chunk + * Purpose: Allocate space for the single chunk * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -231,12 +301,12 @@ done: } /* H5D__single_idx_insert() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_get_addr + * Function: H5D__single_idx_get_addr * - * Purpose: Get the file address of a chunk. - * Save the retrieved information in the udata supplied. + * Purpose: Get the file address of a chunk. + * Save the retrieved information in the udata supplied. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -271,11 +341,31 @@ H5D__single_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *uda } /* H5D__single_idx_get_addr() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_iterate + * Function: H5D__single_idx_load_metadata * - * Purpose: Make callback for the single chunk + * Purpose: Load additional chunk index metadata beyond the chunk index + * itself. Currently a no-op. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__single_idx_load_metadata(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* NO OP */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5D__single_idx_load_metadata() */ + +/*------------------------------------------------------------------------- + * Function: H5D__single_idx_iterate + * + * Purpose: Make callback for the single chunk + * + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -318,11 +408,11 @@ H5D__single_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t } /* end H5D__single_idx_iterate() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_remove + * Function: H5D__single_idx_remove * - * Purpose: Remove the single chunk + * Purpose: Remove the single chunk * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -357,12 +447,13 @@ done: } /* H5D__single_idx_remove() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_delete + * Function: H5D__single_idx_delete * - * Purpose: Delete raw data storage for entire dataset (i.e. the only chunk) + * Purpose: Delete raw data storage for entire dataset (i.e. the only + * chunk) * - * Return: Success: Non-negative - * Failure: negative + * Return: Success: Non-negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -389,11 +480,12 @@ H5D__single_idx_delete(const H5D_chk_idx_info_t *idx_info) } /* end H5D__single_idx_delete() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_copy_setup + * Function: H5D__single_idx_copy_setup * - * Purpose: Set up any necessary information for copying the single chunk + * Purpose: Set up any necessary information for copying the single + * chunk * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -457,11 +549,11 @@ H5D__single_idx_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t } /* end H5D__single_idx_size() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_reset + * Function: H5D__single_idx_reset * - * Purpose: Reset indexing information. + * Purpose: Reset indexing information. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -481,11 +573,11 @@ H5D__single_idx_reset(H5O_storage_chunk_t *storage, bool reset_addr) } /* end H5D__single_idx_reset() */ /*------------------------------------------------------------------------- - * Function: H5D__single_idx_dump + * Function: H5D__single_idx_dump * - * Purpose: Dump the address of the single chunk + * Purpose: Dump the address of the single chunk * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 3f3413c6d0e..c6aea0e327a 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -43,6 +43,9 @@ */ #define ROS3_STATS 0 +/* Max size of the cache, in bytes */ +#define ROS3_MAX_CACHE_SIZE 16777216 + /* The driver identification number, initialized at runtime */ static hid_t H5FD_ROS3_g = 0; @@ -189,6 +192,8 @@ typedef struct H5FD_ros3_t { H5FD_ros3_fapl_t fa; haddr_t eoa; s3r_t *s3r_handle; + uint8_t *cache; + size_t cache_size; #if ROS3_STATS ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1]; ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1]; @@ -1000,6 +1005,18 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics"); #endif /* ROS3_STATS */ + /* Cache the initial bytes of the file */ + { + size_t filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle); + + file->cache_size = (filesize < ROS3_MAX_CACHE_SIZE) ? filesize : ROS3_MAX_CACHE_SIZE; + + if (NULL == (file->cache = (uint8_t *)H5MM_calloc(file->cache_size))) + HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate cache memory"); + if (H5FD_s3comms_s3r_read(file->s3r_handle, 0, file->cache_size, file->cache) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, NULL, "unable to execute read"); + } + ret_value = (H5FD_t *)file; done: @@ -1007,8 +1024,10 @@ done: if (handle != NULL) if (FAIL == H5FD_s3comms_s3r_close(handle)) HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle"); - if (file != NULL) + if (file != NULL) { + H5MM_xfree(file->cache); file = H5FL_FREE(H5FD_ros3_t, file); + } curl_global_cleanup(); /* early cleanup because open failed */ } /* end if null return value (error) */ @@ -1335,6 +1354,7 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file) #endif /* ROS3_STATS */ /* Release the file info */ + H5MM_xfree(file->cache); file = H5FL_FREE(H5FD_ros3_t, file); done: @@ -1666,41 +1686,50 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU fprintf(stdout, "H5FD__ros3_read() called.\n"); #endif - assert(file != NULL); - assert(file->s3r_handle != NULL); - assert(buf != NULL); + assert(file); + assert(file->cache); + assert(file->s3r_handle); + assert(buf); filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle); if ((addr > filesize) || ((addr + size) > filesize)) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address"); - if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) - HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read"); + /* Copy from the cache when accessing the first N bytes of the file. + * Saves network I/O operations when opening files. + */ + if (addr + size < file->cache_size) { + memcpy(buf, file->cache + addr, size); + } + else { + if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read"); #if ROS3_STATS - /* Find which "bin" this read fits in. Can be "overflow" bin. */ - for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) - if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) - break; - bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i]; + /* Find which "bin" this read fits in. Can be "overflow" bin. */ + for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) + if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) + break; + bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i]; - /* Store collected stats in appropriate bin */ - if (bin->count == 0) { - bin->min = size; - bin->max = size; - } - else { - if (size < bin->min) + /* Store collected stats in appropriate bin */ + if (bin->count == 0) { bin->min = size; - if (size > bin->max) bin->max = size; - } - bin->count++; - bin->bytes += (unsigned long long)size; + } + else { + if (size < bin->min) + bin->min = size; + if (size > bin->max) + bin->max = size; + } + bin->count++; + bin->bytes += (unsigned long long)size; #endif /* ROS3_STATS */ + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Fint.c b/src/H5Fint.c index 014f619d8a9..4093b4b7c45 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1968,6 +1968,22 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum raw data fraction of page buffer"); } /* end if */ + /* Get the evict on close setting */ + if (H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value"); + +#ifdef H5_HAVE_PARALLEL + /* Check for evict on close in parallel (currently unsupported) */ + assert(file->shared); + if (H5F_SHARED_HAS_FEATURE(file->shared, H5FD_FEAT_HAS_MPI)) { + int mpi_size = H5F_shared_mpi_get_size(file->shared); + + if ((mpi_size > 1) && evict_on_close) + HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, NULL, + "evict on close is currently not supported in parallel HDF5"); + } +#endif + /* * Read or write the file superblock, depending on whether the file is * empty or not. @@ -2046,8 +2062,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) * or later, verify that the access property list value matches the value * in shared file structure. */ - if (H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value"); if (shared->nrefs == 1) shared->evict_on_close = evict_on_close; else if (shared->nrefs > 1) { diff --git a/src/H5Odeprec.c b/src/H5Odeprec.c index 6e8b34e789b..3de58185675 100644 --- a/src/H5Odeprec.c +++ b/src/H5Odeprec.c @@ -116,9 +116,10 @@ static herr_t H5O__iterate1_adapter(hid_t obj_id, const char *name, const H5O_info2_t *oinfo2, void *op_data) { H5O_visit1_adapter_t *shim_data = (H5O_visit1_adapter_t *)op_data; - H5O_info1_t oinfo; /* Deprecated object info struct */ - unsigned dm_fields; /* Fields for data model query */ - unsigned nat_fields; /* Fields for native query */ + H5O_info1_t oinfo; /* Deprecated object info struct */ + unsigned dm_fields; /* Fields for data model query */ + unsigned nat_fields; /* Fields for native query */ + H5VL_object_t *vol_obj; herr_t ret_value = H5_ITER_CONT; /* Return value */ FUNC_ENTER_PACKAGE @@ -158,7 +159,6 @@ H5O__iterate1_adapter(hid_t obj_id, const char *name, const H5O_info2_t *oinfo2, /* Check for retrieving native information */ nat_fields = shim_data->fields & (H5O_INFO_HDR | H5O_INFO_META_SIZE); if (nat_fields) { - H5VL_object_t *vol_obj; /* Object of obj_id */ H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ H5VL_native_object_optional_args_t obj_opt_args; /* Arguments for optional operation */ H5VL_loc_params_t loc_params; /* Location parameters for VOL callback */ @@ -401,7 +401,8 @@ H5Oget_info1(hid_t loc_id, H5O_info1_t *oinfo /*out*/) { H5VL_object_t *vol_obj = NULL; /* Object of loc_id */ H5VL_loc_params_t loc_params; - herr_t ret_value = SUCCEED; /* Return value */ + bool is_native_vol_obj = false; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE2("e", "ix", loc_id, oinfo); @@ -418,6 +419,15 @@ H5Oget_info1(hid_t loc_id, H5O_info1_t *oinfo /*out*/) if (NULL == (vol_obj = H5VL_vol_object(loc_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL, + "Deprecated H5Oget_info1 is only meant to be used with the native VOL connector"); + /* Retrieve the object's information */ if (H5O__get_info_old(vol_obj, &loc_params, oinfo, H5O_INFO_ALL) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get deprecated info for object"); @@ -441,7 +451,8 @@ H5Oget_info_by_name1(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/, { H5VL_object_t *vol_obj = NULL; /* object of loc_id */ H5VL_loc_params_t loc_params; - herr_t ret_value = SUCCEED; /* Return value */ + bool is_native_vol_obj = false; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE4("e", "i*sxi", loc_id, name, oinfo, lapl_id); @@ -468,6 +479,15 @@ H5Oget_info_by_name1(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/, if (NULL == (vol_obj = H5VL_vol_object(loc_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL, + "Deprecated H5Oget_info_by_name1 is only meant to be used with the native VOL connector"); + /* Retrieve the object's information */ if (H5O__get_info_old(vol_obj, &loc_params, oinfo, H5O_INFO_ALL) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get deprecated info for object"); @@ -493,7 +513,8 @@ H5Oget_info_by_idx1(hid_t loc_id, const char *group_name, H5_index_t idx_type, H { H5VL_object_t *vol_obj = NULL; /* object of loc_id */ H5VL_loc_params_t loc_params; - herr_t ret_value = SUCCEED; /* Return value */ + bool is_native_vol_obj = false; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE7("e", "i*sIiIohxi", loc_id, group_name, idx_type, order, n, oinfo, lapl_id); @@ -524,6 +545,15 @@ H5Oget_info_by_idx1(hid_t loc_id, const char *group_name, H5_index_t idx_type, H if (NULL == (vol_obj = H5VL_vol_object(loc_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL, + "Deprecated H5Oget_info_by_idx1 is only meant to be used with the native VOL connector"); + /* Retrieve the object's information */ if (H5O__get_info_old(vol_obj, &loc_params, oinfo, H5O_INFO_ALL) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get deprecated info for object"); @@ -574,7 +604,7 @@ H5Oget_info2(hid_t loc_id, H5O_info1_t *oinfo /*out*/, unsigned fields) "can't determine if VOL object is native connector object"); if (!is_native_vol_obj) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID, - "H5Oget_info2 is only meant to be used with the native VOL connector"); + "Deprecated H5Oget_info2 is only meant to be used with the native VOL connector"); /* Retrieve deprecated info struct */ if (H5O__get_info_old(vol_obj, &loc_params, oinfo, fields) < 0) @@ -637,7 +667,7 @@ H5Oget_info_by_name2(hid_t loc_id, const char *name, H5O_info1_t *oinfo /*out*/, "can't determine if VOL object is native connector object"); if (!is_native_vol_obj) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID, - "H5Oget_info_by_name2 is only meant to be used with the native VOL connector"); + "Deprecated H5Oget_info_by_name2 is only meant to be used with the native VOL connector"); /* Retrieve deprecated info struct */ if (H5O__get_info_old(vol_obj, &loc_params, oinfo, fields) < 0) @@ -706,7 +736,7 @@ H5Oget_info_by_idx2(hid_t loc_id, const char *group_name, H5_index_t idx_type, H "can't determine if VOL object is native connector object"); if (!is_native_vol_obj) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID, - "H5Oget_info_by_idx2 is only meant to be used with the native VOL connector"); + "Deprecated H5Oget_info_by_idx2 is only meant to be used with the native VOL connector"); /* Retrieve deprecated info struct */ if (H5O__get_info_old(vol_obj, &loc_params, oinfo, fields) < 0) @@ -753,6 +783,7 @@ H5Ovisit1(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1 H5VL_loc_params_t loc_params; /* Location parameters for object access */ H5O_visit1_adapter_t shim_data; /* Adapter for passing app callback & user data */ herr_t ret_value; /* Return value */ + bool is_native_vol_obj = false; FUNC_ENTER_API(FAIL) H5TRACE5("e", "iIiIoOi*x", obj_id, idx_type, order, op, op_data); @@ -769,6 +800,15 @@ H5Ovisit1(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1 if (NULL == (vol_obj = H5VL_vol_object(obj_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL, + "Deprecated H5Ovisit1 is only meant to be used with the native VOL connector"); + /* Set location parameters */ loc_params.type = H5VL_OBJECT_BY_SELF; loc_params.obj_type = H5I_get_type(obj_id); @@ -833,6 +873,7 @@ H5Ovisit_by_name1(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it H5VL_loc_params_t loc_params; /* Location parameters for object access */ H5O_visit1_adapter_t shim_data; /* Adapter for passing app callback & user data */ herr_t ret_value; /* Return value */ + bool is_native_vol_obj = false; FUNC_ENTER_API(FAIL) H5TRACE7("e", "i*sIiIoOi*xi", loc_id, obj_name, idx_type, order, op, op_data, lapl_id); @@ -857,6 +898,15 @@ H5Ovisit_by_name1(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it if (NULL == (vol_obj = H5VL_vol_object(loc_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_OHDR, H5E_VOL, FAIL, + "Deprecated H5Ovisit_by_name1 is only meant to be used with the native VOL connector"); + /* Set location parameters */ loc_params.type = H5VL_OBJECT_BY_NAME; loc_params.loc_data.loc_by_name.name = obj_name; @@ -949,9 +999,10 @@ H5Ovisit2(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate1 if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, H5I_INVALID_HID, "can't determine if VOL object is native connector object"); + if (!is_native_vol_obj) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID, - "H5Ovisit2 is only meant to be used with the native VOL connector"); + "Deprecated H5Ovisit2 is only meant to be used with the native VOL connector"); /* Set location parameters */ loc_params.type = H5VL_OBJECT_BY_SELF; @@ -1053,7 +1104,7 @@ H5Ovisit_by_name2(hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_it "can't determine if VOL object is native connector object"); if (!is_native_vol_obj) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, H5I_INVALID_HID, - "H5Ovisit_by_name2 is only meant to be used with the native VOL connector"); + "Deprecated H5Ovisit_by_name2 is only meant to be used with the native VOL connector"); /* Set location parameters */ loc_params.type = H5VL_OBJECT_BY_NAME; diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 5f5782cae3b..e7c1fb3acb8 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -4848,7 +4848,7 @@ H5P__facc_mdc_log_location_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR *------------------------------------------------------------------------- */ herr_t -H5Pset_evict_on_close(hid_t fapl_id, hbool_t H5_ATTR_PARALLEL_UNUSED evict_on_close) +H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close) { H5P_genplist_t *plist; /* property list pointer */ herr_t ret_value = SUCCEED; /* return value */ @@ -4864,14 +4864,9 @@ H5Pset_evict_on_close(hid_t fapl_id, hbool_t H5_ATTR_PARALLEL_UNUSED evict_on_cl if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for ID"); -#ifndef H5_HAVE_PARALLEL /* Set value */ if (H5P_set(plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set evict on close property"); -#else - HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, - "evict on close is currently not supported in parallel HDF5"); -#endif /* H5_HAVE_PARALLEL */ done: FUNC_LEAVE_API(ret_value) @@ -5174,7 +5169,7 @@ done: * Function: H5Pset_coll_metadata_write * * Purpose: Tell the library whether the metadata write operations will - * be done collectively (1) or not (0). Default is collective. + * be done collectively (1) or not (0). Default is independent. * * Return: Non-negative on success/Negative on failure * diff --git a/src/H5Rdeprec.c b/src/H5Rdeprec.c index 773d8b0c706..1d12ebae25a 100644 --- a/src/H5Rdeprec.c +++ b/src/H5Rdeprec.c @@ -101,14 +101,14 @@ H5R__decode_token_compat(H5VL_object_t *vol_obj, H5I_type_t type, H5R_type_t ref #ifndef NDEBUG { - bool is_native = false; /* Whether the src file is using the native VOL connector */ + bool is_native_vol_obj = false; /* Whether the src file is using the native VOL connector */ /* Check if using native VOL connector */ - if (H5VL_object_is_native(vol_obj, &is_native) < 0) + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't query if file uses native VOL connector"); /* Must use native VOL connector for this operation */ - assert(is_native); + assert(is_native_vol_obj); } #endif /* NDEBUG */ @@ -251,7 +251,8 @@ H5Rget_obj_type1(hid_t id, H5R_type_t ref_type, const void *ref) H5O_token_t obj_token = {0}; /* Object token */ const unsigned char *buf = (const unsigned char *)ref; /* Reference buffer */ H5O_type_t obj_type = H5O_TYPE_UNKNOWN; /* Type of the referenced object */ - H5G_obj_t ret_value; /* Return value */ + bool is_native_vol_obj; /* Whether the native VOL connector is in use */ + H5G_obj_t ret_value; /* Return value */ FUNC_ENTER_API(H5G_UNKNOWN) H5TRACE3("Go", "iRt*x", id, ref_type, ref); @@ -266,6 +267,16 @@ H5Rget_obj_type1(hid_t id, H5R_type_t ref_type, const void *ref) if (NULL == (vol_obj = H5VL_vol_object(id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5G_UNKNOWN, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, + "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, + "H5Rget_obj_type1 is only meant to be used with the native VOL connector"); + /* Get object type */ if ((vol_obj_type = H5I_get_type(id)) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5G_UNKNOWN, "invalid location identifier"); @@ -315,7 +326,8 @@ H5Rdereference1(hid_t obj_id, H5R_type_t ref_type, const void *ref) H5I_type_t opened_type; /* Opened object type */ void *opened_obj = NULL; /* Opened object */ const unsigned char *buf = (const unsigned char *)ref; /* Reference buffer */ - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + bool is_native_vol_obj; /* Whether the native VOL connector is in use */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ FUNC_ENTER_API(H5I_INVALID_HID) H5TRACE3("i", "iRt*x", obj_id, ref_type, ref); @@ -330,6 +342,16 @@ H5Rdereference1(hid_t obj_id, H5R_type_t ref_type, const void *ref) if (NULL == (vol_obj = H5VL_vol_object(obj_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, + "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, + "H5Rdereference1 is only meant to be used with the native VOL connector"); + /* Get object type */ if ((vol_obj_type = H5I_get_type(obj_id)) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier"); @@ -382,8 +404,9 @@ H5Rcreate(void *ref, hid_t loc_id, const char *name, H5R_type_t ref_type, hid_t H5VL_file_get_args_t file_get_vol_cb_args; /* Arguments to VOL callback */ hid_t file_id = H5I_INVALID_HID; /* File ID for region reference */ void *vol_obj_file = NULL; - unsigned char *buf = (unsigned char *)ref; /* Return reference pointer */ - herr_t ret_value = SUCCEED; /* Return value */ + bool is_native_vol_obj = false; /* Whether the src file is using the native VOL connector */ + unsigned char *buf = (unsigned char *)ref; /* Return reference pointer */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE5("e", "*xi*sRti", ref, loc_id, name, ref_type, space_id); @@ -404,18 +427,13 @@ H5Rcreate(void *ref, hid_t loc_id, const char *name, H5R_type_t ref_type, hid_t if (NULL == (vol_obj = H5VL_vol_object(loc_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); -#ifndef NDEBUG - { - bool is_native = false; /* Whether the src file is using the native VOL connector */ + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't query if file uses native VOL connector"); - /* Check if using native VOL connector */ - if (H5VL_object_is_native(vol_obj, &is_native) < 0) - HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't query if file uses native VOL connector"); - - /* Must use native VOL connector for this operation */ - assert(is_native); - } -#endif /* NDEBUG */ + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, "must use native VOL connector to create reference"); /* Get object type */ if ((vol_obj_type = H5I_get_type(loc_id)) < 0) @@ -500,13 +518,14 @@ done: herr_t H5Rget_obj_type2(hid_t id, H5R_type_t ref_type, const void *ref, H5O_type_t *obj_type /*out*/) { - H5VL_object_t *vol_obj = NULL; /* Object of loc_id */ - H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */ - H5VL_object_get_args_t vol_cb_args; /* Arguments to VOL callback */ - H5VL_loc_params_t loc_params; /* Location parameters */ - H5O_token_t obj_token = {0}; /* Object token */ - const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */ - herr_t ret_value = SUCCEED; /* Return value */ + H5VL_object_t *vol_obj = NULL; /* Object of loc_id */ + H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */ + H5VL_object_get_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_loc_params_t loc_params; /* Location parameters */ + H5O_token_t obj_token = {0}; /* Object token */ + const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */ + bool is_native_vol_obj = false; /* Whether the native VOL connector is in use */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE4("e", "iRt*xx", id, ref_type, ref, obj_type); @@ -521,6 +540,16 @@ H5Rget_obj_type2(hid_t id, H5R_type_t ref_type, const void *ref, H5O_type_t *obj if (NULL == (vol_obj = H5VL_vol_object(id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, + "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, + "H5Rget_obj_type2 is only meant to be used with the native VOL connector"); + /* Get object type */ if ((vol_obj_type = H5I_get_type(id)) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier"); @@ -560,14 +589,15 @@ done: hid_t H5Rdereference2(hid_t obj_id, hid_t oapl_id, H5R_type_t ref_type, const void *ref) { - H5VL_object_t *vol_obj = NULL; /* Object of loc_id */ - H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */ - H5VL_loc_params_t loc_params; /* Location parameters */ - H5O_token_t obj_token = {0}; /* Object token */ - H5I_type_t opened_type; /* Opened object type */ - void *opened_obj = NULL; /* Opened object */ - const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */ - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + H5VL_object_t *vol_obj = NULL; /* Object of loc_id */ + H5I_type_t vol_obj_type = H5I_BADID; /* Object type of loc_id */ + H5VL_loc_params_t loc_params; /* Location parameters */ + H5O_token_t obj_token = {0}; /* Object token */ + H5I_type_t opened_type; /* Opened object type */ + void *opened_obj = NULL; /* Opened object */ + const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */ + bool is_native_vol_obj = false; /* Whether the native VOL connector is in use */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ FUNC_ENTER_API(H5I_INVALID_HID) H5TRACE4("i", "iiRt*x", obj_id, oapl_id, ref_type, ref); @@ -588,6 +618,16 @@ H5Rdereference2(hid_t obj_id, hid_t oapl_id, H5R_type_t ref_type, const void *re if (NULL == (vol_obj = H5VL_vol_object(obj_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid file identifier"); + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, + "can't determine if VOL object is native connector object"); + + /* Must use native VOL connector for this operation */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, + "H5Rdereference2 is only meant to be used with the native VOL connector"); + /* Get object type */ if ((vol_obj_type = H5I_get_type(obj_id)) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid location identifier"); @@ -639,7 +679,8 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref) H5S_t *space = NULL; /* Dataspace object */ hid_t file_id = H5I_INVALID_HID; /* File ID for region reference */ const unsigned char *buf = (const unsigned char *)ref; /* Reference pointer */ - hid_t ret_value; /* Return value */ + bool is_native_vol_obj = false; /* Whether the src file is using the native VOL connector */ + hid_t ret_value; /* Return value */ FUNC_ENTER_API(H5I_INVALID_HID) H5TRACE3("i", "iRt*x", id, ref_type, ref); @@ -654,19 +695,14 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref) if (NULL == (vol_obj = H5VL_vol_object(id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "invalid file identifier"); -#ifndef NDEBUG - { - bool is_native = false; /* Whether the src file is using the native VOL connector */ + /* Check if using native VOL connector */ + if (H5VL_object_is_native(vol_obj, &is_native_vol_obj) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, H5I_INVALID_HID, + "can't query if file uses native VOL connector"); - /* Check if using native VOL connector */ - if (H5VL_object_is_native(vol_obj, &is_native) < 0) - HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, H5I_INVALID_HID, - "can't query if file uses native VOL connector"); - - /* Must use native VOL connector for this operation */ - assert(is_native); - } -#endif /* NDEBUG */ + if (!is_native_vol_obj) + HGOTO_ERROR(H5E_REFERENCE, H5E_VOL, FAIL, + "H5Rget_region is only meant to be used with the native VOL connector"); /* Get object type */ if ((vol_obj_type = H5I_get_type(id)) < 0) diff --git a/test/Makefile.am b/test/Makefile.am index 291907ca0f9..fdd83e5bdeb 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -192,7 +192,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 mdset.h5 compact_dataset.h5 dataset.h5 d storage_size.h5 dls_01_strings.h5 power2up.h5 version_bounds.h5 \ alloc_0sized.h5 h5s_block.h5 h5s_plist.h5 \ extend.h5 istore.h5 extlinks*.h5 frspace.h5 links*.h5 \ - sys_file1 tfile[1-7].h5 th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 \ + sys_file1 tfile[1-8].h5 th5s[1-4].h5 lheap.h5 fheap.h5 ohdr.h5 \ stab.h5 extern_[1-5].h5 extern_[1-4][rw].raw gheap[0-4].h5 \ ohdr_min_a.h5 ohdr_min_b.h5 min_dset_ohdr_testfile.h5 \ dt_arith[1-2] links.h5 links[0-6]*.h5 extlinks[0-15].h5 \ @@ -226,7 +226,10 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 mdset.h5 compact_dataset.h5 dataset.h5 d test_swmr*.h5 cache_logging.h5 cache_logging.out vds_swmr.h5 vds_swmr_src_*.h5 \ swmr[0-2].h5 swmr_writer.out swmr_writer.log.* swmr_reader.out.* swmr_reader.log.* \ tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 native_vol_test.h5 \ - splitter*.h5 splitter.log mirror_rw mirror_ro event_set_[0-9].h5 + splitter*.h5 splitter.log mirror_rw mirror_ro event_set_[0-9].h5 \ + cmpd_dtransform.h5 single_latest.h5 source_file.h5 stdio_file.h5 \ + tfile_is_accessible.h5 tfile_is_accessible_non_hdf5.h5 tverbounds_dtype.h5 \ + virtual_file1.h5 tfile_double_open.h5 tfile_incr_filesize.h5 flushrefresh_test # Sources for testhdf5 executable testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ diff --git a/test/evict_on_close.c b/test/evict_on_close.c index 9ca7f9f9cf3..db2a96282ef 100644 --- a/test/evict_on_close.c +++ b/test/evict_on_close.c @@ -32,12 +32,6 @@ #include "H5Ipkg.h" #include "H5VLprivate.h" /* Virtual Object Layer */ -/* Evict on close is not supported under parallel at this time. - * In the meantime, we just run a simple check that EoC can't be - * enabled in parallel HDF5. - */ -#ifndef H5_HAVE_PARALLEL - /* Uncomment to manually inspect cache states */ /* (Requires debug build of the library) */ /* #define EOC_MANUAL_INSPECTION */ @@ -974,89 +968,3 @@ error: exit(EXIT_FAILURE); } /* end main() */ - -#else - -/*------------------------------------------------------------------------- - * Function: check_evict_on_close_parallel_fail() - * - * Purpose: Verify that the H5Pset_evict_on_close() call fails in - * parallel HDF5. - * - * Return: SUCCEED/FAIL - * - *------------------------------------------------------------------------- - */ -static herr_t -check_evict_on_close_parallel_fail(void) -{ - hid_t fapl_id = H5I_INVALID_HID; - bool evict_on_close; - herr_t status; - - TESTING("evict on close fails in parallel"); - - /* Create a fapl */ - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) - TEST_ERROR; - - /* Set the evict on close property (should fail)*/ - evict_on_close = true; - H5E_BEGIN_TRY - { - status = H5Pset_evict_on_close(fapl_id, evict_on_close); - } - H5E_END_TRY - if (status >= 0) - FAIL_PUTS_ERROR("H5Pset_evict_on_close() did not fail in parallel HDF5."); - - /* close fapl */ - if (H5Pclose(fapl_id) < 0) - TEST_ERROR; - - PASSED(); - return SUCCEED; - -error: - H5_FAILED(); - return FAIL; - -} /* check_evict_on_close_parallel_fail() */ - -/*------------------------------------------------------------------------- - * Function: main (parallel version) - * - * Return: EXIT_FAILURE/EXIT_SUCCESS - * - *------------------------------------------------------------------------- - */ -int -main(void) -{ - unsigned nerrors = 0; /* number of test errors */ - - printf("Testing evict-on-close cache behavior\n"); - - /* Initialize */ - h5_reset(); - - /* Test that EoC fails in parallel HDF5 */ - nerrors += check_evict_on_close_parallel_fail() < 0 ? 1 : 0; - - if (nerrors) - goto error; - - printf("All evict-on-close tests passed.\n"); - printf("Note that EoC is not supported under parallel so most tests are skipped.\n"); - - exit(EXIT_SUCCESS); - -error: - - printf("***** %u evict-on-close test%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); - - exit(EXIT_FAILURE); - -} /* main() - parallel */ - -#endif /* H5_HAVE_PARALLEL */ diff --git a/test/tfile.c b/test/tfile.c index d11de11f46a..4335652da6f 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -138,9 +138,15 @@ #define NGROUPS 2 #define NDSETS 4 -/* Declaration for test_incr_filesize() */ +/* Declaration for libver bounds tests */ #define FILE8 "tfile8.h5" /* Test file */ +/* Declaration for test_file_double_file_dataset_open() */ +#define FILE_DOUBLE_OPEN "tfile_double_open" + +/* Declaration for test_incr_filesize() */ +#define FILE_INCR_FILESIZE "tfile_incr_filesize" + /* Files created under 1.6 branch and 1.8 branch--used in test_filespace_compatible() */ static const char *OLD_FILENAME[] = { "filespace_1_6.h5", /* 1.6 HDF5 file */ @@ -2623,8 +2629,8 @@ test_file_double_file_dataset_open(bool new_format) if (new_format) { ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); CHECK(ret, FAIL, "H5Pset_libver_bounds"); - } /* end if */ - h5_fixname(FILE1, fapl, filename, sizeof filename); + } + h5_fixname(FILE_DOUBLE_OPEN, fapl, filename, sizeof filename); /* Create the test file */ fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); @@ -2934,6 +2940,9 @@ test_file_double_file_dataset_open(bool new_format) ret = H5Tclose(tid1); CHECK(ret, FAIL, "H5Tclose"); + /* Delete the test file */ + h5_delete_test_file(filename, fapl); + /* Close FAPL */ ret = H5Pclose(fapl); CHECK(ret, FAIL, "H5Pclose"); @@ -7649,7 +7658,7 @@ test_incr_filesize(void) MESSAGE(5, ("Testing H5Fincrement_filesize() and H5Fget_eoa())\n")); fapl = h5_fileaccess(); - h5_fixname(FILE8, fapl, filename, sizeof filename); + h5_fixname(FILE_INCR_FILESIZE, fapl, filename, sizeof filename); /* Get the VFD feature flags */ driver_id = H5Pget_driver(fapl); @@ -7734,6 +7743,9 @@ test_incr_filesize(void) /* Verify the filesize is the previous stored_eoa + 512 */ VERIFY(filesize, stored_eoa + 512, "file size"); + /* Delete the test file */ + h5_delete_test_file(FILE_INCR_FILESIZE, fapl); + /* Close the file access property list */ ret = H5Pclose(fapl); CHECK(ret, FAIL, "H5Pclose"); @@ -8224,6 +8236,7 @@ cleanup_file(void) H5Fdelete(FILE5, H5P_DEFAULT); H5Fdelete(FILE6, H5P_DEFAULT); H5Fdelete(FILE7, H5P_DEFAULT); + H5Fdelete(FILE8, H5P_DEFAULT); H5Fdelete(DST_FILE, H5P_DEFAULT); } H5E_END_TRY diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 59d47e15ebf..4a8cb826f49 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -58,6 +58,7 @@ LDADD = $(LIBH5TEST) $(LIBHDF5) # after_mpi_fin.h5 is from t_init_term # go is used for debugging. See testphdf5.c. CHECK_CLEANFILES+=MPItest.h5 Para*.h5 bigio_test.h5 CacheTestDummy.h5 \ - ShapeSameTest.h5 shutdown.h5 pmulti_dset.h5 after_mpi_fin.h5 go + ShapeSameTest.h5 shutdown.h5 pmulti_dset.h5 after_mpi_fin.h5 go noflush.h5 \ + mpio_select_test_file.h5 *.btr include $(top_srcdir)/config/conclude.am diff --git a/testpar/t_coll_md.c b/testpar/t_coll_md.c index 1220111a56d..9c6fc7120cf 100644 --- a/testpar/t_coll_md.c +++ b/testpar/t_coll_md.c @@ -43,6 +43,11 @@ #define COLL_GHEAP_WRITE_ATTR_NAME "coll_gheap_write_attr" #define COLL_GHEAP_WRITE_ATTR_DIMS 1 +#define COLL_IO_IND_MD_WRITE_NDIMS 2 +#define COLL_IO_IND_MD_WRITE_CHUNK0 4 +#define COLL_IO_IND_MD_WRITE_CHUNK1 256 +#define COLL_IO_IND_MD_WRITE_NCHUNK1 16384 + /* * A test for issue HDFFV-10501. A parallel hang was reported which occurred * in linked-chunk I/O when collective metadata reads are enabled and some ranks @@ -569,3 +574,101 @@ test_collective_global_heap_write(void) VRFY((H5Pclose(fapl_id) >= 0), "H5Pclose succeeded"); VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); } + +/* + * A test to ensure that hangs don't occur when collective I/O + * is requested at the interface level (by a call to + * H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)), while + * collective metadata writes are NOT requested. + */ +void +test_coll_io_ind_md_write(void) +{ + const char *filename; + long long *data = NULL; + hsize_t dset_dims[COLL_IO_IND_MD_WRITE_NDIMS]; + hsize_t chunk_dims[COLL_IO_IND_MD_WRITE_NDIMS]; + hsize_t sel_dims[COLL_IO_IND_MD_WRITE_NDIMS]; + hsize_t offset[COLL_IO_IND_MD_WRITE_NDIMS]; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t dset_id2 = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + int mpi_rank, mpi_size; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + filename = GetTestParameters(); + + fapl_id = create_faccess_plist(MPI_COMM_WORLD, MPI_INFO_NULL, facc_type); + VRFY((fapl_id >= 0), "create_faccess_plist succeeded"); + + VRFY((H5Pset_all_coll_metadata_ops(fapl_id, false) >= 0), "Unset collective metadata reads succeeded"); + VRFY((H5Pset_coll_metadata_write(fapl_id, false) >= 0), "Unset collective metadata writes succeeded"); + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + + dset_dims[0] = (hsize_t)(mpi_size * COLL_IO_IND_MD_WRITE_CHUNK0); + dset_dims[1] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK1 * COLL_IO_IND_MD_WRITE_NCHUNK1); + + fspace_id = H5Screate_simple(COLL_IO_IND_MD_WRITE_NDIMS, dset_dims, NULL); + VRFY((fspace_id >= 0), "H5Screate_simple succeeded"); + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + VRFY((dcpl_id >= 0), "H5Pcreate succeeded"); + + chunk_dims[0] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK0); + chunk_dims[1] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK1); + + VRFY((H5Pset_chunk(dcpl_id, COLL_IO_IND_MD_WRITE_NDIMS, chunk_dims) >= 0), "H5Pset_chunk succeeded"); + + VRFY((H5Pset_shuffle(dcpl_id) >= 0), "H5Pset_shuffle succeeded"); + + dset_id = H5Dcreate2(file_id, "dset1", H5T_NATIVE_LLONG, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "H5Dcreate2 succeeded"); + + sel_dims[0] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK0); + sel_dims[1] = (hsize_t)(COLL_IO_IND_MD_WRITE_CHUNK1 * COLL_IO_IND_MD_WRITE_NCHUNK1); + + offset[0] = (hsize_t)mpi_rank * sel_dims[0]; + offset[1] = 0; + + VRFY((H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, offset, NULL, sel_dims, NULL) >= 0), + "H5Sselect_hyperslab succeeded"); + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "H5Pcreate succeeded"); + + VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) >= 0), "H5Pset_dxpl_mpio succeeded"); + + data = malloc(sel_dims[0] * sel_dims[1] * sizeof(long long)); + for (size_t i = 0; i < sel_dims[0] * sel_dims[1]; i++) + data[i] = rand(); + + VRFY((H5Dwrite(dset_id, H5T_NATIVE_LLONG, H5S_BLOCK, fspace_id, dxpl_id, data) >= 0), + "H5Dwrite succeeded"); + + dset_id2 = H5Dcreate2(file_id, "dset2", H5T_NATIVE_LLONG, fspace_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + VRFY((dset_id2 >= 0), "H5Dcreate2 succeeded"); + + for (size_t i = 0; i < sel_dims[0] * sel_dims[1]; i++) + data[i] = rand(); + + VRFY((H5Dwrite(dset_id2, H5T_NATIVE_LLONG, H5S_BLOCK, fspace_id, dxpl_id, data) >= 0), + "H5Dwrite succeeded"); + + free(data); + + VRFY((H5Sclose(fspace_id) >= 0), "H5Sclose succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "H5Dclose succeeded"); + VRFY((H5Dclose(dset_id2) >= 0), "H5Dclose succeeded"); + VRFY((H5Pclose(dcpl_id) >= 0), "H5Pclose succeeded"); + VRFY((H5Pclose(dxpl_id) >= 0), "H5Pclose succeeded"); + VRFY((H5Pclose(fapl_id) >= 0), "H5Pclose succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); +} diff --git a/testpar/t_file.c b/testpar/t_file.c index a6a541becf3..700ccc2256f 100644 --- a/testpar/t_file.c +++ b/testpar/t_file.c @@ -1060,3 +1060,62 @@ test_invalid_libver_bounds_file_close_assert(void) ret = H5Pclose(fcpl_id); VRFY((SUCCEED == ret), "H5Pclose"); } + +/* + * Tests that H5Pevict_on_close properly succeeds in serial/one rank and fails when + * called by multiple ranks. + */ +void +test_evict_on_close_parallel_unsupp(void) +{ + const char *filename = NULL; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + hid_t fid = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + herr_t ret; + + filename = (const char *)GetTestParameters(); + + /* set up MPI parameters */ + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + /* setup file access plist */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((fapl_id != H5I_INVALID_HID), "H5Pcreate"); + ret = H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18); + VRFY((SUCCEED == ret), "H5Pset_libver_bounds"); + + ret = H5Pset_evict_on_close(fapl_id, true); + VRFY((SUCCEED == ret), "H5Pset_evict_on_close"); + + /* test on 1 rank */ + ret = H5Pset_fapl_mpio(fapl_id, MPI_COMM_SELF, info); + VRFY((SUCCEED == ret), "H5Pset_fapl_mpio"); + + if (mpi_rank == 0) { + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((SUCCEED == ret), "H5Fcreate"); + ret = H5Fclose(fid); + VRFY((SUCCEED == ret), "H5Fclose"); + } + + VRFY((MPI_SUCCESS == MPI_Barrier(MPI_COMM_WORLD)), "MPI_Barrier"); + + /* test on multiple ranks if we have them */ + if (mpi_size > 1) { + ret = H5Pset_fapl_mpio(fapl_id, comm, info); + VRFY((SUCCEED == ret), "H5Pset_fapl_mpio"); + + H5E_BEGIN_TRY + { + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + } + H5E_END_TRY + VRFY((fid == H5I_INVALID_HID), "H5Fcreate"); + } + + ret = H5Pclose(fapl_id); + VRFY((SUCCEED == ret), "H5Pclose"); +} diff --git a/testpar/t_select_io_dset.c b/testpar/t_select_io_dset.c index 2be2b407236..9d3f1205051 100644 --- a/testpar/t_select_io_dset.c +++ b/testpar/t_select_io_dset.c @@ -221,6 +221,26 @@ check_actual_selection_io_mode(hid_t dxpl, uint32_t sel_io_mode_expected) } } +/* + * Helper routine to check actual selection I/O mode on a dxpl + */ +static void +check_actual_selection_io_mode_either(hid_t dxpl, uint32_t sel_io_mode_expected1, + uint32_t sel_io_mode_expected2) +{ + uint32_t actual_sel_io_mode; + + if (H5Pget_actual_selection_io_mode(dxpl, &actual_sel_io_mode) < 0) + P_TEST_ERROR; + if (actual_sel_io_mode != sel_io_mode_expected1 && actual_sel_io_mode != sel_io_mode_expected2) { + if (MAINPROCESS) + printf("\n Failed: Incorrect selection I/O mode (expected/actual) %u or %u : %u", + (unsigned)sel_io_mode_expected1, (unsigned)sel_io_mode_expected2, + (unsigned)actual_sel_io_mode); + P_TEST_ERROR; + } +} + /* * Case 1: single dataset read/write, no type conversion (null case) */ @@ -327,8 +347,14 @@ test_no_type_conv(hid_t fid, unsigned chunked, unsigned dtrans, unsigned select, exp_io_mode = chunked ? H5D_MPIO_CHUNK_COLLECTIVE : H5D_MPIO_CONTIGUOUS_COLLECTIVE; testing_check_io_mode(dxpl, exp_io_mode); - if (chunked && !dtrans) - check_actual_selection_io_mode(dxpl, H5D_VECTOR_IO); + if (chunked && !dtrans) { + /* If there are more ranks than chunks, then some ranks will not perform vector I/O due to how the + * parallel compression code redistributes data */ + if ((hsize_t)mpi_size > (dims[0] / cdims[0])) + check_actual_selection_io_mode_either(dxpl, H5D_VECTOR_IO, 0); + else + check_actual_selection_io_mode(dxpl, H5D_VECTOR_IO); + } else check_actual_selection_io_mode(dxpl, select ? H5D_SELECTION_IO : H5D_SCALAR_IO); diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index 584ca1f6107..2428c719a1d 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -366,6 +366,9 @@ main(int argc, char **argv) AddTest("invlibverassert", test_invalid_libver_bounds_file_close_assert, NULL, "Invalid libver bounds assertion failure", PARATESTFILE); + AddTest("evictparassert", test_evict_on_close_parallel_unsupp, NULL, "Evict on close in parallel failure", + PARATESTFILE); + AddTest("idsetw", dataset_writeInd, NULL, "dataset independent write", PARATESTFILE); AddTest("idsetr", dataset_readInd, NULL, "dataset independent read", PARATESTFILE); @@ -521,6 +524,8 @@ main(int argc, char **argv) "Collective MD read with link chunk I/O (H5D__sort_chunk)", PARATESTFILE); AddTest("GH_coll_MD_wr", test_collective_global_heap_write, NULL, "Collective MD write of global heap data", PARATESTFILE); + AddTest("COLLIO_INDMDWR", test_coll_io_ind_md_write, NULL, + "Collective I/O with Independent metadata writes", PARATESTFILE); /* Display testing information */ TestInfo(argv[0]); diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index 6ac8080c82a..6bbdb0d23d5 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -233,6 +233,7 @@ void zero_dim_dset(void); void test_file_properties(void); void test_delete(void); void test_invalid_libver_bounds_file_close_assert(void); +void test_evict_on_close_parallel_unsupp(void); void multiple_dset_write(void); void multiple_group_write(void); void multiple_group_read(void); @@ -296,6 +297,7 @@ void test_partial_no_selection_coll_md_read(void); void test_multi_chunk_io_addrmap_issue(void); void test_link_chunk_io_sort_chunk_issue(void); void test_collective_global_heap_write(void); +void test_coll_io_ind_md_write(void); void test_oflush(void); /* commonly used prototypes */ diff --git a/tools/Makefile.am b/tools/Makefile.am index 7db4040d0eb..d0a6c5c5bc4 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -19,7 +19,7 @@ include $(top_srcdir)/config/commence.am if BUILD_TESTS_CONDITIONAL - TESTSERIAL_DIR =test + TESTSERIAL_DIR=libtest test else TESTSERIAL_DIR= endif diff --git a/tools/libtest/Makefile.am b/tools/libtest/Makefile.am index 835667c74b0..45b3f476df7 100644 --- a/tools/libtest/Makefile.am +++ b/tools/libtest/Makefile.am @@ -19,11 +19,11 @@ include $(top_srcdir)/config/commence.am -# Include src and tools/lib directories -AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib +# Include src, test, and tools/lib directories +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -# All programs depend on the hdf5 and h5tools libraries -LDADD=$(LIBH5TOOLS) $(LIBHDF5) +# All programs depend on the hdf5, hdf5 test, and h5tools libraries +LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) # main target diff --git a/tools/test/h5diff/Makefile.am b/tools/test/h5diff/Makefile.am index b561d722275..f920afab74a 100644 --- a/tools/test/h5diff/Makefile.am +++ b/tools/test/h5diff/Makefile.am @@ -60,8 +60,7 @@ endif # Temporary files. *.h5 are generated by h5diff. They should # be copied to the testfiles/ directory if update is required -CHECK_CLEANFILES+=*.h5 expect_sorted actual_sorted - +CHECK_CLEANFILES+=*.h5 *.onion expect_sorted actual_sorted DISTCLEANFILES=testh5diff.sh h5diff_plugin.sh include $(top_srcdir)/config/conclude.am diff --git a/tools/test/h5dump/Makefile.am b/tools/test/h5dump/Makefile.am index a79b0fe8b73..619647c670e 100644 --- a/tools/test/h5dump/Makefile.am +++ b/tools/test/h5dump/Makefile.am @@ -45,7 +45,7 @@ endif # Temporary files. *.h5 are generated by h5dumpgentest. They should # copied to the testfiles/ directory if update is required. -CHECK_CLEANFILES+=*.h5 *.bin +CHECK_CLEANFILES+=*.h5 *.bin *.onion DISTCLEANFILES=testh5dump.sh testh5dumppbits.sh testh5dumpxml.sh h5dump_plugin.sh include $(top_srcdir)/config/conclude.am