mirrors/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2026-09-25 04:09:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add documentation and s3proxy testing for ROS3 VFD (#5560)

Browse Source 
Adds workflow to build ROS3 VFD and optionally build aws-c-s3 library
from source or use package managers

Adds testing of ROS3 VFD with s3proxy and docker

Adds new H5Pset_fapl_ros3_endpoint()/H5Pget_fapl_ros3_endpoint() API
functions to set/get an alternative endpoint URL to use when opening
files with the ROS3 VFD

Cleans up warnings in tools and tests related to ROS3 VFD structure size

Co-authored-by: Larry Knox <lrknox@hdfgroup.org>
Co-authored-by: Allen Byrne <50328838+byrnHDF@users.noreply.github.com>
This commit is contained in:
jhendersonHDF
2025-06-19 16:22:49 -05:00
committed by GitHub
co-authored by Larry Knox Allen Byrne
parent 637c5fd2f7
commit 0743df227f
53 changed files with 127586 additions and 716 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.codespellrc
+1 -1
View File
@@ -1,6 +1,6 @@
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
[codespell]
skip = .git,*.svg,.codespellrc,./bin/trace,./hl/tools/h5watch/h5watch.c,./tools/test/h5jam/tellub.c,./config/sanitizer/LICENSE,./config/sanitizer/sanitizers.cmake,./tools/test/h5import/testfiles/*.conf,./tools/test/h5repack/testfiles/*.dat,./test/API/driver,./c++/src/*.html,./release_docs/HISTORY-*.txt
skip = .git,*.svg,.codespellrc,./bin/trace,./hl/tools/h5watch/h5watch.c,./tools/test/h5jam/tellub.c,./config/sanitizer/LICENSE,./config/sanitizer/sanitizers.cmake,./tools/test/h5import/testfiles/*.conf,./tools/test/h5repack/testfiles/*.dat,./test/API/driver,./test/testfiles/t8.shakespeare.txt,./c++/src/*.html,./release_docs/HISTORY-*.txt
check-hidden = true
# ignore-regex =
ignore-words-list = ot,isnt,inout,nd,parms,parm,ba,offsetP,ser,ois,had,fiter,fo,clude,refere,minnum,offsetp,creat,ans:,eiter,lastr,ans,isn't,ifset,sur,trun,dne,tthe,hda,filname,te,htmp,ake,gord,numer,ro,oce,msdos,TEXTIN,indx,FLE
.github/workflows/cmake.yml
+10
View File
@@ -28,6 +28,16 @@ jobs:
name: "CMake Special Workflows"
uses: ./.github/workflows/main-cmake-spc.yml
call-workflow-ros3-cmake:
name: "CMake ROS3 VFD Workflows"
uses: ./.github/workflows/vfd-ros3.yml
with:
build_mode: "Release"
aws_c_s3_build_type: "package"
# Use latest release for building from source on Ubuntu
# until a package is available to install
aws_c_s3_tag: "v0.8.0"
call-debug-concurrent-cmake:
name: "CMake Debug Concurrency Workflows"
uses: ./.github/workflows/main-cmake.yml
.github/workflows/vfd-main.yml
+9 -7
View File
@@ -30,19 +30,21 @@ jobs:
# with:
# build_mode: "${{ inputs.build_mode }}"
#Test HDF5 Mirror VFD
# Test HDF5 Mirror VFD
#hdf5_vfd_mirror:
# uses: ./.github/workflows/vfd-mirror.yml
# with:
# build_mode: "${{ inputs.build_mode }}"
#Test HDF5 ROS3 VFD
#hdf5_vfd_ros3:
# uses: ./.github/workflows/vfd-ros3.yml
# with:
# build_mode: "${{ inputs.build_mode }}"
# Test HDF5 ROS3 VFD
hdf5_vfd_ros3:
uses: ./.github/workflows/vfd-ros3.yml
with:
build_mode: "${{ inputs.build_mode }}"
aws_c_s3_build_type: "source"
aws_c_s3_tag: "main"
#Test HDF5 HDFS VFD
# Test HDF5 HDFS VFD
#hdf5_vfd_hdfs:
# uses: ./.github/workflows/vfd-hdfs.yml
# with:
.github/workflows/vfd-ros3.yml
+448
View File
@@ -0,0 +1,448 @@
name: Build and test HDF5 ROS3 VFD
on:
workflow_call:
inputs:
build_mode:
description: "CMake build type (CMAKE_BUILD_TYPE)"
required: true
type: string
aws_c_s3_build_type:
description: "Install aws-c-s3 from a package manager ('package') or build from source ('source')"
required: true
type: string
aws_c_s3_tag:
description: "Tag of aws-c-s3 to use when building from source"
required: false
type: string
permissions:
contents: read
jobs:
# Build the aws-c-s3 library from source using the specified tag
# and cache the results, currently only on Ubuntu. The result is
# compressed into a 'libaws-c-s3.tar' archive to preserve permissions
# and then is uploaded as the artifact 'libaws-c-s3' which can later
# be downloaded with 'actions/download-artifact' and then uncompressed
# with 'tar xvf libaws-c-s3.tar -C <directory>'. The uncompressed build
# directory will be called 'aws-c-s3-build'.
build_aws_c_s3:
# Ubuntu doesn't have a package for aws-c-s3 yet
# if: ${{ inputs.aws_c_s3_build_type == 'source' }}
name: "Build aws-c-s3 library"
runs-on: ubuntu-latest
steps:
- name: Get aws-c-s3 sources (main)
if: inputs.aws_c_s3_tag == ''
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-s3
path: aws-c-s3
- name: Get aws-c-s3 sources (tag)
if: inputs.aws_c_s3_tag != ''
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-s3
path: aws-c-s3
ref: ${{ inputs.aws_c_s3_tag }}
- name: Get aws-c-s3 commit hash
shell: bash
id: get-sha
run: |
cd $GITHUB_WORKSPACE/aws-c-s3
export AWSCS3_SHA=$(git rev-parse HEAD)
echo "AWSCS3_SHA=$AWSCS3_SHA" >> $GITHUB_ENV
echo "sha=$AWSCS3_SHA" >> $GITHUB_OUTPUT
# Output SHA for debugging
echo "AWSCS3_SHA=$AWSCS3_SHA"
- name: Cache/Restore aws-c-s3 (GCC) installation
id: cache-aws-c-s3-ubuntu-gcc
uses: actions/cache@v4
with:
path: ${{ runner.workspace }}/aws-c-s3-build
key: ${{ runner.os }}-${{ runner.arch }}-gcc-aws-c-s3-${{ steps.get-sha.outputs.sha }}-${{ inputs.build_mode }}
- name: Get aws-lc sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: aws/aws-lc
path: aws-lc
- name: Get s2n-tls sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: aws/s2n-tls
path: s2n-tls
- name: Get aws-c-common sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-common
path: aws-c-common
- name: Get aws-checksums sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-checksums
path: aws-checksums
- name: Get aws-c-cal sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-cal
path: aws-c-cal
- name: Get aws-c-io sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-io
path: aws-c-io
- name: Get aws-c-compression sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-compression
path: aws-c-compression
- name: Get aws-c-http sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-http
path: aws-c-http
- name: Get aws-c-sdkutils sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-sdkutils
path: aws-c-sdkutils
- name: Get aws-c-auth sources
if: ${{ steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true' }}
uses: actions/checkout@v4.1.7
with:
repository: awslabs/aws-c-auth
path: aws-c-auth
- name: Build aws-c-s3 from source
if: ${{ (steps.cache-aws-c-s3-ubuntu-gcc.outputs.cache-hit != 'true') }}
run: |
# Build aws-lc
echo "Building aws-lc"
cmake -S aws-lc -B aws-lc/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-lc/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build s2n-tls
echo "Building s2n-tls"
cmake -S s2n-tls -B s2n-tls/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build s2n-tls/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-common
echo "Building aws-c-common"
cmake -S aws-c-common -B aws-c-common/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-common/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-checksums
echo "Building aws-checksums"
cmake -S aws-checksums -B aws-checksums/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-checksums/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-cal
echo "Building aws-c-cal"
cmake -S aws-c-cal -B aws-c-cal/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-cal/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-io
echo "Building aws-c-io"
cmake -S aws-c-io -B aws-c-io/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-io/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-compression
echo "Building aws-c-compression"
cmake -S aws-c-compression -B aws-c-compression/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-compression/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-http
echo "Building aws-c-http"
cmake -S aws-c-http -B aws-c-http/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-http/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-sdkutils
echo "Building aws-c-sdkutils"
cmake -S aws-c-sdkutils -B aws-c-sdkutils/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-sdkutils/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-auth
echo "Building aws-c-auth"
cmake -S aws-c-auth -B aws-c-auth/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-auth/build --parallel 3 --config ${{ inputs.build_mode }} --target install
# Build aws-c-s3
echo "Building aws-c-s3"
cmake -S aws-c-s3 -B aws-c-s3/build \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DCMAKE_INSTALL_PREFIX=${{ runner.workspace }}/aws-c-s3-build \
-DCMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build \
-DBUILD_SHARED_LIBS=1
cmake --build aws-c-s3/build --parallel 3 --config ${{ inputs.build_mode }} --target install
- name: Tar aws-c-s3 installation to preserve permissions for artifact
run: tar -cvf libaws-c-s3.tar -C ${{ runner.workspace }} aws-c-s3-build
- name: Save aws-c-s3 installation artifact
uses: actions/upload-artifact@v4
with:
name: libaws-c-s3-${{ inputs.build_mode }}
path: libaws-c-s3.tar
if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
build_from_package_managers:
if: ${{ inputs.aws_c_s3_build_type == 'package' }}
# Ubuntu doesn't have a package for aws-c-s3 yet, so use a
# built from source version until then
needs: build_aws_c_s3
strategy:
# Let jobs run to completion even if one fails
fail-fast: false
matrix:
os_name: ["Windows MSVC", "Ubuntu GCC", "MacOS Clang"]
include:
- os_name: "Windows MSVC"
os: windows-latest
test_ros3: OFF
- os_name: "Ubuntu GCC"
os: ubuntu-latest
test_ros3: ON
- os_name: "MacOS Clang"
os: macos-latest
test_ros3: OFF
name: "Build and test the ROS3 VFD (${{ matrix.os_name }} ${{ inputs.build_mode }})"
runs-on: ${{ matrix.os }}
steps:
- name: Install aws-c-s3 (Windows)
if: ${{ matrix.os_name == 'Windows MSVC' }}
run: vcpkg install aws-c-s3
# Ubuntu doesn't have a package for aws-c-s3 yet, so use a
# built from source version until then
# - name: Install aws-c-s3 (Ubuntu)
# if: ${{ matrix.os_name == 'Ubuntu GCC' }}
# run: |
# sudo apt-get update
# sudo apt-get install aws-c-s3
- name: Install libaws-c-s3 (Ubuntu) (Cached installation)
if: ${{ matrix.os_name == 'Ubuntu GCC' }}
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: libaws-c-s3-${{ inputs.build_mode }}
- name: Untar libaws-c-s3 installation (Ubuntu)
if: ${{ matrix.os_name == 'Ubuntu GCC' }}
run: |
tar xvf libaws-c-s3.tar -C ${{ runner.workspace }}
- name: List contents of libaws-c-s3 installation (Ubuntu)
if: ${{ matrix.os_name == 'Ubuntu GCC' }}
run: |
ls -lR ${{ runner.workspace }}/aws-c-s3-build
- name: Setup environment (Ubuntu)
if: ${{ matrix.os_name == 'Ubuntu GCC' }}
shell: bash
run: |
echo "LD_LIBRARY_PATH=${{ runner.workspace }}/aws-c-s3-build/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
echo "CMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build" >> $GITHUB_ENV
- name: Install aws-c-s3 (MacOS)
if: ${{ matrix.os_name == 'MacOS Clang' }}
run: brew install aws-c-s3
- name: Set environment for MSVC (Windows)
if: ${{ matrix.os_name == 'Windows MSVC' }}
run: |
# Set these environment variables so CMake picks the correct compiler
echo "CXX=cl.exe" >> $GITHUB_ENV
echo "CC=cl.exe" >> $GITHUB_ENV
- name: Get HDF5 sources
uses: actions/checkout@v4.1.7
# For Windows, use vcpkg toolchain file to allow find_package() calls to resolve
- name: "CMake Configure (vcpkg; ROS3 testing: ${{ matrix.test_ros3 }})"
if: ${{ matrix.os_name == 'Windows MSVC' }}
run: |
mkdir "${{ runner.workspace }}/build"
cd "${{ runner.workspace }}/build"
cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake \
--log-level=VERBOSE \
-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DBUILD_SHARED_LIBS=ON \
-DHDF5_ENABLE_ALL_WARNINGS=ON \
-DHDF5_ENABLE_WARNINGS_AS_ERRORS=ON \
-DHDF5_ENABLE_PARALLEL:BOOL=OFF \
-DHDF5_BUILD_FORTRAN:BOOL=OFF \
-DHDF5_BUILD_CPP_LIB:BOOL=ON \
-DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \
-DHDF5_ENABLE_SZIP_SUPPORT:BOOL=ON \
-DHDF5_ENABLE_SZIP_ENCODING:BOOL=ON \
-DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON \
-DLIBAEC_USE_LOCALCONTENT=OFF \
-DZLIB_USE_LOCALCONTENT=OFF \
-DHDF5_ENABLE_ROS3_VFD:BOOL=ON \
-DHDF5_ENABLE_ROS3_VFD_DOCKER_PROXY=${{ matrix.test_ros3 }} \
$GITHUB_WORKSPACE
shell: bash
- name: "CMake Configure (ROS3 testing: ${{ matrix.test_ros3 }})"
if: ${{ matrix.os_name != 'Windows MSVC' }}
run: |
mkdir "${{ runner.workspace }}/build"
cd "${{ runner.workspace }}/build"
cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake \
--log-level=VERBOSE \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DBUILD_SHARED_LIBS=ON \
-DHDF5_ENABLE_ALL_WARNINGS=ON \
-DHDF5_ENABLE_WARNINGS_AS_ERRORS=ON \
-DHDF5_ENABLE_PARALLEL:BOOL=OFF \
-DHDF5_BUILD_FORTRAN:BOOL=OFF \
-DHDF5_BUILD_CPP_LIB:BOOL=ON \
-DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \
-DHDF5_ENABLE_SZIP_SUPPORT:BOOL=ON \
-DHDF5_ENABLE_SZIP_ENCODING:BOOL=ON \
-DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON \
-DLIBAEC_USE_LOCALCONTENT=OFF \
-DZLIB_USE_LOCALCONTENT=OFF \
-DHDF5_ENABLE_ROS3_VFD:BOOL=ON \
-DHDF5_ENABLE_ROS3_VFD_DOCKER_PROXY=${{ matrix.test_ros3 }} \
$GITHUB_WORKSPACE
shell: bash
- name: CMake Build
run: cmake --build . --parallel 3 --config ${{ inputs.build_mode }}
working-directory: ${{ runner.workspace }}/build
- name: CMake Run Tests
if: matrix.test_ros3 == 'ON'
run: |
# For now, just run S3 tests
ctest . -C ${{ inputs.build_mode }} -R "S3TEST" -V
working-directory: ${{ runner.workspace }}/build
# Build and test the ROS3 VFD using aws-c-s3 built from source,
# currently only on Ubuntu.
build_and_test_vfd:
if: ${{ inputs.aws_c_s3_build_type == 'source' }}
needs: build_aws_c_s3
name: "Build and test the ROS3 VFD (Ubuntu ${{ inputs.build_mode }})"
runs-on: ubuntu-latest
steps:
- name: Install libaws-c-s3 (Cached installation)
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: libaws-c-s3-${{ inputs.build_mode }}
- name: Untar libaws-c-s3 installation
run: |
tar xvf libaws-c-s3.tar -C ${{ runner.workspace }}
- name: List contents of libaws-c-s3 installation
run: |
ls -lR ${{ runner.workspace }}/aws-c-s3-build
- name: Setup environment
shell: bash
run: |
echo "LD_LIBRARY_PATH=${{ runner.workspace }}/aws-c-s3-build/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
echo "CMAKE_PREFIX_PATH=${{ runner.workspace }}/aws-c-s3-build" >> $GITHUB_ENV
- name: Get HDF5 sources
uses: actions/checkout@v4.1.7
- name: CMake Configure
run: |
mkdir "${{ runner.workspace }}/build"
cd "${{ runner.workspace }}/build"
cmake -C $GITHUB_WORKSPACE/config/cmake/cacheinit.cmake \
--log-level=VERBOSE \
-DCMAKE_BUILD_TYPE=${{ inputs.build_mode }} \
-DBUILD_SHARED_LIBS=ON \
-DHDF5_ENABLE_ALL_WARNINGS=ON \
-DHDF5_ENABLE_WARNINGS_AS_ERRORS=ON \
-DHDF5_ENABLE_PARALLEL:BOOL=OFF \
-DHDF5_BUILD_FORTRAN:BOOL=OFF \
-DHDF5_BUILD_CPP_LIB:BOOL=ON \
-DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=ON \
-DHDF5_ENABLE_SZIP_SUPPORT:BOOL=ON \
-DHDF5_ENABLE_SZIP_ENCODING:BOOL=ON \
-DHDF5_ENABLE_PLUGIN_SUPPORT:BOOL=ON \
-DLIBAEC_USE_LOCALCONTENT=OFF \
-DZLIB_USE_LOCALCONTENT=OFF \
-DHDF5_ENABLE_ROS3_VFD:BOOL=ON \
-DHDF5_ENABLE_ROS3_VFD_DOCKER_PROXY=ON \
$GITHUB_WORKSPACE
shell: bash
- name: CMake Build
run: cmake --build . --parallel 3 --config ${{ inputs.build_mode }}
working-directory: ${{ runner.workspace }}/build
- name: CMake Run Tests
run: |
# For now, just run S3 tests
ctest . -C ${{ inputs.build_mode }} -R "S3TEST" -V
working-directory: ${{ runner.workspace }}/build
CMakeTests.cmake
+7
View File
@@ -26,6 +26,13 @@
option (HDF5_DISABLE_TESTS_REGEX "Regex pattern to set execution of specific tests to DISABLED" "")
mark_as_advanced (HDF5_DISABLE_TESTS_REGEX)
if (HDF5_ENABLE_ROS3_VFD)
if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
# Create a test credentials file
file (WRITE "${CMAKE_BINARY_DIR}/credentials" "[default]\naws_access_key_id = remote-identity\naws_secret_access_key = remote-credential\nregion = us-east-2\n\n[ros3_vfd_test]\naws_access_key_id = remote-identity\naws_secret_access_key = remote-credential\nregion = us-east-2\n")
endif ()
endif ()
option (HDF5_TEST_API "Execute HDF5 API tests" ON)
mark_as_advanced (HDF5_TEST_API)
if (HDF5_TEST_API)
config/ConfigureChecks.cmake
+32 -2
View File
@@ -607,7 +607,7 @@ endif ()
option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF)
if (HDF5_ENABLE_ROS3_VFD)
# The ROS3 VFD requires the aws-c-s3 library
find_package (aws-c-s3 REQUIRED)
find_package (aws-c-s3 REQUIRED CONFIG)
if (${aws-c-s3_FOUND})
if (NOT TARGET AWS::aws-c-s3)
@@ -617,9 +617,39 @@ if (HDF5_ENABLE_ROS3_VFD)
list (APPEND LINK_LIBS AWS::aws-c-s3)
set (${HDF_PREFIX}_HAVE_ROS3_VFD 1)
option (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY "Use docker for ROS3 VFD S3proxy testing" OFF)
if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
# check if docker is available
find_program (DOCKER_EXECUTABLE docker)
if (DOCKER_EXECUTABLE)
execute_process (
COMMAND ${DOCKER_EXECUTABLE} info
RESULT_VARIABLE DOCKER_CHECK_RESULT
OUTPUT_VARIABLE DOCKER_CHECK_OUTPUT
ERROR_VARIABLE DOCKER_CHECK_ERROR
)
if (DOCKER_CHECK_RESULT EQUAL 0)
message (VERBOSE "Docker is installed and running.")
else()
set (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY OFF CACHE BOOL "Use docker for ROS3 VFD S3proxy testing" FORCE)
message (FATAL_ERROR "Docker is installed but not running or accessible: ${DOCKER_CHECK_ERROR}")
endif ()
else ()
set (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY OFF CACHE BOOL "Use docker for ROS3 VFD S3proxy testing" FORCE)
message (FATAL_ERROR "Docker is not installed.")
endif ()
# check if aws-cli is available
find_program (AWS_CLI_EXECUTABLE aws)
if (NOT AWS_CLI_EXECUTABLE)
set (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY OFF CACHE BOOL "Use docker for ROS3 VFD S3proxy testing" FORCE)
message (FATAL_ERROR "AWS cli is required for ROS3 VFD S3proxy testing, but could not be found.")
endif()
endif ()
else ()
set (HDF5_ENABLE_ROS3_VFD OFF CACHE BOOL "Build the ROS3 Virtual File Driver" FORCE)
message (WARNING "The Read-Only S3 VFD was requested but cannot be built. Please check that the aws-c-s3 library is available on your system, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
message (FATAL_ERROR "The Read-Only S3 VFD was requested but cannot be built. Please check that the aws-c-s3 library is available on your system, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
endif ()
endif ()
config/cmake/runProxy.cmake
+223
View File
@@ -0,0 +1,223 @@
#
# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the LICENSE file, which can be found at the root of the source code
# distribution tree, or in https://www.hdfgroup.org/licenses.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# runProxy.cmake dtarts a docker instance of s3proxy and uploads files.
# Exit status of command can also be compared.
# arguments checking
if (NOT TEST_PROGRAM) # currently this is the docker command
message (FATAL_ERROR "Require TEST_PROGRAM to be defined")
endif ()
if (NOT TEST_PRODUCT) # this is the docker product to be used
message (FATAL_ERROR "Require TEST_PRODUCT to be defined")
endif ()
if (NOT TEST_PORT) # this is the port for the docker instance
message (FATAL_ERROR "Require TEST_PORT to be defined")
endif ()
if (NOT TEST_FOLDER) # this is the folder where the test program is run
message (FATAL_ERROR "Require TEST_FOLDER to be defined")
endif ()
if (NOT TEST_BUCKET)
message (FATAL_ERROR "Require TEST_BUCKET to be defined")
endif ()
if (TEST_ENV_VAR)
set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}")
message (VERBOSE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}")
endif ()
message (STATUS "USING ${TEST_BUCKET} ON COMMAND: docker ${TEST_PRODUCT} ${TEST_ARGS} with creds $ENV{AWS_SHARED_CREDENTIALS_FILE}")
# run the test program to pull the product, capture the stdout/stderr and the result var
execute_process (
COMMAND ${TEST_PROGRAM} pull ${TEST_PRODUCT}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-pull.out
ERROR_FILE s3proxy-pull.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
message (VERBOSE "COMMAND Pull Result: ${TEST_RESULT}")
# run the test program to start an instance of the product, capture the stdout/stderr and the result var
execute_process (
COMMAND ${TEST_PROGRAM} run -d --publish ${TEST_PORT}:80 --restart=always --name ${TEST_ARGS} --env S3PROXY_AUTHORIZATION=aws-v4 --env S3PROXY_ENDPOINT=http://0.0.0.0:80 --env S3PROXY_IDENTITY=remote-identity --env S3PROXY_CREDENTIAL=remote-credential --env S3PROXY_CORS_ALLOW_ALL=true ${TEST_PRODUCT}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-run.out
ERROR_FILE s3proxy-run.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
if (EXISTS "${TEST_FOLDER}/s3proxy-run.out")
file (READ ${TEST_FOLDER}/s3proxy-run.out TEST_STREAM)
message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
message (VERBOSE "COMMAND Run Result: ${TEST_RESULT}")
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (NOT TEST_NOERRDISPLAY)
if (EXISTS "${TEST_FOLDER}/s3proxy-run.err")
file (READ ${TEST_FOLDER}/s3proxy-run.err TEST_STREAM)
message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
endif ()
message (FATAL_ERROR "Failed: Test program ${TEST_PRODUCT} exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
# check that the docker instance is running
execute_process (
COMMAND ${TEST_PROGRAM} inspect --format='{{.State.Running}}' ${TEST_ARGS}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-filter.out
ERROR_FILE s3proxy-filter.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
message (VERBOSE "COMMAND Inspect Result: ${TEST_RESULT}")
if (NOT TEST_NOERRDISPLAY)
if (EXISTS "${TEST_FOLDER}/s3proxy-filter.out")
file (READ ${TEST_FOLDER}/s3proxy-filter.out TEST_STREAM)
message (STATUS "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
string (REGEX MATCH "true" REGEX_MATCH ${TEST_STREAM})
string (COMPARE EQUAL "${REGEX_MATCH}" "true" REGEX_RESULT)
if (NOT REGEX_RESULT)
message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain true")
endif ()
endif ()
endif ()
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (NOT TEST_NOERRDISPLAY)
if (EXISTS "${TEST_FOLDER}/s3proxy-filter.err")
file (READ ${TEST_FOLDER}/s3proxy-filter.err TEST_STREAM)
message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
endif ()
message (FATAL_ERROR "Failed: Test program ${TEST_PRODUCT} exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
# create the bucket to be used
execute_process (
COMMAND aws s3api create-bucket --endpoint-url=http://localhost:${TEST_PORT} --bucket ${TEST_BUCKET}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-bucket.out
ERROR_FILE s3proxy-bucket.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
if (EXISTS "${TEST_FOLDER}/s3proxy-bucket.out")
file (READ ${TEST_FOLDER}/s3proxy-bucket.out TEST_STREAM)
message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
message (VERBOSE "COMMAND Bucket Result: ${TEST_RESULT}")
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (NOT TEST_NOERRDISPLAY)
if (EXISTS "${TEST_FOLDER}/s3proxy-bucket.err")
file (READ ${TEST_FOLDER}/s3proxy-bucket.err TEST_STREAM)
message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
endif ()
message (FATAL_ERROR "Failed: Create-Bucket exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
#upload test files to the bucket
if (TEST_FILES AND TEST_ACLS)
foreach (dfile dacls IN ZIP_LISTS TEST_FILES TEST_ACLS)
if (dacls STREQUAL "anon")
execute_process (
COMMAND aws s3api put-object --acl public-read --endpoint-url=http://localhost:${TEST_PORT} --body ${TEST_FOLDER}/testfiles/${dfile} --bucket ${TEST_BUCKET} --key ${dfile}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-${dfile}.out
ERROR_FILE s3proxy-${dfile}.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
else ()
execute_process (
COMMAND aws s3api put-object --endpoint-url=http://localhost:${TEST_PORT} --body ${TEST_FOLDER}/testfiles/${dfile} --bucket ${TEST_BUCKET} --key ${dfile}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-${dfile}.out
ERROR_FILE s3proxy-${dfile}.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
endif ()
if (EXISTS "${TEST_FOLDER}/s3proxy-${dfile}.out")
file (READ ${TEST_FOLDER}/s3proxy-${dfile}.out TEST_STREAM)
message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
message (VERBOSE "COMMAND Put Result: ${TEST_RESULT}")
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (NOT TEST_NOERRDISPLAY)
if (EXISTS "${TEST_FOLDER}/s3proxy-${dfile}.err")
file (READ ${TEST_FOLDER}/s3proxy-${dfile}.err TEST_STREAM)
message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
endif ()
message (FATAL_ERROR "Failed: Put-Object exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
endforeach ()
elseif (TEST_FILES)
foreach (dfile ${TEST_FILES})
execute_process (
COMMAND aws s3api put-object --endpoint-url=http://localhost:${TEST_PORT} --body ${TEST_FOLDER}/testfiles/${dfile} --bucket ${TEST_BUCKET} --key ${dfile}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE s3proxy-${dfile}.out
ERROR_FILE s3proxy-${dfile}.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
if (EXISTS "${TEST_FOLDER}/s3proxy-${dfile}.out")
file (READ ${TEST_FOLDER}/s3proxy-${dfile}.out TEST_STREAM)
message (VERBOSE "Output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
message (VERBOSE "COMMAND Put Result: ${TEST_RESULT}")
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (NOT TEST_NOERRDISPLAY)
if (EXISTS "${TEST_FOLDER}/s3proxy-${dfile}.err")
file (READ ${TEST_FOLDER}/s3proxy-${dfile}.err TEST_STREAM)
message (STATUS "Error output USING ${TEST_BUCKET}:\n${TEST_STREAM}")
endif ()
endif ()
message (FATAL_ERROR "Failed: Put-Object exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
endforeach ()
endif ()
# cleanup the output files
if (NOT DEFINED ENV{HDF5_NOCLEANUP})
file (GLOB REMOVE_FILES ${TEST_FOLDER}/s3proxy*)
file (REMOVE ${REMOVE_FILES})
endif ()
# everything went fine...
message (STATUS "Passed: The ${TEST_PRODUCT} docker used ${TEST_BUCKET}")
config/cmake/runTest.cmake
+3 -3
View File
@@ -15,13 +15,13 @@ cmake_policy(SET CMP0007 NEW)
cmake_policy(SET CMP0053 NEW)
# arguments checking
if (NOT TEST_PROGRAM)
if (NOT TEST_PROGRAM) #the program to be run
message (FATAL_ERROR "Require TEST_PROGRAM to be defined")
endif ()
if (NOT TEST_FOLDER)
if (NOT TEST_FOLDER) # this is the folder where the test program is run
message (FATAL_ERROR "Require TEST_FOLDER to be defined")
endif ()
if (NOT TEST_OUTPUT)
if (NOT TEST_OUTPUT) # the output file to capture the test program output
message (FATAL_ERROR "Require TEST_OUTPUT to be defined")
endif ()
if (NOT TEST_EXPECT)
config/cmake/stopProxy.cmake
+91
View File
@@ -0,0 +1,91 @@
#
# Copyright by The HDF Group.
# All rights reserved.
#
# This file is part of HDF5. The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the LICENSE file, which can be found at the root of the source code
# distribution tree, or in https://www.hdfgroup.org/licenses.
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
# stopProxy.cmake shuts down a docker instance of s3proxy.
# arguments checking
if (NOT TEST_PROGRAM) # currently this is the docker command
message (FATAL_ERROR "Require TEST_PROGRAM to be defined")
endif ()
if (NOT TEST_FOLDER) # this is the folder where the test program is run
message (FATAL_ERROR "Require TEST_FOLDER to be defined")
endif ()
message (STATUS "Stopping s3proxy instance ${TEST_ARGS}")
# run the test program, capture the stdout/stderr and the result var
execute_process (
COMMAND ${TEST_PROGRAM} stop ${TEST_ARGS}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE docker-stop.out
ERROR_FILE docker-stop.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
message (STATUS "COMMAND Result: ${TEST_RESULT}")
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (EXISTS "${TEST_FOLDER}/docker-stop.out")
file (READ ${TEST_FOLDER}/docker-stop.out TEST_STREAM)
message (STATUS "Output stopping s3proxy:\n${TEST_STREAM}")
endif ()
message (FATAL_ERROR "Failed: s3proxy exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
message (STATUS "COMMAND Error: ${TEST_ERROR}")
# run the test program, capture the stdout/stderr and the result var
execute_process (
COMMAND ${TEST_PROGRAM} rm ${TEST_ARGS}
WORKING_DIRECTORY ${TEST_FOLDER}
RESULT_VARIABLE TEST_RESULT
OUTPUT_FILE docker-rm.out
ERROR_FILE docker-rm.err
OUTPUT_VARIABLE TEST_OUT
ERROR_VARIABLE TEST_ERROR
)
message (STATUS "COMMAND Result: ${TEST_RESULT}")
# if the return value is !=${TEST_EXPECT} bail out
if (NOT TEST_RESULT EQUAL TEST_EXPECT)
if (EXISTS "${TEST_FOLDER}/docker-stop.out")
file (READ ${TEST_FOLDER}/docker-rm.out TEST_STREAM)
message (STATUS "Output removing s3proxy:\n${TEST_STREAM}")
endif ()
message (FATAL_ERROR "Failed: s3proxy exited != ${TEST_EXPECT}.\n${TEST_ERROR}")
endif ()
message (STATUS "COMMAND Error: ${TEST_ERROR}")
# cleanup the output files
if (NOT DEFINED ENV{HDF5_NOCLEANUP})
if (EXISTS "${TEST_FOLDER}/docker-stop.out")
file (REMOVE ${TEST_FOLDER}/docker-stop.out)
endif ()
if (EXISTS "${TEST_FOLDER}/docker-stop.err")
file (REMOVE ${TEST_FOLDER}/docker-stop.err)
endif ()
if (EXISTS "${TEST_FOLDER}/docker-rm.out")
file (REMOVE ${TEST_FOLDER}/docker-rm.out)
endif ()
if (EXISTS "${TEST_FOLDER}/docker-rm.err")
file (REMOVE ${TEST_FOLDER}/docker-rm.err)
endif ()
endif ()
# everything went fine...
message (STATUS "Passed: s3proxy instance ${TEST_ARGS} stopped.")
doxygen/dox/UsersGuide.dox
+1
View File
@@ -78,6 +78,7 @@ Navigate back: \ref index "Main"
<li> \ref subsubsec_file_alternate_drivers_family
<li> \ref subsubsec_file_alternate_drivers_multi
<li> \ref subsubsec_file_alternate_drivers_split
<li> \ref subsubsec_file_alternate_drivers_ros3
<li> \ref subsubsec_file_alternate_drivers_par
</ul>
\li \ref subsec_file_examples
release_docs/INSTALL_S3.txt
+185
View File
@@ -0,0 +1,185 @@
*************************************************************************
* Build and testing instructions for HDF5's ROS3 VFD *
*************************************************************************
Table of Contents
Section I: Preconditions
Section II: Obtaining the aws-c-s3 library
Section III: Building the aws-c-s3 library from source
Section IV: Building HDF5 with the ROS3 VFD enabled
Section V: Testing the ROS3 VFD
************************************************************************
========================================================================
I. Preconditions
========================================================================
- Refer to release_docs/INSTALL for preconditions and instructions for
building HDF5
========================================================================
II. Obtaining the aws-c-s3 library
========================================================================
As of the HDF5 2.0.0 release (or HDF5 commit 75fff3ed0663d3617e3c5cc3c57c1f0c83b38be4),
HDF5's ROS3 VFD requires the aws-c-s3 library (https://github.com/awslabs/aws-c-s3)
in order to be built. This library is distributed by several package
managers, usually as 'aws-c-s3', but consult your particular package
manager for the name of this package if it's available. HDF5 installs
this library in the following ways in CI testing:
Linux (Ubuntu):
- from source, as Ubuntu doesn't have a package for aws-c-s3 yet
Windows:
- vcpkg install aws-c-s3 (make sure to set CMAKE_TOOLCHAIN_FILE to
point to vcpkg.cmake when configuring HDF5)
MacOS:
- brew install aws-c-s3
========================================================================
III. Building the aws-c-s3 library from source
========================================================================
If you are unable to obtain the aws-c-s3 library from a package manager
or other distribution method, the library may also be built from source
code. The following instructions have been reproduced and slightly
modified from the aws-c-s3 repository and may be out of date; be sure to
check https://github.com/awslabs/aws-c-s3 for the latest build
instructions.
1. Obtain the source code for aws-c-s3 and its dependencies
Note that this build process intends for aws-c-s3 and all its
dependencies to be cloned into the same top-level directory.
Only if building on Linux:
- git clone https://github.com/aws/aws-lc.git
- git clone https://github.com/aws/s2n-tls.git
- git clone https://github.com/awslabs/aws-c-common.git
- git clone https://github.com/awslabs/aws-checksums.git
- git clone https://github.com/awslabs/aws-c-cal.git
- git clone https://github.com/awslabs/aws-c-io.git
- git clone https://github.com/awslabs/aws-c-compression.git
- git clone https://github.com/awslabs/aws-c-http.git
- git clone https://github.com/awslabs/aws-c-sdkutils.git
- git clone https://github.com/awslabs/aws-c-auth.git
- git clone https://github.com/awslabs/aws-c-s3.git
2. Build aws-c-s3 and its dependencies
Note that this build process intends for <install-path> to be the
same directory for all commands. '--parallel <num-jobs>' is used
to speed up the build process by building with <num-jobs> concurrent
processes, and can be omitted if desired.
Only if building on Linux:
Build aws-lc:
- cmake -S aws-lc -B aws-lc/build -DCMAKE_INSTALL_PREFIX=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-lc/build --target install --parallel <num-jobs>
Build s2n-tls:
- cmake -S s2n-tls -B s2n-tls/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build s2n-tls/build --target install --parallel <num-jobs>
Build aws-c-common:
- cmake -S aws-c-common -B aws-c-common/build -DCMAKE_INSTALL_PREFIX=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-common/build --target install --parallel <num-jobs>
Build aws-checksums:
- cmake -S aws-checksums -B aws-checksums/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-checksums/build --target install --parallel <num-jobs>
Build aws-c-cal:
- cmake -S aws-c-cal -B aws-c-cal/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-cal/build --target install --parallel <num-jobs>
Build aws-c-io:
- cmake -S aws-c-io -B aws-c-io/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-io/build --target install --parallel <num-jobs>
Build aws-c-compression:
- cmake -S aws-c-compression -B aws-c-compression/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-compression/build --target install --parallel <num-jobs>
Build aws-c-http:
- cmake -S aws-c-http -B aws-c-http/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-http/build --target install --parallel <num-jobs>
Build aws-c-sdkutils:
- cmake -S aws-c-sdkutils -B aws-c-sdkutils/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-sdkutils/build --target install --parallel <num-jobs>
Build aws-c-auth:
- cmake -S aws-c-auth -B aws-c-auth/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-auth/build --target install --parallel <num-jobs>
Build aws-c-s3:
- cmake -S aws-c-s3 -B aws-c-s3/build -DCMAKE_INSTALL_PREFIX=<install-path> -DCMAKE_PREFIX_PATH=<install-path> -DBUILD_SHARED_LIBS=1
- cmake --build aws-c-s3/build --target install --parallel <num-jobs>
Note:
As shown in the above commands, it is suggested to pass the CMake
option BUILD_SHARED_LIBS=1 when building aws-c-s3 and each of its
dependencies if shared libraries will be enabled when building HDF5.
Otherwise, HDF5's CMake configuration logic might find and use a
libcrypto library from the system, which could result in linking errors
depending on the version of that library. If only static libraries will
be enabled when building HDF5, this option can be omitted.
Note:
The instructions above install aws-c-s3 and all of its dependencies
into the same directory with the use of CMAKE_INSTALL_PREFIX. This is
the suggested way of building aws-c-s3 as it simplifies the process of
locating aws-c-s3 and its dependencies with CMAKE_PREFIX_PATH, both
when building aws-c-s3 and HDF5, especially if installing aws-c-s3 to
a non-standard location.
========================================================================
IV. Building HDF5 with the ROS3 VFD enabled
========================================================================
HDF5 uses CMake's find_package() mechanism to locate the aws-c-library,
so as long as it is installed to a standard location, the CMake option
HDF5_ENABLE_ROS3_VFD=ON simply needs to be passed when configuring HDF5
in order to enable the ROS3 VFD, as in:
cmake -DHDF5_ENABLE_ROS3_VFD=ON ..
If the aws-c-s3 library is installed to a non-standard location, the
environment variable CMAKE_PREFIX_PATH should be set to that path when
configuring HDF5, as in:
CMAKE_PREFIX_PATH=<install-path> cmake -DHDF5_ENABLE_ROS3_VFD=ON ..
Refer to release_docs/INSTALL_CMake.txt for more general instructions on
building HDF5 with CMake.
========================================================================
V. Testing the ROS3 VFD
========================================================================
The ROS3 VFD is tested using a combination of docker, s3proxy and the
AWS CLI. If the 'docker' and 'aws' executables are available in standard
locations which CMake can find and the docker daemon is running, the
ROS3 VFD tests can be enabled by passing the CMake option
HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY=ON, as in:
cmake -DHDF5_ENABLE_ROS3_VFD=ON -DHDF5_ENABLE_ROS3_VFD_DOCKER_PROXY=ON ..
If everything configured correctly, the ROS3 VFD tests can be run with:
ctest -R "S3TEST" -VV .
These tests are separated into different groups based on the particular
functionality being tested. For each group of tests, the CMake logic
will bring up s3proxy, create a testing bucket and populate that bucket
with any testing files needed, run all of the tests for the testing
group and then bring down s3proxy.
release_docs/RELEASE.txt
+27
View File
@@ -296,6 +296,20 @@ New Features
If the ROS3 VFD cannot determine an AWS region from one of these locations,
an error will be returned when opening a file.
New API functions H5Pset_fapl_ros3_endpoint() and H5Pget_fapl_ros3_endpoint()
have been added for use with the ROS3 VFD. These functions set/get an
alternate endpoint URL to use when opening files with the ROS3 VFD. This
is useful in cases where the application needs to access files that are
in a location other than the standard s3.<region-code>.amazonaws.com, which
is what the ROS3 VFD uses when an alternate endpoint URL isn't specified.
The ROS3 VFD also checks the AWS_ENDPOINT_URL_S3 and AWS_ENDPOINT_URL
environment variables for an alternate endpoint URL if one isn't specified
with H5Pset_fapl_ros3_endpoint().
Instructions for building the ROS3 VFD with the aws-c-s3 library have been
added to release_docs/INSTALL_S3.txt. The ROS3 VFD and information about
usage of the driver is described in the HDF5 user's guide.
- Renamed some API decorations
Some API decorations (used to hide __declspec on Windows, among other
@@ -608,6 +622,19 @@ New Features
Tools:
------
- Added AWS endpoint command option to allow specifying an alternate endpoint
URL when using the ROS3 VFD
The new option is --endpoint-url, which allows the user to set an alternate
endpoint URL than the standard "protocol://service-code.region-code.amazonaws.com".
If "--endpoint-url" is not specified, the ROS3 VFD will first check the
AWS_ENDPOINT_URL_S3 and AWS_ENDPOINT_URL environment variables for an alternate
endpoint URL before using a default one, with the region-code being supplied by
the FAPL or standard AWS locations/environment variables.
This option is supported by the following tools:
h5dump, h5ls, h5stat
- Deprecated h5dump XML option
The h5dump XML option is deprecated and will be removed in a future
src/H5FDros3.c
+230 -11
View File
@@ -48,6 +48,9 @@ static bool H5FD_ros3_init_s = false;
/* Session/security token property name */
#define ROS3_TOKEN_PROP_NAME "ros3_token_prop"
/* Endpoint URL property name */
#define ROS3_ENDPOINT_PROP_NAME "ros3_endpoint_prop"
#ifdef ROS3_STATS
/* The ros3 VFD can collect some simple I/O stats on a per-file basis. These
@@ -162,6 +165,11 @@ static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2,
static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *_value);
static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value);
static herr_t H5FD__ros3_str_endpoint_copy(const char *name, size_t size, void *_value);
static int H5FD__ros3_str_endpoint_cmp(const void *_value1, const void *_value2, size_t size);
static herr_t H5FD__ros3_str_endpoint_close(const char *name, size_t size, void *_value);
static herr_t H5FD__ros3_str_endpoint_delete(hid_t prop_id, const char *name, size_t size, void *_value);
#ifdef ROS3_STATS
static herr_t H5FD__ros3_reset_stats(H5FD_ros3_t *file);
static herr_t H5FD__ros3_log_read_stats(H5FD_ros3_t *file, H5FD_mem_t type, uint64_t size);
@@ -607,8 +615,6 @@ H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t H5_ATT
* Function: H5FD__ros3_str_token_close
*
* Purpose: Closes/frees the memory associated to the token string.
* Currently, it is an empty implementation since there no
* additional treatment needed for this property.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
@@ -632,8 +638,6 @@ H5FD__ros3_str_token_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSE
*
* Purpose: Deletes the property token from the property list and frees
* the memory associated to the token string.
* Currently, it is an empty implementation since there no
* additional treatment needed for this property.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
@@ -707,6 +711,209 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_fapl_ros3_token() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_endpoint_copy
*
* Purpose: Create a copy of the endpoint string.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_endpoint_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value)
{
char **value = (char **)_value;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
if (*value)
if (NULL == (*value = strdup(*value)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy endpoint URL string");
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_endpoint_copy() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_endpoint_cmp
*
* Purpose: Compares two endpoint strings with each other.
*
* Return: A value like strcmp()
*-------------------------------------------------------------------------
*/
static int
H5FD__ros3_str_endpoint_cmp(const void *_value1, const void *_value2, size_t H5_ATTR_UNUSED size)
{
char *const *value1 = (char *const *)_value1;
char *const *value2 = (char *const *)_value2;
int ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
if (*value1) {
if (*value2)
ret_value = strcmp(*value1, *value2);
else
ret_value = 1;
}
else {
if (*value2)
ret_value = -1;
else
ret_value = 0;
}
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_endpoint_cmp */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_endpoint_close
*
* Purpose: Closes/frees the memory associated to the endpoint string.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_endpoint_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value)
{
char **value = (char **)_value;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
if (*value)
free(*value);
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_endpoint_close */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_str_endpoint_delete
*
* Purpose: Deletes the property endpoint from the property list and frees
* the memory associated to the token string.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
static herr_t
H5FD__ros3_str_endpoint_delete(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name,
size_t H5_ATTR_UNUSED size, void *_value)
{
char **value = (char **)_value;
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE_NOERR
if (*value)
free(*value);
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_str_endpoint_delete */
/*-------------------------------------------------------------------------
* Function: H5Pset_fapl_ros3_endpoint
*
* Purpose: Modify the file access property list by adding or modifying
* the endpoint url property.
*
* Return: Non-negative on success/Negative on failure
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_fapl_ros3_endpoint(hid_t fapl_id, const char *endpoint_url)
{
H5P_genplist_t *plist = NULL;
char *endpoint_src;
htri_t endpoint_exists;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
if (fapl_id == H5P_DEFAULT)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list");
if (!endpoint_url)
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "endpoint URL string was NULL");
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, false)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list");
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
if ((endpoint_exists = H5P_exist_plist(plist, ROS3_ENDPOINT_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if endpoint URL property exists in plist");
if (NULL == (endpoint_src = strdup(endpoint_url)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't copy endpoint URL string");
if (endpoint_exists) {
if (H5P_set(plist, ROS3_ENDPOINT_PROP_NAME, &endpoint_src) < 0) {
free(endpoint_src);
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set endpoint URL value");
}
}
else {
if (H5P_insert(plist, ROS3_ENDPOINT_PROP_NAME, sizeof(char *), &endpoint_src, NULL, NULL, NULL, NULL,
H5FD__ros3_str_endpoint_delete, H5FD__ros3_str_endpoint_copy,
H5FD__ros3_str_endpoint_cmp, H5FD__ros3_str_endpoint_close) < 0) {
free(endpoint_src);
HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist");
}
}
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pset_fapl_ros3_endpoint() */
/*-------------------------------------------------------------------------
* Function: H5Pget_fapl_ros3_endpoint
*
* Purpose: Returns endpoint url of the ros3 file access
* property list though the function arguments.
*
* Return: SUCCEED/FAIL
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_fapl_ros3_endpoint(hid_t fapl_id, size_t size, char *endpoint_dst /*out*/)
{
H5P_genplist_t *plist = NULL;
htri_t endpoint_exists;
herr_t ret_value = SUCCEED;
FUNC_ENTER_API(FAIL)
if (size == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero.");
if (endpoint_dst == NULL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "endpoint_dst is NULL");
if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS, true)))
HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list");
if (H5FD_ROS3 != H5P_peek_driver(plist))
HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver");
if ((endpoint_exists = H5P_exist_plist(plist, ROS3_ENDPOINT_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if endpoint URL property exists in plist");
if (endpoint_exists) {
char *endpoint_src;
if (H5P_get(plist, ROS3_ENDPOINT_PROP_NAME, &endpoint_src) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get endpoint URL value");
if (endpoint_src) {
strncpy(endpoint_dst, endpoint_src, size);
endpoint_dst[size - 1] = '\0';
}
}
else
memset(endpoint_dst, 0, size);
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_fapl_ros3_endpoint() */
/*-------------------------------------------------------------------------
* Function: H5FD__ros3_open
*
@@ -729,12 +936,14 @@ done:
static H5FD_t *
H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
H5FD_ros3_t *file = NULL;
s3r_t *handle = NULL;
const H5FD_ros3_fapl_t *fa = NULL;
H5P_genplist_t *plist = NULL;
char *fapl_token = NULL;
H5FD_t *ret_value = NULL;
H5FD_ros3_t *file = NULL;
s3r_t *handle = NULL;
const H5FD_ros3_fapl_t *fa = NULL;
H5P_genplist_t *plist = NULL;
char *fapl_token = NULL;
char *fapl_endpoint = NULL;
H5FD_t *ret_value = NULL;
htri_t endpt_exists = false;
FUNC_ENTER_PACKAGE
@@ -774,10 +983,20 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
}
}
/* Does the endpoint exist in the fapl? */
if ((endpt_exists = H5P_exist_plist(plist, ROS3_ENDPOINT_PROP_NAME)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "failed check for property endpoint in plist");
/* If so, get it */
if (endpt_exists) {
if (H5P_get(plist, ROS3_ENDPOINT_PROP_NAME, &fapl_endpoint) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to get endpoint value");
}
/* Open file; procedure depends on whether or not the fapl instructs to
* authenticate requests or not.
*/
if (NULL == (handle = H5FD__s3comms_s3r_open(url, fa, fapl_token)))
if (NULL == (handle = H5FD__s3comms_s3r_open(url, fa, fapl_token, fapl_endpoint)))
HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "s3r_open failed");
/* Create new file struct */
src/H5FDros3.h
+67 -8
View File
@@ -40,7 +40,7 @@
/**
* \def H5FD_CURR_ROS3_FAPL_T_VERSION
* The version number of the H5FD_ros3_fapl_t configuration
* structure for the $H5FD_ROS3 driver.
* structure for the #H5FD_ROS3 driver.
*/
#define H5FD_CURR_ROS3_FAPL_T_VERSION 1
@@ -89,9 +89,9 @@
* output will be enabled if this environment variable is defined to
* anything other than one of (case-insensitive):
*
* 'false'
* 'off'
* '0'
* 'false' <br />
* 'off' <br />
* '0' <br />
*
* Debugging output will be printed to stderr.
* \since 2.0.0
@@ -103,10 +103,10 @@
* logging output should be enabled for the ROS3 VFD. This environment
* variable should be specified as one of (case-insensitive):
*
* 'error'
* 'info'
* 'debug'
* 'trace'
* 'error' <br />
* 'info' <br />
* 'debug' <br />
* 'trace' <br />
*
* If specified as one of these values, logging output will be written
* to the file specified by the #HDF5_ROS3_VFD_LOG_FILE environment
@@ -130,6 +130,23 @@
* \since 2.0.0
*/
#define HDF5_ROS3_VFD_LOG_FILE "HDF5_ROS3_VFD_LOG_FILE"
/**
* \def HDF5_ROS3_VFD_FORCE_PATH_STYLE
* Macro for name of the environment variable that forces the VFD
* to use path-style requests rather than virtual-hosted-style
* requests. The VFD attempts to use virtual-hosted-style requests
* by default when possible, but in some cases it may be necessary
* to force it to use path-style requests for compatibility reasons.
* The VFD will use path-style requests if this environment variable
* is defined to anything other than one of (case-insensitive):
*
* 'false' <br />
* 'off' <br />
* '0' <br />
*
* \since 2.0.0
*/
#define HDF5_ROS3_VFD_FORCE_PATH_STYLE "HDF5_ROS3_VFD_FORCE_PATH_STYLE"
/**
* \struct H5FD_ros3_fapl_t
@@ -274,6 +291,48 @@ H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token);
*/
H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token);
/**
* \ingroup FAPL
*
* \brief Queries a File Access Property List for #H5FD_ROS3 file driver endpoint url.
*
* \fapl_id
* \param[in] size Size of the provided char array for storing the endpoint url.
* \param[out] endpoint Alternate endpoint url.
* \returns \herr_t
*
* \since 2.0.0
*/
H5_DLL herr_t H5Pget_fapl_ros3_endpoint(hid_t fapl_id, size_t size, char *endpoint);
/**
* \ingroup FAPL
*
* \brief Modifies the specified File Access Property List to use an alternative endpoint
* URL when opening files with the #H5FD_ROS3 driver.
*
* \fapl_id
* \param[in] endpoint Alternate endpoint url.
* \returns \herr_t
*
* \details H5Pset_fapl_ros3_endpoint() modifies an existing File Access Property List which
* is used by #H5FD_ROS3 driver by adding or updating the endpoint url of the
* property list. When not specified, the #H5FD_ROS3 driver uses the standard
* s3.<region-code>.amazonaws.com, unless an alternative endpoint URL is specified
* in the AWS_ENDPOINT_URL_S3 or AWS_ENDPOINT_URL environment variable. Be aware,
* to set the endpoint first you need to create a proper File Access Property List
* using H5Pset_fapl_ros() and use this list as input argument of the function
* H5Pset_fapl_ros3_endpoint().
*
* Note, the endpoint url is only needed when you want to access a S3 bucket
* using an alternate url. For example, this can be useful when using s3:// object
* URIs to access files which are located somewhere other than the standard
* s3.<region-code>.amazonaws.com.
*
* \since 2.0.0
*/
H5_DLL herr_t H5Pset_fapl_ros3_endpoint(hid_t fapl_id, const char *endpoint);
#ifdef __cplusplus
}
#endif
src/H5FDros3_s3comms.c
+161 -78
View File
@@ -66,6 +66,12 @@
*/
#define MAX_USER_AGENT_STRING_SIZE 128
/*
* The maximum number of entries for the host resolver to
* keep cached
*/
#define HOST_RESOLVER_MAX_NUM_ENTRIES 8
/******************/
/* Local Typedefs */
/******************/
@@ -84,6 +90,7 @@
* - aws_host_resolver_release(host_resolver);
* - aws_event_loop_group_release(event_loop_group);
* - aws_uri_clean_up(&parsed_uri);
* - aws_uri_clean_up(&alt_parsed_uri);
*/
typedef struct H5FD__s3comms_aws_params_t {
struct aws_allocator *allocator;
@@ -94,6 +101,8 @@ typedef struct H5FD__s3comms_aws_params_t {
struct aws_event_loop_group *event_loop_group;
struct aws_signing_config_aws signing_config;
struct aws_uri parsed_uri;
struct aws_uri alt_parsed_uri;
bool force_path_style;
/* Shared MT state for requests */
struct aws_mutex req_mutex; /* Mutex for condition variable */
@@ -176,7 +185,7 @@ static int H5FD__s3comms_s3r_getsize_headers_cb(struct aws_s3_meta_request
void *user_data);
/* Utility functions */
static parsed_url_t *H5FD__s3comms_parse_url(s3r_t *handle, const char *url);
static parsed_url_t *H5FD__s3comms_parse_url(s3r_t *handle, const char *url, struct aws_uri *uri);
static herr_t H5FD__s3comms_free_purl(parsed_url_t *purl);
static herr_t H5FD__s3comms_get_aws_region(const H5FD__s3comms_aws_params_t *aws_params,
@@ -193,10 +202,8 @@ static herr_t H5FD__s3comms_format_http_request_message(const H5FD__s3comms_aws_
static herr_t H5FD__s3comms_format_host_header(const H5FD__s3comms_aws_params_t *aws_params,
struct aws_http_message *message, const char *bucket_name,
const char *host_name, const char *port);
static herr_t H5FD__s3comms_format_range_header(const H5FD__s3comms_aws_params_t *aws_params,
struct aws_http_message *message, haddr_t offset, size_t len);
static herr_t H5FD__s3comms_format_user_agent_header(const H5FD__s3comms_aws_params_t *aws_params,
struct aws_http_message *message);
static herr_t H5FD__s3comms_format_range_header(struct aws_http_message *message, haddr_t offset, size_t len);
static herr_t H5FD__s3comms_format_user_agent_header(struct aws_http_message *message);
static const char *H5FD__s3comms_httpcode_to_str(long httpcode, bool *handled);
@@ -276,7 +283,7 @@ H5FD__s3comms_init(void)
/* Set up host resolver using default options taken from AWS sample code */
host_resolver_opts.el_group = H5FD_ros3_aws_event_loop_group_g;
host_resolver_opts.max_entries = 8;
host_resolver_opts.max_entries = HOST_RESOLVER_MAX_NUM_ENTRIES;
H5FD_ros3_aws_host_resolver_g =
aws_host_resolver_new_default(H5FD_ros3_aws_allocator_g, &host_resolver_opts);
if (!H5FD_ros3_aws_host_resolver_g)
@@ -630,7 +637,8 @@ H5FD__s3comms_cred_provider_pred(void *user_data)
*----------------------------------------------------------------------------
*/
s3r_t *
H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *fapl_token)
H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *fapl_token,
const char *alt_endpoint)
{
struct aws_s3_client *client = NULL;
struct aws_client_bootstrap *client_bootstrap = NULL;
@@ -640,6 +648,8 @@ H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *
struct aws_byte_cursor region_cursor = {0};
H5FD__s3comms_aws_params_t *aws_params = NULL;
s3r_t *handle = NULL;
char *pathstyle_env = NULL;
char *endpoint_env = NULL;
bool mutex_init = false;
bool cond_var_init = false;
s3r_t *ret_value = NULL;
@@ -669,6 +679,14 @@ H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *
aws_error_str(aws_last_error()));
cond_var_init = true;
/* Check if path-style requests should be forced */
pathstyle_env = getenv(HDF5_ROS3_VFD_FORCE_PATH_STYLE);
if (pathstyle_env && (*pathstyle_env != '\0')) {
if (0 != HDstrcasecmp(pathstyle_env, "false") && 0 != HDstrcasecmp(pathstyle_env, "off") &&
0 != strcmp(pathstyle_env, "0"))
aws_params->force_path_style = true;
}
/*
* Set up a new AWS client with default options
*/
@@ -728,8 +746,39 @@ H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *
aws_params->client = client;
/* Parse URL */
if (NULL == (handle->purl = H5FD__s3comms_parse_url(handle, url)))
if (NULL == (handle->purl = H5FD__s3comms_parse_url(handle, url, &aws_params->parsed_uri)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "could not allocate and create parsed URL");
if (!handle->purl->bucket_name)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "invalid URL specified - could not parse bucket name");
if (!handle->purl->key)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "invalid URL specified - could not parse object key");
/* If no alternate endpoint URL was specified in the FAPL, check to see
* if one of the AWS environment variables were specified
*/
if (!alt_endpoint) {
endpoint_env = getenv("AWS_ENDPOINT_URL_S3");
if (endpoint_env && (*endpoint_env != '\0'))
alt_endpoint = endpoint_env;
else {
endpoint_env = getenv("AWS_ENDPOINT_URL");
if (endpoint_env && (*endpoint_env != '\0'))
alt_endpoint = endpoint_env;
}
}
if (alt_endpoint && (*alt_endpoint != '\0')) {
if (H5FD_ros3_debug_g)
fprintf(stderr, " -- parsing alternative endpoint URL\n");
if (NULL == (handle->alternate_purl =
H5FD__s3comms_parse_url(handle, alt_endpoint, &aws_params->alt_parsed_uri)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL,
"could not allocate and create parsed alternate endpoint URL");
if (!handle->alternate_purl->host)
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL,
"invalid alternate endpoint URL specified - could not parse host name");
}
/* Get the S3 object's size. This is the only time we touch the S3 object
* (and thus ensure it exists) during the VFD's open callback.
@@ -752,6 +801,8 @@ done:
aws_mutex_clean_up(&aws_params->req_mutex);
aws_uri_clean_up(&aws_params->parsed_uri);
if (alt_endpoint)
aws_uri_clean_up(&aws_params->alt_parsed_uri);
free(aws_params);
}
@@ -788,15 +839,14 @@ H5FD__s3comms_s3r_close(s3r_t *handle)
free(handle->aws_region);
if (H5FD__s3comms_free_purl(handle->purl) < 0)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to release parsed url structure");
if (handle->priv_data) {
H5FD__s3comms_aws_params_t *aws_params;
aws_params = (H5FD__s3comms_aws_params_t *)handle->priv_data;
aws_uri_clean_up(&aws_params->parsed_uri);
if (handle->alternate_purl)
aws_uri_clean_up(&aws_params->alt_parsed_uri);
aws_s3_client_release(aws_params->client);
aws_params->client = NULL;
@@ -813,6 +863,11 @@ H5FD__s3comms_s3r_close(s3r_t *handle)
free(handle->priv_data);
}
if (H5FD__s3comms_free_purl(handle->purl) < 0)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to release parsed url structure");
if (H5FD__s3comms_free_purl(handle->alternate_purl) < 0)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to release parsed url structure");
free(handle);
done:
@@ -876,16 +931,18 @@ H5FD__s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest, si
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP request message");
/* Add "Host" header to HTTP message */
if (H5FD__s3comms_format_host_header(aws_params, request_http_message, handle->purl->bucket_name,
handle->purl->host, handle->purl->port) < 0)
if (H5FD__s3comms_format_host_header(
aws_params, request_http_message, handle->purl->bucket_name,
(handle->alternate_purl ? handle->alternate_purl->host : handle->purl->host),
(handle->alternate_purl ? handle->alternate_purl->port : handle->purl->port)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP 'Host' header");
/* Add "User-Agent" header to HTTP message */
if (H5FD__s3comms_format_user_agent_header(aws_params, request_http_message) < 0)
if (H5FD__s3comms_format_user_agent_header(request_http_message) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP 'User-Agent' header");
/* Setup "Range: " header for retrieving a specific byte range if desired */
if (H5FD__s3comms_format_range_header(aws_params, request_http_message, offset, len) < 0)
if (H5FD__s3comms_format_range_header(request_http_message, offset, len) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP 'Range' header");
/* Print out request headers if debugging is enabled */
@@ -930,6 +987,10 @@ H5FD__s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest, si
request_opts.finish_callback = H5FD__s3comms_s3r_req_finish_cb;
request_opts.user_data = &read_params;
/* Set alternate endpoint if specified */
if (handle->alternate_purl)
request_opts.endpoint = &aws_params->alt_parsed_uri;
request = aws_s3_client_make_meta_request(aws_params->client, &request_opts);
if (!request)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "couldn't create AWS meta-request: %s",
@@ -1092,12 +1153,14 @@ H5FD__s3comms_s3r_getsize(s3r_t *handle)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP request message");
/* Add "Host" header to HTTP message */
if (H5FD__s3comms_format_host_header(aws_params, request_http_message, handle->purl->bucket_name,
handle->purl->host, handle->purl->port) < 0)
if (H5FD__s3comms_format_host_header(
aws_params, request_http_message, handle->purl->bucket_name,
(handle->alternate_purl ? handle->alternate_purl->host : handle->purl->host),
(handle->alternate_purl ? handle->alternate_purl->port : handle->purl->port)) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP 'Host' header");
/* Add "User-Agent" header to HTTP message */
if (H5FD__s3comms_format_user_agent_header(aws_params, request_http_message) < 0)
if (H5FD__s3comms_format_user_agent_header(request_http_message) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "couldn't create HTTP 'User-Agent' header");
/* Print out request headers if debugging is enabled */
@@ -1145,6 +1208,10 @@ H5FD__s3comms_s3r_getsize(s3r_t *handle)
request_opts.finish_callback = H5FD__s3comms_s3r_req_finish_cb;
request_opts.user_data = &getsize_params;
/* Set alternate endpoint if specified */
if (handle->alternate_purl)
request_opts.endpoint = &aws_params->alt_parsed_uri;
request = aws_s3_client_make_meta_request(aws_params->client, &request_opts);
if (!request)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "couldn't create AWS meta-request: %s",
@@ -1290,14 +1357,17 @@ done:
/*----------------------------------------------------------------------------
* Function: H5FD__s3comms_parse_url
*
* Purpose: Parse a URL into separate components.
* Purpose: Parse a URL into separate components. An aws_uri structure
* is populated after parsing the URL, but a more generic
* parsed_url_t structure is also returned in order to keep
* compatibility with the generic s3comms interface.
*
* Return: Success: A pointer to a parsed_url_t
* Failure: NULL
*----------------------------------------------------------------------------
*/
static parsed_url_t *
H5FD__s3comms_parse_url(s3r_t *handle, const char *url)
H5FD__s3comms_parse_url(s3r_t *handle, const char *url, struct aws_uri *uri)
{
H5FD__s3comms_aws_params_t *aws_params = NULL;
struct aws_byte_cursor url_cursor = {0};
@@ -1311,7 +1381,9 @@ H5FD__s3comms_parse_url(s3r_t *handle, const char *url)
FUNC_ENTER_PACKAGE
assert(handle);
assert(url);
assert(uri);
aws_params = (H5FD__s3comms_aws_params_t *)handle->priv_data;
@@ -1324,21 +1396,21 @@ H5FD__s3comms_parse_url(s3r_t *handle, const char *url)
s3_cursor = aws_byte_cursor_from_c_str("s3");
H5_WARN_AGGREGATE_RETURN_ON
if (AWS_OP_SUCCESS != aws_uri_init_parse(&aws_params->parsed_uri, aws_params->allocator, &url_cursor))
if (AWS_OP_SUCCESS != aws_uri_init_parse(uri, aws_params->allocator, &url_cursor))
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "unable to parse url");
/* Is this a 's3://' url? */
if (aws_byte_cursor_eq_ignore_case(&aws_params->parsed_uri.scheme, &s3_cursor))
if (aws_byte_cursor_eq_ignore_case(&uri->scheme, &s3_cursor))
is_s3_url = true;
/* URI section buffers use library-specific structure and may also not
* be NUL-terminated, so make our own s3comms API-compatible C string
* copies
*/
if (NULL == (purl->scheme = malloc(aws_params->parsed_uri.scheme.len + 1)))
if (NULL == (purl->scheme = malloc(uri->scheme.len + 1)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate space for URL scheme portion");
memcpy(purl->scheme, aws_params->parsed_uri.scheme.ptr, aws_params->parsed_uri.scheme.len);
purl->scheme[aws_params->parsed_uri.scheme.len] = '\0';
memcpy(purl->scheme, uri->scheme.ptr, uri->scheme.len);
purl->scheme[uri->scheme.len] = '\0';
/* Special processing for 's3://' urls */
if (is_s3_url) {
@@ -1349,43 +1421,41 @@ H5FD__s3comms_parse_url(s3r_t *handle, const char *url)
snprintf(purl->host, HOST_NAME_LEN, "s3.%s.amazonaws.com", handle->aws_region);
}
else {
if (NULL == (purl->host = malloc(aws_params->parsed_uri.host_name.len + 1)))
if (NULL == (purl->host = malloc(uri->host_name.len + 1)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate space for URL host portion");
memcpy(purl->host, aws_params->parsed_uri.host_name.ptr, aws_params->parsed_uri.host_name.len);
purl->host[aws_params->parsed_uri.host_name.len] = '\0';
memcpy(purl->host, uri->host_name.ptr, uri->host_name.len);
purl->host[uri->host_name.len] = '\0';
if (aws_params->parsed_uri.port != 0) {
if (uri->port != 0) {
if (NULL == (purl->port = malloc(11)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate space for URL port portion");
snprintf(purl->port, 11, "%" PRIu32, aws_params->parsed_uri.port);
snprintf(purl->port, 11, "%" PRIu32, uri->port);
}
}
if (NULL == (purl->path = malloc(aws_params->parsed_uri.path.len + 1)))
if (NULL == (purl->path = malloc(uri->path.len + 1)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate space for URL path portion");
memcpy(purl->path, aws_params->parsed_uri.path.ptr, aws_params->parsed_uri.path.len);
purl->path[aws_params->parsed_uri.path.len] = '\0';
memcpy(purl->path, uri->path.ptr, uri->path.len);
purl->path[uri->path.len] = '\0';
if (NULL == (purl->query = malloc(aws_params->parsed_uri.query_string.len + 1)))
if (NULL == (purl->query = malloc(uri->query_string.len + 1)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate space for URL query string portion");
memcpy(purl->query, aws_params->parsed_uri.query_string.ptr, aws_params->parsed_uri.query_string.len);
purl->query[aws_params->parsed_uri.query_string.len] = '\0';
memcpy(purl->query, uri->query_string.ptr, uri->query_string.len);
purl->query[uri->query_string.len] = '\0';
/* Parse out the bucket name and object key */
if (is_s3_url) {
if (NULL == (purl->bucket_name = HDstrndup((char *)aws_params->parsed_uri.host_name.ptr,
aws_params->parsed_uri.host_name.len)))
if (NULL == (purl->bucket_name = HDstrndup((char *)uri->host_name.ptr, uri->host_name.len)))
HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't duplicate bucket name");
if (aws_params->parsed_uri.path.len == 0)
if (uri->path.len == 0)
HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid path parsed from URL");
/* Path will always include a leading '/' */
if (NULL == (purl->key = HDstrndup((char *)aws_params->parsed_uri.path.ptr + 1,
aws_params->parsed_uri.path.len - 1)))
if (NULL == (purl->key = HDstrndup((char *)uri->path.ptr + 1, uri->path.len - 1)))
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't duplicate path");
}
else {
else if (uri->path.len > 0) {
/* Attempt to determine whether the URL is a virtual-hosted-style
* or path-style URL using very simple heuristics
*/
@@ -1481,7 +1551,16 @@ H5FD__s3comms_parse_url(s3r_t *handle, const char *url)
}
}
if (H5FD_ros3_debug_g) {
ret_value = purl;
done:
free(host_copy);
if (ret_value == NULL) {
if (H5FD__s3comms_free_purl(purl) < 0)
HDONE_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "unable to free parsed url structure");
}
else if (H5FD_ros3_debug_g && purl) {
fprintf(stderr, " -- parsed URL as:\n");
fprintf(stderr, " - Scheme: %s\n", purl->scheme);
fprintf(stderr, " - Host: %s\n", purl->host);
@@ -1493,16 +1572,6 @@ H5FD__s3comms_parse_url(s3r_t *handle, const char *url)
fprintf(stderr, " - Key: %s\n", purl->key);
}
ret_value = purl;
done:
free(host_copy);
if (ret_value == NULL) {
if (H5FD__s3comms_free_purl(purl) < 0)
HDONE_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "unable to free parsed url structure");
}
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD__s3comms_parse_url() */
@@ -1839,7 +1908,7 @@ H5FD__s3comms_format_http_request_message(const H5FD__s3comms_aws_params_t *aws_
struct aws_byte_cursor key_cursor = {0};
struct aws_byte_cursor slash_cursor = {0};
struct aws_byte_buf path_buf = {0};
bool use_virtual_style = false;
bool use_virtual_style = true; /* Use virtual-hosted-style request by default */
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
@@ -1850,15 +1919,24 @@ H5FD__s3comms_format_http_request_message(const H5FD__s3comms_aws_params_t *aws_
assert(object_key);
assert(message_out);
if (aws_params->force_path_style)
use_virtual_style = false;
/*
* If bucket doesn't have a "." in the name, format the HTTP
* message using a virtual-hosted-style request. Otherwise,
* format it using a path-style request as virtual-hosted-style
* requests don't directly support buckets with "." when using
* HTTPS.
* If an alternate endpoint URL was specified, use a path-style
* request for now. This isn't the most ideal, but is currently
* required for testing.
*/
if (NULL == strstr(bucket_name, "."))
use_virtual_style = true;
if (aws_params->alt_parsed_uri.self_size != 0)
use_virtual_style = false;
/*
* If bucket has a "." in the name, use a path-style request
* as virtual-hosted-style requests don't directly support buckets
* with "." when using HTTPS.
*/
if (strstr(bucket_name, "."))
use_virtual_style = false;
AWS_ZERO_STRUCT(path_buf);
@@ -1941,7 +2019,7 @@ H5FD__s3comms_format_host_header(const H5FD__s3comms_aws_params_t *aws_params,
struct aws_http_header host_header = {0};
struct aws_byte_cursor host_cursor = {0};
struct aws_byte_buf host_buf = {0};
bool use_virtual_style = false;
bool use_virtual_style = true; /* Use virtual-hosted-style request by default */
herr_t ret_value = SUCCEED;
FUNC_ENTER_PACKAGE
@@ -1951,20 +2029,29 @@ H5FD__s3comms_format_host_header(const H5FD__s3comms_aws_params_t *aws_params,
assert(bucket_name);
assert(host_name);
if (aws_params->force_path_style)
use_virtual_style = false;
/*
* If an alternate endpoint URL was specified, use a path-style
* request for now. This isn't the most ideal, but is currently
* required for testing.
*/
if (aws_params->alt_parsed_uri.self_size != 0)
use_virtual_style = false;
/*
* If bucket has a "." in the name, use a path-style request
* as virtual-hosted-style requests don't directly support buckets
* with "." when using HTTPS.
*/
if (strstr(bucket_name, "."))
use_virtual_style = false;
H5_WARN_AGGREGATE_RETURN_OFF
host_cursor = aws_byte_cursor_from_c_str(host_name);
H5_WARN_AGGREGATE_RETURN_ON
/*
* If bucket doesn't have a "." in the name, format the HTTP
* message using a virtual-hosted-style request. Otherwise,
* format it using a path-style request as virtual-hosted-style
* requests don't directly support buckets with "." when using
* HTTPS.
*/
if (NULL == strstr(bucket_name, "."))
use_virtual_style = true;
if (use_virtual_style) {
struct aws_byte_cursor bucket_cursor;
struct aws_byte_cursor period_cursor;
@@ -2057,8 +2144,7 @@ done:
*----------------------------------------------------------------------------
*/
static herr_t
H5FD__s3comms_format_range_header(const H5FD__s3comms_aws_params_t *aws_params,
struct aws_http_message *message, haddr_t offset, size_t len)
H5FD__s3comms_format_range_header(struct aws_http_message *message, haddr_t offset, size_t len)
{
struct aws_http_header range_header = {0};
int ret = 0;
@@ -2067,7 +2153,6 @@ H5FD__s3comms_format_range_header(const H5FD__s3comms_aws_params_t *aws_params,
FUNC_ENTER_PACKAGE
assert(aws_params);
assert(message);
if (offset == 0 && len == 0)
@@ -2110,8 +2195,7 @@ done:
*----------------------------------------------------------------------------
*/
static herr_t
H5FD__s3comms_format_user_agent_header(const H5FD__s3comms_aws_params_t *aws_params,
struct aws_http_message *message)
H5FD__s3comms_format_user_agent_header(struct aws_http_message *message)
{
struct aws_http_header user_agent_header = {0};
int ret = 0;
@@ -2120,7 +2204,6 @@ H5FD__s3comms_format_user_agent_header(const H5FD__s3comms_aws_params_t *aws_par
FUNC_ENTER_PACKAGE
assert(aws_params);
assert(message);
ret = snprintf(user_agent_str, sizeof(user_agent_str), "libhdf5/%u.%u.%u (vfd:ros3) libaws-c-s3",
src/H5FDros3_s3comms.h
+8 -1
View File
@@ -181,6 +181,11 @@ typedef struct {
* key: "myfile.dat"
* }
*
* alternate_purl
*
* Pointer to structure holding the elements of an alternate endpoint
* URL when specified
*
* aws_region
*
* Pointer to NULL-terminated string, specifying S3 "region"
@@ -189,6 +194,7 @@ typedef struct {
*/
typedef struct {
parsed_url_t *purl;
parsed_url_t *alternate_purl;
size_t filesize;
char *aws_region;
@@ -204,7 +210,8 @@ H5_DLL herr_t H5FD__s3comms_init(void);
H5_DLL herr_t H5FD__s3comms_term(void);
/* S3 request buffer routines */
H5_DLL s3r_t *H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *fapl_token);
H5_DLL s3r_t *H5FD__s3comms_s3r_open(const char *url, const H5FD_ros3_fapl_t *fa, const char *fapl_token,
const char *alt_endpoint);
H5_DLL herr_t H5FD__s3comms_s3r_close(s3r_t *handle);
H5_DLL size_t H5FD__s3comms_s3r_get_filesize(s3r_t *handle);
H5_DLL herr_t H5FD__s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest, size_t dest_size);
src/H5Fmodule.h
+45 -1
View File
@@ -858,7 +858,7 @@
* <em>Managing file access for in-memory files</em>
* \code
* herr_t H5Pset_fapl_core (hid_t access_properties, size_t block_size, bool backing_store)
* herr_t H5Pget_fapl_core (hid_t access_properties, size_t *block_size), bool *backing_store)
* herr_t H5Pget_fapl_core (hid_t access_properties, size_t *block_size, bool *backing_store)
* \endcode
*
* #H5Pset_fapl_core sets the file access property list to use the Memory driver; any previously
@@ -1041,6 +1041,50 @@
* additional variable settings associated with the Split driver, there is no H5Pget_fapl_split
* function.
*
* \subsubsection subsubsec_file_alternate_drivers_ros3 The ROS3 Driver
* The ROS3 (read-only S3) driver is used to enable read-only access to HDF5 files which are stored
* in Amazon's S3 web service (https://aws.amazon.com/s3/) or an S3 API-compatible storage system.
* This driver expects that HDF5 files will be stored in the storage system as a single object.
* It translates I/O read requests from the HDF5 library into the appropriate REST API requests
* that will retrieve the relevant parts of the HDF5 file needed by the read requests.
*
* The functions #H5Pset_fapl_ros3 and #H5Pget_fapl_ros3 are used to manage file access properties
* for the #H5FD_ROS3 driver. See the example below.
*
* <em>Managing access properties for ROS3</em>
* \code
* herr_t H5Pset_fapl_ros3(hid_t access_properties, const H5FD_ros3_fapl_t *fa)
* herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out)
*
* typedef struct H5FD_ros3_fapl_t {
* int32_t version;
* hbool_t authenticate;
* char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1];
* char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1];
* char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1];
* } H5FD_ros3_fapl_t;
* \endcode
*
* #H5Pset_fapl_ros3 sets the file access properties managed by the #H5FD_ROS3 driver and
* #H5Pget_fapl_ros3 retrieves those file access properties.
*
* The `version` field is used to specify the revision of the #H5FD_ros3_fapl_t structure
* and currently must always be set to the macro value #H5FD_CURR_ROS3_FAPL_T_VERSION.
*
* The `authenticate` field is a boolean determining whether or not the driver should use
* credentials specified in the `secret_id` and `secret_key` fields of the structure. If
* `true`, credentials will be used from the structure. If `false`, the driver will instead
* look for credentials from standard AWS environment variables, configuration files and
* other locations. In this case, any values specified in the `secret_id` and `secret_key`
* fields will be ignored.
*
* The `aws_region` field is used to specify the AWS region to use when accessing files opened
* through the File Access Property List with these properties set on it. If this field is an
* empty string, the driver will search for a specified AWS region in the standard AWS
* environment variables and configuration files. An AWS region <em>must</em> be specified
* with one of these mechanisms or an error will be returned when attempting to open a file
* with the #H5FD_ROS3 driver.
*
* \subsubsection subsubsec_file_alternate_drivers_par The Parallel Driver
* Parallel environments require a parallel low-level driver. HDF5's default driver for parallel
* systems is called the Parallel driver, #H5FD_MPIO. This driver uses the MPI standard for both
test/CMakeLists.txt
+7 -3
View File
@@ -413,8 +413,6 @@ set (H5_TESTS
enc_dec_plist_cross_platform
getname
vfd
ros3
s3comms
hdfs
#mirror_vfd # multiple source
ntypes
@@ -437,6 +435,12 @@ set (H5_TESTS
event_set
onion
)
set (H5_S3TESTS
ros3
s3comms
)
if (HDF5_BUILD_UTILS)
set (H5_TESTS ${H5_TESTS} mirror_vfd)
endif ()
@@ -472,7 +476,7 @@ set (H5_TESTS_MULTIPLE
mirror_vfd
)
# Only build single source tests here
foreach (h5_test ${H5_TESTS} ${H5_EXPRESS_TESTS})
foreach (h5_test ${H5_TESTS} ${H5_EXPRESS_TESTS} ${H5_S3TESTS})
if (NOT h5_test IN_LIST H5_TESTS_MULTIPLE)
ADD_H5_EXE(${h5_test})
endif ()
test/CMakeTests.cmake
+97
View File
@@ -176,6 +176,18 @@ endforeach ()
add_custom_target(HDF5_TEST_LIB_files ALL COMMENT "Copying files needed by HDF5_TEST_LIB tests" DEPENDS ${HDF5_TEST_LIB_files_list})
set (HDF5_S3PROXY_TEST_FILES
t8.shakespeare.txt
Poe_Raven.txt
charsets.h5
)
foreach (h5_file ${HDF5_S3PROXY_TEST_FILES})
HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${HDF5_TEST_BINARY_DIR}/H5TEST/testfiles/${h5_file}" "HDF5_S3PROXY_files")
endforeach ()
add_custom_target(HDF5_S3PROXY_files ALL COMMENT "Copying files needed by HDF5_S3PROXY tests" DEPENDS ${HDF5_S3PROXY_files_list})
set (testhdf5_CLEANFILES
coord.h5
dtypes10.h5
@@ -665,6 +677,91 @@ if ("H5TEST-tcheck_version-release" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5TEST-tcheck_version-release PROPERTIES DISABLED true)
endif ()
##############################################################
##############################################################
### S 3 T E S T S ###
##############################################################
##############################################################
if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
set (h5test_s3tests_port 9001)
# Setup environment for tests.
# The HDF5_ROS3_TEST_BUCKET_URL environment variable is set
# as a requirement of the ros3 and s3comms tests.
# The AWS_ENDPOINT_URL environment variable is set to work
# around an issue in aws-c-s3 when using localhost URLs
# directly.
# The HDF5_ROS3_VFD_FORCE_PATH_STYLE environment variable is
# set to force the ROS3 VFD to use path-style requests for
# compatibility with s3proxy.
# AWS_PROFILE is set in order to use the correct testing
# credentials created in CMakeTests.cmake
set (h5test_s3tests_env
"srcdir=${HDF5_TEST_BINARY_DIR}/H5TEST"
"HDF5_ROS3_TEST_BUCKET_URL=http://localhost:${h5test_s3tests_port}/hdf5ros3"
"AWS_ENDPOINT_URL=http://localhost:${h5test_s3tests_port}"
"HDF5_ROS3_VFD_FORCE_PATH_STYLE=1"
"AWS_PROFILE=ros3_vfd_test"
)
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/buckets")
add_test (
NAME H5TEST-start-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_PRODUCT=andrewgaul/s3proxy"
-D "TEST_PORT=${h5test_s3tests_port}"
-D "TEST_ARGS:STRING=s3proxy-local-fs"
-D "TEST_BUCKET:STRING=hdf5ros3"
-D "TEST_FILES:STRING=t8.shakespeare.txt;Poe_Raven.txt;charsets.h5"
-D "TEST_ACLS:STRING=skip;anon;anon"
-D "TEST_EXPECT=0"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-D "TEST_FOLDER=${HDF5_TEST_BINARY_DIR}/H5TEST"
-P "${HDF_RESOURCES_DIR}/runProxy.cmake"
)
set_tests_properties (H5TEST-start-proxy PROPERTIES FIXTURES_SETUP s3_proxy)
add_test (
NAME H5TEST-stop-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_ARGS:STRING=s3proxy-local-fs"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${HDF5_TEST_BINARY_DIR}/H5TEST"
-P "${HDF_RESOURCES_DIR}/stopProxy.cmake"
)
set_tests_properties (H5TEST-stop-proxy PROPERTIES FIXTURES_CLEANUP s3_proxy)
foreach (h5_test ${H5_S3TESTS})
if (HDF5_ENABLE_USING_MEMCHECKER)
add_test (NAME H5TEST_S3TESTS-${h5_test} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:${h5_test}>)
else ()
add_test (NAME H5TEST_S3TESTS-${h5_test} COMMAND "${CMAKE_COMMAND}"
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
-D "TEST_PROGRAM=$<TARGET_FILE:${h5_test}>"
-D "TEST_ARGS:STRING="
-D "TEST_EXPECT=0"
-D "TEST_SKIP_COMPARE=TRUE"
-D "TEST_OUTPUT=${h5_test}.txt"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
#-D "TEST_REFERENCE=${h5_test}.out"
-D "TEST_FOLDER=${HDF5_TEST_BINARY_DIR}/H5TEST"
-P "${HDF_RESOURCES_DIR}/runTest.cmake"
)
endif ()
set_tests_properties (H5TEST_S3TESTS-${h5_test} PROPERTIES
FIXTURES_REQUIRED s3_proxy
ENVIRONMENT "${h5test_s3tests_env}"
WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST
)
if ("H5TEST_S3TESTS-${h5_test}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5TEST_S3TESTS-${h5_test} PROPERTIES DISABLED true)
endif ()
endforeach ()
endif ()
##############################################################################
##############################################################################
### A D D I T I O N A L T E S T S ###
test/ros3.c
+16 -12
View File
@@ -43,9 +43,14 @@
#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt"
#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt"
#define S3_TEST_RESOURCE_H5_PUBLIC "GMODO-SVM01.h5"
#define S3_TEST_RESOURCE_H5_PUBLIC "charsets.h5"
#define S3_TEST_RESOURCE_MISSING "missing.csv"
#define S3_TEST_RESOURCE_TEXT_RESTRICTED_SIZE 5458199
#define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE 6464
#define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEOVER 6400
#define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEQUOT 5691
static char url_text_restricted[S3_TEST_MAX_URL_SIZE] = "";
static char url_text_public[S3_TEST_MAX_URL_SIZE] = "";
static char url_h5_public[S3_TEST_MAX_URL_SIZE] = "";
@@ -73,8 +78,8 @@ H5FD_ros3_fapl_t restricted_access_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, /* fapl
H5FD_ros3_fapl_t anonymous_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, false, S3_TEST_DEFAULT_REGION, "", ""};
H5FD_ros3_fapl_t empty_auth_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, true, "", "", ""};
H5FD_ros3_fapl_t empty_id_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, true, "where", "", ""};
H5FD_ros3_fapl_t empty_auth_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, true, S3_TEST_DEFAULT_REGION, "", ""};
H5FD_ros3_fapl_t empty_id_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, true, S3_TEST_DEFAULT_REGION, "", ""};
H5FD_ros3_fapl_t empty_region_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, true, "", "me", ""};
/*---------------------------------------------------------------------------
@@ -508,7 +513,7 @@ error:
static int
test_eof_eoa(void)
{
const haddr_t INITIAL_ADDR = 5458199; /* Fragile! */
const haddr_t INITIAL_ADDR = S3_TEST_RESOURCE_TEXT_RESTRICTED_SIZE;
const haddr_t LOWER_ADDR = INITIAL_ADDR - (1024 * 1024);
const haddr_t HIGHER_ADDR = INITIAL_ADDR + (1024 * 1024);
H5FD_t *fd = NULL;
@@ -608,8 +613,8 @@ test_vfl_read(void)
struct testcase tests[] = {
{
"successful range-get",
6464,
5691,
S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE,
S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEQUOT,
32, /* fancy quotes are three bytes each(?) */
SUCCEED,
"Quoth the Raven “Nevermore.”",
@@ -632,7 +637,7 @@ test_vfl_read(void)
},
{
"read past EOA/EOF fails ((EOA==EOF) < addr)",
6464,
S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE,
7000,
100,
FAIL,
@@ -640,8 +645,8 @@ test_vfl_read(void)
},
{
"read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))",
6464,
6400,
S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE,
S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEOVER,
100,
FAIL,
NULL,
@@ -686,7 +691,7 @@ test_vfl_read(void)
if (NULL == (fd = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
TEST_ERROR;
if (6464 != H5FDget_eof(fd, H5FD_MEM_DEFAULT))
if (S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE != H5FDget_eof(fd, H5FD_MEM_DEFAULT))
FAIL_PUTS_ERROR("incorrect EOF (fragile - make sure the file size didn't change)");
for (int i = 0; i < TESTCASE_COUNT; i++) {
@@ -708,7 +713,6 @@ test_vfl_read(void)
ret = H5FDread(fd, H5FD_MEM_DRAW, H5P_DEFAULT, tests[i].addr, tests[i].len, buffer);
}
H5E_END_TRY
if (tests[i].success != ret)
FAIL_PUTS_ERROR(tests[i].message);
if (ret == SUCCEED)
@@ -1143,7 +1147,7 @@ main(void)
s3_test_aws_secret_access_key[0] = '\0';
s3_test_aws_region[0] = '\0';
if (NULL == (s3_test_aws_session_token = malloc(S3_TEST_SESSION_TOKEN_SIZE))) {
if (NULL == (s3_test_aws_session_token = calloc(1, S3_TEST_SESSION_TOKEN_SIZE))) {
fprintf(stderr, "couldn't allocate buffer for session token\n");
ret_value = EXIT_FAILURE;
goto done;
test/s3comms.c
+31 -23
View File
@@ -31,6 +31,10 @@
#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt"
#define S3_TEST_RESOURCE_MISSING "missing.csv"
#define S3_TEST_RESOURCE_TEXT_RESTRICTED_SIZE 5458199
#define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE 6464
#define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEOVER 6400
/* URL max size */
#define S3_TEST_MAX_URL_SIZE 256
@@ -88,10 +92,10 @@ test_s3r_get_filesize(void)
if (0 != H5FD__s3comms_s3r_get_filesize(NULL))
FAIL_PUTS_ERROR("filesize of the null handle should be 0");
if (NULL == (handle = H5FD__s3comms_s3r_open(url_raven, &anonymous_fa, NULL)))
if (NULL == (handle = H5FD__s3comms_s3r_open(url_raven, &anonymous_fa, NULL, NULL)))
TEST_ERROR;
if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
if (S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE != H5FD__s3comms_s3r_get_filesize(handle))
FAIL_PUTS_ERROR("incorrect file size - fragile, make sure the file size didn't change");
if (H5FD__s3comms_s3r_close(handle) < 0)
@@ -178,7 +182,7 @@ test_s3r_open(void)
fa->authenticate = false;
H5E_BEGIN_TRY
{
handle = H5FD__s3comms_s3r_open(url_missing, fa, NULL);
handle = H5FD__s3comms_s3r_open(url_missing, fa, NULL, NULL);
}
H5E_END_TRY
if (handle != NULL)
@@ -188,7 +192,7 @@ test_s3r_open(void)
fa->authenticate = true;
H5E_BEGIN_TRY
{
handle = H5FD__s3comms_s3r_open(url_missing, fa, s3_test_aws_session_token);
handle = H5FD__s3comms_s3r_open(url_missing, fa, s3_test_aws_session_token, NULL);
}
H5E_END_TRY
if (handle != NULL)
@@ -202,7 +206,7 @@ test_s3r_open(void)
strcpy(fa->secret_id, "I_MADE_UP_MY_ID");
H5E_BEGIN_TRY
{
handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, s3_test_aws_session_token);
handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, s3_test_aws_session_token, NULL);
}
H5E_END_TRY
if (handle != NULL)
@@ -213,7 +217,7 @@ test_s3r_open(void)
strcpy(fa->secret_key, "I_AM_A_FAKE_KEY");
H5E_BEGIN_TRY
{
handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, s3_test_aws_session_token);
handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, s3_test_aws_session_token, NULL);
}
H5E_END_TRY
if (handle != NULL)
@@ -226,10 +230,10 @@ test_s3r_open(void)
/* Attempt with authentication or anonymously (depending on environment) */
fa->authenticate = false;
handle = H5FD__s3comms_s3r_open(url_raven, fa, NULL);
handle = H5FD__s3comms_s3r_open(url_raven, fa, NULL, NULL);
if (handle == NULL)
TEST_ERROR;
if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
if (S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE != H5FD__s3comms_s3r_get_filesize(handle))
FAIL_PUTS_ERROR("did not get expected filesize");
if (H5FD__s3comms_s3r_close(handle) < 0)
TEST_ERROR;
@@ -237,20 +241,20 @@ test_s3r_open(void)
/* Using authentication on anonymously-accessible file? */
fa->authenticate = true;
handle = H5FD__s3comms_s3r_open(url_raven, fa, s3_test_aws_session_token);
handle = H5FD__s3comms_s3r_open(url_raven, fa, s3_test_aws_session_token, NULL);
if (handle == NULL)
TEST_ERROR;
if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
if (S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE != H5FD__s3comms_s3r_get_filesize(handle))
FAIL_PUTS_ERROR("did not get expected filesize");
if (H5FD__s3comms_s3r_close(handle))
TEST_ERROR;
handle = NULL;
/* Authenticating */
handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, s3_test_aws_session_token);
handle = H5FD__s3comms_s3r_open(url_shakespeare, fa, s3_test_aws_session_token, NULL);
if (handle == NULL)
TEST_ERROR;
if (5458199 != H5FD__s3comms_s3r_get_filesize(handle))
if (S3_TEST_RESOURCE_TEXT_RESTRICTED_SIZE != H5FD__s3comms_s3r_get_filesize(handle))
FAIL_PUTS_ERROR("did not get expected filesize");
if (H5FD__s3comms_s3r_close(handle) < 0)
TEST_ERROR;
@@ -309,10 +313,10 @@ test_s3r_read(void)
TEST_ERROR;
/* Open file */
handle = H5FD__s3comms_s3r_open(url_raven, &anonymous_fa, NULL);
handle = H5FD__s3comms_s3r_open(url_raven, &anonymous_fa, NULL, NULL);
if (handle == NULL)
TEST_ERROR;
if (6464 != H5FD__s3comms_s3r_get_filesize(handle))
if (S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE != H5FD__s3comms_s3r_get_filesize(handle))
TEST_ERROR;
/*****************************
@@ -346,10 +350,10 @@ test_s3r_read(void)
memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
if (H5FD__s3comms_s3r_read(handle, (haddr_t)6370, (size_t)0, buffer, S3COMMS_READ_BUFFER_SIZE) < 0)
TEST_ERROR;
if (strncmp(
buffer,
"And my soul from out that shadow that lies floating on the floor\nShall be lifted—nevermore!\n",
94))
if (strncmp(buffer,
"And my soul from out that shadow that lies floating on the floor\nShall be "
"lifted—nevermore!\n",
94))
TEST_ERROR;
/**************************
@@ -360,8 +364,10 @@ test_s3r_read(void)
memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
H5E_BEGIN_TRY
{
ret = H5FD__s3comms_s3r_read(handle, (haddr_t)6400, (size_t)100, /* 6400+100 > 6464 */ buffer,
S3COMMS_READ_BUFFER_SIZE);
ret = H5FD__s3comms_s3r_read(
handle, (haddr_t)S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEOVER, (size_t)100,
/* S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEOVER+100 > S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE */ buffer,
S3COMMS_READ_BUFFER_SIZE);
}
H5E_END_TRY
if (ret == SUCCEED)
@@ -373,7 +379,8 @@ test_s3r_read(void)
memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
H5E_BEGIN_TRY
{
ret = H5FD__s3comms_s3r_read(handle, (haddr_t)1200699, /* 1200699 > 6464 */ (size_t)100, buffer,
ret = H5FD__s3comms_s3r_read(handle, (haddr_t)1200699,
/* 1200699 > S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE */ (size_t)100, buffer,
S3COMMS_READ_BUFFER_SIZE);
}
H5E_END_TRY
@@ -386,7 +393,8 @@ test_s3r_read(void)
memset(buffer, 0, S3COMMS_READ_BUFFER_SIZE);
H5E_BEGIN_TRY
{
ret = H5FD__s3comms_s3r_read(handle, (haddr_t)6464, (size_t)0, buffer, S3COMMS_READ_BUFFER_SIZE);
ret = H5FD__s3comms_s3r_read(handle, (haddr_t)S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE, (size_t)0, buffer,
S3COMMS_READ_BUFFER_SIZE);
}
H5E_END_TRY
if (ret == SUCCEED)
@@ -442,7 +450,7 @@ main(void)
s3_test_aws_region[0] = '\0';
s3_test_bucket_url[0] = '\0';
if (NULL == (s3_test_aws_session_token = malloc(S3_TEST_SESSION_TOKEN_SIZE))) {
if (NULL == (s3_test_aws_session_token = calloc(1, S3_TEST_SESSION_TOKEN_SIZE))) {
fprintf(stderr, "couldn't allocate buffer for session token\n");
ret_value = EXIT_FAILURE;
goto done;
test/testfiles/Poe_Raven.txt
Executable
+125
View File
@@ -0,0 +1,125 @@
Once upon a midnight dreary, while I pondered, weak and weary,
Over many a quaint and curious volume of forgotten lore—
While I nodded, nearly napping, suddenly there came a tapping,
As of some one gently rapping, rapping at my chamber door.
“’Tis some visitor,” I muttered, “tapping at my chamber door—
Only this and nothing more.”
Ah, distinctly I remember it was in the bleak December;
And each separate dying ember wrought its ghost upon the floor.
Eagerly I wished the morrow;—vainly I had sought to borrow
From my books surcease of sorrow—sorrow for the lost Lenore—
For the rare and radiant maiden whom the angels name Lenore—
Nameless here for evermore.
And the silken, sad, uncertain rustling of each purple curtain
Thrilled me—filled me with fantastic terrors never felt before;
So that now, to still the beating of my heart, I stood repeating
“’Tis some visitor entreating entrance at my chamber door—
Some late visitor entreating entrance at my chamber door;—
This it is and nothing more.”
Presently my soul grew stronger; hesitating then no longer,
“Sir,” said I, “or Madam, truly your forgiveness I implore;
But the fact is I was napping, and so gently you came rapping,
And so faintly you came tapping, tapping at my chamber door,
That I scarce was sure I heard you”—here I opened wide the door;—
Darkness there and nothing more.
Deep into that darkness peering, long I stood there wondering, fearing,
Doubting, dreaming dreams no mortal ever dared to dream before;
But the silence was unbroken, and the stillness gave no token,
And the only word there spoken was the whispered word, “Lenore?”
This I whispered, and an echo murmured back the word, “Lenore!”—
Merely this and nothing more.
Back into the chamber turning, all my soul within me burning,
Soon again I heard a tapping somewhat louder than before.
“Surely,” said I, “surely that is something at my window lattice;
Let me see, then, what thereat is, and this mystery explore—
Let my heart be still a moment and this mystery explore;—
’Tis the wind and nothing more!”
Open here I flung the shutter, when, with many a flirt and flutter,
In there stepped a stately Raven of the saintly days of yore;
Not the least obeisance made he; not a minute stopped or stayed he;
But, with mien of lord or lady, perched above my chamber door—
Perched upon a bust of Pallas just above my chamber door—
Perched, and sat, and nothing more.
Then this ebony bird beguiling my sad fancy into smiling,
By the grave and stern decorum of the countenance it wore,
“Though thy crest be shorn and shaven, thou,” I said, “art sure no craven,
Ghastly grim and ancient Raven wandering from the Nightly shore—
Tell me what thy lordly name is on the Night’s Plutonian shore!”
Quoth the Raven “Nevermore.”
Much I marvelled this ungainly fowl to hear discourse so plainly,
Though its answer little meaning—little relevancy bore;
For we cannot help agreeing that no living human being
Ever yet was blessed with seeing bird above his chamber door—
Bird or beast upon the sculptured bust above his chamber door,
With such name as “Nevermore.”
But the Raven, sitting lonely on the placid bust, spoke only
That one word, as if his soul in that one word he did outpour.
Nothing farther then he uttered—not a feather then he fluttered—
Till I scarcely more than muttered “Other friends have flown before—
On the morrow he will leave me, as my Hopes have flown before.”
Then the bird said “Nevermore.”
Startled at the stillness broken by reply so aptly spoken,
“Doubtless,” said I, “what it utters is its only stock and store
Caught from some unhappy master whom unmerciful Disaster
Followed fast and followed faster till his songs one burden bore—
Till the dirges of his Hope that melancholy burden bore
Of ‘Never—nevermore’.”
But the Raven still beguiling all my fancy into smiling,
Straight I wheeled a cushioned seat in front of bird, and bust and door;
Then, upon the velvet sinking, I betook myself to linking
Fancy unto fancy, thinking what this ominous bird of yore—
What this grim, ungainly, ghastly, gaunt, and ominous bird of yore
Meant in croaking “Nevermore.”
This I sat engaged in guessing, but no syllable expressing
To the fowl whose fiery eyes now burned into my bosom’s core;
This and more I sat divining, with my head at ease reclining
On the cushion’s velvet lining that the lamp-light gloated o’er,
But whose velvet-violet lining with the lamp-light gloating o’er,
She shall press, ah, nevermore!
Then, methought, the air grew denser, perfumed from an unseen censer
Swung by Seraphim whose foot-falls tinkled on the tufted floor.
“Wretch,” I cried, “thy God hath lent thee—by these angels he hath sent thee
Respite—respite and nepenthe from thy memories of Lenore;
Quaff, oh quaff this kind nepenthe and forget this lost Lenore!”
Quoth the Raven “Nevermore.”
“Prophet!” said I, “thing of evil!—prophet still, if bird or devil!—
Whether Tempter sent, or whether tempest tossed thee here ashore,
Desolate yet all undaunted, on this desert land enchanted—
On this home by Horror haunted—tell me truly, I implore—
Is there—is there balm in Gilead?—tell me—tell me, I implore!”
Quoth the Raven “Nevermore.”
“Prophet!” said I, “thing of evil!—prophet still, if bird or devil!
By that Heaven that bends above us—by that God we both adore—
Tell this soul with sorrow laden if, within the distant Aidenn,
It shall clasp a sainted maiden whom the angels name Lenore—
Clasp a rare and radiant maiden whom the angels name Lenore.”
Quoth the Raven “Nevermore.”
“Be that word our sign of parting, bird or fiend!” I shrieked, upstarting—
“Get thee back into the tempest and the Night’s Plutonian shore!
Leave no black plume as a token of that lie thy soul hath spoken!
Leave my loneliness unbroken!—quit the bust above my door!
Take thy beak from out my heart, and take thy form from off my door!”
Quoth the Raven “Nevermore.”
And the Raven, never flitting, still is sitting, still is sitting
On the pallid bust of Pallas just above my chamber door;
And his eyes have all the seeming of a demon’s that is dreaming,
And the lamp-light o’er him streaming throws his shadow on the floor;
And my soul from out that shadow that lies floating on the floor
Shall be lifted—nevermore!
test/testfiles/charsets.h5
BIN
View File
Binary file not shown.
test/testfiles/t8.shakespeare.txt
Executable
+124456
View File
File diff suppressed because it is too large Load Diff
tools/lib/h5tools.c
+10 -5
View File
@@ -555,14 +555,19 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
}
else if (!strcmp(vfd_info->u.name, drivernames[ROS3_VFD_IDX])) {
#ifdef H5_HAVE_ROS3_VFD
if (!vfd_info->info)
const H5FD_ros3_fapl_ext_t *ros3_fapl_info = (const H5FD_ros3_fapl_ext_t *)vfd_info->info;
if (!ros3_fapl_info)
H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid");
if (H5Pset_fapl_ros3(fapl_id, &((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->fa) < 0)
if (H5Pset_fapl_ros3(fapl_id, &ros3_fapl_info->fa) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed");
if (H5Pset_fapl_ros3_token(fapl_id, ((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->token) <
0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_token() failed");
if (ros3_fapl_info->token[0] != '\0')
if (H5Pset_fapl_ros3_token(fapl_id, ros3_fapl_info->token) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_token() failed");
if (ros3_fapl_info->ep_url[0] != '\0')
if (H5Pset_fapl_ros3_endpoint(fapl_id, ros3_fapl_info->ep_url) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_endpoint() failed");
#else
H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled");
#endif
tools/lib/h5tools_utils.c
+51 -109
View File
@@ -1041,17 +1041,17 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_e
if (nelems != 3 && nelems != 4)
H5TOOLS_GOTO_ERROR(FAIL, "invalid S3 VFD credentials");
ccred[0] = (const char *)s3cred[0];
ccred[1] = (const char *)s3cred[1];
ccred[2] = (const char *)s3cred[2];
ccred[0] = (const char *)s3cred[0]; /* aws_region */
ccred[1] = (const char *)s3cred[1]; /* aws_access_key_id */
ccred[2] = (const char *)s3cred[2]; /* aws_secret_access_key */
if (nelems == 3) {
ccred[3] = "";
}
else {
ccred[3] = (const char *)s3cred[3];
ccred[3] = (const char *)s3cred[3]; /* aws_session_token */
}
if (0 == h5tools_populate_ros3_fapl(fapl_config_out, ccred))
if (FAIL == h5tools_populate_ros3_fapl(fapl_config_out, ccred, 4))
H5TOOLS_GOTO_ERROR(FAIL, "failed to populate S3 VFD FAPL config");
done:
@@ -1079,11 +1079,11 @@ done:
* { aws_region,
* secret_id,
* secret_key,
* session_token (optional)
* }
* If all three strings are empty (""), the default fapl will be default.
* Both aws_region and secret_id values must be both empty or both
* populated. If
* Only secret_key is allowed to be empty (the empty string, "").
* If all three (or four) strings are empty (""), the default fapl will be
* default. The aws_region and secret_id values must be both empty or both
* populated. Only secret_key is allowed to be empty (the empty string, "").
* All values are checked against overflow as defined in the ros3 vfd
* header file; if a value overruns the permitted space, FAIL is returned
* and the function aborts without resetting the fapl to values initially
@@ -1091,7 +1091,7 @@ done:
*
* Return:
*
* 0 (failure) if...
* FAIL if...
* * Read-Only S3 VFD is not enabled.
* * NULL fapl pointer: (NULL, {...} )
* * Warning: In all cases below, fapl will be set as "default"
@@ -1100,7 +1100,7 @@ done:
* * Incomplete fapl info:
* * empty region, non-empty id, key either way, token either way
* * (&fa, token, {"", "...", "?", "?"})
* * empty id, non-empty region, key either way, token either way
* * non-empty region, empty id, key either way, token either way
* * (&fa, token, {"...", "", "?", "?"})
* * "non-empty key, token either way and either id or region empty
* * (&fa, token, {"", "", "...", "?")
@@ -1108,7 +1108,7 @@ done:
* * (&fa, token, {"...", "", "...", "?")
* * Any string would overflow allowed space in fapl definition.
* or
* 1 (success)
* SUCCEED
* * Sets components in fapl_t pointer, copying strings as appropriate.
* * "Default" fapl (valid version, authenticate->False, empty strings)
* * `values` pointer is NULL
@@ -1121,30 +1121,23 @@ done:
* * (&fa, token, {"...", "...", "...", ""})
* * (&fa, token, {"...", "...", "...", "..."})
*
* Return: SUCCEED/FAIL
*
*----------------------------------------------------------------------------
*/
int
h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values)
herr_t
h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values, size_t num_values)
{
int show_progress = 0; /* set to 1 for debugging */
int ret_value = 1; /* 1 for success, 0 for failure */
/* e.g.? if (!populate()) { then failed } */
herr_t ret_value = SUCCEED;
if (show_progress) {
printf("called h5tools_populate_ros3_fapl\n");
}
H5TOOLS_START_DEBUG("");
if (fa == NULL) {
if (show_progress) {
printf(" ERROR: null pointer to fapl_t\n");
}
ret_value = 0;
goto done;
}
if (fa == NULL)
H5TOOLS_GOTO_ERROR(FAIL, "ERROR: null pointer to fapl_t\n");
if (values && (num_values < 3))
H5TOOLS_GOTO_ERROR(FAIL, "ERROR: too few values (< 3) provided\n");
if (show_progress) {
printf(" preset fapl with default values\n");
}
H5TOOLS_DEBUG(" preset fapl with default values\n");
fa->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
fa->fa.authenticate = false;
*(fa->fa.aws_region) = '\0';
@@ -1152,105 +1145,54 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values)
*(fa->fa.secret_key) = '\0';
*(fa->token) = '\0';
/* sanity-check supplied values
*/
/* sanity-check supplied values */
if (values != NULL) {
if (values[0] == NULL) {
if (show_progress) {
printf(" ERROR: aws_region value cannot be NULL\n");
}
ret_value = 0;
goto done;
}
if (values[1] == NULL) {
if (show_progress) {
printf(" ERROR: secret_id value cannot be NULL\n");
}
ret_value = 0;
goto done;
}
if (values[2] == NULL) {
if (show_progress) {
printf(" ERROR: secret_key value cannot be NULL\n");
}
ret_value = 0;
goto done;
}
if (values[3] == NULL) {
if (show_progress) {
printf(" ERROR: token value cannot be NULL\n");
}
ret_value = 0;
goto done;
}
if (values[0] == NULL)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: aws_region value cannot be NULL\n");
if (values[1] == NULL)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: secret_id value cannot be NULL\n");
if (values[2] == NULL)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: secret_key value cannot be NULL\n");
if ((num_values > 3) && (values[3] == NULL))
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: token value cannot be NULL\n");
/* if region and ID are supplied (key optional), write to fapl...
* fail if value would overflow
*/
if (*values[0] != '\0' && *values[1] != '\0') {
if (strlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN) {
if (show_progress) {
printf(" ERROR: aws_region value too long\n");
}
ret_value = 0;
goto done;
}
if (strlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: aws_region value too long\n");
memcpy(fa->fa.aws_region, values[0], (strlen(values[0]) + 1));
if (show_progress) {
printf(" aws_region set\n");
}
H5TOOLS_DEBUG(" aws_region set\n");
if (strlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN) {
if (show_progress) {
printf(" ERROR: secret_id value too long\n");
}
ret_value = 0;
goto done;
}
if (strlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: secret_id value too long\n");
memcpy(fa->fa.secret_id, values[1], (strlen(values[1]) + 1));
if (show_progress) {
printf(" secret_id set\n");
}
H5TOOLS_DEBUG(" secret_id set\n");
if (strlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN) {
if (show_progress) {
printf(" ERROR: secret_key value too long\n");
}
ret_value = 0;
goto done;
}
if (strlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: secret_key value too long\n");
memcpy(fa->fa.secret_key, values[2], (strlen(values[2]) + 1));
if (show_progress) {
printf(" secret_key set\n");
}
H5TOOLS_DEBUG(" secret_key set\n");
if (strlen(values[3]) > H5FD_ROS3_MAX_SECRET_TOK_LEN) {
if (show_progress) {
printf(" ERROR: token value too long\n");
}
ret_value = 0;
goto done;
}
memcpy(fa->token, values[3], (strlen(values[3]) + 1));
if (show_progress) {
printf(" token set\n");
if (num_values > 3) {
if (strlen(values[3]) > H5FD_ROS3_MAX_SECRET_TOK_LEN)
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: token value too long\n");
memcpy(fa->token, values[3], (strlen(values[3]) + 1));
H5TOOLS_DEBUG(" token set\n");
}
fa->fa.authenticate = true;
if (show_progress) {
printf(" set to authenticate\n");
}
H5TOOLS_DEBUG(" set to authenticate\n");
}
else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0' || *values[3] != '\0') {
if (show_progress) {
printf(" ERROR: invalid assortment of empty/non-empty values\n");
}
ret_value = 0;
goto done;
else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0' ||
((num_values > 3) && (*values[3] != '\0'))) {
H5TOOLS_GOTO_ERROR(FAIL, " ERROR: invalid assortment of empty/non-empty values\n");
}
} /* values != NULL */
done:
H5TOOLS_ENDDEBUG("");
return ret_value;
} /* h5tools_populate_ros3_fapl */
#endif /* H5_HAVE_ROS3_VFD */
tools/lib/h5tools_utils.h
+9 -3
View File
@@ -26,6 +26,10 @@ extern "C" {
#define PRINT_DATA_MAX_SIZE 512
#define OUTBUFF_SIZE (PRINT_DATA_MAX_SIZE * 4)
#ifdef H5_HAVE_ROS3_VFD
#define H5FD_ROS3_MAX_ENDPOINT_URL_LEN 256
#endif /* H5_HAVE_ROS3_VFD */
H5TOOLS_DLLVAR int g_nTasks;
H5TOOLS_DLLVAR unsigned char g_Parallel;
H5TOOLS_DLLVAR char outBuff[];
@@ -64,8 +68,9 @@ typedef struct find_objs_t {
#ifdef H5_HAVE_ROS3_VFD
/*extended configuration struct for holding the configuration data to the #H5FD_ROS3 driver */
typedef struct H5FD_ros3_fapl_ext_t {
H5FD_ros3_fapl_t fa; /* ROS3 configuration struct*/
char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; /* Session/security token*/
H5FD_ros3_fapl_t fa; /* ROS3 configuration struct */
char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; /* Session/security token */
char ep_url[H5FD_ROS3_MAX_ENDPOINT_URL_LEN + 1]; /* Optional endpoint url */
} H5FD_ros3_fapl_ext_t;
#endif /* H5_HAVE_ROS3_VFD */
@@ -135,7 +140,8 @@ H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void);
#ifdef H5_HAVE_ROS3_VFD
H5TOOLS_DLL herr_t h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim,
H5FD_ros3_fapl_ext_t *fapl_config_out);
H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values);
H5TOOLS_DLL herr_t h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values,
size_t num_values);
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
H5TOOLS_DLL herr_t h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim,
tools/libtest/h5tools_test_utils.c
+335 -174
View File
@@ -534,9 +534,10 @@ test_populate_ros3_fa(void)
* TEST-LOCAL VARIABLES *
************************/
bool show_progress = false;
int bad_version = 0xf87a; /* arbitrarily wrong version number */
#endif /* H5_HAVE_ROS3_VFD */
H5FD_ros3_fapl_ext_t *fa = NULL; /* Structure is too large for stack */
bool show_progress = false;
int bad_version = 0xf87a; /* arbitrarily wrong version number */
#endif
TESTING("programmatic ros3 fapl population");
@@ -552,6 +553,11 @@ test_populate_ros3_fa(void)
assert(bad_version != H5FD_CURR_ROS3_FAPL_T_VERSION);
if (NULL == (fa = malloc(sizeof(*fa)))) {
fprintf(stderr, "couldn't allocate ROS3 VFD FAPL structure\n");
goto error;
}
/*********
* TESTS *
*********/
@@ -565,115 +571,168 @@ test_populate_ros3_fa(void)
printf("NULL fapl pointer\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(NULL, values), "fapl pointer cannot be null")
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(NULL, values, 3), "fapl pointer cannot be null")
}
/* NULL values pointer yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, true, "u", "v", "w"}, "x"};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = true;
strcpy(fa->fa.aws_region, "u");
strcpy(fa->fa.secret_id, "v");
strcpy(fa->fa.secret_key, "w");
strcpy(fa->token, "x");
strcpy(fa->ep_url, "y");
if (show_progress) {
printf("NULL values pointer\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, NULL), "NULL values pointer yields \"default\" fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(SUCCEED, h5tools_populate_ros3_fapl(fa, NULL, 0),
"NULL values pointer yields \"default\" fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("y", fa->ep_url, (char *)NULL)
}
/* all-empty values
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, true, "u", "v", "w"}, "x"};
const char *values[] = {"", "", "", ""};
const char *values[] = {"", "", "", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = true;
strcpy(fa->fa.aws_region, "u");
strcpy(fa->fa.secret_id, "v");
strcpy(fa->fa.secret_key, "w");
strcpy(fa->token, "x");
strcpy(fa->ep_url, "y");
if (show_progress) {
printf("all empty values\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "empty values yields \"default\" fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(SUCCEED, h5tools_populate_ros3_fapl(fa, values, 4), "empty values yields \"default\" fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("y", fa->ep_url, (char *)NULL)
}
/* successfully set fapl with values
* excess value is ignored
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", "z", "a", "b"};
const char *values[] = {"x", "y", "z", "a", "b"};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("successful full set\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "four values")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(true, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("z", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("a", fa.token, (char *)NULL)
JSVERIFY(SUCCEED, h5tools_populate_ros3_fapl(fa, values, 5), "four values")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(true, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("z", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("a", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* NULL region
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {NULL, "y", "z", ""};
const char *values[] = {NULL, "y", "z", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("NULL region\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* empty region
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"", "y", "z", ""};
const char *values[] = {"", "y", "z", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("empty region; non-empty id, key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* region overflow
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"somewhere over the rainbow not too high "
"there is another rainbow bounding some darkened sky",
"y", "z", ""};
const char *values[] = {"somewhere over the rainbow not too high "
"there is another rainbow bounding some darkened sky",
"y", "z", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("region overflow\n");
@@ -681,72 +740,99 @@ test_populate_ros3_fa(void)
assert(strlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN);
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* NULL id
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", NULL, "z", ""};
const char *values[] = {"x", NULL, "z", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("NULL id\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* empty id (non-empty region, key)
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "", "z", ""};
const char *values[] = {"x", "", "z", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("empty id; non-empty region and key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* id overflow
* partial set: region
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x",
"Why is it necessary to solve the problem? "
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
"z", ""};
const char *values[] = {"x",
"Why is it necessary to solve the problem? "
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
"z", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("id overflow\n");
@@ -754,132 +840,186 @@ test_populate_ros3_fa(void)
assert(strlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN);
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* NULL key
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", NULL, ""};
const char *values[] = {"x", "y", NULL, ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("NULL key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* NULL token
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", "z", NULL};
const char *values[] = {"x", "y", "z", NULL};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("NULL key\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill token")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill token")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* empty key (non-empty region, id)
* yields authenticating fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y", "", ""};
const char *values[] = {"x", "y", "", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("empty key; non-empty region and id\n");
}
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(true, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(SUCCEED, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(true, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* empty key, region (non-empty id)
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"", "y", "", ""};
const char *values[] = {"", "y", "", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("empty key and region; non-empty id\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* empty key, id (non-empty region)
* yields default fapl
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "", "", ""};
const char *values[] = {"x", "", "", ""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("empty key and id; non-empty region\n");
}
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* key overflow
* partial set: region, id
*/
{
H5FD_ros3_fapl_ext_t fa = {{bad_version, false, "a", "b", "c"}, "d"};
const char *values[] = {"x", "y",
"Why is it necessary to solve the problem? "
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
""};
const char *values[] = {"x", "y",
"Why is it necessary to solve the problem? "
"What benefits will you receive by solving the problem? "
"What is the unknown? "
"What is it you don't yet understand? "
"What is the information you have? "
"What isn't the problem? "
"Is the information insufficient, redundant, or contradictory? "
"Should you draw a diagram or figure of the problem? "
"What are the boundaries of the problem? "
"Can you separate the various parts of the problem?",
""};
memset(fa, 0, sizeof(*fa));
fa->fa.version = bad_version;
fa->fa.authenticate = false;
strcpy(fa->fa.aws_region, "a");
strcpy(fa->fa.secret_id, "b");
strcpy(fa->fa.secret_key, "c");
strcpy(fa->token, "d");
strcpy(fa->ep_url, "e");
if (show_progress) {
printf("key overflow\n");
@@ -887,30 +1027,38 @@ test_populate_ros3_fa(void)
assert(strlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN);
JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL)
JSVERIFY(false, fa.fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa.token, (char *)NULL)
JSVERIFY(FAIL, h5tools_populate_ros3_fapl(fa, values, 4), "could not fill fapl")
JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa->fa.version, (char *)NULL)
JSVERIFY(false, fa->fa.authenticate, (char *)NULL)
JSVERIFY_STR("x", fa->fa.aws_region, (char *)NULL)
JSVERIFY_STR("y", fa->fa.secret_id, (char *)NULL)
JSVERIFY_STR("", fa->fa.secret_key, (char *)NULL)
JSVERIFY_STR("", fa->token, (char *)NULL)
JSVERIFY_STR("e", fa->ep_url, (char *)NULL)
}
/* use case
*/
{
H5FD_ros3_fapl_ext_t fa = {{0, 0, "", "", ""}, ""};
const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA",
"ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+", ""};
JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "unable to set use case")
JSVERIFY(1, fa.fa.version, "version check")
JSVERIFY(1, fa.fa.authenticate, "should authenticate")
const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA",
"ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+", ""};
memset(fa, 0, sizeof(*fa));
JSVERIFY(SUCCEED, h5tools_populate_ros3_fapl(fa, values, 4), "unable to set use case")
JSVERIFY(1, fa->fa.version, "version check")
JSVERIFY(1, fa->fa.authenticate, "should authenticate")
}
free(fa);
fa = NULL;
PASSED();
return 0;
error:
free(fa);
/***********
* CLEANUP *
***********/
@@ -963,7 +1111,7 @@ test_set_configured_fapl(void)
hid_t fapl_id = H5I_INVALID_HID;
other_fa_t wrong_fa = {0x432, 0xf82, 0x9093};
#ifdef H5_HAVE_ROS3_VFD
H5FD_ros3_fapl_ext_t ros3_anon_fa = {{1, false, "", "", ""}, ""};
H5FD_ros3_fapl_ext_t *ros3_anon_fa = NULL;
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
H5FD_hdfs_fapl_t hdfs_fa = {
@@ -1031,11 +1179,9 @@ test_set_configured_fapl(void)
NULL,
},
{
"(ROS3) successful set",
1,
UTIL_TEST_CREATE,
"ros3",
&ros3_anon_fa,
"(ROS3) successful set", 1, UTIL_TEST_CREATE, "ros3",
NULL, /* NOTE: assigned below after FAPL structure is allocated - must keep testcase index in sync
*/
},
#endif /* H5_HAVE_ROS3_VFD */
@@ -1069,7 +1215,14 @@ test_set_configured_fapl(void)
unsigned int i;
#ifdef H5_HAVE_ROS3_VFD
if (NULL == (ros3_anon_fa = calloc(1, sizeof(*ros3_anon_fa))))
goto error;
ros3_anon_fa->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
ros3_anon_fa->fa.authenticate = false;
n_cases += 3;
cases[n_cases - 1].conf_fa = ros3_anon_fa;
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
@@ -1151,6 +1304,11 @@ test_set_configured_fapl(void)
fflush(stderr);
#endif /* UTIL_TEST_DEBUG */
#ifdef H5_HAVE_ROS3_VFD
free(ros3_anon_fa);
ros3_anon_fa = NULL;
#endif /* H5_HAVE_ROS3_VFD */
PASSED();
return 0;
@@ -1158,6 +1316,9 @@ error:
/***********
* CLEANUP *
***********/
#ifdef H5_HAVE_ROS3_VFD
free(ros3_anon_fa);
#endif /* H5_HAVE_ROS3_VFD */
#if UTIL_TEST_DEBUG
fprintf(stderr, "ERROR\n");
tools/src/h5dump/h5dump.c
+69 -43
View File
@@ -34,30 +34,11 @@ static h5tools_vfd_info_t vfd_info_g = {0};
static bool get_onion_revision_count = false;
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
static H5FD_ros3_fapl_ext_t ros3_fa_g = {
{
1, /* Structure Version */
false, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
},
"", /* Session/security token */
};
static H5FD_ros3_fapl_ext_t *ros3_fa_g = NULL;
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
/* "Default" HDFS configuration */
static H5FD_hdfs_fapl_t hdfs_fa_g = {
1, /* Structure Version */
"localhost", /* Namenode Name */
0, /* Namenode Port */
"", /* Kerberos ticket cache */
"", /* User name */
2048, /* Stream buffer size */
};
#endif /* H5_HAVE_LIBHDFS */
static H5FD_hdfs_fapl_t *hdfs_fa_g = NULL;
#endif
static H5FD_onion_fapl_info_t onion_fa_g = {
H5FD_ONION_FAPL_INFO_VERSION_CURR,
@@ -93,11 +74,6 @@ struct handler_t {
struct subset_t *subset_info;
};
/*
* Command-line options: The user can specify short or long-named
* parameters. The long-named ones can be partially spelled. When
* adding more, make sure that they don't clash with each other.
*/
/* The following initialization makes use of C language concatenating */
/* "xxx" "yyy" into "xxxyyy". */
static const char *s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HK:L:M:N:O*RS:VX:";
@@ -144,6 +120,7 @@ static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'},
{"stride", require_arg, 'S'},
{"version", no_arg, 'V'},
{"xml-ns", require_arg, 'X'},
{"endpoint-url", require_arg, '!'},
{"s3-cred", require_arg, '$'},
{"hdfs-attrs", require_arg, '#'},
{"vol-value", require_arg, '1'},
@@ -205,6 +182,10 @@ usage(const char *prog)
" Use blank(empty) filename F to suppress ddl display\n");
PRINTVALSTREAM(rawoutstream,
" --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
PRINTVALSTREAM(rawoutstream,
" --endpoint-url=P Supply S3 endpoint url information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream, " P is the AWS service endpoint.\n");
PRINTVALSTREAM(rawoutstream, " Has no effect if filedriver is not \"ros3\".\n");
PRINTVALSTREAM(rawoutstream,
" --s3-cred=<cred> Supply S3 authentication information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream,
@@ -891,18 +872,6 @@ parse_start:
vfd_info_g.type = VFD_BY_NAME;
vfd_info_g.u.name = H5_optarg;
use_custom_vfd_g = true;
#ifdef H5_HAVE_ROS3_VFD
if (0 == strcmp(vfd_info_g.u.name, drivernames[ROS3_VFD_IDX]))
if (!vfd_info_g.info)
vfd_info_g.info = &ros3_fa_g;
#endif
#ifdef H5_HAVE_LIBHDFS
if (0 == strcmp(vfd_info_g.u.name, drivernames[HDFS_VFD_IDX]))
if (!vfd_info_g.info)
vfd_info_g.info = &hdfs_fa_g;
#endif
break;
case 'g':
dump_opts.display_all = 0;
@@ -1211,10 +1180,22 @@ end_collect:
hand = NULL;
h5tools_setstatus(EXIT_SUCCESS);
goto done;
break;
case '!':
#ifdef H5_HAVE_ROS3_VFD
snprintf(ros3_fa_g->ep_url, H5FD_ROS3_MAX_ENDPOINT_URL_LEN + 1, "%s", H5_optarg);
#else
error_msg(
"Read-Only S3 VFD is not available unless enabled when HDF5 is configured and built.\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
#endif
break;
case '$':
#ifdef H5_HAVE_ROS3_VFD
if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa_g) < 0) {
if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', ros3_fa_g) < 0) {
error_msg("failed to parse S3 VFD credential info\n");
usage(h5tools_getprogname());
free_handler(hand, argc);
@@ -1223,7 +1204,7 @@ end_collect:
goto done;
}
vfd_info_g.info = &ros3_fa_g;
vfd_info_g.info = ros3_fa_g;
#else
error_msg(
"Read-Only S3 VFD is not available unless enabled when HDF5 is configured and built.\n");
@@ -1234,7 +1215,7 @@ end_collect:
case '#':
#ifdef H5_HAVE_LIBHDFS
if (h5tools_parse_hdfs_fapl_tuple(H5_optarg, ',', &hdfs_fa_g) < 0) {
if (h5tools_parse_hdfs_fapl_tuple(H5_optarg, ',', hdfs_fa_g) < 0) {
error_msg("failed to parse HDFS VFD configuration info\n");
usage(h5tools_getprogname());
free_handler(hand, argc);
@@ -1243,7 +1224,7 @@ end_collect:
goto done;
}
vfd_info_g.info = &hdfs_fa_g;
vfd_info_g.info = hdfs_fa_g;
#else
error_msg("HDFS VFD is not available unless enabled when HDF5 is configured and built.\n");
h5tools_setstatus(EXIT_FAILURE);
@@ -1290,6 +1271,19 @@ end_collect:
}
}
#ifdef H5_HAVE_ROS3_VFD
if (use_custom_vfd_g && !vfd_info_g.info) {
if (vfd_info_g.type == VFD_BY_NAME && 0 == strcmp(vfd_info_g.u.name, drivernames[ROS3_VFD_IDX]))
vfd_info_g.info = ros3_fa_g;
}
#endif
#ifdef H5_HAVE_LIBHDFS
if (use_custom_vfd_g && !vfd_info_g.info) {
if (vfd_info_g.type == VFD_BY_NAME && 0 == strcmp(vfd_info_g.u.name, drivernames[HDFS_VFD_IDX]))
vfd_info_g.info = hdfs_fa_g;
}
#endif
/* If the file uses the onion VFD, get the revision number */
if (vfd_info_g.type == VFD_BY_NAME && vfd_info_g.u.name && !strcmp(vfd_info_g.u.name, "onion")) {
@@ -1363,6 +1357,31 @@ main(int argc, char *argv[])
/* Initialize h5tools lib */
h5tools_init();
/* Initialize VFD-specific structures */
#ifdef H5_HAVE_ROS3_VFD
if (NULL == (ros3_fa_g = calloc(1, sizeof(*ros3_fa_g)))) {
error_msg("unable to allocate space for configuration structure\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
/* Default "anonymous" S3 configuration */
ros3_fa_g->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
ros3_fa_g->fa.authenticate = false;
#endif
#ifdef H5_HAVE_LIBHDFS
if (NULL == (hdfs_fa_g = calloc(1, sizeof(*hdfs_fa_g)))) {
error_msg("unable to allocate space for configuration structure\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
/* "Default" HDFS configuration */
hdfs_fa_g->version = H5FD__CURR_HDFS_FAPL_T_VERSION;
hdfs_fa_g->stream_buffer_size = 2048;
strcpy(hdfs_fa_g->namenode_name, "localhost");
#endif
if ((hand = parse_command_line(argc, (const char *const *)argv)) == NULL) {
goto done;
}
@@ -1676,6 +1695,13 @@ done:
if (hand)
free_handler(hand, argc);
#ifdef H5_HAVE_ROS3_VFD
free(ros3_fa_g);
#endif
#ifdef H5_HAVE_LIBHDFS
free(hdfs_fa_g);
#endif
/* To Do: clean up XML table */
leave(h5tools_getstatus());
tools/src/h5dump/h5dump.h
+3
View File
@@ -44,6 +44,9 @@
* \li <strong>--ddl=F</strong> Output ddl text into file F
* Use blank(empty) filename F to suppress ddl display
* \li <strong>--page-buffer-size=N</strong> Set the page buffer cache size, N=non-negative integers
* \li <strong>--endpoint-url=P</strong> Supply S3 endpoint url information to "ros3" vfd.
* P is the AWS service endpoint.
* Has no effect if filedriver is not "ros3".
* \li <strong>--s3-cred=\<cred\></strong> Supply S3 authentication information to "ros3" vfd.
* \code <cred> :: "(<aws-region>,<access-id>,<access-key>)" \endcode
* \code <cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)" \endcode
tools/src/h5ls/h5ls.c
+67 -35
View File
@@ -220,6 +220,9 @@ usage(void)
" --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
PRINTVALSTREAM(rawoutstream, " --vfd=DRIVER Use the specified virtual file driver\n");
PRINTVALSTREAM(rawoutstream, " -x, --hexdump Show raw data in hexadecimal format\n");
PRINTVALSTREAM(rawoutstream, " --endpoint-url=P Supply S3 endpoint url information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream, " P is the AWS service endpoint.\n");
PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag not set to \"ros3\".\n");
PRINTVALSTREAM(rawoutstream,
" --s3-cred=C Supply S3 authentication information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream,
@@ -2739,32 +2742,12 @@ main(int argc, char *argv[])
bool custom_vfd_fapl = false;
h5tools_vol_info_t vol_info = {0};
h5tools_vfd_info_t vfd_info = {0};
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
H5FD_ros3_fapl_ext_t ros3_fa = {
{
1, /* Structure Version */
false, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
},
"", /* Session/security token */
};
#endif /* H5_HAVE_ROS3_VFD */
H5FD_ros3_fapl_ext_t *ros3_fa = NULL;
#endif
#ifdef H5_HAVE_LIBHDFS
/* "Default" HDFS configuration */
H5FD_hdfs_fapl_t hdfs_fa = {
1, /* Structure Version */
"localhost", /* Namenode Name */
0, /* Namenode Port */
"", /* Kerberos ticket cache */
"", /* User name */
2048, /* Stream buffer size */
};
#endif /* H5_HAVE_LIBHDFS */
H5FD_hdfs_fapl_t *hdfs_fa = NULL;
#endif
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -2776,6 +2759,29 @@ main(int argc, char *argv[])
memset(&vol_info, 0, sizeof(h5tools_vol_info_t));
memset(&vfd_info, 0, sizeof(h5tools_vfd_info_t));
/* Initialize other VFD-specific structs */
#ifdef H5_HAVE_ROS3_VFD
if (NULL == (ros3_fa = calloc(1, sizeof(*ros3_fa)))) {
fprintf(rawerrorstream, "Error: Unable to allocate space for configuration structure\n");
leave(EXIT_FAILURE);
}
/* Default "anonymous" S3 configuration */
ros3_fa->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
ros3_fa->fa.authenticate = false;
#endif /* H5_HAVE_ROS3_VFD */
#ifdef H5_HAVE_LIBHDFS
if (NULL == (hdfs_fa = calloc(1, sizeof(*hdfs_fa)))) {
fprintf(rawerrorstream, "Error: Unable to allocate space for configuration structure\n");
leave(EXIT_FAILURE);
}
/* "Default" HDFS configuration */
hdfs_fa->version = H5FD__CURR_HDFS_FAPL_T_VERSION;
hdfs_fa->stream_buffer_size = 2048;
strcpy(hdfs_fa->namenode_name, "localhost");
#endif /* H5_HAVE_LIBHDFS */
/* Build object display table */
DISPATCH(H5O_TYPE_GROUP, "Group", NULL, NULL);
DISPATCH(H5O_TYPE_DATASET, "Dataset", dataset_list1, dataset_list2);
@@ -2802,7 +2808,7 @@ main(int argc, char *argv[])
data_g = true;
}
else if (!strcmp(argv[argno], "--enable-error-stack")) {
enable_error_stack = 1;
enable_error_stack = 2;
}
else if (!strcmp(argv[argno], "--errors")) {
/* deprecated --errors */
@@ -2939,13 +2945,32 @@ main(int argc, char *argv[])
}
start++;
if (h5tools_parse_ros3_fapl_tuple(start, ',', &ros3_fa) < 0) {
if (h5tools_parse_ros3_fapl_tuple(start, ',', ros3_fa) < 0) {
fprintf(rawerrorstream, "Error: failed to parse S3 VFD credential info\n\n");
usage();
leave(EXIT_FAILURE);
}
vfd_info.info = &ros3_fa;
vfd_info.info = ros3_fa;
#else
fprintf(rawerrorstream, "Error: Read-Only S3 VFD is not available unless enabled when HDF5 is "
"configured and built.\n\n");
usage();
leave(EXIT_FAILURE);
#endif
}
else if (!strncmp(argv[argno], "--endpoint-url=", (size_t)15)) {
#ifdef H5_HAVE_ROS3_VFD
char const *start = NULL;
start = strchr(argv[argno], '=');
if (start == NULL) {
fprintf(rawerrorstream, "Error: Unable to parse null endpoint url\n");
usage();
leave(EXIT_FAILURE);
}
start++;
snprintf(ros3_fa->ep_url, H5FD_ROS3_MAX_ENDPOINT_URL_LEN + 1, "%s", start);
#else
fprintf(rawerrorstream, "Error: Read-Only S3 VFD is not available unless enabled when HDF5 is "
"configured and built.\n\n");
@@ -2963,13 +2988,13 @@ main(int argc, char *argv[])
leave(EXIT_FAILURE);
}
if (h5tools_parse_hdfs_fapl_tuple(start, ',', &hdfs_fa) < 0) {
if (h5tools_parse_hdfs_fapl_tuple(start, ',', hdfs_fa) < 0) {
fprintf(rawerrorstream, "Error: failed to parse HDFS VFD configuration info\n\n");
usage();
leave(EXIT_FAILURE);
}
vfd_info.info = &hdfs_fa;
vfd_info.info = hdfs_fa;
#else
fprintf(
rawerrorstream,
@@ -3074,15 +3099,15 @@ main(int argc, char *argv[])
/* Setup a custom fapl for file accesses */
#ifdef H5_HAVE_ROS3_VFD
if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[ROS3_VFD_IDX]))) {
if (!vfd_info.info)
vfd_info.info = &ros3_fa;
if (custom_vfd_fapl && !vfd_info.info) {
if (vfd_info.type == VFD_BY_NAME && 0 == strcmp(vfd_info.u.name, drivernames[ROS3_VFD_IDX]))
vfd_info.info = ros3_fa;
}
#endif
#ifdef H5_HAVE_LIBHDFS
if (custom_vfd_fapl && (0 == strcmp(vfd_info.u.name, drivernames[HDFS_VFD_IDX]))) {
if (!vfd_info.info)
vfd_info.info = &hdfs_fa;
if (custom_vfd_fapl && !vfd_info.info) {
if (vfd_info.type == VFD_BY_NAME && 0 == strcmp(vfd_info.u.name, drivernames[HDFS_VFD_IDX]))
vfd_info.info = hdfs_fa;
}
#endif
@@ -3257,6 +3282,13 @@ main(int argc, char *argv[])
}
}
#ifdef H5_HAVE_ROS3_VFD
free(ros3_fa);
#endif
#ifdef H5_HAVE_LIBHDFS
free(hdfs_fa);
#endif
if (err_exit)
leave(EXIT_FAILURE);
else
tools/src/h5ls/h5ls.h
+3
View File
@@ -73,6 +73,9 @@
* \li <strong>--page-buffer-size=N</strong> Set the page buffer cache size, N=non-negative integers
* \li <strong>--vfd=DRIVER</strong> Use the specified virtual file driver
* \li <strong>--hexdump</strong> Show raw data in hexadecimal format
* \li <strong>--endpoint-url=P</strong> Supply S3 endpoint url information to "ros3" vfd.
* P is the AWS service endpoint.
* Has no effect if vfd flag not set to "ros3".
* \li <strong>--s3-cred=C</strong> Supply S3 authentication information to "ros3" vfd.
* Accepts tuple of \code (<aws-region>,<access-id>,<access-key>) \endcode
* or \code (<aws-region>,<access-id>,<access-key>,<session-token>) \endcode.
tools/src/h5stat/h5stat.c
+260 -136
View File
@@ -115,35 +115,20 @@ typedef struct iter_t {
int local; /* Flag to indicate iteration over the object*/
} iter_t;
static const char *drivername = NULL;
size_t page_cache = 0;
#ifdef H5_HAVE_ROS3_VFD
/* Default "anonymous" S3 configuration */
static H5FD_ros3_fapl_ext_t ros3_fa = {
{
1, /* Structure Version */
false, /* Authenticate? */
"", /* AWS Region */
"", /* Access Key ID */
"", /* Secret Access Key */
},
"", /* Session/security token */
};
#endif /* H5_HAVE_ROS3_VFD */
static bool use_custom_vol_g = false;
static bool use_custom_vfd_g = false;
static h5tools_vol_info_t vol_info_g = {0};
static h5tools_vfd_info_t vfd_info_g = {0};
#ifdef H5_HAVE_ROS3_VFD
static H5FD_ros3_fapl_ext_t *ros3_fa_g = NULL;
#endif
#ifdef H5_HAVE_LIBHDFS
/* "Default" HDFS configuration */
static H5FD_hdfs_fapl_t hdfs_fa = {
1, /* Structure Version */
"localhost", /* Namenode Name */
0, /* Namenode Port */
"", /* Kerberos ticket cache */
"", /* User name */
2048, /* Stream buffer size */
};
#endif /* H5_HAVE_LIBHDFS */
static H5FD_hdfs_fapl_t *hdfs_fa_g = NULL;
#endif
static int display_all = true;
@@ -173,7 +158,7 @@ struct handler_t {
char **obj;
};
static const char *s_opts = "Aa:Ddm:E*FfhGgl:K:sSTO:Vw:H:";
static const char *s_opts = "a:dfghl:m:sw:ADE*FGH:K:O:STV";
/* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */
static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'},
{"filemetadata", no_arg, 'F'},
@@ -192,76 +177,133 @@ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'},
{"page-buffer-size", require_arg, 'K'},
{"s3-cred", require_arg, 'w'},
{"hdfs-attrs", require_arg, 'H'},
{"endpoint-url", require_arg, 'y'},
{"vol-value", require_arg, '1'},
{"vol-name", require_arg, '2'},
{"vol-info", require_arg, '3'},
{"vfd-value", require_arg, '4'},
{"vfd-name", require_arg, '5'},
{"vfd-info", require_arg, '6'},
{NULL, 0, '\0'}};
/*-------------------------------------------------------------------------
* Function: leave
*
* Purpose: Shutdown HDF5 and call exit()
*
* Return: Does not return
*-------------------------------------------------------------------------
*/
static void
leave(int ret)
{
h5tools_close();
exit(ret);
}
/*-------------------------------------------------------------------------
* Function: usage
* Function: usage
*
* Purpose: Compute the ceiling of log_10(x)
*
* Return: >0 on success, 0 on failure
* Purpose: Print the usage message about stat
*
* Return: void
*-------------------------------------------------------------------------
*/
static void
usage(const char *prog)
{
fflush(rawoutstream);
fprintf(rawoutstream, "usage: %s [OPTIONS] file\n", prog);
fprintf(rawoutstream, "\n");
fprintf(rawoutstream, " ERROR\n");
fprintf(rawoutstream,
" --enable-error-stack Prints messages from the HDF5 error stack as they occur\n");
fprintf(rawoutstream, " Optional value 2 also prints file open errors\n");
fprintf(rawoutstream, " OPTIONS\n");
fprintf(rawoutstream, " -h, --help Print a usage message and exit\n");
fprintf(rawoutstream, " -V, --version Print version number and exit\n");
fprintf(rawoutstream, " -f, --file Print file information\n");
fprintf(rawoutstream, " -F, --filemetadata Print file space information for file's metadata\n");
fprintf(rawoutstream, " -g, --group Print group information\n");
fprintf(rawoutstream, " -l N, --links=N Set the threshold for the # of links when printing\n");
fprintf(rawoutstream,
" information for small groups. N is an integer greater\n");
fprintf(rawoutstream, " than 0. The default threshold is 10.\n");
fprintf(rawoutstream, " -G, --groupmetadata Print file space information for groups' metadata\n");
fprintf(rawoutstream, " -d, --dset Print dataset information\n");
fprintf(rawoutstream,
" -m N, --dims=N Set the threshold for the dimension sizes when printing\n");
fprintf(rawoutstream,
" information for small datasets. N is an integer greater\n");
fprintf(rawoutstream, " than 0. The default threshold is 10.\n");
fprintf(rawoutstream, " -D, --dsetmetadata Print file space information for datasets' metadata\n");
fprintf(rawoutstream, " -T, --dtypemetadata Print datasets' datatype information\n");
fprintf(rawoutstream, " -A, --attribute Print attribute information\n");
fprintf(rawoutstream,
" -a N, --numattrs=N Set the threshold for the # of attributes when printing\n");
fprintf(rawoutstream,
" information for small # of attributes. N is an integer greater\n");
fprintf(rawoutstream, " than 0. The default threshold is 10.\n");
fprintf(rawoutstream, " -s, --freespace Print free space information\n");
fprintf(rawoutstream, " -S, --summary Print summary of file space information\n");
fprintf(rawoutstream,
" --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
fprintf(rawoutstream, " --s3-cred=<cred> Access file on S3, using provided credentials\n");
fprintf(
FLUSHSTREAM(rawoutstream);
PRINTSTREAM(rawoutstream, "usage: %s [OPTIONS] file\n", prog);
PRINTVALSTREAM(rawoutstream, " OPTIONS\n");
PRINTVALSTREAM(rawoutstream, " -h, --help Print a usage message and exit\n");
PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n");
PRINTVALSTREAM(rawoutstream, "--------------- Error Options ---------------\n");
PRINTVALSTREAM(rawoutstream,
" --enable-error-stack Prints messages from the HDF5 error stack as they occur.\n");
PRINTVALSTREAM(rawoutstream,
" Optional value 2 also prints file open errors.\n");
PRINTVALSTREAM(rawoutstream, " Default setting disables any error reporting.\n");
PRINTVALSTREAM(rawoutstream, "--------------- File Options ---------------\n");
PRINTVALSTREAM(rawoutstream, " -f, --file Print file information\n");
PRINTVALSTREAM(rawoutstream,
" -F, --filemetadata Print file space information for file's metadata\n");
PRINTVALSTREAM(rawoutstream, " -s, --freespace Print free space information\n");
PRINTVALSTREAM(rawoutstream, " -S, --summary Print summary of file space information\n");
PRINTVALSTREAM(rawoutstream,
" --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
PRINTVALSTREAM(rawoutstream,
" --endpoint-url=P Supply S3 endpoint url information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream, " P is the AWS service endpoint.\n");
PRINTVALSTREAM(rawoutstream, " Has no effect if filedriver is not \"ros3\".\n");
PRINTVALSTREAM(rawoutstream,
" --s3-cred=<cred> Supply S3 authentication information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream,
" <cred> :: \"(<aws-region>,<access-id>,<access-key>)\"\n");
PRINTVALSTREAM(
rawoutstream,
" <cred> :: (region,id,key) or <cred> :: (region,id,key,session token)\n");
fprintf(rawoutstream, " If <cred> == \"(,,)\" or <cred> == \"(,,,)\", no "
"authentication is used.\n");
fprintf(rawoutstream, " --hdfs-attrs=<attrs> Access a file on HDFS with given configuration\n");
fprintf(rawoutstream, " attributes.\n");
fprintf(rawoutstream, " <attrs> :: (<namenode name>,<namenode port>,\n");
fprintf(rawoutstream, " <kerberos cache path>,<username>,\n");
fprintf(rawoutstream, " <buffer size>)\n");
fprintf(rawoutstream, " If an attribute is empty, a default value will be\n");
fprintf(rawoutstream, " used.\n");
" <cred> :: \"(<aws-region>,<access-id>,<access-key>,<session-token>)\"\n");
PRINTVALSTREAM(rawoutstream, " If absent, <cred> -> \"(,,)\" or <cred> -> "
"\"(,,,)\", no authentication.\n");
PRINTVALSTREAM(rawoutstream, " Has no effect if filedriver is not \"ros3\".\n");
PRINTVALSTREAM(rawoutstream,
" --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.\n");
PRINTVALSTREAM(rawoutstream, " For use with \"--filedriver=hdfs\"\n");
PRINTVALSTREAM(rawoutstream, " <attrs> :: (<namenode name>,<namenode port>,\n");
PRINTVALSTREAM(rawoutstream, " <kerberos cache path>,<username>,\n");
PRINTVALSTREAM(rawoutstream, " <buffer size>)\n");
PRINTVALSTREAM(rawoutstream,
" Any absent attribute will use a default value.\n");
PRINTVALSTREAM(rawoutstream,
" --vol-value Value (ID) of the VOL connector to use for opening the\n");
PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n");
PRINTVALSTREAM(rawoutstream,
" --vol-name Name of the VOL connector to use for opening the\n");
PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n");
PRINTVALSTREAM(rawoutstream,
" --vol-info VOL-specific info to pass to the VOL connector used for\n");
PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n");
PRINTVALSTREAM(
rawoutstream,
" If none of the above options are used to specify a VOL, then\n");
PRINTVALSTREAM(
rawoutstream,
" the VOL named by HDF5_VOL_CONNECTOR (or the native VOL connector,\n");
PRINTVALSTREAM(rawoutstream,
" if that environment variable is unset) will be used\n");
PRINTVALSTREAM(rawoutstream,
" --vfd-value Value (ID) of the VFL driver to use for opening the\n");
PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n");
PRINTVALSTREAM(rawoutstream, " --vfd-name Name of the VFL driver to use for opening the\n");
PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n");
PRINTVALSTREAM(rawoutstream,
" --vfd-info VFD-specific info to pass to the VFL driver used for\n");
PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n");
PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n");
PRINTVALSTREAM(rawoutstream, " -g, --group Print group information\n");
PRINTVALSTREAM(rawoutstream,
" -l N, --links=N Set the threshold for the # of links when printing\n");
PRINTVALSTREAM(rawoutstream,
" information for small groups. N is an integer greater\n");
PRINTVALSTREAM(rawoutstream, " than 0. The default threshold is 10.\n");
PRINTVALSTREAM(rawoutstream,
" -G, --groupmetadata Print file space information for groups' metadata\n");
PRINTVALSTREAM(rawoutstream, " -d, --dset Print dataset information\n");
PRINTVALSTREAM(rawoutstream,
" -m N, --dims=N Set the threshold for the dimension sizes when printing\n");
PRINTVALSTREAM(rawoutstream,
" information for small datasets. N is an integer greater\n");
PRINTVALSTREAM(rawoutstream, " than 0. The default threshold is 10.\n");
PRINTVALSTREAM(rawoutstream,
" -D, --dsetmetadata Print file space information for datasets' metadata\n");
PRINTVALSTREAM(rawoutstream, " -T, --dtypemetadata Print datasets' datatype information\n");
PRINTVALSTREAM(rawoutstream, " -A, --attribute Print attribute information\n");
PRINTVALSTREAM(rawoutstream,
" -a N, --numattrs=N Set the threshold for the # of attributes when printing\n");
PRINTVALSTREAM(
rawoutstream,
" information for small # of attributes. N is an integer greater\n");
PRINTVALSTREAM(rawoutstream, " than 0. The default threshold is 10.\n");
}
/*-------------------------------------------------------------------------
@@ -282,7 +324,7 @@ ceil_log10(unsigned long x)
while (x >= pow10) {
pow10 *= 10;
ret++;
} /* end while */
}
return ret;
} /* ceil_log10() */
@@ -326,7 +368,7 @@ attribute_stats(iter_t *iter, const H5O_info2_t *oi, const H5O_native_info_t *na
/* Initialize count for new bin */
iter->attr_bins[bin] = 1;
} /* end if */
}
else
(iter->attr_bins[bin])++;
@@ -384,7 +426,7 @@ group_stats(iter_t *iter, const char *name, const H5O_info2_t *oi, const H5O_nat
/* Initialize count for new bin */
iter->group_bins[bin] = 1;
} /* end if */
}
else
(iter->group_bins[bin])++;
@@ -517,10 +559,10 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info2_t *oi, const H5O_n
/* Initialize count for this bin */
iter->dset_dim_bins[bin] = 1;
} /* end if */
}
else
(iter->dset_dim_bins[bin])++;
} /* end if */
}
if (H5Sclose(sid) < 0)
H5TOOLS_GOTO_ERROR(FAIL, "H5Sclose() failed");
@@ -534,7 +576,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info2_t *oi, const H5O_n
if (H5Tequal(iter->dset_type_info[u].tid, tid) > 0) {
type_found = true;
break;
} /* end for */
}
if (type_found)
(iter->dset_type_info[u].count)++;
@@ -557,7 +599,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info2_t *oi, const H5O_n
/* Set index for later */
u = curr_ntype;
} /* end else */
}
/* Check if the datatype is a named datatype */
if (H5Tcommitted(tid) > 0)
@@ -670,8 +712,8 @@ obj_stats(const char *path, const H5O_info2_t *oi, const char *already_visited,
/* Gather statistics about this type of object */
iter->uniq_others++;
break;
} /* end switch */
} /* end if */
}
}
done:
return ret_value;
@@ -706,7 +748,7 @@ lnk_stats(const char H5_ATTR_UNUSED *path, const H5L_info2_t *li, void *_iter)
/* Gather statistics about this type of object */
iter->uniq_others++;
break;
} /* end switch() */
}
return 0;
} /* end lnk_stats() */
@@ -736,7 +778,7 @@ freespace_stats(hid_t fid, iter_t *iter)
return (FAIL);
nsects = H5Fget_free_sections(fid, H5FD_MEM_DEFAULT, (size_t)nsects, sect_info);
assert(nsects);
} /* end else-if */
}
for (u = 0; u < (size_t)nsects; u++) {
unsigned bin; /* "bin" the number of objects falls in */
@@ -758,7 +800,7 @@ freespace_stats(hid_t fid, iter_t *iter)
/* Initialize count for this bin */
iter->sect_bins[bin] = 1;
} /* end if */
}
else
(iter->sect_bins[bin])++;
} /* end for */
@@ -774,10 +816,7 @@ freespace_stats(hid_t fid, iter_t *iter)
*
* Purpose: Free handler structure
*
* Return: Success: 0
*
* Failure: Never fails
*
* Return: Nothing
*-------------------------------------------------------------------------
*/
static void
@@ -790,11 +829,11 @@ hand_free(struct handler_t *hand)
if (hand->obj[u]) {
free(hand->obj[u]);
hand->obj[u] = NULL;
} /* end if */
}
hand->obj_count = 0;
free(hand->obj);
free(hand);
} /* end if */
}
} /* end hand_free() */
/*-------------------------------------------------------------------------
@@ -815,6 +854,12 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re
unsigned u;
struct handler_t *hand = NULL;
/* no arguments */
if (argc == 1) {
usage(h5tools_getprogname());
goto error;
}
/* parse command line options */
while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) {
switch ((char)opt) {
@@ -934,31 +979,42 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re
if (NULL == (hand = (struct handler_t *)calloc((size_t)1, sizeof(struct handler_t)))) {
error_msg("unable to allocate memory for object struct\n");
goto error;
} /* end if */
}
/* Allocate space to hold the object strings */
hand->obj_count = (size_t)argc;
if (NULL == (hand->obj = (char **)calloc((size_t)argc, sizeof(char *)))) {
error_msg("unable to allocate memory for object array\n");
goto error;
} /* end if */
}
/* Store object names */
for (u = 0; u < hand->obj_count; u++)
if (NULL == (hand->obj[u] = strdup(H5_optarg))) {
error_msg("unable to allocate memory for object name\n");
goto error;
} /* end if */
}
break;
case 'y':
#ifdef H5_HAVE_ROS3_VFD
snprintf(ros3_fa_g->ep_url, H5FD_ROS3_MAX_ENDPOINT_URL_LEN + 1, "%s", H5_optarg);
#else
error_msg(
"Read-Only S3 VFD is not available unless enabled when HDF5 is configured and built.\n");
goto error;
#endif
break;
case 'w':
#ifdef H5_HAVE_ROS3_VFD
if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa) < 0) {
if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', ros3_fa_g) < 0) {
error_msg("failed to parse S3 VFD credential info\n");
usage(h5tools_getprogname());
goto error;
}
drivername = drivernames[ROS3_VFD_IDX];
vfd_info_g.info = ros3_fa_g;
#else
error_msg(
"Read-Only S3 VFD is not available unless enabled when HDF5 is configured and built.\n");
@@ -968,12 +1024,13 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re
case 'H':
#ifdef H5_HAVE_LIBHDFS
if (h5tools_parse_hdfs_fapl_tuple(H5_optarg, ',', &hdfs_fa) < 0) {
if (h5tools_parse_hdfs_fapl_tuple(H5_optarg, ',', hdfs_fa_g) < 0) {
error_msg("failed to parse HDFS VFD configuration info\n");
usage(h5tools_getprogname());
goto error;
}
drivername = drivernames[HDFS_VFD_IDX];
vfd_info_g.info = hdfs_fa_g;
#else
error_msg("HDFS VFD is not available unless enabled when HDF5 is configured and built.\n");
goto error;
@@ -984,18 +1041,63 @@ parse_command_line(int argc, const char *const *argv, struct handler_t **hand_re
page_cache = strtoul(H5_optarg, NULL, 0);
break;
case '1':
vol_info_g.type = VOL_BY_VALUE;
vol_info_g.u.value = (H5VL_class_value_t)atoi(H5_optarg);
use_custom_vol_g = true;
break;
case '2':
vol_info_g.type = VOL_BY_NAME;
vol_info_g.u.name = H5_optarg;
use_custom_vol_g = true;
break;
case '3':
vol_info_g.info_string = H5_optarg;
break;
case '4':
vfd_info_g.type = VFD_BY_VALUE;
vfd_info_g.u.value = (H5FD_class_value_t)atoi(H5_optarg);
use_custom_vfd_g = true;
break;
case '5':
vfd_info_g.type = VFD_BY_NAME;
vfd_info_g.u.name = H5_optarg;
use_custom_vfd_g = true;
break;
case '6':
vfd_info_g.info = (const void *)H5_optarg;
break;
default:
usage(h5tools_getprogname());
goto error;
} /* end switch */
} /* end while */
}
}
#ifdef H5_HAVE_ROS3_VFD
if (use_custom_vfd_g && !vfd_info_g.info) {
if (vfd_info_g.type == VFD_BY_NAME && 0 == strcmp(vfd_info_g.u.name, drivernames[ROS3_VFD_IDX]))
vfd_info_g.info = ros3_fa_g;
}
#endif
#ifdef H5_HAVE_LIBHDFS
if (use_custom_vfd_g && !vfd_info_g.info) {
if (vfd_info_g.type == VFD_BY_NAME && 0 == strcmp(vfd_info_g.u.name, drivernames[HDFS_VFD_IDX]))
vfd_info_g.info = hdfs_fa_g;
}
#endif
/* check for file name to be processed */
if (argc <= H5_optind) {
error_msg("missing file name\n");
usage(h5tools_getprogname());
goto error;
} /* end if */
}
/* Set handler structure */
*hand_ret = hand;
@@ -1635,10 +1737,35 @@ main(int argc, char *argv[])
memset(&iter, 0, sizeof(iter));
/* Initialize VFD-specific structures */
#ifdef H5_HAVE_ROS3_VFD
if (NULL == (ros3_fa_g = calloc(1, sizeof(*ros3_fa_g)))) {
error_msg("unable to allocate space for configuration structure\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
/* Default "anonymous" S3 configuration */
ros3_fa_g->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION;
ros3_fa_g->fa.authenticate = false;
#endif
#ifdef H5_HAVE_LIBHDFS
if (NULL == (hdfs_fa_g = calloc(1, sizeof(*hdfs_fa_g)))) {
error_msg("unable to allocate space for configuration structure\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
/* "Default" HDFS configuration */
hdfs_fa_g->version = H5FD__CURR_HDFS_FAPL_T_VERSION;
hdfs_fa_g->stream_buffer_size = 2048;
strcpy(hdfs_fa_g->namenode_name, "localhost");
#endif
if (parse_command_line(argc, (const char *const *)argv, &hand) < 0)
goto done;
/* enable error reporting if command line option */
/* Enable error reporting if command line option */
h5tools_error_report();
if ((fapl_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) {
@@ -1646,24 +1773,17 @@ main(int argc, char *argv[])
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
if (drivername) {
h5tools_vfd_info_t vfd_info;
vfd_info.type = VFD_BY_NAME;
vfd_info.info = NULL;
vfd_info.u.name = drivername;
#ifdef H5_HAVE_ROS3_VFD
if (!strcmp(drivername, drivernames[ROS3_VFD_IDX]))
vfd_info.info = &ros3_fa;
#endif
#ifdef H5_HAVE_LIBHDFS
if (!strcmp(drivername, drivernames[HDFS_VFD_IDX]))
vfd_info.info = &hdfs_fa;
#endif
/* Set non-default virtual file driver, if requested */
if (h5tools_set_fapl_vfd(fapl_id, &vfd_info) < 0) {
/* Set non-default VOL connector, if requested */
if (use_custom_vol_g) {
if (h5tools_set_fapl_vol(fapl_id, &vol_info_g) < 0) {
error_msg("unable to set VOL on fapl for file\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
}
/* Set non-default virtual file driver, if requested */
if (use_custom_vfd_g) {
if (h5tools_set_fapl_vfd(fapl_id, &vfd_info_g) < 0) {
error_msg("unable to set VFD on fapl for file\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
@@ -1686,13 +1806,13 @@ main(int argc, char *argv[])
fprintf(rawoutstream, "Filename: %s\n", fname);
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (drivername != NULL), NULL, 0);
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (use_custom_vol_g || use_custom_vfd_g), NULL, 0);
if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
goto done;
} /* end if */
}
/* Initialize iter structure */
iter.fid = fid;
@@ -1771,17 +1891,21 @@ done:
/* Free iter structure */
iter_free(&iter);
if (fapl_id != H5P_DEFAULT) {
if (H5Pclose(fapl_id) < 0) {
error_msg("unable to close fapl entry\n");
h5tools_setstatus(EXIT_FAILURE);
}
#ifdef H5_HAVE_ROS3_VFD
free(ros3_fa_g);
#endif
#ifdef H5_HAVE_LIBHDFS
free(hdfs_fa_g);
#endif
if (fapl_id != H5P_DEFAULT && H5Pclose(fapl_id) < 0) {
error_msg("unable to close fapl entry\n");
h5tools_setstatus(EXIT_FAILURE);
}
if (fid >= 0 && H5Fclose(fid) < 0) {
error_msg("unable to close file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
} /* end if */
if (fid >= 0)
if (H5Fclose(fid) < 0)
h5tools_setstatus(EXIT_FAILURE);
leave(h5tools_getstatus());
} /* end main() */
tools/src/h5stat/h5stat.h
+18
View File
@@ -52,6 +52,9 @@
* \li <strong>--freespace</strong> Print free space information
* \li <strong>--summary</strong> Print summary of file space information
* \li <strong>--page-buffer-size=N</strong> Set the page buffer cache size, N=non-negative integers
* \li <strong>--endpoint-url=P</strong> Supply S3 endpoint url information to "ros3" vfd.
* P is the AWS service endpoint.
* Has no effect if vfd flag not set to "ros3".
* \li <strong>--s3-cred=C</strong> Supply S3 authentication information to "ros3" vfd.
* Accepts tuple of \code (<aws-region>,<access-id>,<access-key>) \endcode
* or \code (<aws-region>,<access-id>,<access-key>,<session-token>) \endcode.
@@ -62,6 +65,21 @@
* If absent or A == \code (,,,,) \endcode all default values are used.
* Has no effect if vfd flag is not 'hdfs'.<br />
* If an attribute is empty, a default value will be used.
* \li <strong>--vol-value</strong> Value (ID) of the VOL connector to use for opening the
* HDF5 file specified
* \li <strong>--vol-name</strong> Name of the VOL connector to use for opening the
* HDF5 file specified
* \li <strong>--vol-info</strong> VOL-specific info to pass to the VOL connector used for
* opening the HDF5 file specified
* If none of the above options are used to specify a VOL, then
* the VOL named by HDF5_VOL_CONNECTOR (or the native VOL connector,
* if that environment variable is unset) will be used
* \li <strong>--vfd-value</strong> Value (ID) of the VFL driver to use for opening the
* HDF5 file specified
* \li <strong>--vfd-name</strong> Name of the VFL driver to use for opening the
* HDF5 file specified
* \li <strong>--vfd-info</strong> VFD-specific info to pass to the VFL driver used for
* opening the HDF5 file specified
*
* Previous Chapter \ref sec_cltools_h5repack - Next Chapter \ref sec_cltools_h5clear
*
tools/test/h5dump/CMakeTests.cmake
+108
View File
@@ -377,8 +377,14 @@
tst_onion_dset_1d.h5.onion
)
set (H5DUMP_S3PROXY_TEST_FILES
tattrintsize.h5
)
# make test dir
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST")
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST/testfiles")
#
# copy test files from source dir to test dir
@@ -399,6 +405,10 @@
HDFTEST_COPY_FILE("${HDF5_TOOLS_TST_DIR}/h5dump/expected/${tst_h5N_file}" "${PROJECT_BINARY_DIR}/testfiles/std/${tst_h5N_file}-N" "h5dump_std_files")
endforeach ()
foreach (tst_s3_file ${H5DUMP_S3PROXY_TEST_FILES})
HDFTEST_COPY_FILE("${HDF5_TOOLS_TST_DIR}/testfiles/${tst_s3_file}" "${PROJECT_BINARY_DIR}/S3TEST/testfiles/${tst_s3_file}" "h5dump_std_files")
endforeach ()
# --------------------------------------------------------------------
# Special file handling
# --------------------------------------------------------------------
@@ -988,6 +998,42 @@
endif ()
endmacro ()
macro (ADD_H5_S3TEST resultfile resultcode credtype urlscheme urlpath)
# If using memchecker add tests without using scripts
if (HDF5_ENABLE_USING_MEMCHECKER)
add_test (NAME H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5dump> ${ARGN})
if (${resultcode})
set_tests_properties (H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES WILL_FAIL "true")
endif ()
set_tests_properties (H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES
WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST"
)
else ()
add_test (
NAME H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5dump>"
-D "TEST_ARGS:STRING=--enable-error-stack=2;${ARGN};${urlscheme}://${urlpath}/${resultfile}.h5"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-D "TEST_OUTPUT=${resultfile}_${urlscheme}_${credtype}.out"
-D "TEST_EXPECT=${resultcode}"
-D "TEST_REFERENCE=${resultfile}.ddl"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-P "${HDF_RESOURCES_DIR}/runTest.cmake"
)
endif ()
set_tests_properties (H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES
FIXTURES_REQUIRED h5dump_s3_proxy
ENVIRONMENT "${h5dump_s3tests_env}"
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/S3TEST
)
if ("H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5DUMP_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES DISABLED true)
endif ()
endmacro ()
##############################################################################
##############################################################################
### T H E T E S T S ###
@@ -1461,3 +1507,65 @@ endif ()
if (HDF5_TEST_VFD)
include (CMakeVFDTests.cmake)
endif ()
##############################################################
##############################################################
### S 3 T E S T S ###
##############################################################
##############################################################
if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/buckets")
set (h5dump_s3tests_port 9002)
# Setup environment for tests.
# The AWS_ENDPOINT_URL environment variable is set to work
# around an issue in aws-c-s3 when using localhost URLs
# directly.
# The HDF5_ROS3_VFD_FORCE_PATH_STYLE environment variable is
# set to force the ROS3 VFD to use path-style requests for
# compatibility with s3proxy.
# AWS region is required by the ROS3 VFD - set a default to
# use when one isn't supplied
# AWS_PROFILE is set in order to use the correct testing
# credentials created in CMakeTests.cmake
set (h5dump_s3tests_env
"AWS_ENDPOINT_URL=http://localhost:${h5dump_s3tests_port}"
"HDF5_ROS3_VFD_FORCE_PATH_STYLE=1"
"AWS_REGION=us-east-2"
"AWS_PROFILE=ros3_vfd_test"
)
add_test (
NAME H5DUMP-start-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_PRODUCT=andrewgaul/s3proxy"
-D "TEST_PORT=${h5dump_s3tests_port}"
-D "TEST_ARGS:STRING=s3proxy-local-h5dump"
-D "TEST_BUCKET:STRING=h5dumpros3"
-D "TEST_FILES:STRING=tattrintsize.h5"
-D "TEST_ACLS:STRING=anon"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-P "${HDF_RESOURCES_DIR}/runProxy.cmake"
)
set_tests_properties (H5DUMP-start-proxy PROPERTIES FIXTURES_SETUP h5dump_s3_proxy)
add_test (
NAME H5DUMP-stop-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_ARGS:STRING=s3proxy-local-h5dump"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-P "${HDF_RESOURCES_DIR}/stopProxy.cmake"
)
set_tests_properties (H5DUMP-stop-proxy PROPERTIES FIXTURES_CLEANUP h5dump_s3_proxy)
ADD_H5_S3TEST (tattrintsize 0 anon http localhost:${h5dump_s3tests_port}/h5dumpros3 --vfd-name=ros3 --s3-cred=\(,,\))
ADD_H5_S3TEST (tattrintsize 0 anon s3 h5dumpros3 --filedriver=ros3 --vfd-name=ros3 --s3-cred=\(,,\) --endpoint-url=http://localhost:${h5dump_s3tests_port})
ADD_H5_S3TEST (tattrintsize 0 profile http localhost:${h5dump_s3tests_port}/h5dumpros3 --vfd-name=ros3)
ADD_H5_S3TEST (tattrintsize 0 profile s3 h5dumpros3 --filedriver=ros3 --vfd-name=ros3 --endpoint-url=http://localhost:${h5dump_s3tests_port})
endif ()
tools/test/h5dump/expected/h5dump-help.txt
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tnofilename-with-packed-bits.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tpbitsIncomplete.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tpbitsLengthExceeded.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tpbitsLengthPositive.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tpbitsMaxExceeded.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tpbitsOffsetExceeded.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5dump/expected/pbits/tpbitsOffsetNegative.ddl
+3
View File
@@ -17,6 +17,9 @@ usage: h5dump [OPTIONS] files
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
tools/test/h5ls/CMakeTests.cmake
+107
View File
@@ -128,7 +128,13 @@
tvldtypes2be.ls
)
set (H5LS_S3PROXY_TEST_FILES
tgroup.h5
)
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST")
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST/testfiles")
# copy the list of test files
foreach (listlsfiles ${LIST_HDF5_TESTLS_FILES})
@@ -140,8 +146,12 @@
foreach (listothers ${LIST_OTHER_TEST_FILES})
HDFTEST_COPY_FILE("${HDF5_TOOLS_TST_DIR}/h5ls/expected/${listothers}" "${PROJECT_BINARY_DIR}/testfiles/${listothers}" "h5ls_files")
endforeach ()
foreach (lists3file ${H5LS_S3PROXY_TEST_FILES})
HDFTEST_COPY_FILE("${HDF5_TOOLS_TST_DIR}/testfiles/${lists3file}" "${PROJECT_BINARY_DIR}/S3TEST/testfiles/${lists3file}" "h5ls_files")
endforeach ()
add_custom_target(h5ls_files ALL COMMENT "Copying files needed by h5ls tests" DEPENDS ${h5ls_files_list})
##############################################################################
##############################################################################
### T H E T E S T S M A C R O S ###
@@ -239,6 +249,42 @@
endif ()
endmacro ()
macro (ADD_H5_S3TEST resultfile resultcode credtype urlscheme urlpath)
# If using memchecker add tests without using scripts
if (HDF5_ENABLE_USING_MEMCHECKER)
add_test (NAME H5LS_S3TEST-${resultfile}_${urlscheme}_${credtype} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5ls> ${ARGN})
set_tests_properties (H5LS-${resultfile}_${urlscheme}_${credtype} PROPERTIES
WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST"
)
if ("${resultcode}" STREQUAL "1")
set_tests_properties (H5LS_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES WILL_FAIL "true")
endif ()
else ()
add_test (
NAME H5LS_S3TEST-${resultfile}_${urlscheme}_${credtype}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5ls>"
-D "TEST_ARGS=--enable-error-stack;${ARGN};${urlscheme}://${urlpath}/${resultfile}.h5"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-D "TEST_OUTPUT=${resultfile}_${urlscheme}_${credtype}.out"
-D "TEST_EXPECT=${resultcode}"
-D "TEST_REFERENCE=${resultfile}.ls"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-P "${HDF_RESOURCES_DIR}/runTest.cmake"
)
endif ()
set_tests_properties (H5LS_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES
FIXTURES_REQUIRED h5ls_s3_proxy
ENVIRONMENT "${h5ls_s3tests_env}"
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/S3TEST
)
if ("H5LS_S3TEST-${resultfile}_${urlscheme}_${credtype}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5LS_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES DISABLED true)
endif ()
endmacro ()
##############################################################################
##############################################################################
### T H E T E S T S ###
@@ -472,6 +518,67 @@
endif ()
##############################################################
##############################################################
### S 3 T E S T S ###
##############################################################
##############################################################
if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/buckets")
set (h5ls_s3tests_port 9003)
# Setup environment for tests.
# The AWS_ENDPOINT_URL environment variable is set to work
# around an issue in aws-c-s3 when using localhost URLs
# directly.
# The HDF5_ROS3_VFD_FORCE_PATH_STYLE environment variable is
# set to force the ROS3 VFD to use path-style requests for
# compatibility with s3proxy.
# AWS region is required by the ROS3 VFD - set a default to
# use when one isn't supplied
# AWS_PROFILE is set in order to use the correct testing
# credentials created in CMakeTests.cmake
set (h5ls_s3tests_env
"AWS_ENDPOINT_URL=http://localhost:${h5ls_s3tests_port}"
"HDF5_ROS3_VFD_FORCE_PATH_STYLE=1"
"AWS_REGION=us-east-2"
"AWS_PROFILE=ros3_vfd_test"
)
add_test (
NAME H5LS-start-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_PRODUCT=andrewgaul/s3proxy"
-D "TEST_PORT=${h5ls_s3tests_port}"
-D "TEST_ARGS:STRING=s3proxy-local-h5ls"
-D "TEST_BUCKET:STRING=h5lsros3"
-D "TEST_FILES:STRING=tgroup.h5"
-D "TEST_ACLS:STRING=anon"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-P "${HDF_RESOURCES_DIR}/runProxy.cmake"
)
set_tests_properties (H5LS-start-proxy PROPERTIES FIXTURES_SETUP h5ls_s3_proxy)
add_test (
NAME H5LS-stop-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_ARGS:STRING=s3proxy-local-h5ls"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-P "${HDF_RESOURCES_DIR}/stopProxy.cmake"
)
set_tests_properties (H5LS-stop-proxy PROPERTIES FIXTURES_CLEANUP h5ls_s3_proxy)
ADD_H5_S3TEST (tgroup 0 anon http localhost:${h5ls_s3tests_port}/h5lsros3 --vfd-name=ros3 --s3-cred=\(,,\))
ADD_H5_S3TEST (tgroup 0 anon s3 h5lsros3 --vfd-name=ros3 --s3-cred=\(,,\) --endpoint-url=http://localhost:${h5ls_s3tests_port})
ADD_H5_S3TEST (tgroup 0 profile http localhost:${h5ls_s3tests_port}/h5lsros3 --vfd-name=ros3)
ADD_H5_S3TEST (tgroup 0 profile s3 h5lsros3 --vfd-name=ros3 --endpoint-url=http://localhost:${h5ls_s3tests_port})
endif ()
##############################################################################
### P L U G I N T E S T S
##############################################################################
tools/test/h5ls/expected/help-1.ls
+3
View File
@@ -38,6 +38,9 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if vfd flag not set to "ros3".
--s3-cred=C Supply S3 authentication information to "ros3" vfd.
Accepts tuple of "(<aws-region>,<access-id>,<access-key>)" or
"(<aws-region>,<access-id>,<access-key>,<session-token>)".
tools/test/h5ls/expected/help-2.ls
+3
View File
@@ -38,6 +38,9 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if vfd flag not set to "ros3".
--s3-cred=C Supply S3 authentication information to "ros3" vfd.
Accepts tuple of "(<aws-region>,<access-id>,<access-key>)" or
"(<aws-region>,<access-id>,<access-key>,<session-token>)".
tools/test/h5ls/expected/help-3.ls
+3
View File
@@ -38,6 +38,9 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if vfd flag not set to "ros3".
--s3-cred=C Supply S3 authentication information to "ros3" vfd.
Accepts tuple of "(<aws-region>,<access-id>,<access-key>)" or
"(<aws-region>,<access-id>,<access-key>,<session-token>)".
tools/test/h5ls/expected/textlinksrc-nodangle-1.ls
+3
View File
@@ -38,6 +38,9 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if vfd flag not set to "ros3".
--s3-cred=C Supply S3 authentication information to "ros3" vfd.
Accepts tuple of "(<aws-region>,<access-id>,<access-key>)" or
"(<aws-region>,<access-id>,<access-key>,<session-token>)".
tools/test/h5ls/expected/tgroup-1.ls
+3
View File
@@ -38,6 +38,9 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if vfd flag not set to "ros3".
--s3-cred=C Supply S3 authentication information to "ros3" vfd.
Accepts tuple of "(<aws-region>,<access-id>,<access-key>)" or
"(<aws-region>,<access-id>,<access-key>,<session-token>)".
tools/test/h5stat/CMakeTests.cmake
+104 -1
View File
@@ -64,13 +64,22 @@
h5stat_threshold.h5
)
set (H5STAT_S3PROXY_TEST_FILES
h5stat_threshold.h5
)
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST")
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/S3TEST/testfiles")
foreach (ddl_file ${HDF5_REFERENCE_FILES})
HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/expected/${ddl_file}.ddl" "${PROJECT_BINARY_DIR}/${ddl_file}.ddl" "h5stat_files")
endforeach ()
foreach (h5_file ${HDF5_REFERENCE_TEST_FILES})
HDFTEST_COPY_FILE("${HDF5_TOOLS_TST_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "h5stat_files")
endforeach ()
foreach (lists3file ${H5STAT_S3PROXY_TEST_FILES})
HDFTEST_COPY_FILE("${HDF5_TOOLS_TST_DIR}/testfiles/${lists3file}" "${PROJECT_BINARY_DIR}/S3TEST/testfiles/${lists3file}" "h5stat_files")
endforeach ()
add_custom_target(h5stat_files ALL COMMENT "Copying files needed by h5stat tests" DEPENDS ${h5stat_files_list})
##############################################################################
@@ -170,6 +179,39 @@
endif ()
endmacro ()
macro (ADD_H5_S3TEST resultfile resultcode credtype urlscheme urlpath)
# If using memchecker add tests without using scripts
if (HDF5_ENABLE_USING_MEMCHECKER)
add_test (NAME H5STAT_S3TEST-${resultfile}_${urlscheme}_${credtype} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5stat> ${ARGN})
if (${resultcode})
set_tests_properties (H5STAT_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES WILL_FAIL "true")
endif ()
else ()
add_test (
NAME H5STAT_S3TEST-${resultfile}_${urlscheme}_${credtype}
COMMAND "${CMAKE_COMMAND}"
-D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
-D "TEST_PROGRAM=$<TARGET_FILE:h5stat>"
-D "TEST_ARGS=--enable-error-stack=2;${ARGN};${urlscheme}://${urlpath}/${resultfile}.h5"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-D "TEST_OUTPUT=${resultfile}_${urlscheme}_${credtype}.out"
-D "TEST_EXPECT=${resultcode}"
-D "TEST_REFERENCE=${resultfile}.ddl"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-P "${HDF_RESOURCES_DIR}/runTest.cmake"
)
endif ()
set_tests_properties (H5STAT_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES
FIXTURES_REQUIRED h5stat_s3_proxy
ENVIRONMENT "${h5stat_s3tests_env}"
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/S3TEST
)
if ("H5STAT_S3TEST-${resultfile}_${urlscheme}_${credtype}" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
set_tests_properties (H5STAT_S3TEST-${resultfile}_${urlscheme}_${credtype} PROPERTIES DISABLED true)
endif ()
endmacro ()
##############################################################################
##############################################################################
### T H E T E S T S ###
@@ -254,3 +296,64 @@
ADD_H5_CMP_TEST (h5stat_err_old_fill 1 "unable to traverse objects" h5stat_err_old_fill.h5)
#
#
##############################################################
##############################################################
### S 3 T E S T S ###
##############################################################
##############################################################
if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/buckets")
set (h5stat_s3tests_port 9004)
# Setup environment for tests.
# The AWS_ENDPOINT_URL environment variable is set to work
# around an issue in aws-c-s3 when using localhost URLs
# directly.
# The HDF5_ROS3_VFD_FORCE_PATH_STYLE environment variable is
# set to force the ROS3 VFD to use path-style requests for
# compatibility with s3proxy.
# AWS region is required by the ROS3 VFD - set a default to
# use when one isn't supplied
# AWS_PROFILE is set in order to use the correct testing
# credentials created in CMakeTests.cmake
set (h5stat_s3tests_env
"AWS_ENDPOINT_URL=http://localhost:${h5stat_s3tests_port}"
"HDF5_ROS3_VFD_FORCE_PATH_STYLE=1"
"AWS_REGION=us-east-2"
"AWS_PROFILE=ros3_vfd_test"
)
add_test (
NAME H5STAT-start-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_PRODUCT=andrewgaul/s3proxy"
-D "TEST_PORT=${h5stat_s3tests_port}"
-D "TEST_ARGS:STRING=s3proxy-local-h5stat"
-D "TEST_BUCKET:STRING=h5statros3"
-D "TEST_FILES:STRING=h5stat_threshold.h5"
-D "TEST_ACLS:STRING=anon"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
-D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"
-P "${HDF_RESOURCES_DIR}/runProxy.cmake"
)
set_tests_properties (H5STAT-start-proxy PROPERTIES FIXTURES_SETUP h5stat_s3_proxy)
add_test (
NAME H5STAT-stop-proxy
COMMAND "${CMAKE_COMMAND}"
-D "TEST_PROGRAM=${DOCKER_EXECUTABLE}"
-D "TEST_ARGS:STRING=s3proxy-local-h5stat"
-D "TEST_EXPECT=0"
-D "TEST_FOLDER=${PROJECT_BINARY_DIR}/S3TEST"
-P "${HDF_RESOURCES_DIR}/stopProxy.cmake"
)
set_tests_properties (H5STAT-stop-proxy PROPERTIES FIXTURES_CLEANUP h5stat_s3_proxy)
ADD_H5_S3TEST (h5stat_threshold 0 anon http localhost:${h5stat_s3tests_port}/h5statros3 --vfd-name=ros3 --s3-cred=\(,,\))
ADD_H5_S3TEST (h5stat_threshold 0 anon s3 h5statros3 --vfd-name=ros3 --s3-cred=\(,,\) --endpoint-url=http://localhost:${h5stat_s3tests_port})
ADD_H5_S3TEST (h5stat_threshold 0 profile http localhost:${h5stat_s3tests_port}/h5statros3 --vfd-name=ros3)
ADD_H5_S3TEST (h5stat_threshold 0 profile s3 h5statros3 --vfd-name=ros3 --endpoint-url=http://localhost:${h5stat_s3tests_port})
endif ()
tools/test/h5stat/expected/h5stat_help1.ddl
+41 -20
View File
@@ -1,13 +1,47 @@
usage: h5stat [OPTIONS] file
ERROR
--enable-error-stack Prints messages from the HDF5 error stack as they occur
Optional value 2 also prints file open errors
OPTIONS
-h, --help Print a usage message and exit
-V, --version Print version number and exit
OPTIONS
-h, --help Print a usage message and exit
-V, --version Print version number and exit
--------------- Error Options ---------------
--enable-error-stack Prints messages from the HDF5 error stack as they occur.
Optional value 2 also prints file open errors.
Default setting disables any error reporting.
--------------- File Options ---------------
-f, --file Print file information
-F, --filemetadata Print file space information for file's metadata
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
If absent, <cred> -> "(,,)" or <cred> -> "(,,,)", no authentication.
Has no effect if filedriver is not "ros3".
--hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
For use with "--filedriver=hdfs"
<attrs> :: (<namenode name>,<namenode port>,
<kerberos cache path>,<username>,
<buffer size>)
Any absent attribute will use a default value.
--vol-value Value (ID) of the VOL connector to use for opening the
HDF5 file specified
--vol-name Name of the VOL connector to use for opening the
HDF5 file specified
--vol-info VOL-specific info to pass to the VOL connector used for
opening the HDF5 file specified
If none of the above options are used to specify a VOL, then
the VOL named by HDF5_VOL_CONNECTOR (or the native VOL connector,
if that environment variable is unset) will be used
--vfd-value Value (ID) of the VFL driver to use for opening the
HDF5 file specified
--vfd-name Name of the VFL driver to use for opening the
HDF5 file specified
--vfd-info VFD-specific info to pass to the VFL driver used for
opening the HDF5 file specified
--------------- Object Options ---------------
-g, --group Print group information
-l N, --links=N Set the threshold for the # of links when printing
information for small groups. N is an integer greater
@@ -23,16 +57,3 @@ usage: h5stat [OPTIONS] file
-a N, --numattrs=N Set the threshold for the # of attributes when printing
information for small # of attributes. N is an integer greater
than 0. The default threshold is 10.
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--s3-cred=<cred> Access file on S3, using provided credentials
<cred> :: (region,id,key) or <cred> :: (region,id,key,session token)
If <cred> == "(,,)" or <cred> == "(,,,)", no authentication is used.
--hdfs-attrs=<attrs> Access a file on HDFS with given configuration
attributes.
<attrs> :: (<namenode name>,<namenode port>,
<kerberos cache path>,<username>,
<buffer size>)
If an attribute is empty, a default value will be
used.
tools/test/h5stat/expected/h5stat_help2.ddl
+41 -20
View File
@@ -1,13 +1,47 @@
usage: h5stat [OPTIONS] file
ERROR
--enable-error-stack Prints messages from the HDF5 error stack as they occur
Optional value 2 also prints file open errors
OPTIONS
-h, --help Print a usage message and exit
-V, --version Print version number and exit
OPTIONS
-h, --help Print a usage message and exit
-V, --version Print version number and exit
--------------- Error Options ---------------
--enable-error-stack Prints messages from the HDF5 error stack as they occur.
Optional value 2 also prints file open errors.
Default setting disables any error reporting.
--------------- File Options ---------------
-f, --file Print file information
-F, --filemetadata Print file space information for file's metadata
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
If absent, <cred> -> "(,,)" or <cred> -> "(,,,)", no authentication.
Has no effect if filedriver is not "ros3".
--hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
For use with "--filedriver=hdfs"
<attrs> :: (<namenode name>,<namenode port>,
<kerberos cache path>,<username>,
<buffer size>)
Any absent attribute will use a default value.
--vol-value Value (ID) of the VOL connector to use for opening the
HDF5 file specified
--vol-name Name of the VOL connector to use for opening the
HDF5 file specified
--vol-info VOL-specific info to pass to the VOL connector used for
opening the HDF5 file specified
If none of the above options are used to specify a VOL, then
the VOL named by HDF5_VOL_CONNECTOR (or the native VOL connector,
if that environment variable is unset) will be used
--vfd-value Value (ID) of the VFL driver to use for opening the
HDF5 file specified
--vfd-name Name of the VFL driver to use for opening the
HDF5 file specified
--vfd-info VFD-specific info to pass to the VFL driver used for
opening the HDF5 file specified
--------------- Object Options ---------------
-g, --group Print group information
-l N, --links=N Set the threshold for the # of links when printing
information for small groups. N is an integer greater
@@ -23,16 +57,3 @@ usage: h5stat [OPTIONS] file
-a N, --numattrs=N Set the threshold for the # of attributes when printing
information for small # of attributes. N is an integer greater
than 0. The default threshold is 10.
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--s3-cred=<cred> Access file on S3, using provided credentials
<cred> :: (region,id,key) or <cred> :: (region,id,key,session token)
If <cred> == "(,,)" or <cred> == "(,,,)", no authentication is used.
--hdfs-attrs=<attrs> Access a file on HDFS with given configuration
attributes.
<attrs> :: (<namenode name>,<namenode port>,
<kerberos cache path>,<username>,
<buffer size>)
If an attribute is empty, a default value will be
used.
tools/test/h5stat/expected/h5stat_nofile.ddl
+41 -20
View File
@@ -1,13 +1,47 @@
usage: h5stat [OPTIONS] file
ERROR
--enable-error-stack Prints messages from the HDF5 error stack as they occur
Optional value 2 also prints file open errors
OPTIONS
-h, --help Print a usage message and exit
-V, --version Print version number and exit
OPTIONS
-h, --help Print a usage message and exit
-V, --version Print version number and exit
--------------- Error Options ---------------
--enable-error-stack Prints messages from the HDF5 error stack as they occur.
Optional value 2 also prints file open errors.
Default setting disables any error reporting.
--------------- File Options ---------------
-f, --file Print file information
-F, --filemetadata Print file space information for file's metadata
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--endpoint-url=P Supply S3 endpoint url information to "ros3" vfd.
P is the AWS service endpoint.
Has no effect if filedriver is not "ros3".
--s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
<cred> :: "(<aws-region>,<access-id>,<access-key>)"
<cred> :: "(<aws-region>,<access-id>,<access-key>,<session-token>)"
If absent, <cred> -> "(,,)" or <cred> -> "(,,,)", no authentication.
Has no effect if filedriver is not "ros3".
--hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
For use with "--filedriver=hdfs"
<attrs> :: (<namenode name>,<namenode port>,
<kerberos cache path>,<username>,
<buffer size>)
Any absent attribute will use a default value.
--vol-value Value (ID) of the VOL connector to use for opening the
HDF5 file specified
--vol-name Name of the VOL connector to use for opening the
HDF5 file specified
--vol-info VOL-specific info to pass to the VOL connector used for
opening the HDF5 file specified
If none of the above options are used to specify a VOL, then
the VOL named by HDF5_VOL_CONNECTOR (or the native VOL connector,
if that environment variable is unset) will be used
--vfd-value Value (ID) of the VFL driver to use for opening the
HDF5 file specified
--vfd-name Name of the VFL driver to use for opening the
HDF5 file specified
--vfd-info VFD-specific info to pass to the VFL driver used for
opening the HDF5 file specified
--------------- Object Options ---------------
-g, --group Print group information
-l N, --links=N Set the threshold for the # of links when printing
information for small groups. N is an integer greater
@@ -23,16 +57,3 @@ usage: h5stat [OPTIONS] file
-a N, --numattrs=N Set the threshold for the # of attributes when printing
information for small # of attributes. N is an integer greater
than 0. The default threshold is 10.
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--page-buffer-size=N Set the page buffer cache size, N=non-negative integers
--s3-cred=<cred> Access file on S3, using provided credentials
<cred> :: (region,id,key) or <cred> :: (region,id,key,session token)
If <cred> == "(,,)" or <cred> == "(,,,)", no authentication is used.
--hdfs-attrs=<attrs> Access a file on HDFS with given configuration
attributes.
<attrs> :: (<namenode name>,<namenode port>,
<kerberos cache path>,<username>,
<buffer size>)
If an attribute is empty, a default value will be
used.