Fix HTTP 403 errors in ROS3 VFD for object keys that need URI encoding (#6441)

The ROS3 VFD appended the raw object key to the HTTP request path.
Because the signing configuration disables use_double_uri_encode (the
correct setting for S3), the SigV4 signer uses the request path
verbatim, so keys containing characters that AWS requires to be
percent-encoded -- such as '=' in Hive-style "key=value" partition
prefixes, '+', or spaces -- produced signatures that disagree with
S3's server-side recomputation. S3 rejects such requests with
SignatureDoesNotMatch, surfaced as a bodyless HTTP 403 that is
indistinguishable from a permissions error on a HEAD request, even
though other S3 clients (AWS CLI, boto3, s3fs) could read the same
objects.

test/CMakeTests.cmake

@@ -664,7 +664,7 @@ endif ()
 add_test (NAME H5TEST-tcheck_version-release COMMAND $<TARGET_FILE:tcheck_version> "-tr")
 set_tests_properties (H5TEST-tcheck_version-release PROPERTIES
     WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST
-) 
+)
 if ("H5TEST-tcheck_version-release" MATCHES "${HDF5_DISABLE_TESTS_REGEX}")
   set_tests_properties (H5TEST-tcheck_version-release PROPERTIES DISABLED true)
 endif ()
@@ -705,8 +705,9 @@ if (HDF5_ENABLE_ROS3_VFD_DOCKER_PROXY)
           -D "TEST_PORT=${h5test_s3tests_port}"
           -D "TEST_ARGS:STRING=s3proxy-local-fs"
           -D "TEST_BUCKET:STRING=hdf5ros3"
-          -D "TEST_FILES:STRING=t8.shakespeare.txt;Poe_Raven.txt;charsets.h5"
-          -D "TEST_ACLS:STRING=skip;anon;anon"
+          -D "TEST_FILES:STRING=t8.shakespeare.txt;Poe_Raven.txt;charsets.h5;charsets.h5"
+          -D "TEST_ACLS:STRING=skip;anon;anon;skip"
+          -D "TEST_KEYS:STRING=t8.shakespeare.txt;Poe_Raven.txt;charsets.h5;src=h5test/fmt=hive+style@cloud/charsets.h5"
           -D "TEST_EXPECT=0"
           -D "TEST_ENV_VAR:STRING=AWS_SHARED_CREDENTIALS_FILE"
           -D "TEST_ENV_VALUE:STRING=${CMAKE_BINARY_DIR}/credentials"

test/ros3.c

@@ -46,6 +46,12 @@
 #define S3_TEST_RESOURCE_H5_PUBLIC       "charsets.h5"
 #define S3_TEST_RESOURCE_MISSING         "missing.csv"
 
+/* Object key containing characters ('=', '+') that must be URI-encoded
+ * when request paths are built and signed; mimics Hive-style "key=value"
+ * partition naming
+ */
+#define S3_TEST_RESOURCE_H5_HIVE_KEY "src=h5test/fmt=hive+style@cloud/charsets.h5"
+
 #define S3_TEST_RESOURCE_TEXT_RESTRICTED_SIZE 5458199
 #define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZE     6464
 #define S3_TEST_RESOURCE_TEXT_PUBLIC_SIZEOVER 6400
@@ -54,6 +60,7 @@
 static char url_text_restricted[S3_TEST_MAX_URL_SIZE] = "";
 static char url_text_public[S3_TEST_MAX_URL_SIZE]     = "";
 static char url_h5_public[S3_TEST_MAX_URL_SIZE]       = "";
+static char url_h5_hive_key[S3_TEST_MAX_URL_SIZE]     = "";
 static char url_missing[S3_TEST_MAX_URL_SIZE]         = "";
 static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE]  = "";
 static bool s3_test_bucket_defined                    = false;
@@ -1068,6 +1075,75 @@ error:
     return 1;
 
 } /* end test_ros3_access_modes() */
+
+/*---------------------------------------------------------------------------
+ * Function:    test_hive_style_object_key
+ *
+ * Purpose:     Test opening a file whose object key contains characters,
+ *              such as the '=' in Hive-style "key=value" partition names,
+ *              that must be URI-encoded when request paths are built.
+ *              If the key is not URI-encoded exactly once, S3 rejects
+ *              the requests with HTTP 403 (SignatureDoesNotMatch).
+ *
+ * Return:      PASS : 0
+ *              FAIL : 1
+ *---------------------------------------------------------------------------
+ */
+static int
+test_hive_style_object_key(void)
+{
+    hid_t fid     = H5I_INVALID_HID;
+    hid_t fapl_id = H5I_INVALID_HID;
+
+    TESTING("ros3 object keys with characters requiring URI encoding");
+
+    if (s3_test_credentials_loaded == 0) {
+        SKIPPED();
+        puts("    s3 credentials are not loaded");
+        fflush(stdout);
+        return 0;
+    }
+
+    if (false == s3_test_bucket_defined) {
+        SKIPPED();
+        puts("    environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+        fflush(stdout);
+        return 0;
+    }
+
+    if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+        TEST_ERROR;
+    if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0)
+        TEST_ERROR;
+    if (*s3_test_aws_session_token != '\0')
+        if (H5Pset_fapl_ros3_token(fapl_id, s3_test_aws_session_token) < 0)
+            TEST_ERROR;
+
+    /* The object is not public, so all of the requests made when opening
+     * and reading the file must be signed
+     */
+    if ((fid = H5Fopen(url_h5_hive_key, H5F_ACC_RDONLY, fapl_id)) < 0)
+        TEST_ERROR;
+
+    if (H5Fclose(fid) < 0)
+        TEST_ERROR;
+    if (H5Pclose(fapl_id) < 0)
+        TEST_ERROR;
+
+    PASSED();
+    return 0;
+
+error:
+    H5E_BEGIN_TRY
+    {
+        H5Fclose(fid);
+        H5Pclose(fapl_id);
+    }
+    H5E_END_TRY
+
+    return 1;
+
+} /* end test_hive_style_object_key() */
 #endif /* H5_HAVE_ROS3_VFD */
 
 /*-------------------------------------------------------------------------
@@ -1130,6 +1206,13 @@ main(void)
         ret_value = EXIT_FAILURE;
         goto done;
     }
+    if (S3_TEST_MAX_URL_SIZE < snprintf(url_h5_hive_key, (size_t)S3_TEST_MAX_URL_SIZE, "%s/%s",
+                                        (const char *)s3_test_bucket_url,
+                                        (const char *)S3_TEST_RESOURCE_H5_HIVE_KEY)) {
+        printf("* ros3 setup failed (h5_hive_key) ! *\n");
+        ret_value = EXIT_FAILURE;
+        goto done;
+    }
     if (S3_TEST_MAX_URL_SIZE < snprintf(url_missing, S3_TEST_MAX_URL_SIZE, "%s/%s",
                                         (const char *)s3_test_bucket_url,
                                         (const char *)S3_TEST_RESOURCE_MISSING)) {
@@ -1231,6 +1314,7 @@ main(void)
         nerrors += test_noops_and_autofails();
         nerrors += test_cmp();
         nerrors += test_ros3_access_modes();
+        nerrors += test_hive_style_object_key();
     }
 
     if (H5FD__s3comms_term() < 0) {