Support HSDS 1.0 responses and request parsing

.github/workflows/main.yml

@@ -86,9 +86,21 @@ jobs:
         shell: bash
         working-directory: ${{github.workspace}}/vol-rest/build/
 
+      - name: Get latest HSDS release tag
+        id: hsds-release
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag=$(gh release view --repo HDFGroup/hsds --json tagName --jq .tagName)
+          test -n "$tag"
+          echo "Using HSDS release $tag"
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
       - uses: actions/checkout@v4
         with:
           repository: HDFGroup/hsds
+          ref: ${{ steps.hsds-release.outputs.tag }}
           path: ${{github.workspace}}/hsds
 
       - name: Set up Python ${{ matrix.python-version }}
@@ -115,11 +127,6 @@ jobs:
           cd ${{github.workspace}}/hsds
           pytest
 
-      # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
-      - name: Fix requests version
-        run: |
-          pip install requests==2.31.0
-
       - name: Start HSDS
         run: | 
               cd ${{github.workspace}}/hsds
@@ -180,10 +187,22 @@ jobs:
         with:
           path: ${{github.workspace}}/vol-rest
 
+      - name: Get latest HSDS release tag
+        id: hsds-release
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag=$(gh release view --repo HDFGroup/hsds --json tagName --jq .tagName)
+          test -n "$tag"
+          echo "Using HSDS release $tag"
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+
       - name: Get HSDS
         uses: actions/checkout@v3
         with:
           repository: HDFGroup/hsds
+          ref: ${{ steps.hsds-release.outputs.tag }}
           path: ${{github.workspace}}/hsds
 
       - name: Set up Python ${{ matrix.python-version }}

docs/users_guide/sections/usage.tex

@@ -161,7 +161,9 @@ \subsubsection{Connection Information}
 \begin{itemize}
   \item \texttt{HSDS\_USERNAME} - (optional) The username to use for authentication
   \item \texttt{HSDS\_PASSWORD} - (optional) The password to use for authentication
-  \item \texttt{HSDS\_ENDPOINT} - The base URL of the HDF Kita instance (e.g. http://hsdshdflab.hdfgroup.org)
+  \item \texttt{HSDS\_ENDPOINT} - The base URL of the HDF Kita instance (e.g. http://hsdshdflab.hdfgroup.org).
+    To connect through a UNIX domain socket, use \texttt{http+unix://} followed by the
+    percent-encoded path of the socket (e.g. \texttt{http+unix://\%2Ftmp\%2Fhs\%2Fsn\_1.sock}).
 \end{itemize}
 
 \textbf{Configuration File}

src/rest_vol.c

@@ -156,14 +156,16 @@ const char *h5paths_keys[] = {"h5paths", (const char *)0};
 /* JSON key to retrieve the version of server from a request to a file. */
 const char *server_version_keys[] = {"version", (const char *)0};
 
-/* Used for cURL's base URL if the connection is through a local socket */
-const char *socket_base_url = "0";
+/* Base URL for requests made through a UNIX domain socket. cURL connects to the
+ * socket rather than resolving this host, so the host only fills the Host header. */
+static const char *socket_base_url = "http://localhost";
 
 /* Internal initialization/termination functions which are called by
  * the public functions H5rest_init() and H5rest_term() */
 static herr_t H5_rest_init(hid_t vipl_id);
 static herr_t H5_rest_term(void);
 
+static herr_t H5_rest_parse_endpoint(const char *endpoint, const char **base_URL, char **socket_path);
 static herr_t H5_rest_authenticate_with_AD(H5_rest_ad_info_t *ad_info, const char *base_URL);
 
 /* Introspection callbacks */
@@ -563,16 +565,6 @@ H5_rest_init(hid_t vipl_id)
     if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent))
         FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option (CURLOPT_USERAGENT)");
 
-    const char *URL = getenv("HSDS_ENDPOINT");
-
-    if (URL && !strncmp(URL, UNIX_SOCKET_PREFIX, strlen(UNIX_SOCKET_PREFIX))) {
-        const char *socket_path = "/tmp/hs/sn_1.sock";
-
-        if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_UNIX_SOCKET_PATH, socket_path))
-            FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL socket path header: %s",
-                            curl_err_buf);
-    }
-
 #ifdef RV_CURL_DEBUG
     /* Enable cURL debugging output if desired */
     curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
@@ -843,9 +835,12 @@ H5_rest_set_connection_information(server_info_t *server_info)
     FILE             *config_file = NULL;
     herr_t            ret_value   = SUCCEED;
 
-    const char *username = NULL;
-    const char *password = NULL;
-    const char *base_URL = NULL;
+    const char *username      = NULL;
+    const char *password      = NULL;
+    const char *endpoint      = NULL;
+    const char *base_URL      = NULL;
+    char       *endpoint_copy = NULL;
+    char       *socket_path   = NULL;
 
     memset(&ad_info, 0, sizeof(ad_info));
 
@@ -854,14 +849,13 @@ H5_rest_set_connection_information(server_info_t *server_info)
      * the environment.
      */
 
-    if (base_URL = getenv("HSDS_ENDPOINT")) {
+    if ((endpoint = getenv("HSDS_ENDPOINT"))) {
 
         username = getenv("HSDS_USERNAME");
         password = getenv("HSDS_PASSWORD");
 
-        if (!strncmp(base_URL, UNIX_SOCKET_PREFIX, strlen(UNIX_SOCKET_PREFIX))) {
-            base_URL = socket_base_url;
-        }
+        if (H5_rest_parse_endpoint(endpoint, &base_URL, &socket_path) < 0)
+            FUNC_GOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "invalid HSDS_ENDPOINT '%s'", endpoint);
 
         if (!username && !password) {
             const char *clientID      = getenv("HSDS_AD_CLIENT_ID");
@@ -955,14 +949,20 @@ H5_rest_set_connection_information(server_info_t *server_info)
             key = strtok(file_line, " =\n");
             val = strtok(NULL, " =\n");
 
+            /* Skip lines with no key, such as blank lines */
+            if (!key)
+                continue;
+
             if (!strcmp(key, "hs_endpoint")) {
                 if (val) {
-                    if (!strncmp(base_URL, UNIX_SOCKET_PREFIX, strlen(UNIX_SOCKET_PREFIX))) {
-                        base_URL = socket_base_url;
-                    }
-                    else {
-                        base_URL = val;
-                    }
+                    /* file_line is overwritten by each fgets call, so keep a copy */
+                    RV_free(endpoint_copy);
+
+                    if (NULL == (endpoint_copy = RV_malloc(strlen(val) + 1)))
+                        FUNC_GOTO_ERROR(H5E_VOL, H5E_CANTALLOC, FAIL,
+                                        "can't allocate space for config file endpoint");
+
+                    strcpy(endpoint_copy, val);
                 } /* end if */
             }     /* end if */
             else if (!strcmp(key, "hs_ad_app_id")) {
@@ -985,6 +985,10 @@ H5_rest_set_connection_information(server_info_t *server_info)
             }     /* end else if */
         }         /* end while */
 
+        if (endpoint_copy && H5_rest_parse_endpoint(endpoint_copy, &base_URL, &socket_path) < 0)
+            FUNC_GOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "invalid hs_endpoint '%s' in config file",
+                            endpoint_copy);
+
         /* Attempt authentication with Active Directory if ID values are present */
         if (ad_info.clientID[0] != '\0' && ad_info.tenantID[0] != '\0' && ad_info.resourceID[0] != '\0')
             if (!base_URL || (H5_rest_authenticate_with_AD(&ad_info, base_URL) < 0))
@@ -1020,6 +1024,11 @@ H5_rest_set_connection_information(server_info_t *server_info)
         }
     }
 
+    /* Connect through the endpoint's UNIX domain socket, if it has one. Setting a NULL
+     * path also clears a socket left on the global handle by an earlier connection. */
+    if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_UNIX_SOCKET_PATH, socket_path))
+        FUNC_GOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set cURL UNIX socket path: %s", curl_err_buf);
+
     /* TODO - Global curl handle is only used for dataset read/writes right now.
      * Once that is removed, this will be unnecessary. */
     if (username && password) {
@@ -1033,6 +1042,9 @@ H5_rest_set_connection_information(server_info_t *server_info)
     if (config_file)
         fclose(config_file);
 
+    curl_free(socket_path);
+    RV_free(endpoint_copy);
+
     if (ret_value < 0 && server_info) {
         RV_free(server_info->username);
         server_info->username = NULL;
@@ -1049,6 +1061,67 @@ H5_rest_set_connection_information(server_info_t *server_info)
     return ret_value;
 } /* end H5_rest_set_connection_information() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5_rest_parse_endpoint
+ *
+ * Purpose:     Splits a server endpoint into the base URL that requests
+ *              are made against and, for an endpoint of the form
+ *              "http+unix://<percent-encoded socket path>", the path of
+ *              the UNIX domain socket to connect through.
+ *
+ *              *base_URL either points into endpoint or is a static
+ *              string. *socket_path is NULL for a TCP endpoint and must
+ *              otherwise be freed with curl_free().
+ *
+ * Return:      Non-negative on success/Negative on failure
+ */
+static herr_t
+H5_rest_parse_endpoint(const char *endpoint, const char **base_URL, char **socket_path)
+{
+    const char *encoded_path;
+    size_t      encoded_path_len;
+    int         socket_path_len = 0;
+    herr_t      ret_value       = SUCCEED;
+
+    *base_URL    = endpoint;
+    *socket_path = NULL;
+
+    if (strncmp(endpoint, UNIX_SOCKET_PREFIX, strlen(UNIX_SOCKET_PREFIX)))
+        FUNC_GOTO_DONE(SUCCEED);
+
+    /* The socket path takes the place of the host, so it must be percent-encoded and
+     * can only be followed by an optional trailing slash */
+    encoded_path     = endpoint + strlen(UNIX_SOCKET_PREFIX);
+    encoded_path_len = strcspn(encoded_path, "/");
+
+    if (encoded_path_len == 0 ||
+        (encoded_path[encoded_path_len] != '\0' && strcmp(encoded_path + encoded_path_len, "/")))
+        FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+                        "UNIX socket endpoint must be '%s' followed by the percent-encoded socket path, "
+                        "e.g. '%s%%2Ftmp%%2Fhs%%2Fsn_1.sock'",
+                        UNIX_SOCKET_PREFIX, UNIX_SOCKET_PREFIX);
+
+    if (encoded_path_len > INT_MAX)
+        FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "UNIX socket path is too long");
+
+    if (NULL ==
+        (*socket_path = curl_easy_unescape(curl, encoded_path, (int)encoded_path_len, &socket_path_len)))
+        FUNC_GOTO_ERROR(H5E_ARGS, H5E_CANTDECODE, FAIL, "can't decode UNIX socket path");
+
+    if (strlen(*socket_path) != (size_t)socket_path_len)
+        FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "UNIX socket path contains a null byte");
+
+    *base_URL = socket_base_url;
+
+done:
+    if (ret_value < 0) {
+        curl_free(*socket_path);
+        *socket_path = NULL;
+    }
+
+    return ret_value;
+} /* end H5_rest_parse_endpoint() */
+
 /*-------------------------------------------------------------------------
  * Function:    H5_rest_authenticate_with_AD
  *

src/rest_vol.h

@@ -116,7 +116,9 @@ const char *RV_json_obj_key_at(yyjson_val *obj, size_t idx);
 
 #define HDF5_VOL_REST_NAME "REST"
 
-#define UNIX_SOCKET_PREFIX "http+unix"
+/* Scheme of an endpoint that reaches the server through a UNIX domain socket,
+ * e.g. "http+unix://%2Ftmp%2Fhs%2Fsn_1.sock". The socket path is percent-encoded. */
+#define UNIX_SOCKET_PREFIX "http+unix://"
 
 /* Defines for the use of HTTP status codes */
 #define HTTP_INFORMATIONAL_MIN 100 /* Minimum and maximum values for the 100 class of */

src/rest_vol_attr.c

@@ -1707,23 +1707,28 @@ RV_attr_specific(void *obj, const H5VL_loc_params_t *loc_params, H5VL_attr_speci
                             strncpy(attr_iter_obj->u.file.filepath_name, loc_obj->u.file.filepath_name,
                                     strlen(loc_obj->u.file.filepath_name) + 1);
 
-                            if ((attr_iter_obj->u.file.server_info.username =
-                                     RV_malloc(strlen(loc_obj->u.file.server_info.username) + 1)) == NULL)
-                                FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL,
-                                                "can't allocate space for copied username");
+                            /* Not set when connecting without credentials */
+                            if (loc_obj->u.file.server_info.username) {
+                                if ((attr_iter_obj->u.file.server_info.username =
+                                         RV_malloc(strlen(loc_obj->u.file.server_info.username) + 1)) == NULL)
+                                    FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL,
+                                                    "can't allocate space for copied username");
+
+                                strncpy(attr_iter_obj->u.file.server_info.username,
+                                        loc_obj->u.file.server_info.username,
+                                        strlen(loc_obj->u.file.server_info.username) + 1);
+                            }
 
-                            strncpy(attr_iter_obj->u.file.server_info.username,
-                                    loc_obj->u.file.server_info.username,
-                                    strlen(loc_obj->u.file.server_info.username) + 1);
+                            if (loc_obj->u.file.server_info.password) {
+                                if ((attr_iter_obj->u.file.server_info.password =
+                                         RV_malloc(strlen(loc_obj->u.file.server_info.password) + 1)) == NULL)
+                                    FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL,
+                                                    "can't allocate space for copied password");
 
-                            if ((attr_iter_obj->u.file.server_info.password =
-                                     RV_malloc(strlen(loc_obj->u.file.server_info.password) + 1)) == NULL)
-                                FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL,
-                                                "can't allocate space for copied password");
-
-                            strncpy(attr_iter_obj->u.file.server_info.password,
-                                    loc_obj->u.file.server_info.password,
-                                    strlen(loc_obj->u.file.server_info.password) + 1);
+                                strncpy(attr_iter_obj->u.file.server_info.password,
+                                        loc_obj->u.file.server_info.password,
+                                        strlen(loc_obj->u.file.server_info.password) + 1);
+                            }
 
                             if ((attr_iter_obj->u.file.server_info.base_URL =
                                      RV_malloc(strlen(loc_obj->u.file.server_info.base_URL) + 1)) == NULL)

src/rest_vol_dataset.c

@@ -1284,9 +1284,6 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa
                 FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL,
                                 "point selection write buffer exceeded allocated buffer size");
 
-            transfer_info[i].curl_headers =
-                curl_slist_append(transfer_info[i].curl_headers, "Content-Type: application/json");
-
 #ifdef RV_CONNECTOR_DEBUG
             printf("-> Setup cURL to POST point list for dataset write\n\n");
 #endif
@@ -2469,7 +2466,11 @@ RV_parse_dataset_creation_properties_callback(char *HTTP_response, const void *c
      * Determine the layout information of the Dataset and set this on the DCPL *
      *                                                                          *
      ****************************************************************************/
-    if ((key_obj = RV_json_get(creation_properties_obj, layout_keys, RV_JSON_OBJECT))) {
+    /* HSDS 1.0 and later may return an empty layout object (e.g. for a dataset with a NULL
+     * dataspace). In that case, leave the DCPL's default layout in place.
+     */
+    if ((key_obj = RV_json_get(creation_properties_obj, layout_keys, RV_JSON_OBJECT)) &&
+        RV_json_get(key_obj, layout_class_keys, RV_JSON_STRING)) {
         yyjson_val *sub_obj;
         size_t      i;
         char       *layout_class;

src/rest_vol_file.c

@@ -83,13 +83,12 @@ RV_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, h
     if (fapl_id == H5I_INVALID_HID)
         FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid FAPL");
 
-    /* Allocate and setup internal File struct */
-    if (NULL == (new_file = (RV_object_t *)RV_malloc(sizeof(*new_file))))
+    /* Allocate and setup internal File struct. Zero it so that fields which may never be
+     * set (e.g. a username when connecting without credentials) are safe for RV_file_close
+     * to free, and set up everything RV_file_close needs before anything can fail */
+    if (NULL == (new_file = (RV_object_t *)RV_calloc(sizeof(*new_file))))
         FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for file object");
 
-    if (H5_rest_set_connection_information(&new_file->u.file.server_info) < 0)
-        FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set REST VOL connector connection information");
-
     new_file->URI[0]               = '\0';
     new_file->obj_type             = H5I_FILE;
     new_file->u.file.intent        = H5F_ACC_RDWR;
@@ -98,6 +97,9 @@ RV_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, h
     new_file->u.file.fcpl_id       = FAIL;
     new_file->u.file.ref_count     = 1;
 
+    if (H5_rest_set_connection_information(&new_file->u.file.server_info) < 0)
+        FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set REST VOL connector connection information");
+
     /* Allocate root "path" on heap for consistency with other RV_object_t types */
     if ((new_file->handle_path = RV_malloc(2)) == NULL)
         FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for filepath");
@@ -279,13 +281,12 @@ RV_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, voi
     if (fapl_id == H5I_INVALID_HID)
         FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid FAPL");
 
-    /* Allocate and setup internal File struct */
-    if (NULL == (file = (RV_object_t *)RV_malloc(sizeof(*file))))
+    /* Allocate and setup internal File struct. Zero it so that fields which may never be
+     * set (e.g. a username when connecting without credentials) are safe for RV_file_close
+     * to free, and set up everything RV_file_close needs before anything can fail */
+    if (NULL == (file = (RV_object_t *)RV_calloc(sizeof(*file))))
         FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for file object");
 
-    if (H5_rest_set_connection_information(&file->u.file.server_info) < 0)
-        FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set REST VOL connector connection information");
-
     file->URI[0]               = '\0';
     file->obj_type             = H5I_FILE;
     file->u.file.intent        = flags;
@@ -294,6 +295,9 @@ RV_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, voi
     file->u.file.fcpl_id       = FAIL;
     file->u.file.ref_count     = 1;
 
+    if (H5_rest_set_connection_information(&file->u.file.server_info) < 0)
+        FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set REST VOL connector connection information");
+
     /* Allocate root "path" on heap for consistency with other RV_object_t types */
     if ((file->handle_path = RV_malloc(2)) == NULL)
         FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for filepath");