Add support for custom headers for REST via config. (#5104)

[SC-49141](https://app.shortcut.com/tiledb-inc/story/49141/add-support-for-custom-rest-headers-via-config)

Similar to S3 custom header support (#4400) this PR adds support for
custom headers in REST request. This allows users to set header via a
config parameter instead of the current `tildb_context_set_tag`. The
context set tag does not work in all cases because its not always
possible to use the same context, i.e SOMA and VCF python APIs can't
pass a TileDB-Py ctx to their respective C++ code.

Big thanks to @Shelnutt2 for implementing this. I added a test and
documentation.

---
TYPE: CONFIG
DESC: Add `rest.custom_headers.*` config option to set custom headers on
REST requests.

---------

Co-authored-by: Seth Shelnutt <[email protected]>
Co-authored-by: KiterLuc <[email protected]>
Co-authored-by: eric-hughes-tiledb <[email protected]>
Co-authored-by: Luc Rancourt <[email protected]>

test/src/unit-curl.cc

@@ -128,3 +128,19 @@ TEST_CASE(
               .ok());
   CHECK(rest_client.rest_server() == "http://localhost:8080");
 }
+
+TEST_CASE(
+    "RestClient: Ensure custom headers are set",
+    "[rest-client][custom-headers]") {
+  tiledb::sm::Config cfg;
+  REQUIRE(cfg.set("rest.custom_headers.abc", "def").ok());
+  REQUIRE(cfg.set("rest.custom_headers.ghi", "jkl").ok());
+
+  ContextResources resources(
+      cfg, tiledb::test::g_helper_logger(), 1, 1, "test");
+  // We copy because the [] operator is not const-qualified.
+  auto extra_headers = resources.rest_client()->extra_headers();
+  CHECK(extra_headers.size() == 2);
+  CHECK(extra_headers["abc"] == "def");
+  CHECK(extra_headers["ghi"] == "jkl");
+}

tiledb/api/c_api/config/config_api_external.h

@@ -709,6 +709,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT;
  * - `rest.capnp_traversal_limit` <br>
  *    CAPNP traversal limit used in the deserialization of messages(bytes) <br>
  *    **Default**: 536870912 (512MB)
+ * - `rest.custom_headers.*` <br>
+ *    (Optional) Prefix for custom headers on REST requests. For each custom
+ *    header, use "rest.custom_headers.header_key" = "header_value" <br>
+ *    **Optional. No Default**
  * - `filestore.buffer_size` <br>
  *    Specifies the size in bytes of the internal buffers used in the filestore
  *    API. The size should be bigger than the minimum tile size filestore

tiledb/sm/cpp_api/config.h

@@ -889,6 +889,10 @@ class Config {
    *    CAPNP traversal limit used in the deserialization of messages(bytes)
    * <br>
    *    **Default**: 536870912 (512MB)
+   * - `rest.custom_headers.*` <br>
+   *    (Optional) Prefix for custom headers on REST requests. For each custom
+   *    header, use "rest.custom_headers.header_key" = "header_value" <br>
+   *    **Optional. No Default**
    * - `filestore.buffer_size` <br>
    *    Specifies the size in bytes of the internal buffers used in the
    *    filestore API. The size should be bigger than the minimum tile size

tiledb/sm/misc/constants.cc

@@ -773,6 +773,9 @@ const unsigned concurrent_attr_reads = 2;
 /** The redirection header key in REST response. */
 extern const std::string redirection_header_key = "location";
 
+/** The config key prefix for REST custom headers. */
+const std::string rest_header_prefix = "rest.custom_headers.";
+
 /** String describing MIME_AUTODETECT. */
 const std::string mime_autodetect_str = "AUTODETECT";
 

tiledb/sm/misc/constants.h

@@ -760,6 +760,9 @@ const void* fill_value(Datatype type);
 /** The redirection header key in REST response. */
 extern const std::string redirection_header_key;
 
+/** The REST custom headers config key prefix. */
+extern const std::string rest_header_prefix;
+
 /** Delimiter for lists passed as config parameter */
 extern const std::string config_delimiter;
 

tiledb/sm/rest/rest_client.cc

@@ -60,6 +60,7 @@
 #include "tiledb/common/logger.h"
 #include "tiledb/common/memory_tracker.h"
 #include "tiledb/sm/array/array.h"
+#include "tiledb/sm/config/config_iter.h"
 #include "tiledb/sm/enums/query_type.h"
 #include "tiledb/sm/group/group.h"
 #include "tiledb/sm/misc/constants.h"
@@ -148,6 +149,8 @@ Status RestClient::init(
   RETURN_NOT_OK(config_->get<bool>(
       "rest.resubmit_incomplete", &resubmit_incomplete_, &found));
 
+  load_headers(*config_);
+
   return Status::Ok();
 }
 
@@ -1661,6 +1664,17 @@ RestClient::post_consolidation_plan_from_rest(
       serialization_type_, returned_data);
 }
 
+void RestClient::load_headers(const Config& cfg) {
+  for (auto iter = ConfigIter(cfg, constants::rest_header_prefix); !iter.end();
+       iter.next()) {
+    auto key = iter.param();
+    if (key.size() == 0) {
+      continue;
+    }
+    extra_headers_[key] = iter.value();
+  }
+}
+
 #else
 
 RestClient::RestClient() {
@@ -1832,6 +1846,10 @@ RestClient::post_consolidation_plan_from_rest(
   throw RestClientDisabledException();
 }
 
+void RestClient::load_headers(const Config&) {
+  throw RestClientDisabledException();
+}
+
 #endif  // TILEDB_SERIALIZATION
 
 }  // namespace tiledb::sm

tiledb/sm/rest/rest_client.h

@@ -418,6 +418,13 @@ class RestClient {
   std::vector<std::vector<std::string>> post_consolidation_plan_from_rest(
       const URI& uri, const Config& config, uint64_t fragment_size);
 
+  /**
+   * Constant accessor to `extra_headers_` member variable.
+   */
+  const std::unordered_map<std::string, std::string>& extra_headers() const {
+    return extra_headers_;
+  }
+
  private:
   /* ********************************* */
   /*        PRIVATE ATTRIBUTES         */
@@ -566,6 +573,9 @@ class RestClient {
    * @return Status
    */
   Status ensure_json_null_delimited_string(Buffer* buffer);
+
+  /** Load all custom headers from the given config.  */
+  void load_headers(const Config& cfg);
 };
 
 }  // namespace tiledb::sm