From 3eeba4b8460a1c8b4a2af4eb1ad207e8d7ba95b1 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Thu, 17 Oct 2024 22:46:26 +0300 Subject: [PATCH] Remove `Array::get_max_buffer_sizes`. (#5292) This PR removes all the code associated with `Array::get_max_buffer_sizes`. This means Array bookkeeping members, functions for calculating maximum buffer sizes based on a subarray, capnp struct, rest client functions. The `tiledb_serialize_array_max_buffer_sizes` C API was kept until downstreams migrate, but its implementation was updated to always fail. Soon will follow a PR in the REST server for removing the handling code of this request. --- TYPE: NO_HISTORY --------- Co-authored-by: Theodore Tsirpanis Co-authored-by: Shaun Reed --- test/src/unit-capi-rest-dense_array.cc | 30 -- tiledb/sm/array/array.cc | 267 ------------------ tiledb/sm/array/array.h | 55 ---- tiledb/sm/c_api/tiledb.cc | 32 +-- tiledb/sm/c_api/tiledb_serialization.h | 6 +- tiledb/sm/rest/rest_client.h | 9 - tiledb/sm/rest/rest_client_remote.cc | 41 --- tiledb/sm/rest/rest_client_remote.h | 16 -- tiledb/sm/serialization/array_schema.cc | 172 ----------- tiledb/sm/serialization/array_schema.h | 13 - tiledb/sm/serialization/tiledb-rest.capnp | 5 - tiledb/sm/serialization/tiledb-rest.capnp.c++ | 65 ----- tiledb/sm/serialization/tiledb-rest.capnp.h | 207 -------------- 13 files changed, 10 insertions(+), 908 deletions(-) diff --git a/test/src/unit-capi-rest-dense_array.cc b/test/src/unit-capi-rest-dense_array.cc index 2f16fd4bb23..e2373a1144f 100644 --- a/test/src/unit-capi-rest-dense_array.cc +++ b/test/src/unit-capi-rest-dense_array.cc @@ -1794,36 +1794,6 @@ TEST_CASE_METHOD( tiledb_array_free(&array); } -TEST_CASE_METHOD( - DenseArrayRESTFx, - "C API: REST Test dense array, get max buffer sizes", - "[capi][rest][dense]") { - array_uri_ = vfs_test_setup_.array_uri("max_buffer_sizes_array"); - create_dense_array(array_uri_); - - // Check max buffer sizes with empty array - tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_READ); - CHECK(rc == TILEDB_OK); - REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); - tiledb_array_free(&array); - - // Write array - write_dense_array(array_uri_); - - // Check max buffer sizes for whole domain - rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_READ); - CHECK(rc == TILEDB_OK); - - // Clean up - REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); - tiledb_array_free(&array); -} - TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, error without rest server configured", diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index e323fef94a4..f7a708fd2de 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -600,8 +600,6 @@ Status Array::close() { return Status::Ok(); } - clear_last_max_buffer_sizes(); - try { set_array_closed(); @@ -990,131 +988,6 @@ QueryType Array::get_query_type() const { return query_type_; } -Status Array::get_max_buffer_size( - const char* name, const void* subarray, uint64_t* buffer_size) { - // Check if array is open - if (!is_open_) { - return LOG_STATUS( - Status_ArrayError("Cannot get max buffer size; Array is not open")); - } - - // Error if the array was not opened in read mode - if (query_type_ != QueryType::READ) { - return LOG_STATUS( - Status_ArrayError("Cannot get max buffer size; " - "Array was not opened in read mode")); - } - - // Check if name is null - if (name == nullptr) { - return LOG_STATUS(Status_ArrayError( - "Cannot get max buffer size; Attribute/Dimension name is null")); - } - - // Not applicable to heterogeneous domains - if (!array_schema_latest().domain().all_dims_same_type()) { - return LOG_STATUS( - Status_ArrayError("Cannot get max buffer size; Function not " - "applicable to heterogeneous domains")); - } - - // Not applicable to variable-sized dimensions - if (!array_schema_latest().domain().all_dims_fixed()) { - return LOG_STATUS(Status_ArrayError( - "Cannot get max buffer size; Function not " - "applicable to domains with variable-sized dimensions")); - } - - // Check if name is attribute or dimension - bool is_dim = array_schema_latest().is_dim(name); - bool is_attr = array_schema_latest().is_attr(name); - - // Check if attribute/dimension exists - if (name != constants::coords && !is_dim && !is_attr) { - return LOG_STATUS(Status_ArrayError( - std::string("Cannot get max buffer size; Attribute/Dimension '") + - name + "' does not exist")); - } - - // Check if attribute/dimension is fixed sized - if (array_schema_latest().var_size(name)) { - return LOG_STATUS(Status_ArrayError( - std::string("Cannot get max buffer size; Attribute/Dimension '") + - name + "' is var-sized")); - } - - RETURN_NOT_OK(compute_max_buffer_sizes(subarray)); - - // Retrieve buffer size - auto it = last_max_buffer_sizes_.find(name); - assert(it != last_max_buffer_sizes_.end()); - *buffer_size = it->second.first; - - return Status::Ok(); -} - -Status Array::get_max_buffer_size( - const char* name, - const void* subarray, - uint64_t* buffer_off_size, - uint64_t* buffer_val_size) { - // Check if array is open - if (!is_open_) { - return LOG_STATUS( - Status_ArrayError("Cannot get max buffer size; Array is not open")); - } - - // Error if the array was not opened in read mode - if (query_type_ != QueryType::READ) { - return LOG_STATUS( - Status_ArrayError("Cannot get max buffer size; " - "Array was not opened in read mode")); - } - - // Check if name is null - if (name == nullptr) { - return LOG_STATUS(Status_ArrayError( - "Cannot get max buffer size; Attribute/Dimension name is null")); - } - - // Not applicable to heterogeneous domains - if (!array_schema_latest().domain().all_dims_same_type()) { - return LOG_STATUS( - Status_ArrayError("Cannot get max buffer size; Function not " - "applicable to heterogeneous domains")); - } - - // Not applicable to variable-sized dimensions - if (!array_schema_latest().domain().all_dims_fixed()) { - return LOG_STATUS(Status_ArrayError( - "Cannot get max buffer size; Function not " - "applicable to domains with variable-sized dimensions")); - } - - RETURN_NOT_OK(compute_max_buffer_sizes(subarray)); - - // Check if attribute/dimension exists - auto it = last_max_buffer_sizes_.find(name); - if (it == last_max_buffer_sizes_.end()) { - return LOG_STATUS(Status_ArrayError( - std::string("Cannot get max buffer size; Attribute/Dimension '") + - name + "' does not exist")); - } - - // Check if attribute/dimension is var-sized - if (!array_schema_latest().var_size(name)) { - return LOG_STATUS(Status_ArrayError( - std::string("Cannot get max buffer size; Attribute/Dimension '") + - name + "' is fixed-sized")); - } - - // Retrieve buffer sizes - *buffer_off_size = it->second.first; - *buffer_val_size = it->second.second; - - return Status::Ok(); -} - Status Array::reopen() { // Note: Array will only reopen for reads. This is why we are checking the // timestamp for the array directory and not new components. This needs to be @@ -1155,9 +1028,6 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { } array_dir_timestamp_start_ = timestamp_start; - // Reset the last max buffer sizes. - clear_last_max_buffer_sizes(); - // Reopen metadata. auto key = opened_array_->encryption_key(); opened_array_ = make_shared( @@ -1902,143 +1772,6 @@ Array::open_for_writes() { return {array_schema_latest, array_schemas_all}; } -void Array::clear_last_max_buffer_sizes() { - last_max_buffer_sizes_.clear(); - last_max_buffer_sizes_subarray_.clear(); - last_max_buffer_sizes_subarray_.shrink_to_fit(); -} - -Status Array::compute_max_buffer_sizes(const void* subarray) { - // Applicable only to domains where all dimensions have the same type - if (!array_schema_latest().domain().all_dims_same_type()) { - return LOG_STATUS( - Status_ArrayError("Cannot compute max buffer sizes; Inapplicable when " - "dimension domains have different types")); - } - - // Allocate space for max buffer sizes subarray - auto dim_num = array_schema_latest().dim_num(); - auto coord_size{ - array_schema_latest().domain().dimension_ptr(0)->coord_size()}; - auto subarray_size = 2 * dim_num * coord_size; - last_max_buffer_sizes_subarray_.resize(subarray_size); - - // Compute max buffer sizes - if (last_max_buffer_sizes_.empty() || - std::memcmp( - &last_max_buffer_sizes_subarray_[0], subarray, subarray_size) != 0) { - last_max_buffer_sizes_.clear(); - - // Get all attributes and coordinates - auto& attributes = array_schema_latest().attributes(); - last_max_buffer_sizes_.clear(); - for (const auto& attr : attributes) - last_max_buffer_sizes_[attr->name()] = - std::pair(0, 0); - last_max_buffer_sizes_[constants::coords] = - std::pair(0, 0); - for (unsigned d = 0; d < dim_num; ++d) - last_max_buffer_sizes_ - [array_schema_latest().domain().dimension_ptr(d)->name()] = - std::pair(0, 0); - - RETURN_NOT_OK(compute_max_buffer_sizes(subarray, &last_max_buffer_sizes_)); - } - - // Update subarray - std::memcpy(&last_max_buffer_sizes_subarray_[0], subarray, subarray_size); - - return Status::Ok(); -} - -Status Array::compute_max_buffer_sizes( - const void* subarray, - std::unordered_map>* - buffer_sizes) const { - if (remote_) { - auto rest_client = resources_.rest_client(); - if (rest_client == nullptr) { - return LOG_STATUS(Status_ArrayError( - "Cannot get max buffer sizes; remote array with no REST client.")); - } - - return rest_client->get_array_max_buffer_sizes( - array_uri_, array_schema_latest(), subarray, buffer_sizes); - } - - // Keep the current opened array alive for the duration of this call. - auto opened_array = opened_array_; - auto& fragment_metadata = opened_array->fragment_metadata(); - auto& array_schema_latest = opened_array->array_schema_latest(); - - // Return if there are no metadata - if (fragment_metadata.empty()) { - return Status::Ok(); - } - - // First we calculate a rough upper bound. Especially for dense - // arrays, this will not be accurate, as it accounts only for the - // non-empty regions of the subarray. - for (auto& meta : fragment_metadata) { - meta->add_max_buffer_sizes(*encryption_key(), subarray, buffer_sizes); - } - - // Prepare an NDRange for the subarray - auto dim_num = array_schema_latest.dim_num(); - NDRange sub(dim_num); - auto sub_ptr = (const unsigned char*)subarray; - uint64_t offset = 0; - for (unsigned d = 0; d < dim_num; ++d) { - auto r_size{2 * array_schema_latest.dimension_ptr(d)->coord_size()}; - sub[d] = Range(&sub_ptr[offset], r_size); - offset += r_size; - } - - // Rectify bound for dense arrays - if (array_schema_latest.dense()) { - auto cell_num = array_schema_latest.domain().cell_num(sub); - // `cell_num` becomes 0 when `subarray` is huge, leading to a - // `uint64_t` overflow. - if (cell_num != 0) { - for (auto& it : *buffer_sizes) { - if (array_schema_latest.var_size(it.first)) { - it.second.first = cell_num * constants::cell_var_offset_size; - it.second.second += - cell_num * datatype_size(array_schema_latest.type(it.first)); - } else { - it.second.first = cell_num * array_schema_latest.cell_size(it.first); - } - } - } - } - - // Rectify bound for sparse arrays with integer domain, without duplicates - if (!array_schema_latest.dense() && !array_schema_latest.allows_dups() && - array_schema_latest.domain().all_dims_int()) { - auto cell_num = array_schema_latest.domain().cell_num(sub); - // `cell_num` becomes 0 when `subarray` is huge, leading to a - // `uint64_t` overflow. - if (cell_num != 0) { - for (auto& it : *buffer_sizes) { - if (!array_schema_latest.var_size(it.first)) { - // Check for overflow - uint64_t new_size = - cell_num * array_schema_latest.cell_size(it.first); - if (new_size / array_schema_latest.cell_size((it.first)) != - cell_num) { - continue; - } - - // Potentially rectify size - it.second.first = std::min(it.second.first, new_size); - } - } - } - } - - return Status::Ok(); -} - void Array::do_load_metadata() { if (!array_directory().loaded()) { throw ArrayException( diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 76c1de5baf7..babc3009a43 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -632,23 +632,6 @@ class Array { /** Retrieves the query type. Throws if the array is not open. */ QueryType get_query_type() const; - /** - * Returns the max buffer size given a fixed-sized attribute/dimension and - * a subarray. Errors if the array is not open. - */ - Status get_max_buffer_size( - const char* name, const void* subarray, uint64_t* buffer_size); - - /** - * Returns the max buffer size given a var-sized attribute/dimension and - * a subarray. Errors if the array is not open. - */ - Status get_max_buffer_size( - const char* name, - const void* subarray, - uint64_t* buffer_off_size, - uint64_t* buffer_val_size); - /** * Returns a reference to the private encryption key. */ @@ -1132,16 +1115,6 @@ class Array { /** The array config. */ Config config_; - /** Stores the max buffer sizes requested last time by the user .*/ - std::unordered_map> - last_max_buffer_sizes_; - - /** - * This is the last subarray used by the user to retrieve the - * max buffer sizes. - */ - std::vector last_max_buffer_sizes_subarray_; - /** True if the array is remote (has `tiledb://` URI scheme). */ bool remote_; @@ -1214,34 +1187,6 @@ class Array { std::unordered_map>> open_for_writes(); - /** Clears the cached max buffer sizes and subarray. */ - void clear_last_max_buffer_sizes(); - - /** - * Computes the maximum buffer sizes for all attributes given a subarray, - * which are cached locally in the instance. - */ - Status compute_max_buffer_sizes(const void* subarray); - - /** - * Computes an upper bound on the buffer sizes required for a read - * query, for a given subarray and set of attributes. Note that - * the attributes are already set in `max_buffer_sizes` - * - * @param subarray The subarray to focus on. Note that it must have the same - * underlying type as the array domain. - * @param max_buffer_sizes The buffer sizes to be retrieved. This is a map - * from the attribute (or coordinates) name to a pair of sizes (in bytes). For - * fixed-sized attributes, the second size is ignored. For var-sized - * attributes, the first is the offsets size and the second is the - * values size. - * @return Status - */ - Status compute_max_buffer_sizes( - const void* subarray, - std::unordered_map>* - max_buffer_sizes_) const; - /** * Load non-remote array metadata. */ diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 475c580f3a7..b23150f54af 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1806,31 +1806,13 @@ int32_t tiledb_deserialize_array_non_empty_domain_all_dimensions( } int32_t tiledb_serialize_array_max_buffer_sizes( - tiledb_ctx_t* ctx, - const tiledb_array_t* array, - const void* subarray, - tiledb_serialization_type_t serialize_type, - tiledb_buffer_t** buffer) { - ensure_array_is_valid(array); - - auto buf = tiledb_buffer_handle_t::make_handle( - ctx->resources().serialization_memory_tracker()); - - // Serialize - if (SAVE_ERROR_CATCH( - ctx, - tiledb::sm::serialization::max_buffer_sizes_serialize( - array->array().get(), - subarray, - (tiledb::sm::SerializationType)serialize_type, - buf->buffer()))) { - tiledb_buffer_handle_t::break_handle(buf); - return TILEDB_ERR; - } - - *buffer = buf; - - return TILEDB_OK; + tiledb_ctx_t*, + const tiledb_array_t*, + const void*, + tiledb_serialization_type_t, + tiledb_buffer_t**) { + throw CAPIException( + "tiledb_serialize_array_max_buffer_sizes is no longer supported."); } capi_return_t tiledb_handle_array_delete_fragments_timestamps_request( diff --git a/tiledb/sm/c_api/tiledb_serialization.h b/tiledb/sm/c_api/tiledb_serialization.h index 821a7ed1d38..d9d4afde6fd 100644 --- a/tiledb/sm/c_api/tiledb_serialization.h +++ b/tiledb/sm/c_api/tiledb_serialization.h @@ -418,7 +418,7 @@ TILEDB_EXPORT int32_t tiledb_deserialize_array_non_empty_domain_all_dimensions( /** * Serializes the array max buffer sizes information into the given buffer. * - * @note The caller must free the returned `tiledb_buffer_t`. + * This API is deprecated and using it will always return an error. * * @param ctx The TileDB context. * @param array Array to which the subarray belongs to @@ -426,9 +426,9 @@ TILEDB_EXPORT int32_t tiledb_deserialize_array_non_empty_domain_all_dimensions( * @param serialization_type Type of serialization to use * @param buffer Will be set to a newly allocated buffer containing the * serialized max buffer sizes. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * @return `TILEDB_ERR` */ -TILEDB_EXPORT int32_t tiledb_serialize_array_max_buffer_sizes( +TILEDB_DEPRECATED_EXPORT int32_t tiledb_serialize_array_max_buffer_sizes( tiledb_ctx_t* ctx, const tiledb_array_t* array, const void* subarray, diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 026afb86528..cc456e258fe 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -364,15 +364,6 @@ class RestClient { throw RestClientDisabledException(); } - /// Operation disabled in base class. - inline virtual Status get_array_max_buffer_sizes( - const URI&, - const ArraySchema&, - const void*, - std::unordered_map>*) { - throw RestClientDisabledException(); - } - /// Operation disabled in base class. inline virtual Status get_array_metadata_from_rest( const URI&, uint64_t, uint64_t, Array*) { diff --git a/tiledb/sm/rest/rest_client_remote.cc b/tiledb/sm/rest/rest_client_remote.cc index 76d717bcba4..87926f01efe 100644 --- a/tiledb/sm/rest/rest_client_remote.cc +++ b/tiledb/sm/rest/rest_client_remote.cc @@ -494,47 +494,6 @@ Status RestClientRemote::get_array_non_empty_domain( array, returned_data, serialization_type_); } -Status RestClientRemote::get_array_max_buffer_sizes( - const URI& uri, - const ArraySchema& schema, - const void* subarray, - std::unordered_map>* - buffer_sizes) { - // Convert subarray to string for query parameter - std::string subarray_str; - RETURN_NOT_OK(subarray_to_str(schema, subarray, &subarray_str)); - std::string subarray_query_param = - subarray_str.empty() ? "" : ("?subarray=" + subarray_str); - - // Init curl and form the URL - Curl curlc(logger_); - std::string array_ns, array_uri; - RETURN_NOT_OK(uri.get_rest_components(&array_ns, &array_uri)); - const std::string cache_key = array_ns + ":" + array_uri; - RETURN_NOT_OK( - curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); - const std::string url = redirect_uri(cache_key) + "/v1/arrays/" + array_ns + - "/" + curlc.url_escape(array_uri) + - "/max_buffer_sizes" + subarray_query_param; - - // Get the data - Buffer returned_data; - RETURN_NOT_OK(curlc.get_data( - stats_, url, serialization_type_, &returned_data, cache_key)); - - if (returned_data.data() == nullptr || returned_data.size() == 0) - return LOG_STATUS( - Status_RestError("Error getting array max buffer sizes " - "from REST; server returned no data.")); - - // Ensure data has a null delimiter for cap'n proto if using JSON - RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); - - // Deserialize data returned - return serialization::max_buffer_sizes_deserialize( - schema, returned_data, serialization_type_, buffer_sizes); -} - Status RestClientRemote::get_array_metadata_from_rest( const URI& uri, uint64_t timestamp_start, diff --git a/tiledb/sm/rest/rest_client_remote.h b/tiledb/sm/rest/rest_client_remote.h index 6717023fe1f..bc2f52c312a 100644 --- a/tiledb/sm/rest/rest_client_remote.h +++ b/tiledb/sm/rest/rest_client_remote.h @@ -242,22 +242,6 @@ class RestClientRemote : public RestClient { Status get_array_non_empty_domain( Array* array, uint64_t timestamp_start, uint64_t timestamp_end) override; - /** - * Get array's max buffer sizes from rest server. - * - * @param uri URI of array - * @param schema Array schema of array - * @param subarray Subrray to get max buffer sizes for - * @param buffer_sizes Will be populated with max buffer sizes - * @return Status - */ - Status get_array_max_buffer_sizes( - const URI& uri, - const ArraySchema& schema, - const void* subarray, - std::unordered_map>* - buffer_sizes) override; - /** * Gets the array's metadata from the REST server (and updates the in-memory * Metadata of the array to match the returned values). diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index b4696a0b422..e8fe0f962ca 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -1630,163 +1630,6 @@ Status nonempty_domain_deserialize( return Status::Ok(); } -Status max_buffer_sizes_serialize( - Array* array, - const void* subarray, - SerializationType serialize_type, - SerializationBuffer& serialized_buffer) { - const auto& schema = array->array_schema_latest(); - - try { - // Serialize - ::capnp::MallocMessageBuilder message; - auto builder = message.initRoot(); - - // Get all attribute names including coords - const auto& attrs = schema.attributes(); - std::set attr_names; - attr_names.insert(constants::coords); - for (const auto& a : attrs) - attr_names.insert(a->name()); - - // Get max buffer size for each attribute from the given Array instance - // and serialize it. - auto max_buffer_sizes_builder = - builder.initMaxBufferSizes(attr_names.size()); - size_t i = 0; - for (const auto& attr_name : attr_names) { - bool var_size = - attr_name != constants::coords && schema.var_size(attr_name); - auto max_buffer_size_builder = max_buffer_sizes_builder[i++]; - max_buffer_size_builder.setAttribute(attr_name); - - if (var_size) { - uint64_t offset_bytes, data_bytes; - RETURN_NOT_OK(array->get_max_buffer_size( - attr_name.c_str(), subarray, &offset_bytes, &data_bytes)); - max_buffer_size_builder.setOffsetBytes(offset_bytes); - max_buffer_size_builder.setDataBytes(data_bytes); - } else { - uint64_t data_bytes; - RETURN_NOT_OK(array->get_max_buffer_size( - attr_name.c_str(), subarray, &data_bytes)); - max_buffer_size_builder.setOffsetBytes(0); - max_buffer_size_builder.setDataBytes(data_bytes); - } - } - - switch (serialize_type) { - case SerializationType::JSON: { - ::capnp::JsonCodec json; - kj::String capnp_json = json.encode(builder); - serialized_buffer.assign_null_terminated(capnp_json); - break; - } - case SerializationType::CAPNP: { - kj::Array<::capnp::word> protomessage = messageToFlatArray(message); - serialized_buffer.assign(protomessage.asChars()); - break; - } - default: { - return LOG_STATUS(Status_SerializationError( - "Error serializing max buffer sizes; Unknown serialization type " - "passed")); - } - } - - } catch (kj::Exception& e) { - return LOG_STATUS(Status_SerializationError( - "Error serializing max buffer sizes; kj::Exception: " + - std::string(e.getDescription().cStr()))); - } catch (std::exception& e) { - return LOG_STATUS(Status_SerializationError( - "Error serializing max buffer sizes; exception " + - std::string(e.what()))); - } - - return Status::Ok(); -} - -Status max_buffer_sizes_deserialize( - const ArraySchema& schema, - span serialized_buffer, - SerializationType serialize_type, - std::unordered_map>* - buffer_sizes) { - try { - switch (serialize_type) { - case SerializationType::JSON: { - ::capnp::JsonCodec json; - ::capnp::MallocMessageBuilder message_builder; - auto builder = message_builder.initRoot(); - json.decode( - kj::StringPtr(static_cast(serialized_buffer.data())), - builder); - auto reader = builder.asReader(); - - // Deserialize - auto max_buffer_sizes_reader = reader.getMaxBufferSizes(); - const size_t num_max_buffer_sizes = max_buffer_sizes_reader.size(); - for (size_t i = 0; i < num_max_buffer_sizes; i++) { - auto max_buffer_size_reader = max_buffer_sizes_reader[i]; - std::string attribute = max_buffer_size_reader.getAttribute(); - uint64_t offset_size = max_buffer_size_reader.getOffsetBytes(); - uint64_t data_size = max_buffer_size_reader.getDataBytes(); - - if (attribute == constants::coords || !schema.var_size(attribute)) { - (*buffer_sizes)[attribute] = std::make_pair(data_size, 0); - } else { - (*buffer_sizes)[attribute] = std::make_pair(offset_size, data_size); - } - } - - break; - } - case SerializationType::CAPNP: { - const auto mBytes = - reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); - auto reader = msg_reader.getRoot(); - - // Deserialize - auto max_buffer_sizes_reader = reader.getMaxBufferSizes(); - const size_t num_max_buffer_sizes = max_buffer_sizes_reader.size(); - for (size_t i = 0; i < num_max_buffer_sizes; i++) { - auto max_buffer_size_reader = max_buffer_sizes_reader[i]; - std::string attribute = max_buffer_size_reader.getAttribute(); - uint64_t offset_size = max_buffer_size_reader.getOffsetBytes(); - uint64_t data_size = max_buffer_size_reader.getDataBytes(); - - if (attribute == constants::coords || !schema.var_size(attribute)) { - (*buffer_sizes)[attribute] = std::make_pair(data_size, 0); - } else { - (*buffer_sizes)[attribute] = std::make_pair(offset_size, data_size); - } - } - - break; - } - default: { - return LOG_STATUS(Status_SerializationError( - "Error deserializing max buffer sizes; Unknown serialization type " - "passed")); - } - } - } catch (kj::Exception& e) { - return LOG_STATUS(Status_SerializationError( - "Error deserializing max buffer sizes; kj::Exception: " + - std::string(e.getDescription().cStr()))); - } catch (std::exception& e) { - return LOG_STATUS(Status_SerializationError( - "Error deserializing max buffer sizes; exception " + - std::string(e.what()))); - } - - return Status::Ok(); -} - void load_array_schema_request_to_capnp( capnp::LoadArraySchemaRequest::Builder& builder, const Config& config, @@ -2060,21 +1903,6 @@ Status nonempty_domain_deserialize( "Cannot serialize; serialization not enabled.")); } -Status max_buffer_sizes_serialize( - Array*, const void*, SerializationType, SerializationBuffer&) { - return LOG_STATUS(Status_SerializationError( - "Cannot serialize; serialization not enabled.")); -} - -Status max_buffer_sizes_deserialize( - const ArraySchema&, - span, - SerializationType, - std::unordered_map>*) { - return LOG_STATUS(Status_SerializationError( - "Cannot serialize; serialization not enabled.")); -} - void serialize_load_array_schema_request( const Config&, const LoadArraySchemaRequest&, diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index 8bb3d199360..03ce35664fc 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -189,19 +189,6 @@ Status nonempty_domain_deserialize( span serialized_buffer, SerializationType serialize_type); -Status max_buffer_sizes_serialize( - Array* array, - const void* subarray, - SerializationType serialize_type, - SerializationBuffer& serialized_buffer); - -Status max_buffer_sizes_deserialize( - const ArraySchema& schema, - span serialized_buffer, - SerializationType serialize_type, - std::unordered_map>* - buffer_sizes); - void serialize_load_array_schema_request( const Config& config, const LoadArraySchemaRequest& req, diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 82c72a49466..2d30e7528a2 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -830,11 +830,6 @@ struct AttributeBufferSize { # size (in bytes) of data buffer } -struct MaxBufferSizes { - maxBufferSizes @0 :List(AttributeBufferSize); - # a list of max buffer sizes, one per attribute -} - struct ArrayMetadata { # object representing array metadata diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index b37c6653039..874fe7ebe0d 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -6215,59 +6215,6 @@ const ::capnp::_::RawSchema s_9be1921b07e6cd2d = { 0, 4, i_9be1921b07e6cd2d, nullptr, nullptr, { &s_9be1921b07e6cd2d, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<39> b_f01116579e9ea98e = { - { 0, 0, 0, 0, 5, 0, 6, 0, - 142, 169, 158, 158, 87, 22, 17, 240, - 18, 0, 0, 0, 1, 0, 0, 0, - 127, 216, 135, 181, 36, 146, 125, 181, - 1, 0, 7, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 45, 114, - 101, 115, 116, 46, 99, 97, 112, 110, - 112, 58, 77, 97, 120, 66, 117, 102, - 102, 101, 114, 83, 105, 122, 101, 115, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 3, 0, 1, 0, - 40, 0, 0, 0, 2, 0, 1, 0, - 109, 97, 120, 66, 117, 102, 102, 101, - 114, 83, 105, 122, 101, 115, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, - 16, 0, 0, 0, 0, 0, 0, 0, - 45, 205, 230, 7, 27, 146, 225, 155, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, } -}; -::capnp::word const* const bp_f01116579e9ea98e = b_f01116579e9ea98e.words; -#if !CAPNP_LITE -static const ::capnp::_::RawSchema* const d_f01116579e9ea98e[] = { - &s_9be1921b07e6cd2d, -}; -static const uint16_t m_f01116579e9ea98e[] = {0}; -static const uint16_t i_f01116579e9ea98e[] = {0}; -const ::capnp::_::RawSchema s_f01116579e9ea98e = { - 0xf01116579e9ea98e, b_f01116579e9ea98e.words, 39, d_f01116579e9ea98e, m_f01116579e9ea98e, - 1, 1, i_f01116579e9ea98e, nullptr, nullptr, { &s_f01116579e9ea98e, nullptr, nullptr, 0, 0, nullptr }, false -}; -#endif // !CAPNP_LITE static const ::capnp::_::AlignedData<41> b_9737dcafdfce31bb = { { 0, 0, 0, 0, 5, 0, 6, 0, 187, 49, 206, 223, 175, 220, 55, 151, @@ -11348,18 +11295,6 @@ constexpr ::capnp::_::RawSchema const* AttributeBufferSize::_capnpPrivate::schem #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE -// MaxBufferSizes -#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint16_t MaxBufferSizes::_capnpPrivate::dataWordSize; -constexpr uint16_t MaxBufferSizes::_capnpPrivate::pointerCount; -#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -#if !CAPNP_LITE -#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr ::capnp::Kind MaxBufferSizes::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* MaxBufferSizes::_capnpPrivate::schema; -#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -#endif // !CAPNP_LITE - // ArrayMetadata #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr uint16_t ArrayMetadata::_capnpPrivate::dataWordSize; diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 03a4092139b..011410d65e7 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -74,7 +74,6 @@ CAPNP_DECLARE_SCHEMA(96ba49d0f8b23ccc); CAPNP_DECLARE_SCHEMA(9df6f2a42c4e5f0b); CAPNP_DECLARE_SCHEMA(a18264549448ece3); CAPNP_DECLARE_SCHEMA(9be1921b07e6cd2d); -CAPNP_DECLARE_SCHEMA(f01116579e9ea98e); CAPNP_DECLARE_SCHEMA(9737dcafdfce31bb); CAPNP_DECLARE_SCHEMA(926fe1c3b12ed651); CAPNP_DECLARE_SCHEMA(9317f20ce509d918); @@ -1116,23 +1115,6 @@ struct AttributeBufferSize { }; }; -struct MaxBufferSizes { - MaxBufferSizes() = delete; - - class Reader; - class Builder; - class Pipeline; - - struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(f01116579e9ea98e, 0, 1) -#if !CAPNP_LITE - static constexpr ::capnp::_::RawBrandedSchema const* brand() { - return &schema->defaultBrand; - } -#endif // !CAPNP_LITE - }; -}; - struct ArrayMetadata { ArrayMetadata() = delete; @@ -10085,121 +10067,6 @@ class AttributeBufferSize::Pipeline { }; #endif // !CAPNP_LITE -class MaxBufferSizes::Reader { - public: - typedef MaxBufferSizes Reads; - - Reader() = default; - inline explicit Reader(::capnp::_::StructReader base) - : _reader(base) { - } - - inline ::capnp::MessageSize totalSize() const { - return _reader.totalSize().asPublic(); - } - -#if !CAPNP_LITE - inline ::kj::StringTree toString() const { - return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); - } -#endif // !CAPNP_LITE - - inline bool hasMaxBufferSizes() const; - inline ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Reader - getMaxBufferSizes() const; - - private: - ::capnp::_::StructReader _reader; - template - friend struct ::capnp::ToDynamic_; - template - friend struct ::capnp::_::PointerHelpers; - template - friend struct ::capnp::List; - friend class ::capnp::MessageBuilder; - friend class ::capnp::Orphanage; -}; - -class MaxBufferSizes::Builder { - public: - typedef MaxBufferSizes Builds; - - Builder() = delete; // Deleted to discourage incorrect usage. - // You can explicitly initialize to nullptr instead. - inline Builder(decltype(nullptr)) { - } - inline explicit Builder(::capnp::_::StructBuilder base) - : _builder(base) { - } - inline operator Reader() const { - return Reader(_builder.asReader()); - } - inline Reader asReader() const { - return *this; - } - - inline ::capnp::MessageSize totalSize() const { - return asReader().totalSize(); - } -#if !CAPNP_LITE - inline ::kj::StringTree toString() const { - return asReader().toString(); - } -#endif // !CAPNP_LITE - - inline bool hasMaxBufferSizes(); - inline ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Builder - getMaxBufferSizes(); - inline void setMaxBufferSizes( - ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Reader value); - inline ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Builder - initMaxBufferSizes(unsigned int size); - inline void adoptMaxBufferSizes( - ::capnp::Orphan<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>&& value); - inline ::capnp::Orphan<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>> - disownMaxBufferSizes(); - - private: - ::capnp::_::StructBuilder _builder; - template - friend struct ::capnp::ToDynamic_; - friend class ::capnp::Orphanage; - template - friend struct ::capnp::_::PointerHelpers; -}; - -#if !CAPNP_LITE -class MaxBufferSizes::Pipeline { - public: - typedef MaxBufferSizes Pipelines; - - inline Pipeline(decltype(nullptr)) - : _typeless(nullptr) { - } - inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) - : _typeless(kj::mv(typeless)) { - } - - private: - ::capnp::AnyPointer::Pipeline _typeless; - friend class ::capnp::PipelineHook; - template - friend struct ::capnp::ToDynamic_; -}; -#endif // !CAPNP_LITE - class ArrayMetadata::Reader { public: typedef ArrayMetadata Reads; @@ -27270,80 +27137,6 @@ inline void AttributeBufferSize::Builder::setValidityBytes(::uint64_t value) { ::capnp::bounded<2>() * ::capnp::ELEMENTS, value); } -inline bool MaxBufferSizes::Reader::hasMaxBufferSizes() const { - return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) - .isNull(); -} -inline bool MaxBufferSizes::Builder::hasMaxBufferSizes() { - return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) - .isNull(); -} -inline ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Reader -MaxBufferSizes::Reader::getMaxBufferSizes() const { - return ::capnp::_::PointerHelpers<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>::get(_reader - .getPointerField( - ::capnp::bounded<0>() * - ::capnp::POINTERS)); -} -inline ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Builder -MaxBufferSizes::Builder::getMaxBufferSizes() { - return ::capnp::_::PointerHelpers<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>::get(_builder - .getPointerField( - ::capnp::bounded<0>() * - ::capnp::POINTERS)); -} -inline void MaxBufferSizes::Builder::setMaxBufferSizes( - ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Reader value) { - ::capnp::_::PointerHelpers<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>:: - set(_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - value); -} -inline ::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>::Builder -MaxBufferSizes::Builder::initMaxBufferSizes(unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>:: - init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); -} -inline void MaxBufferSizes::Builder::adoptMaxBufferSizes( - ::capnp::Orphan<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>&& value) { - ::capnp::_::PointerHelpers<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>:: - adopt( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - kj::mv(value)); -} -inline ::capnp::Orphan<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>> -MaxBufferSizes::Builder::disownMaxBufferSizes() { - return ::capnp::_::PointerHelpers<::capnp::List< - ::tiledb::sm::serialization::capnp::AttributeBufferSize, - ::capnp::Kind::STRUCT>>::disown(_builder - .getPointerField( - ::capnp::bounded<0>() * - ::capnp::POINTERS)); -} - inline bool ArrayMetadata::Reader::hasEntries() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull();