From e29d57db221562b8c9f86e80737e2de6c4fd1e3d Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 15 Feb 2024 17:16:49 -0800 Subject: [PATCH] Enumeration feature first draft --- test/src/unit-enumerations.cc | 80 +++++++++++++++++-- test/src/unit-request-handlers.cc | 1 + .../api/c_api/enumeration/enumeration_api.cc | 6 ++ tiledb/common/memory_tracker.h | 2 + tiledb/sm/array/array_directory.cc | 2 +- tiledb/sm/array_schema/enumeration.cc | 11 ++- tiledb/sm/array_schema/enumeration.h | 23 +++++- tiledb/sm/c_api/tiledb.cc | 5 +- tiledb/sm/rest/rest_client.cc | 2 +- tiledb/sm/serialization/array_schema.cc | 3 +- .../serialization/array_schema_evolution.cc | 21 +++-- .../sm/serialization/array_schema_evolution.h | 4 +- tiledb/sm/serialization/enumeration.cc | 16 ++-- tiledb/sm/serialization/enumeration.h | 5 +- 14 files changed, 148 insertions(+), 33 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 584d88983aca..2a2afea9f003 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -158,7 +158,15 @@ QueryCondition create_qc( TEST_CASE_METHOD( EnumerationFx, "Create Empty Enumeration", "[enumeration][empty]") { Enumeration::create( - default_enmr_name, Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + tiledb::test::create_test_memory_tracker(), + default_enmr_name, + Datatype::INT32, + 1, + false, + nullptr, + 0, + nullptr, + 0); } TEST_CASE_METHOD( @@ -166,6 +174,7 @@ TEST_CASE_METHOD( "Create Empty Var Sized Enumeration", "[enumeration][empty]") { Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -246,6 +255,7 @@ TEST_CASE_METHOD( "[enumeration][error][invalid-offsets-args]") { uint64_t offsets = 0; auto enmr = Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -315,6 +325,7 @@ TEST_CASE_METHOD( "[enumeration][basic][fixed][multi-cell-val-num]") { std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; auto enmr = Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 2, @@ -334,6 +345,7 @@ TEST_CASE_METHOD( "Invalid data buffer must not be nullptr for fixed sized data."); REQUIRE_THROWS_WITH( Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 1, @@ -354,7 +366,15 @@ TEST_CASE_METHOD( "Invalid data size; must be non-zero for fixed size data."); REQUIRE_THROWS_WITH( Enumeration::create( - default_enmr_name, Datatype::INT32, 1, false, &val, 0, nullptr, 0), + tiledb::test::create_test_memory_tracker(), + default_enmr_name, + Datatype::INT32, + 1, + false, + &val, + 0, + nullptr, + 0), matcher); } @@ -367,6 +387,7 @@ TEST_CASE_METHOD( "Var sized enumeration values require a non-null offsets pointer."); REQUIRE_THROWS_WITH( Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -388,6 +409,7 @@ TEST_CASE_METHOD( "Var sized enumeration values require a non-zero offsets size."); REQUIRE_THROWS_WITH( Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -409,6 +431,7 @@ TEST_CASE_METHOD( "is non-zero."); REQUIRE_THROWS_WITH( Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -430,6 +453,7 @@ TEST_CASE_METHOD( "require data."); REQUIRE_THROWS_WITH( Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -452,6 +476,7 @@ TEST_CASE_METHOD( "offset."); REQUIRE_THROWS_WITH( Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -469,6 +494,7 @@ TEST_CASE_METHOD( "[enumeration][error][invalid-name]") { std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), std::string(), Datatype::INT32, 2, @@ -485,6 +511,7 @@ TEST_CASE_METHOD( "[enumeration][error][invalid-name]") { std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), "", Datatype::INT32, 2, @@ -501,6 +528,7 @@ TEST_CASE_METHOD( "[enumeration][error][invalid-name]") { std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, "an/bad/path", Datatype::INT32, @@ -518,6 +546,7 @@ TEST_CASE_METHOD( "[enumeration][error][invalid-cell-val-num]") { std::vector values = {1, 2, 3}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 0, @@ -534,6 +563,7 @@ TEST_CASE_METHOD( "[enumeration][error][data-nullptr]") { std::vector values = {1, 2, 3}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 1, @@ -550,6 +580,7 @@ TEST_CASE_METHOD( "[enumeration][error][data-zero-size]") { std::vector values = {1, 2, 3}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 1, @@ -567,6 +598,7 @@ TEST_CASE_METHOD( auto data = "foobarbazbam"; std::vector offsets = {0, 3, 6, 9}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -584,6 +616,7 @@ TEST_CASE_METHOD( auto data = "foobarbazbam"; std::vector offsets = {0, 3, 6, 9}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -601,6 +634,7 @@ TEST_CASE_METHOD( std::vector values = {0, 1, 2, 3, 4}; std::vector offsets = {0, 3, 6, 9}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 1, @@ -617,6 +651,7 @@ TEST_CASE_METHOD( "[enumeration][error][offsets-not-required]") { std::vector values = {0, 1, 2, 3, 4}; REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 1, @@ -636,6 +671,7 @@ TEST_CASE_METHOD( // Passing 3 for the offsets size is incorrect because the offsets size has // to be a multiple of `sizeof(uint64_t)` REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -654,6 +690,7 @@ TEST_CASE_METHOD( std::vector offsets = {0, 3, 6, 100}; // The last offset is larger than data_size REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::STRING_ASCII, constants::var_num, @@ -672,6 +709,7 @@ TEST_CASE_METHOD( // Passing 3 for the data size is invalid as its not a multiple of // sizeof(int) REQUIRE_THROWS(Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 1, @@ -742,6 +780,7 @@ TEST_CASE_METHOD( std::vector extend_values = {5, 6, 7, 8, 9, 10}; std::vector final_values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; auto enmr1 = Enumeration::create( + tiledb::test::create_test_memory_tracker(), default_enmr_name, Datatype::INT32, 2, @@ -946,7 +985,8 @@ TEST_CASE_METHOD( memset(data, 1, 4); Deserializer deserializer(tile.data(), tile.size()); - REQUIRE_THROWS(Enumeration::deserialize(deserializer)); + REQUIRE_THROWS(Enumeration::deserialize( + tiledb::test::create_test_memory_tracker(), deserializer)); } TEST_CASE_METHOD( @@ -1391,6 +1431,7 @@ TEST_CASE_METHOD( std::vector data(1024 * 1024 * 10 + 1); std::vector offsets = {0}; auto enmr = Enumeration::create( + tiledb::test::create_test_memory_tracker(), "enmr_name", Datatype::STRING_ASCII, constants::var_num, @@ -1420,6 +1461,7 @@ TEST_CASE_METHOD( // Create more than 50MiB of enumeration data for (size_t i = 0; i < 10; i++) { auto enmr = Enumeration::create( + tiledb::test::create_test_memory_tracker(), "enmr_name_" + std::to_string(i), Datatype::STRING_ASCII, constants::var_num, @@ -1548,6 +1590,7 @@ TEST_CASE_METHOD( // We have to force this condition by hand auto enmr3 = tiledb::sm::Enumeration::create( + tiledb::test::create_test_memory_tracker(), enmr2->name(), // Notice we're reusing the existing path name from enmr1 enmr1->path_name(), @@ -2146,8 +2189,17 @@ TEST_CASE_METHOD( auto schema1 = create_schema(); auto enmr1 = Enumeration::create( - "empty_fixed", Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + tiledb::test::create_test_memory_tracker(), + "empty_fixed", + Datatype::INT32, + 1, + false, + nullptr, + 0, + nullptr, + 0); auto enmr2 = Enumeration::create( + tiledb::test::create_test_memory_tracker(), "empty_var", Datatype::STRING_ASCII, constants::var_num, @@ -2442,6 +2494,7 @@ shared_ptr EnumerationFx::create_enumeration( raw_values[i] = values[i] ? 1 : 0; } return Enumeration::create( + tiledb::test::create_test_memory_tracker(), name, tp.type_, tp.cell_val_num_, @@ -2452,6 +2505,7 @@ shared_ptr EnumerationFx::create_enumeration( 0); } else if constexpr (std::is_pod_v) { return Enumeration::create( + tiledb::test::create_test_memory_tracker(), name, tp.type_, tp.cell_val_num_, @@ -2478,6 +2532,7 @@ shared_ptr EnumerationFx::create_enumeration( } return Enumeration::create( + tiledb::test::create_test_memory_tracker(), name, tp.type_, tp.cell_val_num_, @@ -2492,7 +2547,15 @@ shared_ptr EnumerationFx::create_enumeration( shared_ptr EnumerationFx::create_empty_enumeration( Datatype type, uint32_t cell_val_num, bool ordered, std::string name) { return Enumeration::create( - name, type, cell_val_num, ordered, nullptr, 0, nullptr, 0); + tiledb::test::create_test_memory_tracker(), + name, + type, + cell_val_num, + ordered, + nullptr, + 0, + nullptr, + 0); } template @@ -2566,7 +2629,8 @@ void EnumerationFx::check_storage_deserialization( auto tile = serialize_to_tile(enmr); Deserializer deserializer(tile.data(), tile.size()); - auto deserialized = Enumeration::deserialize(deserializer); + auto deserialized = Enumeration::deserialize( + tiledb::test::create_test_memory_tracker(), deserializer); REQUIRE(deserialized->name() == enmr->name()); REQUIRE(deserialized->path_name().empty() == false); @@ -2762,8 +2826,8 @@ shared_ptr EnumerationFx::ser_des_array_schema_evolution( ase, stype, &buf, client_side)); ArraySchemaEvolution* ret; - throw_if_not_ok( - serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + throw_if_not_ok(serialization::array_schema_evolution_deserialize( + &ret, stype, buf, tiledb::test::create_test_memory_tracker())); return shared_ptr(ret); } diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index da00e664f69e..db5fd7a7dc23 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -385,6 +385,7 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( } return Enumeration::create( + tiledb::test::create_test_memory_tracker(), name, Datatype::STRING_ASCII, constants::var_num, diff --git a/tiledb/api/c_api/enumeration/enumeration_api.cc b/tiledb/api/c_api/enumeration/enumeration_api.cc index d73759bef31c..da64162edfcb 100644 --- a/tiledb/api/c_api/enumeration/enumeration_api.cc +++ b/tiledb/api/c_api/enumeration/enumeration_api.cc @@ -34,10 +34,12 @@ #include "../string/string_api_internal.h" #include "enumeration_api_experimental.h" #include "enumeration_api_internal.h" +#include "tiledb/common/memory_tracker.h" namespace tiledb::api { capi_return_t tiledb_enumeration_alloc( + shared_ptr memory_tracker, const char* name, tiledb_datatype_t type, uint32_t cell_val_num, @@ -66,6 +68,7 @@ capi_return_t tiledb_enumeration_alloc( try { *enumeration = tiledb_enumeration_handle_t::make_handle( + memory_tracker, std::string(name), datatype, cell_val_num, @@ -191,8 +194,11 @@ CAPI_INTERFACE( const void* offsets, uint64_t offsets_size, tiledb_enumeration_t** enumeration) { + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ENUMERATION_CREATE); return api_entry_context( ctx, + memory_tracker, name, type, cell_val_num, diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index f06d3dc29ea5..a4e1857600dd 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -121,9 +121,11 @@ enum class MemoryType { enum class MemoryTrackerType { ANONYMOUS, ARRAY_CREATE, + ARRAY_EVOLVE, ARRAY_LOAD, ARRAY_READ, ARRAY_WRITE, + ENUMERATION_CREATE, FRAGMENT_INFO_LOAD, QUERY_READ, QUERY_WRITE, diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index a0042b2bdde8..0bc7c84633d2 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -1335,7 +1335,7 @@ shared_ptr ArrayDirectory::load_enumeration( } Deserializer deserializer(tile.data(), tile.size()); - return Enumeration::deserialize(deserializer); + return Enumeration::deserialize(memory_tracker, deserializer); } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index c6b2ab9633f7..4855dcfb6fe6 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -48,6 +48,7 @@ class EnumerationException : public StatusException { }; Enumeration::Enumeration( + shared_ptr memory_tracker, const std::string& name, const std::string& path_name, Datatype type, @@ -57,13 +58,15 @@ Enumeration::Enumeration( uint64_t data_size, const void* offsets, uint64_t offsets_size) - : name_(name) + : memory_tracker_(memory_tracker) + , name_(name) , path_name_(path_name) , type_(type) , cell_val_num_(cell_val_num) , ordered_(ordered) , data_(data_size) - , offsets_(offsets_size) { + , offsets_(offsets_size) + , value_map_(memory_tracker_->get_resource(MemoryType::ENUMERATION)) { ensure_datatype_is_valid(type); if (name.empty()) { @@ -178,7 +181,7 @@ Enumeration::Enumeration( } shared_ptr Enumeration::deserialize( - Deserializer& deserializer) { + shared_ptr memory_tracker, Deserializer& deserializer) { auto disk_version = deserializer.read(); if (disk_version > constants::enumerations_version) { throw EnumerationException( @@ -216,6 +219,7 @@ shared_ptr Enumeration::deserialize( } return create( + memory_tracker, name, path_name, static_cast(type), @@ -296,6 +300,7 @@ shared_ptr Enumeration::extend( } return create( + memory_tracker_, name_, "", type_, diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h index 950ca447dc92..30dcd18a5ea2 100644 --- a/tiledb/sm/array_schema/enumeration.h +++ b/tiledb/sm/array_schema/enumeration.h @@ -36,6 +36,8 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/types/untyped_datum.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" @@ -87,6 +89,7 @@ class Enumeration { * @return shared_ptr The created enumeration. */ static shared_ptr create( + shared_ptr memory_tracker, const std::string& name, Datatype type, uint32_t cell_val_num, @@ -96,6 +99,7 @@ class Enumeration { const void* offsets, uint64_t offsets_size) { return create( + memory_tracker, name, "", type, @@ -125,6 +129,7 @@ class Enumeration { * @return shared_ptr The created enumeration. */ static shared_ptr create( + shared_ptr memory_tracker, const std::string& name, const std::string& path_name, Datatype type, @@ -136,6 +141,7 @@ class Enumeration { uint64_t offsets_size) { struct EnableMakeShared : public Enumeration { EnableMakeShared( + shared_ptr memory_tracker, const std::string& name, const std::string& path_name, Datatype type, @@ -146,6 +152,7 @@ class Enumeration { const void* offsets, uint64_t offsets_size) : Enumeration( + memory_tracker, name, path_name, type, @@ -159,6 +166,7 @@ class Enumeration { }; return make_shared( HERE(), + memory_tracker, name, path_name, type, @@ -173,13 +181,15 @@ class Enumeration { /** * Deserialize an enumeration * + * @param memory_tracker The memory tracker associated with this Enumeration. * @param deserializer The deserializer to deserialize from. * @return A new Enumeration. */ - static shared_ptr deserialize(Deserializer& deserializer); + static shared_ptr deserialize( + shared_ptr memory_tracker, Deserializer& deserializer); /** - * Create a new enumeration by extending an existing enumeration's + * Create a new enumeration by extending an existinsg enumeration's * list of values. * * The returned Enumeration can then be used by the @@ -346,6 +356,7 @@ class Enumeration { /** Constructor * + * @param memory_tracker The memory tracker. * @param name The name of this Enumeration as referenced by attributes. * @param path_name The last URI path component of the Enumeration. * @param type The datatype of the enumeration values. @@ -361,6 +372,7 @@ class Enumeration { * zero of cell_var_num is not var_num. */ Enumeration( + shared_ptr memory_tracker, const std::string& name, const std::string& path_name, Datatype type, @@ -375,6 +387,11 @@ class Enumeration { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * The memory tracker of the Enumeration. + */ + shared_ptr memory_tracker_; + /** The name of this Enumeration stored in the enumerations directory. */ std::string name_; @@ -397,7 +414,7 @@ class Enumeration { Buffer offsets_; /** Map of values to indices */ - std::unordered_map value_map_; + tdb::pmr::unordered_map value_map_; /* ********************************* */ /* PRIVATE METHODS */ diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 6736ee3fa954..bc483e4a6b9a 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3659,12 +3659,15 @@ int32_t tiledb_deserialize_array_schema_evolution( return TILEDB_OOM; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_EVOLVE); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_schema_evolution_deserialize( &((*array_schema_evolution)->array_schema_evolution_), (tiledb::sm::SerializationType)serialize_type, - buffer->buffer()))) { + buffer->buffer(), + memory_tracker))) { delete *array_schema_evolution; *array_schema_evolution = nullptr; return TILEDB_ERR; diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index f170c56d77da..dc8d1c5d03ad 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -691,7 +691,7 @@ RestClient::post_enumerations_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); return serialization::deserialize_load_enumerations_response( - serialization_type_, returned_data); + serialization_type_, returned_data, memory_tracker_); } Status RestClient::submit_query_to_rest(const URI& uri, Query* query) { diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 57c3d25ac96c..fccae42b20b6 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -1087,7 +1087,8 @@ shared_ptr array_schema_from_capnp( enumerations.reserve(enmr_readers.size()); try { for (auto&& enmr_reader : enmr_readers) { - enumerations.emplace_back(enumeration_from_capnp(enmr_reader)); + enumerations.emplace_back( + enumeration_from_capnp(memory_tracker, enmr_reader)); } } catch (const std::exception& e) { std::throw_with_nested(std::runtime_error( diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index c60b9882a674..a84b26f66c4c 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -156,6 +156,7 @@ Status array_schema_evolution_to_capnp( } tdb_unique_ptr array_schema_evolution_from_capnp( + shared_ptr memory_tracker, const capnp::ArraySchemaEvolution::Reader& evolution_reader) { // Create attributes to add std::unordered_map> attrs_to_add; @@ -177,7 +178,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( std::unordered_map> enmrs_to_add; auto enmrs_to_add_reader = evolution_reader.getEnumerationsToAdd(); for (auto enmr_reader : enmrs_to_add_reader) { - auto enmr = enumeration_from_capnp(enmr_reader); + auto enmr = enumeration_from_capnp(memory_tracker, enmr_reader); enmrs_to_add[enmr->name()] = enmr; } @@ -186,7 +187,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_extend; auto enmrs_to_extend_reader = evolution_reader.getEnumerationsToExtend(); for (auto enmr_reader : enmrs_to_extend_reader) { - auto enmr = enumeration_from_capnp(enmr_reader); + auto enmr = enumeration_from_capnp(memory_tracker, enmr_reader); enmrs_to_extend[enmr->name()] = enmr; } @@ -275,7 +276,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer) { + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { try { tdb_unique_ptr decoded_array_schema_evolution = nullptr; @@ -291,8 +293,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + memory_tracker, array_schema_evolution_reader); break; } case SerializationType::CAPNP: { @@ -303,8 +305,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + memory_tracker, array_schema_evolution_reader); break; } default: { @@ -343,7 +345,10 @@ Status array_schema_evolution_serialize( } Status array_schema_evolution_deserialize( - ArraySchemaEvolution**, SerializationType, const Buffer&) { + ArraySchemaEvolution**, + SerializationType, + const Buffer&, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index c1b1d8d25841..1a6e66b959a0 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -35,6 +35,7 @@ #include +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" using namespace tiledb::common; @@ -69,7 +70,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer); + const Buffer& serialized_buffer, + shared_ptr memory_tracker); } // namespace serialization } // namespace sm diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 16eb87916636..cc0eb4a6f0c8 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -71,6 +71,7 @@ void enumeration_to_capnp( } shared_ptr enumeration_from_capnp( + shared_ptr memory_tracker, const capnp::Enumeration::Reader& reader) { auto name = reader.getName(); auto path_name = reader.getPathName(); @@ -96,6 +97,7 @@ shared_ptr enumeration_from_capnp( } return Enumeration::create( + memory_tracker, name, path_name, datatype, @@ -150,12 +152,13 @@ void load_enumerations_response_to_capnp( std::vector> load_enumerations_response_from_capnp( + shared_ptr memory_tracker, const capnp::LoadEnumerationsResponse::Reader& reader) { std::vector> ret; if (reader.hasEnumerations()) { auto enmr_readers = reader.getEnumerations(); for (auto enmr_reader : enmr_readers) { - ret.push_back(enumeration_from_capnp(enmr_reader)); + ret.push_back(enumeration_from_capnp(memory_tracker, enmr_reader)); } } return ret; @@ -306,7 +309,9 @@ void serialize_load_enumerations_response( std::vector> deserialize_load_enumerations_response( - SerializationType serialize_type, const Buffer& response) { + SerializationType serialize_type, + const Buffer& response, + shared_ptr memory_tracker) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -317,7 +322,7 @@ deserialize_load_enumerations_response( json.decode( kj::StringPtr(static_cast(response.data())), builder); capnp::LoadEnumerationsResponse::Reader reader = builder.asReader(); - return load_enumerations_response_from_capnp(reader); + return load_enumerations_response_from_capnp(memory_tracker, reader); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(response.data()); @@ -326,7 +331,7 @@ deserialize_load_enumerations_response( response.size() / sizeof(::capnp::word))); capnp::LoadEnumerationsResponse::Reader reader = array_reader.getRoot(); - return load_enumerations_response_from_capnp(reader); + return load_enumerations_response_from_capnp(memory_tracker, reader); } default: { throw Status_SerializationError( @@ -371,7 +376,8 @@ void serialize_load_enumerations_response( } std::vector> -deserialize_load_enumerations_response(SerializationType, const Buffer&) { +deserialize_load_enumerations_response( + SerializationType, const Buffer&, shared_ptr) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } diff --git a/tiledb/sm/serialization/enumeration.h b/tiledb/sm/serialization/enumeration.h index 85e5faaaf7a3..9f8cd4ee9287 100644 --- a/tiledb/sm/serialization/enumeration.h +++ b/tiledb/sm/serialization/enumeration.h @@ -67,6 +67,7 @@ void enumeration_to_capnp( * @return A new Enumeration */ shared_ptr enumeration_from_capnp( + shared_ptr memory_tracker, const capnp::Enumeration::Reader& reader); #endif @@ -87,7 +88,9 @@ void serialize_load_enumerations_response( std::vector> deserialize_load_enumerations_response( - SerializationType serialization_type, const Buffer& response); + SerializationType serialization_type, + const Buffer& response, + shared_ptr memory_tracker); } // namespace serialization } // namespace tiledb::sm