diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index ca85b4d5528..b5bb38f0e8f 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -797,7 +797,8 @@ TEST_CASE( TEST_CASE( "SchemaEvolution Error Handling Tests", "[cppapi][schema][evolution][errors]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); REQUIRE_THROWS(ase->evolve_schema(nullptr)); REQUIRE_THROWS(ase->add_attribute(nullptr)); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index f528b0b8712..6f841fa278b 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1576,7 +1576,7 @@ TEST_CASE_METHOD( array->load_all_enumerations(); auto orig_schema = array->array_schema_latest_ptr(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); ase->add_attribute(attr3); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); @@ -1588,7 +1588,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][simple]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1609,7 +1609,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][drop-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1630,7 +1630,7 @@ TEST_CASE_METHOD( create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1649,7 +1649,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values); @@ -1675,7 +1675,7 @@ TEST_CASE_METHOD( REQUIRE(old_enmr != nullptr); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); } @@ -1684,7 +1684,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration", "[enumeration][array-schema-evolution][enmr-to-drop]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1694,7 +1694,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-drop]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase1 = make_shared(HERE()); + auto ase1 = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1702,7 +1702,7 @@ TEST_CASE_METHOD( auto new_schema = ase1->evolve_schema(orig_schema); - auto ase2 = make_shared(HERE()); + auto ase2 = make_shared(HERE(), memory_tracker_); ase2->drop_enumeration("enmr"); CHECK_NOTHROW(ase2->evolve_schema(new_schema)); @@ -1712,7 +1712,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration Repeated", "[enumeration][array-schema-evolution][enmr-to-drop-repeated]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1721,7 +1721,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration After Add", "[enumeration][array-schema-evolution][enmr-add-drop]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1736,7 +1736,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - nullptr", "[enumeration][array-schema-evolution][enmr-nullptr]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); REQUIRE_THROWS(ase->add_enumeration(nullptr)); } @@ -1745,7 +1745,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Already Added", "[enumeration][array-schema-evolution][enmr-already-added]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1759,7 +1759,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Missing Name", "[enumeration][array-schema-evolution][missing-name]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); REQUIRE(ase->enumeration_to_add("foo") == nullptr); } @@ -1769,7 +1769,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-still-in-use]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->drop_enumeration("test_enmr"); REQUIRE_THROWS(ase->evolve_schema(orig_schema)); @@ -1784,7 +1784,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); attr3->set_enumeration_name("test_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_attribute(attr3); auto orig_schema = get_array_schema_latest(); @@ -1805,7 +1805,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::INT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1827,7 +1827,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1839,7 +1839,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration nullptr", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); auto matcher = Catch::Matchers::ContainsSubstring( "Cannot extend enumeration; Input enumeration is null"); REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher); @@ -1849,7 +1849,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration Already Extended", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); auto matcher = Catch::Matchers::ContainsSubstring( @@ -1928,7 +1928,7 @@ TEST_CASE_METHOD( auto old_enmr = schema->get_enumeration("test_enmr"); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); auto st = ctx_.storage_manager()->array_evolve_schema( array->array_uri(), ase.get(), array->get_encryption_key()); @@ -2182,7 +2182,7 @@ TEST_CASE_METHOD( auto attr = make_shared(HERE(), "ohai", Datatype::INT64); attr->set_enumeration_name("enmr2"); - ArraySchemaEvolution ase1; + ArraySchemaEvolution ase1(memory_tracker_); ase1.add_attribute(attr); ase1.add_enumeration(enmr1); ase1.add_enumeration(enmr2); @@ -2216,7 +2216,7 @@ TEST_CASE_METHOD( std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); - ArraySchemaEvolution ase1; + ArraySchemaEvolution ase1(memory_tracker_); ase1.extend_enumeration(enmr1); ase1.extend_enumeration(enmr2); @@ -2416,6 +2416,7 @@ EnumerationFx::EnumerationFx() , memory_tracker_(tiledb::test::create_test_memory_tracker()) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); + memory_tracker_ = tiledb::test::create_test_memory_tracker(); } EnumerationFx::~EnumerationFx() { @@ -2762,8 +2763,8 @@ shared_ptr EnumerationFx::ser_des_array_schema_evolution( ase, stype, &buf, client_side)); ArraySchemaEvolution* ret; - throw_if_not_ok( - serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + throw_if_not_ok(serialization::array_schema_evolution_deserialize( + &ret, stype, buf, memory_tracker_)); return shared_ptr(ret); } diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 65246947170..833d7adac0b 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -68,6 +68,12 @@ std::string memory_type_to_str(MemoryType type) { return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; + case MemoryType::ATTRIBUTES: + return "Attributes"; + case MemoryType::DIMENSION_LABELS: + return "DimensionLabels"; + case MemoryType::DIMENSIONS: + return "Dimensions"; case MemoryType::TILE_SUMS: return "TileSums"; case MemoryType::TILE_WRITER_DATA: @@ -82,16 +88,28 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { switch (type) { case MemoryTrackerType::ANONYMOUS: return "Anonymous"; + case MemoryTrackerType::ARRAY_CREATE: + return "ArrayCreate"; + case MemoryTrackerType::ARRAY_LOAD: + return "ArrayLoad"; case MemoryTrackerType::ARRAY_READ: return "ArrayRead"; case MemoryTrackerType::ARRAY_WRITE: return "ArrayWrite"; + case MemoryTrackerType::FRAGMENT_INFO_LOAD: + return "FragmentInfoLoad"; case MemoryTrackerType::QUERY_READ: return "QueryRead"; case MemoryTrackerType::QUERY_WRITE: return "QueryWrite"; case MemoryTrackerType::CONSOLIDATOR: return "Consolidator"; + case MemoryTrackerType::REST_CLIENT: + return "RestClient"; + case MemoryTrackerType::EPHEMERAL: + return "Ephemeral"; + case MemoryTrackerType::SCHEMA_EVOLUTION: + return "SchemaEvolution"; default: auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory tracker type: " + val); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 29b34fcb259..e9f7e5632fe 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -131,7 +131,8 @@ enum class MemoryTrackerType { QUERY_WRITE, CONSOLIDATOR, REST_CLIENT, - EPHEMERAL + EPHEMERAL, + SCHEMA_EVOLUTION }; class MemoryTrackerResource : public tdb::pmr::memory_resource { diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index edfdfc7e70f..117dee9607b 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" @@ -71,7 +72,15 @@ class ArraySchemaEvolutionException : public StatusException { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -ArraySchemaEvolution::ArraySchemaEvolution() { +ArraySchemaEvolution::ArraySchemaEvolution( + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , attributes_to_add_map_( + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) + , enumerations_to_add_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumerations_to_extend_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) { } ArraySchemaEvolution::ArraySchemaEvolution( @@ -81,13 +90,29 @@ ArraySchemaEvolution::ArraySchemaEvolution( std::unordered_map> enmrs_to_extend, std::unordered_set enmrs_to_drop, - std::pair timestamp_range) - : attributes_to_add_map_(attrs_to_add) + std::pair timestamp_range, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , attributes_to_add_map_( + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) , attributes_to_drop_(attrs_to_drop) - , enumerations_to_add_map_(enmrs_to_add) - , enumerations_to_extend_map_(enmrs_to_extend) + , enumerations_to_add_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumerations_to_extend_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) , enumerations_to_drop_(enmrs_to_drop) , timestamp_range_(timestamp_range) { + for (auto& elem : attrs_to_add) { + attributes_to_add_map_.insert(elem); + } + + for (auto& elem : enmrs_to_add) { + enumerations_to_add_map_.insert(elem); + } + + for (auto& elem : enmrs_to_extend) { + enumerations_to_extend_map_.insert(elem); + } } ArraySchemaEvolution::~ArraySchemaEvolution() { diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 24818c598ca..bc2160698f1 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -36,8 +36,8 @@ #include #include - #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" @@ -53,6 +53,7 @@ class ConstBuffer; class Dimension; class Domain; class Enumeration; +class MemoryTracker; class ArraySchema; enum class ArrayType : uint8_t; @@ -68,13 +69,17 @@ class ArraySchemaEvolution { /* ********************************* */ /** Constructor. */ - ArraySchemaEvolution(); + ArraySchemaEvolution() = delete; + + /** Constructor with memory tracker. */ + ArraySchemaEvolution(shared_ptr memory_tracker); /** Constructor. * @param attrs_to_add Attributes to add to the schema. * @param enmrs_to_add Enumerations to add to the schema. * @param attrs_to_drop Attributes to remove from the schema. * @param timestamp_range Timestamp range to use for the new schema. + * @param memory_tracker Memory tracker to use for the new schema. */ ArraySchemaEvolution( std::unordered_map> attrs_to_add, @@ -84,7 +89,11 @@ class ArraySchemaEvolution { std::unordered_map> enmrs_to_extend, std::unordered_set enmrs_to_drop, - std::pair timestamp_range); + std::pair timestamp_range, + shared_ptr memory_tracker); + + DISABLE_COPY_AND_COPY_ASSIGN(ArraySchemaEvolution); + DISABLE_MOVE_AND_MOVE_ASSIGN(ArraySchemaEvolution); /** Destructor. */ ~ArraySchemaEvolution(); @@ -184,19 +193,25 @@ class ArraySchemaEvolution { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * The memory tracker of the ArraySchema. + */ + shared_ptr memory_tracker_; + /** The array attributes to be added. */ /** It maps each attribute name to the corresponding attribute object. */ - std::unordered_map> attributes_to_add_map_; + tdb::pmr::unordered_map> + attributes_to_add_map_; /** The names of array attributes to be dropped. */ std::unordered_set attributes_to_drop_; /** Enumerations to add with any attribute. */ - std::unordered_map> + tdb::pmr::unordered_map> enumerations_to_add_map_; /** Enumerations to extend. */ - std::unordered_map> + tdb::pmr::unordered_map> enumerations_to_extend_map_; /** The names of array enumerations to be dropped. */ diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9f8c290541f..ebf5888ea0f 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -829,8 +829,10 @@ int32_t tiledb_array_schema_evolution_alloc( } // Create a new SchemaEvolution object + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); (*array_schema_evolution)->array_schema_evolution_ = - new (std::nothrow) tiledb::sm::ArraySchemaEvolution(); + new (std::nothrow) tiledb::sm::ArraySchemaEvolution(memory_tracker); if ((*array_schema_evolution)->array_schema_evolution_ == nullptr) { delete *array_schema_evolution; *array_schema_evolution = nullptr; @@ -3665,12 +3667,15 @@ int32_t tiledb_deserialize_array_schema_evolution( return TILEDB_OOM; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_schema_evolution_deserialize( &((*array_schema_evolution)->array_schema_evolution_), (tiledb::sm::SerializationType)serialize_type, - buffer->buffer()))) { + buffer->buffer(), + memory_tracker))) { delete *array_schema_evolution; *array_schema_evolution = nullptr; return TILEDB_ERR; diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index c60b9882a67..66c18a7635b 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -156,7 +156,8 @@ Status array_schema_evolution_to_capnp( } tdb_unique_ptr array_schema_evolution_from_capnp( - const capnp::ArraySchemaEvolution::Reader& evolution_reader) { + const capnp::ArraySchemaEvolution::Reader& evolution_reader, + shared_ptr memory_tracker) { // Create attributes to add std::unordered_map> attrs_to_add; auto attrs_to_add_reader = evolution_reader.getAttributesToAdd(); @@ -213,7 +214,8 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_add, enmrs_to_extend, enmrs_to_drop, - ts_range)); + ts_range, + memory_tracker)); } Status array_schema_evolution_serialize( @@ -275,7 +277,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer) { + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { try { tdb_unique_ptr decoded_array_schema_evolution = nullptr; @@ -291,8 +294,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + array_schema_evolution_reader, memory_tracker); break; } case SerializationType::CAPNP: { @@ -303,8 +306,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + array_schema_evolution_reader, memory_tracker); break; } default: { @@ -343,7 +346,10 @@ Status array_schema_evolution_serialize( } Status array_schema_evolution_deserialize( - ArraySchemaEvolution**, SerializationType, const Buffer&) { + ArraySchemaEvolution**, + SerializationType, + const Buffer&, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index c1b1d8d2584..e48ebe37bc8 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -69,7 +69,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer); + const Buffer& serialized_buffer, + shared_ptr memory_tracker); } // namespace serialization } // namespace sm