From 3011f9388f0ceb5c242129567e209ae9da9174f4 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 15 Feb 2024 23:10:21 -0800 Subject: [PATCH 1/4] First draft of ArraySchemaEvolution PR --- test/src/unit-cppapi-schema-evolution.cc | 3 +- test/src/unit-enumerations.cc | 71 ++++++++++++------- tiledb/common/memory_tracker.cc | 18 +++++ tiledb/common/memory_tracker.h | 3 +- .../sm/array_schema/array_schema_evolution.cc | 14 +++- .../sm/array_schema/array_schema_evolution.h | 17 ++++- tiledb/sm/c_api/tiledb.cc | 9 ++- .../serialization/array_schema_evolution.cc | 18 +++-- .../sm/serialization/array_schema_evolution.h | 3 +- 9 files changed, 115 insertions(+), 41 deletions(-) diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index ca85b4d5528..b5bb38f0e8f 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -797,7 +797,8 @@ TEST_CASE( TEST_CASE( "SchemaEvolution Error Handling Tests", "[cppapi][schema][evolution][errors]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); REQUIRE_THROWS(ase->evolve_schema(nullptr)); REQUIRE_THROWS(ase->add_attribute(nullptr)); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index f528b0b8712..23263790c63 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1576,7 +1576,8 @@ TEST_CASE_METHOD( array->load_all_enumerations(); auto orig_schema = array->array_schema_latest_ptr(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); ase->add_attribute(attr3); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); @@ -1588,7 +1589,8 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][simple]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1609,7 +1611,8 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][drop-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1630,7 +1633,8 @@ TEST_CASE_METHOD( create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1649,7 +1653,8 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values); @@ -1675,7 +1680,8 @@ TEST_CASE_METHOD( REQUIRE(old_enmr != nullptr); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase->extend_enumeration(new_enmr); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); } @@ -1684,7 +1690,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration", "[enumeration][array-schema-evolution][enmr-to-drop]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1694,7 +1701,8 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-drop]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase1 = make_shared(HERE()); + auto ase1 = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1702,7 +1710,8 @@ TEST_CASE_METHOD( auto new_schema = ase1->evolve_schema(orig_schema); - auto ase2 = make_shared(HERE()); + auto ase2 = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase2->drop_enumeration("enmr"); CHECK_NOTHROW(ase2->evolve_schema(new_schema)); @@ -1712,7 +1721,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration Repeated", "[enumeration][array-schema-evolution][enmr-to-drop-repeated]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1721,7 +1731,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration After Add", "[enumeration][array-schema-evolution][enmr-add-drop]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1736,7 +1747,8 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - nullptr", "[enumeration][array-schema-evolution][enmr-nullptr]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); REQUIRE_THROWS(ase->add_enumeration(nullptr)); } @@ -1745,7 +1757,8 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Already Added", "[enumeration][array-schema-evolution][enmr-already-added]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1759,7 +1772,8 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Missing Name", "[enumeration][array-schema-evolution][missing-name]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); REQUIRE(ase->enumeration_to_add("foo") == nullptr); } @@ -1769,7 +1783,8 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-still-in-use]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase->drop_enumeration("test_enmr"); REQUIRE_THROWS(ase->evolve_schema(orig_schema)); @@ -1784,7 +1799,8 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); attr3->set_enumeration_name("test_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase->add_attribute(attr3); auto orig_schema = get_array_schema_latest(); @@ -1805,7 +1821,8 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::INT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1827,7 +1844,8 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1839,7 +1857,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration nullptr", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); auto matcher = Catch::Matchers::ContainsSubstring( "Cannot extend enumeration; Input enumeration is null"); REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher); @@ -1849,7 +1868,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration Already Extended", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); auto matcher = Catch::Matchers::ContainsSubstring( @@ -1928,7 +1948,8 @@ TEST_CASE_METHOD( auto old_enmr = schema->get_enumeration("test_enmr"); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); ase->extend_enumeration(new_enmr); auto st = ctx_.storage_manager()->array_evolve_schema( array->array_uri(), ase.get(), array->get_encryption_key()); @@ -2182,7 +2203,7 @@ TEST_CASE_METHOD( auto attr = make_shared(HERE(), "ohai", Datatype::INT64); attr->set_enumeration_name("enmr2"); - ArraySchemaEvolution ase1; + ArraySchemaEvolution ase1(tiledb::test::create_test_memory_tracker()); ase1.add_attribute(attr); ase1.add_enumeration(enmr1); ase1.add_enumeration(enmr2); @@ -2216,7 +2237,7 @@ TEST_CASE_METHOD( std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); - ArraySchemaEvolution ase1; + ArraySchemaEvolution ase1(tiledb::test::create_test_memory_tracker()); ase1.extend_enumeration(enmr1); ase1.extend_enumeration(enmr2); @@ -2762,8 +2783,8 @@ shared_ptr EnumerationFx::ser_des_array_schema_evolution( ase, stype, &buf, client_side)); ArraySchemaEvolution* ret; - throw_if_not_ok( - serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + throw_if_not_ok(serialization::array_schema_evolution_deserialize( + &ret, stype, buf, tiledb::test::create_test_memory_tracker())); return shared_ptr(ret); } diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 65246947170..f7497c77539 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -68,6 +68,12 @@ std::string memory_type_to_str(MemoryType type) { return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; + case MemoryType::ATTRIBUTES: + return "Attributes"; + case MemoryType::DIMENSION_LABELS: + return "DimensionLabels"; + case MemoryType::DIMENSIONS: + return "Dimensions"; case MemoryType::TILE_SUMS: return "TileSums"; case MemoryType::TILE_WRITER_DATA: @@ -82,16 +88,28 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { switch (type) { case MemoryTrackerType::ANONYMOUS: return "Anonymous"; + case MemoryTrackerType::ARRAY_CREATE: + return "ArrayCreate"; + case MemoryTrackerType::ARRAY_LOAD: + return "ArrayLoad"; case MemoryTrackerType::ARRAY_READ: return "ArrayRead"; case MemoryTrackerType::ARRAY_WRITE: return "ArrayWrite"; + case MemoryTrackerType::FRAGMENT_INFO_LOAD: + return "FragmentInfoLoad"; case MemoryTrackerType::QUERY_READ: return "QueryRead"; case MemoryTrackerType::QUERY_WRITE: return "QueryWrite"; case MemoryTrackerType::CONSOLIDATOR: return "Consolidator"; + case MemoryTrackerType::REST_CLIENT: + return "RestClient"; + case MemoryTrackerType::EPHEMERAL: + return "Ephemeral"; + case MemoryTrackerType::SCHEMA_EVOLUTION: + return "SchemaEvolution"; default: auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory tracker type: " + val); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 29b34fcb259..e9f7e5632fe 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -131,7 +131,8 @@ enum class MemoryTrackerType { QUERY_WRITE, CONSOLIDATOR, REST_CLIENT, - EPHEMERAL + EPHEMERAL, + SCHEMA_EVOLUTION }; class MemoryTrackerResource : public tdb::pmr::memory_resource { diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index edfdfc7e70f..df1be309a4b 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -71,10 +71,15 @@ class ArraySchemaEvolutionException : public StatusException { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -ArraySchemaEvolution::ArraySchemaEvolution() { +ArraySchemaEvolution::ArraySchemaEvolution( + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , attributes_to_add_map_( + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) { } ArraySchemaEvolution::ArraySchemaEvolution( + shared_ptr memory_tracker, std::unordered_map> attrs_to_add, std::unordered_set attrs_to_drop, std::unordered_map> enmrs_to_add, @@ -82,12 +87,17 @@ ArraySchemaEvolution::ArraySchemaEvolution( enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range) - : attributes_to_add_map_(attrs_to_add) + : memory_tracker_(memory_tracker) + , attributes_to_add_map_( + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) , attributes_to_drop_(attrs_to_drop) , enumerations_to_add_map_(enmrs_to_add) , enumerations_to_extend_map_(enmrs_to_extend) , enumerations_to_drop_(enmrs_to_drop) , timestamp_range_(timestamp_range) { + for (auto& elem : attrs_to_add) { + attributes_to_add_map_.insert(elem); + } } ArraySchemaEvolution::~ArraySchemaEvolution() { diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 24818c598ca..f5919f652e0 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -36,8 +36,9 @@ #include #include - #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" @@ -68,7 +69,10 @@ class ArraySchemaEvolution { /* ********************************* */ /** Constructor. */ - ArraySchemaEvolution(); + ArraySchemaEvolution() = delete; + + /** Constructor with memory tracker. */ + ArraySchemaEvolution(shared_ptr memory_tracker); /** Constructor. * @param attrs_to_add Attributes to add to the schema. @@ -77,6 +81,7 @@ class ArraySchemaEvolution { * @param timestamp_range Timestamp range to use for the new schema. */ ArraySchemaEvolution( + shared_ptr memory_tracker, std::unordered_map> attrs_to_add, std::unordered_set attrs_to_drop, std::unordered_map> @@ -184,9 +189,15 @@ class ArraySchemaEvolution { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * The memory tracker of the ArraySchema. + */ + shared_ptr memory_tracker_; + /** The array attributes to be added. */ /** It maps each attribute name to the corresponding attribute object. */ - std::unordered_map> attributes_to_add_map_; + tdb::pmr::unordered_map> + attributes_to_add_map_; /** The names of array attributes to be dropped. */ std::unordered_set attributes_to_drop_; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9f8c290541f..ebf5888ea0f 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -829,8 +829,10 @@ int32_t tiledb_array_schema_evolution_alloc( } // Create a new SchemaEvolution object + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); (*array_schema_evolution)->array_schema_evolution_ = - new (std::nothrow) tiledb::sm::ArraySchemaEvolution(); + new (std::nothrow) tiledb::sm::ArraySchemaEvolution(memory_tracker); if ((*array_schema_evolution)->array_schema_evolution_ == nullptr) { delete *array_schema_evolution; *array_schema_evolution = nullptr; @@ -3665,12 +3667,15 @@ int32_t tiledb_deserialize_array_schema_evolution( return TILEDB_OOM; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_schema_evolution_deserialize( &((*array_schema_evolution)->array_schema_evolution_), (tiledb::sm::SerializationType)serialize_type, - buffer->buffer()))) { + buffer->buffer(), + memory_tracker))) { delete *array_schema_evolution; *array_schema_evolution = nullptr; return TILEDB_ERR; diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index c60b9882a67..360a07cfba5 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -156,6 +156,7 @@ Status array_schema_evolution_to_capnp( } tdb_unique_ptr array_schema_evolution_from_capnp( + shared_ptr memory_tracker, const capnp::ArraySchemaEvolution::Reader& evolution_reader) { // Create attributes to add std::unordered_map> attrs_to_add; @@ -208,6 +209,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( return tdb_unique_ptr(tdb_new( ArraySchemaEvolution, + memory_tracker, attrs_to_add, attrs_to_drop, enmrs_to_add, @@ -275,7 +277,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer) { + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { try { tdb_unique_ptr decoded_array_schema_evolution = nullptr; @@ -291,8 +294,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + memory_tracker, array_schema_evolution_reader); break; } case SerializationType::CAPNP: { @@ -303,8 +306,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + memory_tracker, array_schema_evolution_reader); break; } default: { @@ -343,7 +346,10 @@ Status array_schema_evolution_serialize( } Status array_schema_evolution_deserialize( - ArraySchemaEvolution**, SerializationType, const Buffer&) { + ArraySchemaEvolution**, + SerializationType, + const Buffer&, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index c1b1d8d2584..e48ebe37bc8 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -69,7 +69,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer); + const Buffer& serialized_buffer, + shared_ptr memory_tracker); } // namespace serialization } // namespace sm From 7e8af37e6dc63fb34e2c73903c7ce776d3a4a2c9 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 16 Feb 2024 19:19:59 -0800 Subject: [PATCH 2/4] changes --- test/src/unit-enumerations.cc | 70 +++++++------------ .../sm/array_schema/array_schema_evolution.cc | 5 +- .../sm/array_schema/array_schema_evolution.h | 7 +- .../serialization/array_schema_evolution.cc | 12 ++-- 4 files changed, 38 insertions(+), 56 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 23263790c63..6f841fa278b 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1576,8 +1576,7 @@ TEST_CASE_METHOD( array->load_all_enumerations(); auto orig_schema = array->array_schema_latest_ptr(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); ase->add_attribute(attr3); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); @@ -1589,8 +1588,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][simple]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1611,8 +1609,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][drop-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1633,8 +1630,7 @@ TEST_CASE_METHOD( create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1653,8 +1649,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values); @@ -1680,8 +1675,7 @@ TEST_CASE_METHOD( REQUIRE(old_enmr != nullptr); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); } @@ -1690,8 +1684,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration", "[enumeration][array-schema-evolution][enmr-to-drop]") { - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1701,8 +1694,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-drop]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase1 = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase1 = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1710,8 +1702,7 @@ TEST_CASE_METHOD( auto new_schema = ase1->evolve_schema(orig_schema); - auto ase2 = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase2 = make_shared(HERE(), memory_tracker_); ase2->drop_enumeration("enmr"); CHECK_NOTHROW(ase2->evolve_schema(new_schema)); @@ -1721,8 +1712,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration Repeated", "[enumeration][array-schema-evolution][enmr-to-drop-repeated]") { - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1731,8 +1721,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration After Add", "[enumeration][array-schema-evolution][enmr-add-drop]") { - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1747,8 +1736,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - nullptr", "[enumeration][array-schema-evolution][enmr-nullptr]") { create_array(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); REQUIRE_THROWS(ase->add_enumeration(nullptr)); } @@ -1757,8 +1745,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Already Added", "[enumeration][array-schema-evolution][enmr-already-added]") { create_array(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1772,8 +1759,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Missing Name", "[enumeration][array-schema-evolution][missing-name]") { create_array(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); REQUIRE(ase->enumeration_to_add("foo") == nullptr); } @@ -1783,8 +1769,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-still-in-use]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); ase->drop_enumeration("test_enmr"); REQUIRE_THROWS(ase->evolve_schema(orig_schema)); @@ -1799,8 +1784,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); attr3->set_enumeration_name("test_enmr"); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_attribute(attr3); auto orig_schema = get_array_schema_latest(); @@ -1821,8 +1805,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::INT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1844,8 +1827,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1857,8 +1839,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration nullptr", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); auto matcher = Catch::Matchers::ContainsSubstring( "Cannot extend enumeration; Input enumeration is null"); REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher); @@ -1868,8 +1849,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration Already Extended", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); auto matcher = Catch::Matchers::ContainsSubstring( @@ -1948,8 +1928,7 @@ TEST_CASE_METHOD( auto old_enmr = schema->get_enumeration("test_enmr"); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared( - HERE(), tiledb::test::create_test_memory_tracker()); + auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); auto st = ctx_.storage_manager()->array_evolve_schema( array->array_uri(), ase.get(), array->get_encryption_key()); @@ -2203,7 +2182,7 @@ TEST_CASE_METHOD( auto attr = make_shared(HERE(), "ohai", Datatype::INT64); attr->set_enumeration_name("enmr2"); - ArraySchemaEvolution ase1(tiledb::test::create_test_memory_tracker()); + ArraySchemaEvolution ase1(memory_tracker_); ase1.add_attribute(attr); ase1.add_enumeration(enmr1); ase1.add_enumeration(enmr2); @@ -2237,7 +2216,7 @@ TEST_CASE_METHOD( std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); - ArraySchemaEvolution ase1(tiledb::test::create_test_memory_tracker()); + ArraySchemaEvolution ase1(memory_tracker_); ase1.extend_enumeration(enmr1); ase1.extend_enumeration(enmr2); @@ -2437,6 +2416,7 @@ EnumerationFx::EnumerationFx() , memory_tracker_(tiledb::test::create_test_memory_tracker()) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); + memory_tracker_ = tiledb::test::create_test_memory_tracker(); } EnumerationFx::~EnumerationFx() { @@ -2784,7 +2764,7 @@ shared_ptr EnumerationFx::ser_des_array_schema_evolution( ArraySchemaEvolution* ret; throw_if_not_ok(serialization::array_schema_evolution_deserialize( - &ret, stype, buf, tiledb::test::create_test_memory_tracker())); + &ret, stype, buf, memory_tracker_)); return shared_ptr(ret); } diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index df1be309a4b..edbbfec4bfb 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" @@ -79,14 +80,14 @@ ArraySchemaEvolution::ArraySchemaEvolution( } ArraySchemaEvolution::ArraySchemaEvolution( - shared_ptr memory_tracker, std::unordered_map> attrs_to_add, std::unordered_set attrs_to_drop, std::unordered_map> enmrs_to_add, std::unordered_map> enmrs_to_extend, std::unordered_set enmrs_to_drop, - std::pair timestamp_range) + std::pair timestamp_range, + shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , attributes_to_add_map_( memory_tracker->get_resource(MemoryType::ATTRIBUTES)) diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index f5919f652e0..235e1b414c7 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -37,7 +37,6 @@ #include #include #include "tiledb/common/common.h" -#include "tiledb/common/memory_tracker.h" #include "tiledb/common/pmr.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" @@ -54,6 +53,7 @@ class ConstBuffer; class Dimension; class Domain; class Enumeration; +class MemoryTracker; class ArraySchema; enum class ArrayType : uint8_t; @@ -79,9 +79,9 @@ class ArraySchemaEvolution { * @param enmrs_to_add Enumerations to add to the schema. * @param attrs_to_drop Attributes to remove from the schema. * @param timestamp_range Timestamp range to use for the new schema. + * @param memory_tracker Memory tracker to use for the new schema. */ ArraySchemaEvolution( - shared_ptr memory_tracker, std::unordered_map> attrs_to_add, std::unordered_set attrs_to_drop, std::unordered_map> @@ -89,7 +89,8 @@ class ArraySchemaEvolution { std::unordered_map> enmrs_to_extend, std::unordered_set enmrs_to_drop, - std::pair timestamp_range); + std::pair timestamp_range, + shared_ptr memory_tracker); /** Destructor. */ ~ArraySchemaEvolution(); diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index 360a07cfba5..66c18a7635b 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -156,8 +156,8 @@ Status array_schema_evolution_to_capnp( } tdb_unique_ptr array_schema_evolution_from_capnp( - shared_ptr memory_tracker, - const capnp::ArraySchemaEvolution::Reader& evolution_reader) { + const capnp::ArraySchemaEvolution::Reader& evolution_reader, + shared_ptr memory_tracker) { // Create attributes to add std::unordered_map> attrs_to_add; auto attrs_to_add_reader = evolution_reader.getAttributesToAdd(); @@ -209,13 +209,13 @@ tdb_unique_ptr array_schema_evolution_from_capnp( return tdb_unique_ptr(tdb_new( ArraySchemaEvolution, - memory_tracker, attrs_to_add, attrs_to_drop, enmrs_to_add, enmrs_to_extend, enmrs_to_drop, - ts_range)); + ts_range, + memory_tracker)); } Status array_schema_evolution_serialize( @@ -295,7 +295,7 @@ Status array_schema_evolution_deserialize( capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); decoded_array_schema_evolution = array_schema_evolution_from_capnp( - memory_tracker, array_schema_evolution_reader); + array_schema_evolution_reader, memory_tracker); break; } case SerializationType::CAPNP: { @@ -307,7 +307,7 @@ Status array_schema_evolution_deserialize( capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); decoded_array_schema_evolution = array_schema_evolution_from_capnp( - memory_tracker, array_schema_evolution_reader); + array_schema_evolution_reader, memory_tracker); break; } default: { From 0ef3fd6ba6d7edb6c1b821c2eb9f2a9180ed40d3 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 16 Feb 2024 19:31:02 -0800 Subject: [PATCH 3/4] removed copy/move constructors, and added other fields --- .../sm/array_schema/array_schema_evolution.cc | 20 ++++++++++++++++--- .../sm/array_schema/array_schema_evolution.h | 7 +++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index edbbfec4bfb..117dee9607b 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -76,7 +76,11 @@ ArraySchemaEvolution::ArraySchemaEvolution( shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , attributes_to_add_map_( - memory_tracker->get_resource(MemoryType::ATTRIBUTES)) { + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) + , enumerations_to_add_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumerations_to_extend_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) { } ArraySchemaEvolution::ArraySchemaEvolution( @@ -92,13 +96,23 @@ ArraySchemaEvolution::ArraySchemaEvolution( , attributes_to_add_map_( memory_tracker->get_resource(MemoryType::ATTRIBUTES)) , attributes_to_drop_(attrs_to_drop) - , enumerations_to_add_map_(enmrs_to_add) - , enumerations_to_extend_map_(enmrs_to_extend) + , enumerations_to_add_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumerations_to_extend_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) , enumerations_to_drop_(enmrs_to_drop) , timestamp_range_(timestamp_range) { for (auto& elem : attrs_to_add) { attributes_to_add_map_.insert(elem); } + + for (auto& elem : enmrs_to_add) { + enumerations_to_add_map_.insert(elem); + } + + for (auto& elem : enmrs_to_extend) { + enumerations_to_extend_map_.insert(elem); + } } ArraySchemaEvolution::~ArraySchemaEvolution() { diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 235e1b414c7..bc2160698f1 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -92,6 +92,9 @@ class ArraySchemaEvolution { std::pair timestamp_range, shared_ptr memory_tracker); + DISABLE_COPY_AND_COPY_ASSIGN(ArraySchemaEvolution); + DISABLE_MOVE_AND_MOVE_ASSIGN(ArraySchemaEvolution); + /** Destructor. */ ~ArraySchemaEvolution(); @@ -204,11 +207,11 @@ class ArraySchemaEvolution { std::unordered_set attributes_to_drop_; /** Enumerations to add with any attribute. */ - std::unordered_map> + tdb::pmr::unordered_map> enumerations_to_add_map_; /** Enumerations to extend. */ - std::unordered_map> + tdb::pmr::unordered_map> enumerations_to_extend_map_; /** The names of array enumerations to be dropped. */ From d8f81017a445fbb0df6979760b999696700b10c8 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Tue, 27 Feb 2024 14:57:08 -0800 Subject: [PATCH 4/4] stuff --- tiledb/common/memory_tracker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index f7497c77539..833d7adac0b 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -68,7 +68,7 @@ std::string memory_type_to_str(MemoryType type) { return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; - case MemoryType::ATTRIBUTES: + case MemoryType::ATTRIBUTES: return "Attributes"; case MemoryType::DIMENSION_LABELS: return "DimensionLabels";