From 2520965d0fc4e1273a8060da90c99ca7bc685972 Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Tue, 20 Feb 2024 10:51:20 -0500 Subject: [PATCH 01/12] Plumbing for MemoryTracker --- test/src/unit-Reader.cc | 2 +- test/src/unit-capi-array.cc | 9 +- test/src/unit-cppapi-schema-evolution.cc | 4 +- test/src/unit-enumerations.cc | 15 +- test/src/unit-request-handlers.cc | 13 +- test/support/src/helpers.cc | 3 +- tiledb/api/c_api/domain/domain_api.cc | 11 +- tiledb/api/c_api/domain/domain_api_internal.h | 9 +- .../test/compile_capi_domain_stub_main.cc | 2 +- tiledb/common/memory_tracker.h | 6 +- tiledb/sm/array/array.cc | 5 +- tiledb/sm/array/array_directory.cc | 24 +-- tiledb/sm/array/array_directory.h | 18 ++- tiledb/sm/array/test/CMakeLists.txt | 1 + tiledb/sm/array/test/unit_consistency.h | 13 +- tiledb/sm/array_schema/array_schema.cc | 20 ++- tiledb/sm/array_schema/array_schema.h | 6 +- tiledb/sm/array_schema/dimension_label.cc | 9 +- tiledb/sm/array_schema/dimension_label.h | 4 +- tiledb/sm/array_schema/domain.cc | 15 +- tiledb/sm/array_schema/domain.h | 20 ++- tiledb/sm/array_schema/test/CMakeLists.txt | 1 + .../test/array_schema_test_support.h | 18 ++- .../sm/array_schema/test/unit_domain_data.cc | 7 +- tiledb/sm/c_api/tiledb.cc | 32 +++- tiledb/sm/c_api/tiledb_filestore.cc | 3 +- tiledb/sm/consolidator/test/CMakeLists.txt | 2 +- .../test/unit_fragment_consolidator.cc | 4 +- tiledb/sm/fragment/fragment_info.cc | 7 +- tiledb/sm/query/test/unit_query_condition.cc | 137 ++++++++++-------- tiledb/sm/query_plan/test/unit_query_plan.cc | 10 +- tiledb/sm/rest/rest_client.cc | 31 +++- tiledb/sm/rest/rest_client.h | 7 +- tiledb/sm/rtree/test/unit_rtree.cc | 65 +++++---- tiledb/sm/serialization/array.cc | 22 ++- tiledb/sm/serialization/array.h | 7 +- tiledb/sm/serialization/array_schema.cc | 47 +++--- tiledb/sm/serialization/array_schema.h | 18 ++- tiledb/sm/serialization/fragment_info.cc | 26 ++-- tiledb/sm/serialization/fragment_info.h | 8 +- tiledb/sm/serialization/query.cc | 21 ++- tiledb/sm/serialization/query.h | 4 +- tiledb/sm/serialization/test/CMakeLists.txt | 2 +- .../test/unit_capnp_array_schema.cc | 8 +- .../sm/storage_manager/context_resources.cc | 2 +- tiledb/sm/storage_manager/storage_manager.cc | 11 +- .../sm/subarray/test/unit_add_ranges_list.cc | 6 +- 47 files changed, 491 insertions(+), 224 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 26d13cc93f6..25a5d16d685 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -246,7 +246,7 @@ TEST_CASE_METHOD( auto d2{make_shared(HERE(), "d2", Datatype::INT32)}; CHECK(d2->set_domain(&domain_vec[2]).ok()); CHECK(d2->set_tile_extent(&tile_extents_vec[1]).ok()); - auto dom{make_shared(HERE())}; + auto dom{make_shared(HERE(), tracker_)}; CHECK(dom->add_dimension(d1).ok()); CHECK(dom->add_dimension(d2).ok()); diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 3da9120f911..6b656fb6fe3 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -79,6 +79,9 @@ using namespace tiledb::common; using namespace tiledb::sm; struct ArrayFx { + // The memory tracker + shared_ptr memory_tracker_; + // TileDB context tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; @@ -116,7 +119,8 @@ static const std::string test_ca_file = std::string(TILEDB_TEST_INPUTS_DIR) + "/test_certs/public.crt"; ArrayFx::ArrayFx() - : fs_vec_(vfs_test_get_fs_vec()) { + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , fs_vec_(vfs_test_get_fs_vec()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); } @@ -2590,7 +2594,8 @@ TEST_CASE_METHOD( array->array_.get(), tiledb::sm::SerializationType::CAPNP, buff->buffer(), - ctx_->storage_manager()); + ctx_->storage_manager(), + memory_tracker_); REQUIRE(st.ok()); // 6. Server: Close array and clean up diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 651b667759b..4f529cb1117 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -31,6 +31,7 @@ */ #include +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/attribute.h" @@ -796,6 +797,7 @@ TEST_CASE( TEST_CASE( "SchemaEvolution Error Handling Tests", "[cppapi][schema][evolution][errors]") { + auto memory_tracker = tiledb::test::create_test_memory_tracker(); auto ase = make_shared(HERE()); REQUIRE_THROWS(ase->evolve_schema(nullptr)); REQUIRE_THROWS(ase->add_attribute(nullptr)); @@ -814,7 +816,7 @@ TEST_CASE( int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 107e8005361..3958dfce3c7 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -32,6 +32,7 @@ #include +#include "test/support/src/mem_helpers.h" #include "test/support/tdb_catch.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" @@ -140,6 +141,7 @@ struct EnumerationFx { void rm_array(); + shared_ptr memory_tracker_; URI uri_; Config cfg_; Context ctx_; @@ -2398,7 +2400,8 @@ struct TypeParams { }; EnumerationFx::EnumerationFx() - : uri_("enumeration_test_array") + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , uri_("enumeration_test_array") , ctx_(cfg_) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); @@ -2678,7 +2681,7 @@ shared_ptr EnumerationFx::create_schema() { int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); @@ -2724,7 +2727,7 @@ shared_ptr EnumerationFx::get_array_directory() { shared_ptr EnumerationFx::get_array_schema_latest() { auto array_dir = get_array_directory(); - return array_dir->load_array_schema_latest(enc_key_); + return array_dir->load_array_schema_latest(enc_key_, memory_tracker_); } #ifdef TILEDB_SERIALIZATION @@ -2736,7 +2739,7 @@ ArraySchema EnumerationFx::ser_des_array_schema( Buffer buf; throw_if_not_ok(serialization::array_schema_serialize( *(schema.get()), stype, &buf, client_side)); - return serialization::array_schema_deserialize(stype, buf); + return serialization::array_schema_deserialize(stype, buf, memory_tracker_); } shared_ptr EnumerationFx::ser_des_array_schema_evolution( @@ -2779,8 +2782,8 @@ void EnumerationFx::ser_des_array( SerializationType stype) { Buffer buf; throw_if_not_ok(serialization::array_serialize(in, stype, &buf, client_side)); - throw_if_not_ok( - serialization::array_deserialize(out, stype, buf, ctx.storage_manager())); + throw_if_not_ok(serialization::array_deserialize( + out, stype, buf, ctx.storage_manager(), memory_tracker_)); } #else // No TILEDB_SERIALIZATION diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index f3378026adc..1154456844e 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -32,6 +32,7 @@ #ifdef TILEDB_SERIALIZATION +#include "test/support/src/mem_helpers.h" #include "test/support/tdb_catch.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" #include "tiledb/api/c_api/string/string_api_internal.h" @@ -60,6 +61,7 @@ struct RequestHandlerFx { shared_ptr get_array(QueryType type); + shared_ptr memory_tracker_; URI uri_; Config cfg_; Context ctx_; @@ -98,7 +100,7 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { auto dim = make_shared(HERE(), "dim1", Datatype::INT32); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); return schema; @@ -333,7 +335,8 @@ TEST_CASE_METHOD( /* ********************************* */ RequestHandlerFx::RequestHandlerFx(const std::string uri) - : uri_(uri) + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , uri_(uri) , ctx_(cfg_) { delete_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); @@ -399,7 +402,7 @@ shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); @@ -436,7 +439,7 @@ ArraySchema HandleLoadArraySchemaRequestFx::call_handler( REQUIRE(rval == TILEDB_OK); return serialization::deserialize_load_array_schema_response( - stype, resp_buf->buffer()); + stype, resp_buf->buffer(), memory_tracker_); } shared_ptr HandleQueryPlanRequestFx::create_schema() { @@ -451,7 +454,7 @@ shared_ptr HandleQueryPlanRequestFx::create_schema() { auto dim2 = make_shared(HERE(), "dim2", Datatype::INT32); throw_if_not_ok(dim2->set_domain(&dim_domain[2])); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim1)); throw_if_not_ok(dom->add_dimension(dim2)); throw_if_not_ok(schema->set_domain(dom)); diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 251e6cfbd65..7f2789d8bb0 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1593,7 +1593,8 @@ int array_open_wrapper( (*open_array)->array_.get(), tiledb::sm::SerializationType::CAPNP, buff->buffer(), - client_ctx->storage_manager()); + client_ctx->storage_manager(), + tiledb::test::create_test_memory_tracker()); REQUIRE(st.ok()); // 6. Server: Close array and clean up diff --git a/tiledb/api/c_api/domain/domain_api.cc b/tiledb/api/c_api/domain/domain_api.cc index 3a694a97c11..00238c143fc 100644 --- a/tiledb/api/c_api/domain/domain_api.cc +++ b/tiledb/api/c_api/domain/domain_api.cc @@ -30,12 +30,16 @@ #include "../dimension/dimension_api_internal.h" #include "domain_api_external.h" #include "domain_api_internal.h" +#include "tiledb/common/memory_tracker.h" namespace tiledb::api { -int32_t tiledb_domain_alloc(tiledb_domain_handle_t** domain) { +int32_t tiledb_domain_alloc( + tiledb_ctx_t* ctx, tiledb_domain_handle_t** domain) { ensure_output_pointer_is_valid(domain); - *domain = tiledb_domain_handle_t::make_handle(); + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_CREATE); + *domain = tiledb_domain_handle_t::make_handle(memory_tracker); return TILEDB_OK; } @@ -147,7 +151,8 @@ int32_t tiledb_domain_dump(const tiledb_domain_t* domain, FILE* out) { using tiledb::api::api_entry_context; CAPI_INTERFACE(domain_alloc, tiledb_ctx_t* ctx, tiledb_domain_t** domain) { - return api_entry_context(ctx, domain); + return tiledb::api::api_entry_with_context( + ctx, domain); } CAPI_INTERFACE_VOID(domain_free, tiledb_domain_t** domain) { diff --git a/tiledb/api/c_api/domain/domain_api_internal.h b/tiledb/api/c_api/domain/domain_api_internal.h index 76a427ce8ac..dabd36e402c 100644 --- a/tiledb/api/c_api/domain/domain_api_internal.h +++ b/tiledb/api/c_api/domain/domain_api_internal.h @@ -35,6 +35,10 @@ #include "tiledb/api/c_api_support/handle/handle.h" #include "tiledb/sm/array_schema/domain.h" +namespace tiledb::sm { +class MemoryTracker; +} + struct tiledb_domain_handle_t : public tiledb::api::CAPIHandle { private: @@ -54,8 +58,9 @@ struct tiledb_domain_handle_t * `class Domain` is principally a container for `Dimension` objects. Domain * handles are first constructed as empty containers. */ - tiledb_domain_handle_t() - : domain_{make_shared(HERE())} { + explicit tiledb_domain_handle_t( + shared_ptr memory_tracker) + : domain_{make_shared(HERE(), memory_tracker)} { } /** diff --git a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc index c5959eecd7b..ba46e5cf2bb 100644 --- a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc +++ b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc @@ -30,6 +30,6 @@ int main() { // Domain is easy to deal with because it has a default constructor. - tiledb_domain_handle_t x{}; + tiledb_domain_handle_t x{(shared_ptr)nullptr}; return 0; } diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 81c5a74cc03..037d923dccc 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -116,11 +116,15 @@ enum class MemoryType { /** The type of MemoryTracker. */ enum class MemoryTrackerType { ANONYMOUS, + ARRAY_CREATE, + ARRAY_LOAD, ARRAY_READ, ARRAY_WRITE, + FRAGMENT_INFO_LOAD, QUERY_READ, QUERY_WRITE, - CONSOLIDATOR + CONSOLIDATOR, + REST_CLIENT, }; class MemoryTrackerResource : public tdb::pmr::memory_resource { diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 7094cd3e6f5..35854cba841 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1265,7 +1265,8 @@ Array::open_for_reads_without_fragments() { "array_open_read_without_fragments_load_schemas"); // Load array schemas - auto result = array_directory().load_array_schemas(*encryption_key()); + auto result = + array_directory().load_array_schemas(*encryption_key(), memory_tracker_); auto version = std::get<0>(result)->version(); ensure_supported_schema_version_for_read(version); @@ -1290,7 +1291,7 @@ Array::open_for_writes() { // Load array schemas auto&& [array_schema_latest, array_schemas_all] = - array_directory().load_array_schemas(*encryption_key()); + array_directory().load_array_schemas(*encryption_key(), memory_tracker_); // If building experimentally, this library should not be able to // write to newer-versioned or older-versioned arrays diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 72615f67468..d046f325e53 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -92,7 +92,8 @@ ArrayDirectory::ArrayDirectory( shared_ptr ArrayDirectory::load_array_schema_from_uri( ContextResources& resources, const URI& schema_uri, - const EncryptionKey& encryption_key) { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) { auto timer_se = resources.stats().start_timer("sm_load_array_schema_from_uri"); @@ -103,11 +104,13 @@ shared_ptr ArrayDirectory::load_array_schema_from_uri( // Deserialize Deserializer deserializer(tile.data(), tile.size()); return make_shared( - HERE(), ArraySchema::deserialize(deserializer, schema_uri)); + HERE(), + ArraySchema::deserialize(deserializer, schema_uri, memory_tracker)); } shared_ptr ArrayDirectory::load_array_schema_latest( - const EncryptionKey& encryption_key) const { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const { auto timer_se = resources_.get().stats().start_timer("sm_load_array_schema_latest"); @@ -118,8 +121,8 @@ shared_ptr ArrayDirectory::load_array_schema_latest( // Load schema from URI const URI& schema_uri = latest_array_schema_uri(); - auto&& array_schema = - load_array_schema_from_uri(resources_.get(), schema_uri, encryption_key); + auto&& array_schema = load_array_schema_from_uri( + resources_.get(), schema_uri, encryption_key, memory_tracker); array_schema->set_array_uri(uri_); @@ -129,9 +132,11 @@ shared_ptr ArrayDirectory::load_array_schema_latest( tuple< shared_ptr, std::unordered_map>> -ArrayDirectory::load_array_schemas(const EncryptionKey& encryption_key) const { +ArrayDirectory::load_array_schemas( + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const { // Load all array schemas - auto&& array_schemas = load_all_array_schemas(encryption_key); + auto&& array_schemas = load_all_array_schemas(encryption_key, memory_tracker); // Locate the latest array schema const auto& array_schema_latest_name = @@ -144,7 +149,8 @@ ArrayDirectory::load_array_schemas(const EncryptionKey& encryption_key) const { std::unordered_map> ArrayDirectory::load_all_array_schemas( - const EncryptionKey& encryption_key) const { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const { auto timer_se = resources_.get().stats().start_timer("sm_load_all_array_schemas"); @@ -168,7 +174,7 @@ ArrayDirectory::load_all_array_schemas( auto& schema_uri = schema_uris[schema_ith]; try { auto&& array_schema = load_array_schema_from_uri( - resources_.get(), schema_uri, encryption_key); + resources_.get(), schema_uri, encryption_key, memory_tracker); array_schema->set_array_uri(uri_); schema_vector[schema_ith] = array_schema; } catch (std::exception& e) { diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 9f29563b981..e58cdf5b552 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -323,12 +323,14 @@ class ArrayDirectory { * * @param array_schema_uri The URI path of the array schema. * @param encryption_key The encryption key to use. + * @param memory_tracker The memory tracker to use. * @return Status, the loaded array schema */ static shared_ptr load_array_schema_from_uri( ContextResources& resources, const URI& array_schema_uri, - const EncryptionKey& encryption_key); + const EncryptionKey& encryption_key, + shared_ptr memory_tracker); /** * Get the full vac uri using the base URI and a vac uri that might be @@ -347,10 +349,12 @@ class ArrayDirectory { * @param array_dir The ArrayDirectory object used to retrieve the * various URIs in the array directory. * @param encryption_key The encryption key to use. + * @param memory_tracker The memory tracker to use. * @return Status, a new ArraySchema */ shared_ptr load_array_schema_latest( - const EncryptionKey& encryption_key) const; + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const; /** * It loads and returns the latest schema and all the array schemas @@ -359,6 +363,7 @@ class ArrayDirectory { * @param array_dir The ArrayDirectory object used to retrieve the * various URIs in the array directory. * @param encryption_key The encryption key to use. + * @param memory_tracker The memory tracker to use. * @return tuple of Status, latest array schema and all array schemas. * Status Ok on success, else error * ArraySchema The latest array schema. @@ -367,19 +372,24 @@ class ArrayDirectory { tuple< shared_ptr, std::unordered_map>> - load_array_schemas(const EncryptionKey& encryption_key) const; + load_array_schemas( + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const; /** * Loads all schemas of an array from persistent storage into memory. * * @param encryption_key The encryption key to use. + * @param memory_tracker The memory tracker to use. * @return tuple of Status and optional unordered map. If Status is an error * the unordered_map will be nullopt * Status Ok on success, else error * ArraySchemaMap Map of all array schemas found keyed by name */ std::unordered_map> - load_all_array_schemas(const EncryptionKey& encryption_key) const; + load_all_array_schemas( + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const; /** * Load the enumerations from the provided list of paths. diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 94b34c77bf7..147bf93ea23 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -36,4 +36,5 @@ commence(unit_test consistency) this_target_sources(main.cc unit_consistency.cc) this_target_link_libraries(TILEDB_CORE_OBJECTS) this_target_link_libraries(TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 8e2aea539fe..e078cd1ba67 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -39,6 +39,7 @@ #include "../array.h" #include "../consistency.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" @@ -62,8 +63,14 @@ using array_entry = std::tuple; using entry_type = std::multimap::const_iterator; class WhiteboxConsistencyController : public ConsistencyController { + /** The memory tracker to use for the fixture */ + shared_ptr memory_tracker_; + public: - WhiteboxConsistencyController() = default; + WhiteboxConsistencyController() + : memory_tracker_(tiledb::test::create_test_memory_tracker()) { + } + ~WhiteboxConsistencyController() = default; entry_type register_array( @@ -102,8 +109,8 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(dim->set_tile_extent(&tile_extent)); std::vector> dims = {dim}; - shared_ptr domain = - make_shared(HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); + shared_ptr domain = make_shared( + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker_); // Create the ArraySchema shared_ptr schema = diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 95fc59ace89..d32c4a3fb79 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1023,7 +1023,14 @@ void ArraySchema::add_dimension_label( // Create the dimension label reference. auto dim_label_ref = make_shared( - HERE(), dim_id, name, uri, dim, label_order, label_type); + HERE(), + dim_id, + name, + uri, + dim, + label_order, + label_type, + domain_->memory_tracker()); dimension_labels_.emplace_back(dim_label_ref); dimension_label_map_[name] = dim_label_ref.get(); } catch (...) { @@ -1251,7 +1258,9 @@ void ArraySchema::drop_enumeration(const std::string& enmr_name) { // #TODO Add security validation on incoming URI ArraySchema ArraySchema::deserialize( - Deserializer& deserializer, const URI& uri) { + Deserializer& deserializer, + const URI& uri, + shared_ptr memory_tracker) { Status st; // Load version // #TODO Add security validation @@ -1325,7 +1334,12 @@ ArraySchema ArraySchema::deserialize( // Note: Security validation delegated to invoked API // #TODO Add security validation auto domain{Domain::deserialize( - deserializer, version, cell_order, tile_order, coords_filters)}; + deserializer, + version, + cell_order, + tile_order, + coords_filters, + memory_tracker)}; // Load attributes // Note: Security validation delegated to invoked API diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index f41308eec53..233de4f76ca 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -466,9 +466,13 @@ class ArraySchema { * * @param deserializer The deserializer to deserialize from. * @param uri The uri of the Array. + * @param memory_tracker The memory tracker to use. * @return A new ArraySchema. */ - static ArraySchema deserialize(Deserializer& deserializer, const URI& uri); + static ArraySchema deserialize( + Deserializer& deserializer, + const URI& uri, + shared_ptr memory_tracker); /** Returns the array domain. */ inline const Domain& domain() const { diff --git a/tiledb/sm/array_schema/dimension_label.cc b/tiledb/sm/array_schema/dimension_label.cc index 0d41a3ff200..6d273faa453 100644 --- a/tiledb/sm/array_schema/dimension_label.cc +++ b/tiledb/sm/array_schema/dimension_label.cc @@ -128,7 +128,8 @@ DimensionLabel::DimensionLabel( const URI& uri, const Dimension* dim, DataOrder label_order, - Datatype label_type) + Datatype label_type, + shared_ptr memory_tracker) : dim_id_(dim_id) , dim_label_name_(dim_label_name) , uri_(uri) @@ -174,7 +175,11 @@ DimensionLabel::DimensionLabel( throw_if_not_ok( index_dims.back()->set_tile_extent(dim->tile_extent().data())); throw_if_not_ok(schema_->set_domain(make_shared( - HERE(), Layout::ROW_MAJOR, index_dims, Layout::ROW_MAJOR))); + HERE(), + Layout::ROW_MAJOR, + index_dims, + Layout::ROW_MAJOR, + memory_tracker))); // Create and set dimension label attribute. auto label_attr = make_shared( diff --git a/tiledb/sm/array_schema/dimension_label.h b/tiledb/sm/array_schema/dimension_label.h index 3d91c8ffeda..1e7a41b1faf 100644 --- a/tiledb/sm/array_schema/dimension_label.h +++ b/tiledb/sm/array_schema/dimension_label.h @@ -48,6 +48,7 @@ class ArraySchema; class Buffer; class ConstBuffer; class Dimension; +class MemoryTracker; enum class Datatype : uint8_t; enum class DataOrder : uint8_t; @@ -120,7 +121,8 @@ class DimensionLabel { const URI& uri, const Dimension* dim, DataOrder label_order, - Datatype label_type); + Datatype label_type, + shared_ptr memory_tracker); /** * Populates the object members from the data in the input binary buffer. diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 139cca4ecac..b2251ea1f32 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -37,6 +37,7 @@ #include "tiledb/common/blank.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/misc/tdb_math.h" @@ -57,7 +58,8 @@ namespace tiledb::sm { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ -Domain::Domain() { +Domain::Domain(shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) { cell_order_ = Layout::ROW_MAJOR; tile_order_ = Layout::ROW_MAJOR; dim_num_ = 0; @@ -67,8 +69,10 @@ Domain::Domain() { Domain::Domain( Layout cell_order, const std::vector> dimensions, - Layout tile_order) - : cell_order_(cell_order) + Layout tile_order, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , cell_order_(cell_order) , dimensions_(dimensions) , dim_num_(static_cast(dimensions.size())) , tile_order_(tile_order) { @@ -321,7 +325,8 @@ shared_ptr Domain::deserialize( uint32_t version, Layout cell_order, Layout tile_order, - FilterPipeline& coords_filters) { + FilterPipeline& coords_filters, + shared_ptr memory_tracker) { Status st; // Load type Datatype type = Datatype::INT32; @@ -339,7 +344,7 @@ shared_ptr Domain::deserialize( } return tiledb::common::make_shared( - HERE(), cell_order, dimensions, tile_order); + HERE(), cell_order, dimensions, tile_order, memory_tracker); } const Range& Domain::domain(unsigned i) const { diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 80e98dd0cbf..6c7b51fe88f 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/macros.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/common/types/dynamic_typed_datum.h" #include "tiledb/common/types/untyped_datum.h" @@ -58,6 +59,7 @@ class ConstBuffer; class Dimension; class DomainTypedDataView; class FilterPipeline; +class MemoryTracker; enum class Datatype : uint8_t; enum class Layout : uint8_t; @@ -74,14 +76,15 @@ class Domain { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Empty constructor. */ - Domain(); + /** Constructor. */ + Domain(shared_ptr memory_tracker); /** Constructor.*/ Domain( Layout cell_order, const std::vector> dimensions, - Layout tile_order); + Layout tile_order, + shared_ptr memory_tracker); /** Copy constructor. */ Domain(const Domain&) = default; @@ -106,6 +109,10 @@ class Domain { /* API */ /* ********************************* */ + inline shared_ptr memory_tracker() { + return memory_tracker_; + } + /** * Adds a dimension to the domain. * @@ -186,6 +193,7 @@ class Domain { * @param cell_order Cell order. * @param tile_order Tile order. * @param coords_filters Coords filters to replace empty coords pipelines. + * @param memory_tracker The memory tracker to use. * @return Status and Domain */ static shared_ptr deserialize( @@ -193,7 +201,8 @@ class Domain { uint32_t version, Layout cell_order, Layout tile_order, - FilterPipeline& coords_filters); + FilterPipeline& coords_filters, + shared_ptr memory_tracker); /** Returns the cell order. */ Layout cell_order() const; @@ -505,6 +514,9 @@ class Domain { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker for this Domain. */ + shared_ptr memory_tracker_; + /** The number of cells per tile. Meaningful only for the **dense** case. */ uint64_t cell_num_per_tile_; diff --git a/tiledb/sm/array_schema/test/CMakeLists.txt b/tiledb/sm/array_schema/test/CMakeLists.txt index e8a03ec32b9..efb004a35ee 100644 --- a/tiledb/sm/array_schema/test/CMakeLists.txt +++ b/tiledb/sm/array_schema/test/CMakeLists.txt @@ -38,4 +38,5 @@ commence(unit_test array_schema) unit_domain_data.cc unit_tile_domain.cc ) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/array_schema/test/array_schema_test_support.h b/tiledb/sm/array_schema/test/array_schema_test_support.h index 5728a96b492..e536eeb7db3 100644 --- a/tiledb/sm/array_schema/test/array_schema_test_support.h +++ b/tiledb/sm/array_schema/test/array_schema_test_support.h @@ -72,6 +72,7 @@ #include #include +#include "src/mem_helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" @@ -235,6 +236,8 @@ class TestAttribute { * Array Schema wrapper */ class TestArraySchema { + shared_ptr memory_tracker_; + ArraySchema schema_; /** @@ -259,9 +262,14 @@ class TestArraySchema { static shared_ptr make_domain( std::initializer_list dimensions, Layout cell_order, - Layout tile_order) { + Layout tile_order, + shared_ptr memory_tracker) { return make_shared( - HERE(), cell_order, make_dimension_vector(dimensions), tile_order); + HERE(), + cell_order, + make_dimension_vector(dimensions), + tile_order, + memory_tracker); } /** @@ -303,7 +311,11 @@ class TestArraySchema { "", // name array_type, false, // allow duplicates - make_domain(dimensions, cell_order, tile_order), + make_domain( + dimensions, + cell_order, + tile_order, + tiledb::test::create_test_memory_tracker()), cell_order, tile_order, 10000, // capacity diff --git a/tiledb/sm/array_schema/test/unit_domain_data.cc b/tiledb/sm/array_schema/test/unit_domain_data.cc index e5b9968b703..e65f4093c38 100644 --- a/tiledb/sm/array_schema/test/unit_domain_data.cc +++ b/tiledb/sm/array_schema/test/unit_domain_data.cc @@ -33,6 +33,7 @@ #include #include "../domain_typed_data_view.h" #include "../dynamic_array.h" +#include "test/support/src/mem_helpers.h" /* * Instantiating the class `Domain` requires a full definition of `Dimension` so * that its destructor is visible. The need to include this header indicates @@ -109,7 +110,8 @@ struct TestNullInitializer { }; TEST_CASE("DomainTypedDataView::DomainTypedDataView, null initializer") { - Domain d{}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain d{memory_tracker}; // tiledb::sm::Dimension dim{"", tiledb::sm::Datatype::INT32}; auto dim{make_shared( HERE(), "", tiledb::sm::Datatype::INT32)}; @@ -131,7 +133,8 @@ TEST_CASE("DomainTypedDataView::DomainTypedDataView, simple initializer") { } }; - Domain d{}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain d{memory_tracker}; auto dim{make_shared( HERE(), "", tiledb::sm::Datatype::INT32)}; CHECK(d.add_dimension(dim).ok()); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 7bafd132752..0a0c69bb945 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -52,6 +52,7 @@ #include "tiledb/common/dynamic_memory/dynamic_memory.h" #include "tiledb/common/heap_profiler.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension_label.h" @@ -513,7 +514,10 @@ int32_t tiledb_array_schema_load( } // Load latest array schema - auto&& array_schema_latest = array_dir->load_array_schema_latest(key); + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); + auto&& array_schema_latest = + array_dir->load_array_schema_latest(key, memory_tracker); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; @@ -604,7 +608,10 @@ int32_t tiledb_array_schema_load_with_key( } // Load latest array schema - auto&& array_schema_latest = array_dir->load_array_schema_latest(key); + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); + auto&& array_schema_latest = + array_dir->load_array_schema_latest(key, memory_tracker); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; @@ -3438,13 +3445,16 @@ int32_t tiledb_deserialize_array( return TILEDB_OOM; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_deserialize( (*array)->array_.get(), (tiledb::sm::SerializationType)serialize_type, buffer->buffer(), - ctx->storage_manager()))) { + ctx->storage_manager(), + memory_tracker))) { delete *array; *array = nullptr; return TILEDB_ERR; @@ -3500,10 +3510,14 @@ int32_t tiledb_deserialize_array_schema( } try { + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); (*array_schema)->array_schema_ = make_shared( HERE(), tiledb::sm::serialization::array_schema_deserialize( - (tiledb::sm::SerializationType)serialize_type, buffer->buffer())); + (tiledb::sm::SerializationType)serialize_type, + buffer->buffer(), + memory_tracker)); } catch (...) { delete *array_schema; *array_schema = nullptr; @@ -3766,11 +3780,14 @@ int32_t tiledb_deserialize_query_and_array( } // First deserialize the array included in the query + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_LOAD); throw_if_not_ok(tiledb::sm::serialization::array_from_query_deserialize( buffer->buffer(), (tiledb::sm::SerializationType)serialize_type, *(*array)->array_, - ctx->storage_manager())); + ctx->storage_manager(), + memory_tracker)); // Create query struct *query = new (std::nothrow) tiledb_query_t; @@ -4271,13 +4288,16 @@ int32_t tiledb_deserialize_fragment_info( return TILEDB_ERR; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::FRAGMENT_INFO_LOAD); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::fragment_info_deserialize( fragment_info->fragment_info_, (tiledb::sm::SerializationType)serialize_type, uri, - buffer->buffer()))) { + buffer->buffer(), + memory_tracker))) { return TILEDB_ERR; } diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 9c0a38342a5..d0264031509 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -138,7 +138,8 @@ int32_t tiledb_filestore_schema_create( tiledb::sm::FilterPipeline{}, tiledb::sm::ByteVecValue(std::move(tile_extent_vec))); - auto domain = make_shared(HERE()); + auto memory_tracker = ctx->resources().create_memory_tracker(); + auto domain = make_shared(HERE(), memory_tracker); throw_if_not_ok(domain->add_dimension(dim)); auto attr = make_shared( diff --git a/tiledb/sm/consolidator/test/CMakeLists.txt b/tiledb/sm/consolidator/test/CMakeLists.txt index 3d97a4d2095..1b926f96d40 100644 --- a/tiledb/sm/consolidator/test/CMakeLists.txt +++ b/tiledb/sm/consolidator/test/CMakeLists.txt @@ -29,5 +29,5 @@ commence(unit_test consolidator) this_target_sources(main.cc unit_fragment_consolidator.cc) # Not actually testing a unit yet, but some things that ought to be units - this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index 3e2f9d9e2fb..e3a08c5486f 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -32,6 +32,7 @@ #include #include "../fragment_consolidator.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" @@ -50,7 +51,8 @@ shared_ptr make_schema( HERE(), sparse ? ArrayType::SPARSE : ArrayType::DENSE); // Create the domain/dimensions. - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); for (uint64_t d = 0; d < dim_types.size(); d++) { auto dim{make_shared( HERE(), "d" + std::to_string(d + 1), dim_types[d])}; diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 941b9313487..32f4b522a00 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/fragment/fragment_info.h" #include "tiledb/common/common.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/dimension.h" @@ -750,8 +751,10 @@ shared_ptr FragmentInfo::get_array_schema(uint32_t fid) { } EncryptionKey encryption_key; + auto memory_tracker = resources_->create_memory_tracker(); + memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); return ArrayDirectory::load_array_schema_from_uri( - *resources_, schema_uri, encryption_key); + *resources_, schema_uri, encryption_key, memory_tracker); } Status FragmentInfo::get_array_schema_name( @@ -1003,7 +1006,7 @@ FragmentInfo::load_array_schemas_and_fragment_metadata( std::unordered_map> array_schemas_all; std::tie(array_schema_latest, array_schemas_all) = - array_dir.load_array_schemas(enc_key); + array_dir.load_array_schemas(enc_key, memory_tracker); const auto filtered_fragment_uris = [&]() { auto timer_se = diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index b4a8c6b0fb9..b8f95270b34 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1554,13 +1554,20 @@ void test_apply_tile( */ template void test_apply( - const Datatype type, bool var_size = false, bool nullable = false); + shared_ptr memory_tracker, + const Datatype type, + bool var_size = false, + bool nullable = false); /** * C-string template-specialization for `test_apply`. */ template <> -void test_apply(const Datatype type, bool var_size, bool nullable) { +void test_apply( + shared_ptr memory_tracker, + const Datatype type, + bool var_size, + bool nullable) { REQUIRE((type == Datatype::STRING_ASCII || type == Datatype::STRING_UTF8)); const std::string field_name = "foo"; @@ -1579,7 +1586,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + Domain domain(memory_tracker); auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); @@ -1591,7 +1598,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), + memory_tracker, array_schema, URI(), std::make_pair(0, 0), @@ -1623,7 +1630,11 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { * Non-specialized template type for `test_apply`. */ template -void test_apply(const Datatype type, bool var_size, bool nullable) { +void test_apply( + shared_ptr memory_tracker, + const Datatype type, + bool var_size, + bool nullable) { const std::string field_name = "foo"; const uint64_t cells = 10; const T fill_value = 3; @@ -1635,7 +1646,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + Domain domain(memory_tracker); auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); @@ -1647,7 +1658,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), + memory_tracker, array_schema, URI(), std::make_pair(0, 0), @@ -1674,45 +1685,46 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { } TEST_CASE("QueryCondition: Test apply", "[QueryCondition][apply]") { - test_apply(Datatype::INT8); - test_apply(Datatype::UINT8); - test_apply(Datatype::INT16); - test_apply(Datatype::UINT16); - test_apply(Datatype::INT32); - test_apply(Datatype::UINT32); - test_apply(Datatype::INT64); - test_apply(Datatype::UINT64); - test_apply(Datatype::FLOAT32); - test_apply(Datatype::FLOAT64); - test_apply(Datatype::CHAR); - test_apply(Datatype::DATETIME_YEAR); - test_apply(Datatype::DATETIME_MONTH); - test_apply(Datatype::DATETIME_WEEK); - test_apply(Datatype::DATETIME_DAY); - test_apply(Datatype::DATETIME_HR); - test_apply(Datatype::DATETIME_MIN); - test_apply(Datatype::DATETIME_SEC); - test_apply(Datatype::DATETIME_MS); - test_apply(Datatype::DATETIME_US); - test_apply(Datatype::DATETIME_NS); - test_apply(Datatype::DATETIME_PS); - test_apply(Datatype::DATETIME_FS); - test_apply(Datatype::DATETIME_AS); - test_apply(Datatype::TIME_HR); - test_apply(Datatype::TIME_MIN); - test_apply(Datatype::TIME_SEC); - test_apply(Datatype::TIME_MS); - test_apply(Datatype::TIME_US); - test_apply(Datatype::TIME_NS); - test_apply(Datatype::TIME_PS); - test_apply(Datatype::TIME_FS); - test_apply(Datatype::TIME_AS); - test_apply(Datatype::STRING_ASCII); - test_apply(Datatype::STRING_ASCII, true); - test_apply(Datatype::STRING_ASCII, false, true); - test_apply(Datatype::STRING_UTF8); - test_apply(Datatype::STRING_UTF8, true); - test_apply(Datatype::STRING_UTF8, false, true); + auto tracker = tiledb::test::create_test_memory_tracker(); + test_apply(tracker, Datatype::INT8); + test_apply(tracker, Datatype::UINT8); + test_apply(tracker, Datatype::INT16); + test_apply(tracker, Datatype::UINT16); + test_apply(tracker, Datatype::INT32); + test_apply(tracker, Datatype::UINT32); + test_apply(tracker, Datatype::INT64); + test_apply(tracker, Datatype::UINT64); + test_apply(tracker, Datatype::FLOAT32); + test_apply(tracker, Datatype::FLOAT64); + test_apply(tracker, Datatype::CHAR); + test_apply(tracker, Datatype::DATETIME_YEAR); + test_apply(tracker, Datatype::DATETIME_MONTH); + test_apply(tracker, Datatype::DATETIME_WEEK); + test_apply(tracker, Datatype::DATETIME_DAY); + test_apply(tracker, Datatype::DATETIME_HR); + test_apply(tracker, Datatype::DATETIME_MIN); + test_apply(tracker, Datatype::DATETIME_SEC); + test_apply(tracker, Datatype::DATETIME_MS); + test_apply(tracker, Datatype::DATETIME_US); + test_apply(tracker, Datatype::DATETIME_NS); + test_apply(tracker, Datatype::DATETIME_PS); + test_apply(tracker, Datatype::DATETIME_FS); + test_apply(tracker, Datatype::DATETIME_AS); + test_apply(tracker, Datatype::TIME_HR); + test_apply(tracker, Datatype::TIME_MIN); + test_apply(tracker, Datatype::TIME_SEC); + test_apply(tracker, Datatype::TIME_MS); + test_apply(tracker, Datatype::TIME_US); + test_apply(tracker, Datatype::TIME_NS); + test_apply(tracker, Datatype::TIME_PS); + test_apply(tracker, Datatype::TIME_FS); + test_apply(tracker, Datatype::TIME_AS); + test_apply(tracker, Datatype::STRING_ASCII); + test_apply(tracker, Datatype::STRING_ASCII, true); + test_apply(tracker, Datatype::STRING_ASCII, false, true); + test_apply(tracker, Datatype::STRING_UTF8); + test_apply(tracker, Datatype::STRING_UTF8, true); + test_apply(tracker, Datatype::STRING_UTF8, false, true); } TEST_CASE( @@ -1742,7 +1754,8 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2291,7 +2304,8 @@ void test_apply_dense( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2350,7 +2364,8 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2461,7 +2476,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2990,7 +3006,8 @@ void test_apply_sparse( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -3049,7 +3066,8 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -3819,7 +3837,8 @@ TEST_CASE( Attribute attr(field_name, type); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -4108,7 +4127,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); @@ -4462,7 +4482,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); @@ -4781,7 +4802,8 @@ TEST_CASE( attr.set_nullable(true); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -4883,7 +4905,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain domain(memory_tracker); auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 23ce164c092..e48ac18ca58 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -34,6 +34,7 @@ #include #include "../query_plan.h" #include "external/include/nlohmann/json.hpp" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" @@ -58,6 +59,8 @@ struct QueryPlanFx { URI array_uri(const std::string& uri); + shared_ptr memory_tracker_; + TemporaryLocalDirectory temp_dir_; Config cfg_; @@ -76,8 +79,8 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { throw_if_not_ok(dim->set_tile_extent(&tile_extent)); std::vector> dims = {dim}; - shared_ptr domain = - make_shared(HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); + shared_ptr domain = make_shared( + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker_); // Create the ArraySchema shared_ptr schema = @@ -109,7 +112,8 @@ URI QueryPlanFx::array_uri(const std::string& array_name) { } QueryPlanFx::QueryPlanFx() - : logger_(make_shared(HERE(), "foo")) + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , logger_(make_shared(HERE(), "foo")) , resources_(cfg_, logger_, 1, 1, "") , sm_(make_shared(resources_, logger_, cfg_)) { } diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index f69b55c4d23..84eb2c286e7 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -57,6 +57,7 @@ #include #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/group/group.h" @@ -81,7 +82,8 @@ RestClient::RestClient() : stats_(nullptr) , config_(nullptr) , compute_tp_(nullptr) - , resubmit_incomplete_(true) { + , resubmit_incomplete_(true) + , memory_tracker_(nullptr) { auto st = utils::parse::convert( Config::REST_SERIALIZATION_DEFAULT_FORMAT, &serialization_type_); throw_if_not_ok(st); @@ -91,7 +93,8 @@ Status RestClient::init( stats::Stats* const parent_stats, const Config* config, ThreadPool* compute_tp, - const std::shared_ptr& logger) { + const std::shared_ptr& logger, + ContextResources& resources) { if (config == nullptr) return LOG_STATUS( Status_RestError("Error initializing rest client; config is null.")); @@ -103,6 +106,12 @@ Status RestClient::init( config_ = config; compute_tp_ = compute_tp; + // Setting the type of the memory tracker as MemoryTrackerType::REST_CLIENT + // for now. This is because the class is used in many places not directly tied + // to an array. + memory_tracker_ = resources.create_memory_tracker(); + memory_tracker_->set_type(MemoryTrackerType::REST_CLIENT); + const char* c_str; RETURN_NOT_OK(config_->get("rest.server_address", &c_str)); if (c_str != nullptr) @@ -242,7 +251,7 @@ RestClient::get_array_schema_from_rest(const URI& uri) { make_shared( HERE(), serialization::array_schema_deserialize( - serialization_type_, returned_data))}; + serialization_type_, returned_data, memory_tracker_))}; } shared_ptr RestClient::post_array_schema_from_rest( @@ -292,7 +301,7 @@ shared_ptr RestClient::post_array_schema_from_rest( return make_shared( HERE(), serialization::deserialize_load_array_schema_response( - serialization_type_, returned_data)); + serialization_type_, returned_data, memory_tracker_)); } Status RestClient::post_array_schema_to_rest( @@ -376,7 +385,11 @@ Status RestClient::post_array_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); return serialization::array_deserialize( - array, serialization_type_, returned_data, storage_manager); + array, + serialization_type_, + returned_data, + storage_manager, + memory_tracker_); } void RestClient::delete_array_from_rest(const URI& uri) { @@ -1265,7 +1278,7 @@ Status RestClient::post_fragment_info_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); return serialization::fragment_info_deserialize( - fragment_info, serialization_type_, uri, returned_data); + fragment_info, serialization_type_, uri, returned_data, memory_tracker_); } Status RestClient::post_group_metadata_from_rest(const URI& uri, Group* group) { @@ -1525,7 +1538,11 @@ RestClient::RestClient() { } Status RestClient::init( - stats::Stats*, const Config*, ThreadPool*, const std::shared_ptr&) { + stats::Stats*, + const Config*, + ThreadPool*, + const std::shared_ptr&, + ContextResources&) { return LOG_STATUS( Status_RestError("Cannot use rest client; serialization not enabled.")); } diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 63d9044db9e..1d9f476f320 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -53,6 +53,7 @@ class ArraySchemaEvolution; class Config; class FragmentInfo; class Query; +class MemoryTracker; enum class SerializationType : uint8_t; @@ -66,7 +67,8 @@ class RestClient { stats::Stats* parent_stats, const Config* config, ThreadPool* compute_tp, - const std::shared_ptr& logger); + const std::shared_ptr& logger, + ContextResources& resources); /** Sets a header that will be attached to all requests. */ Status set_header(const std::string& name, const std::string& value); @@ -434,6 +436,9 @@ class RestClient { /** UID of the logger instance */ inline static std::atomic logger_id_ = 0; + /** The class MemoryTracker. */ + shared_ptr memory_tracker_; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index 6ca42b686d7..0f172e1d270 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -90,7 +90,8 @@ Domain create_domain( const std::vector& dim_names, const std::vector& dim_types, const std::vector& dim_domains, - const std::vector& dim_tile_extents) { + const std::vector& dim_tile_extents, + shared_ptr memory_tracker) { assert(!dim_names.empty()); assert(dim_names.size() == dim_types.size()); assert(dim_names.size() == dim_domains.size()); @@ -121,7 +122,8 @@ Domain create_domain( dimensions.emplace_back(std::move(dim)); } - return Domain(Layout::ROW_MAJOR, dimensions, Layout::ROW_MAJOR); + return Domain( + Layout::ROW_MAJOR, dimensions, Layout::ROW_MAJOR, memory_tracker); } TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { @@ -138,8 +140,8 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { // 1D int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom1 = - create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); + Domain dom1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); const Domain d1{dom1}; RTree rtree1(&d1, 3, tracker); @@ -199,7 +201,8 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { {"d1", "d2"}, {Datatype::INT64, Datatype::INT64}, {dim_dom_2, dim_dom_2}, - {&dim_extent_2, &dim_extent_2}); + {&dim_extent_2, &dim_extent_2}, + tracker); auto mbrs_2d = create_mbrs( {1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}, tracker); const Domain d2{dom2}; @@ -239,10 +242,9 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { float dim_extent_f = 10.0; auto mbrs_f = create_mbrs({1.0f, 3.0f, 5.0f, 10.0f, 20.0f, 22.0f}, tracker); - Domain dom2f = - create_domain({"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}); - const Domain d2f{dom2f}; - RTree rtreef(&d2f, 5, tracker); + Domain dom2f = create_domain( + {"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}, tracker); + RTree rtreef(&dom2f, 5, tracker); CHECK(rtreef.set_leaves(mbrs_f).ok()); rtreef.build_tree(); @@ -282,8 +284,8 @@ TEST_CASE("RTree: Test 1D R-tree, height 2", "[rtree][1d][2h]") { std::vector is_default(1, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom1 = - create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); + Domain dom1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); const Domain d1{dom1}; RTree rtree(&d1, 3, tracker); @@ -331,10 +333,9 @@ TEST_CASE("RTree: Test 1D R-tree, height 3", "[rtree][1d][3h]") { int32_t dim_extent = 10; auto mbrs = create_mbrs( {1, 3, 5, 10, 20, 22, 30, 35, 36, 38, 40, 49, 50, 51, 65, 69}, tracker); - Domain dom1 = - create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); - const Domain d1(dom1); - RTree rtree(&d1, 3, tracker); + Domain dom1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -400,7 +401,8 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { {"d1", "d2"}, {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, - {&dim_extent, &dim_extent}); + {&dim_extent, &dim_extent}, + tracker); auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}, tracker); const Domain d2{dom2}; @@ -454,7 +456,8 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { {"d1", "d2"}, {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, - {&dim_extent, &dim_extent}); + {&dim_extent, &dim_extent}, + tracker); auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15, 11, 15, 20, 22, 16, 16, 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}, @@ -535,7 +538,8 @@ TEST_CASE( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, - {&uint8_extent, &int32_extent}); + {&uint8_extent, &int32_extent}, + tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}, tracker); const Domain d1{dom}; @@ -592,7 +596,8 @@ TEST_CASE( {"d1", "d2"}, {Datatype::UINT64, Datatype::FLOAT32}, {uint64_dom, float_dom}, - {&uint64_extent, &float_extent}); + {&uint64_extent, &float_extent}, + tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}, tracker); const Domain d1{dom}; @@ -649,7 +654,8 @@ TEST_CASE( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, - {&uint8_extent, &int32_extent}); + {&uint8_extent, &int32_extent}, + tracker); auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20}, {5, 6, 7, 9, 11, 30}, tracker); const Domain d1{dom}; @@ -719,7 +725,8 @@ TEST_CASE( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, - {&uint8_extent, &int32_extent}); + {&uint8_extent, &int32_extent}, + tracker); auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20, 21, 26}, {5, 6, 7, 9, 11, 30, 31, 40}, tracker); const Domain d1{dom}; @@ -855,8 +862,8 @@ TEST_CASE( // Build tree auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); - Domain dom1 = - create_domain({"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}); + Domain dom1 = create_domain( + {"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}, tracker); auto mbrs = create_str_mbrs<1>({"aa", "b", "eee", "g", "gggg", "ii"}, tracker); @@ -934,8 +941,8 @@ TEST_CASE( // Build tree auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); - Domain dom1 = - create_domain({"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}); + Domain dom1 = create_domain( + {"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}, tracker); auto mbrs = create_str_mbrs<1>( {"aa", "b", @@ -1032,7 +1039,8 @@ TEST_CASE( {"d1", "d2"}, {Datatype::STRING_ASCII, Datatype::STRING_ASCII}, {nullptr, nullptr}, - {nullptr, nullptr}); + {nullptr, nullptr}, + tracker); auto mbrs = create_str_mbrs<2>( {"aa", "b", "eee", "g", "gggg", "ii", "jj", "lll", "m", "n", "oo", "qqq"}, tracker); @@ -1135,12 +1143,13 @@ TEST_CASE( std::vector is_default(2, false); int32_t dom_int32[] = {1, 20}; int32_t tile_extent = 5; + auto tracker = create_test_memory_tracker(); Domain dom = create_domain( {"d1", "d2"}, {Datatype::STRING_ASCII, Datatype::INT32}, {nullptr, dom_int32}, - {nullptr, &tile_extent}); - auto tracker = create_test_memory_tracker(); + {nullptr, &tile_extent}, + tracker); auto mbrs = create_str_int32_mbrs( {"aa", "b", "eee", "g", "gggg", "ii"}, {1, 5, 7, 8, 10, 14}, tracker); diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 6e1e4f04aa7..955c3bf6834 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -232,7 +232,8 @@ Status array_from_capnp( const capnp::Array::Reader& array_reader, StorageManager* storage_manager, Array* array, - const bool client_side) { + const bool client_side, + shared_ptr memory_tracker) { // The serialized URI is set if it exists // this is used for backwards compatibility with pre TileDB 2.5 clients that // want to serialized a query object TileDB >= 2.5 no longer needs to receive @@ -270,7 +271,7 @@ Status array_from_capnp( auto entries = array_reader.getArraySchemasAll().getEntries(); for (auto array_schema_build : entries) { auto schema{array_schema_from_capnp( - array_schema_build.getValue(), array->array_uri())}; + array_schema_build.getValue(), array->array_uri(), memory_tracker)}; schema.set_array_uri(array->array_uri()); all_schemas[array_schema_build.getKey()] = make_shared(HERE(), schema); @@ -282,7 +283,7 @@ Status array_from_capnp( if (array_reader.hasArraySchemaLatest()) { auto array_schema_latest_reader = array_reader.getArraySchemaLatest(); auto array_schema_latest{array_schema_from_capnp( - array_schema_latest_reader, array->array_uri())}; + array_schema_latest_reader, array->array_uri(), memory_tracker)}; array_schema_latest.set_array_uri(array->array_uri()); array->set_array_schema_latest( make_shared(HERE(), array_schema_latest)); @@ -543,7 +544,8 @@ Status array_deserialize( Array* array, SerializationType serialize_type, const Buffer& serialized_buffer, - StorageManager* storage_manager) { + StorageManager* storage_manager, + shared_ptr memory_tracker) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -555,7 +557,8 @@ Status array_deserialize( kj::StringPtr(static_cast(serialized_buffer.data())), array_builder); capnp::Array::Reader array_reader = array_builder.asReader(); - RETURN_NOT_OK(array_from_capnp(array_reader, storage_manager, array)); + RETURN_NOT_OK(array_from_capnp( + array_reader, storage_manager, array, true, memory_tracker)); break; } case SerializationType::CAPNP: { @@ -575,7 +578,8 @@ Status array_deserialize( serialized_buffer.size() / sizeof(::capnp::word)), readerOptions); capnp::Array::Reader array_reader = reader.getRoot(); - RETURN_NOT_OK(array_from_capnp(array_reader, storage_manager, array)); + RETURN_NOT_OK(array_from_capnp( + array_reader, storage_manager, array, true, memory_tracker)); break; } default: { @@ -708,7 +712,11 @@ Status array_serialize(Array*, SerializationType, Buffer*, const bool) { } Status array_deserialize( - Array*, SerializationType, const Buffer&, StorageManager*) { + Array*, + SerializationType, + const Buffer&, + StorageManager*, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/array.h b/tiledb/sm/serialization/array.h index bef9d5143de..46d22d22657 100644 --- a/tiledb/sm/serialization/array.h +++ b/tiledb/sm/serialization/array.h @@ -72,6 +72,7 @@ Status array_to_capnp( * @param storage_manager the storage manager associated with the array * @param array Array to deserialize into * @param client_side Allows to specify different behavior depending on who is + * @param memory_tracker Memory tracker to use for memory allocations. * serializing, the client (1) or the Cloud server (0). This is sometimes needed * since they are both using the same Core library APIs for serialization. * @return Status @@ -80,7 +81,8 @@ Status array_from_capnp( const capnp::Array::Reader& array_reader, StorageManager* storage_manager, Array* array, - const bool client_side = true); + const bool client_side, + shared_ptr memory_tracker); /** * Convert info for opening and array to Cap'n Proto message @@ -135,7 +137,8 @@ Status array_deserialize( Array* array, SerializationType serialize_type, const Buffer& serialized_buffer, - StorageManager* storage_manager); + StorageManager* storage_manager, + shared_ptr memory_tracker); /** * Serialize an open array request via Cap'n Proto diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index bc36b4ed970..37efbed07ae 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -705,7 +705,8 @@ Status domain_to_capnp( /* Deserialize a domain from a cap'n proto object. */ shared_ptr domain_from_capnp( - const capnp::Domain::Reader& domain_reader) { + const capnp::Domain::Reader& domain_reader, + shared_ptr memory_tracker) { Status st; // Deserialize and validate cell order @@ -748,7 +749,8 @@ shared_ptr domain_from_capnp( dims.emplace_back(dimension_from_capnp(dimension)); } - return make_shared(HERE(), cell_order, dims, tile_order); + return make_shared( + HERE(), cell_order, dims, tile_order, memory_tracker); } void dimension_label_to_capnp( @@ -780,7 +782,8 @@ void dimension_label_to_capnp( } shared_ptr dimension_label_from_capnp( - const capnp::DimensionLabel::Reader& dim_label_reader) { + const capnp::DimensionLabel::Reader& dim_label_reader, + shared_ptr memory_tracker) { // Get datatype Datatype datatype = Datatype::ANY; throw_if_not_ok(datatype_enum(dim_label_reader.getType(), &datatype)); @@ -789,7 +792,7 @@ shared_ptr dimension_label_from_capnp( if (dim_label_reader.hasSchema()) { auto schema_reader = dim_label_reader.getSchema(); schema = make_shared( - HERE(), array_schema_from_capnp(schema_reader, URI())); + HERE(), array_schema_from_capnp(schema_reader, URI(), memory_tracker)); } auto is_relative = dim_label_reader.getRelative(); @@ -915,7 +918,9 @@ Status array_schema_to_capnp( // #TODO Add security validation on incoming URI ArraySchema array_schema_from_capnp( - const capnp::ArraySchema::Reader& schema_reader, const URI& uri) { + const capnp::ArraySchema::Reader& schema_reader, + const URI& uri, + shared_ptr memory_tracker) { // Deserialize and validate array_type ArrayType array_type = ArrayType::DENSE; Status st = array_type_enum(schema_reader.getArrayType(), &array_type); @@ -996,7 +1001,7 @@ ArraySchema array_schema_from_capnp( // Deserialize domain // Note: Security validation delegated to invoked API auto domain_reader = schema_reader.getDomain(); - auto domain{domain_from_capnp(domain_reader)}; + auto domain{domain_from_capnp(domain_reader, memory_tracker)}; // Set coords filter pipelines // Note: Security validation delegated to invoked API @@ -1069,7 +1074,7 @@ ArraySchema array_schema_from_capnp( try { for (auto dim_label_reader : dim_labels_reader) { dimension_labels.emplace_back( - dimension_label_from_capnp(dim_label_reader)); + dimension_label_from_capnp(dim_label_reader, memory_tracker)); } } catch (const std::exception& e) { std::throw_with_nested(std::runtime_error( @@ -1202,7 +1207,9 @@ Status array_schema_serialize( } ArraySchema array_schema_deserialize( - SerializationType serialize_type, const Buffer& serialized_buffer) { + SerializationType serialize_type, + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { capnp::ArraySchema::Reader array_schema_reader; ::capnp::MallocMessageBuilder message_builder; @@ -1216,7 +1223,8 @@ ArraySchema array_schema_deserialize( kj::StringPtr(static_cast(serialized_buffer.data())), array_schema_builder); array_schema_reader = array_schema_builder.asReader(); - return array_schema_from_capnp(array_schema_reader, URI()); + return array_schema_from_capnp( + array_schema_reader, URI(), memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = @@ -1225,7 +1233,8 @@ ArraySchema array_schema_deserialize( reinterpret_cast(mBytes), serialized_buffer.size() / sizeof(::capnp::word))); array_schema_reader = reader.getRoot(); - return array_schema_from_capnp(array_schema_reader, URI()); + return array_schema_from_capnp( + array_schema_reader, URI(), memory_tracker); } default: { throw StatusException(Status_SerializationError( @@ -1964,13 +1973,16 @@ void serialize_load_array_schema_response( } ArraySchema load_array_schema_response_from_capnp( - capnp::LoadArraySchemaResponse::Reader& reader) { + capnp::LoadArraySchemaResponse::Reader& reader, + shared_ptr memory_tracker) { auto schema_reader = reader.getSchema(); - return array_schema_from_capnp(schema_reader, URI()); + return array_schema_from_capnp(schema_reader, URI(), memory_tracker); } ArraySchema deserialize_load_array_schema_response( - SerializationType serialization_type, const Buffer& data) { + SerializationType serialization_type, + const Buffer& data, + shared_ptr memory_tracker) { try { switch (serialization_type) { case SerializationType::JSON: { @@ -1981,7 +1993,7 @@ ArraySchema deserialize_load_array_schema_response( json.decode( kj::StringPtr(static_cast(data.data())), builder); auto reader = builder.asReader(); - return load_array_schema_response_from_capnp(reader); + return load_array_schema_response_from_capnp(reader, memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(data.data()); @@ -1989,7 +2001,7 @@ ArraySchema deserialize_load_array_schema_response( reinterpret_cast(mBytes), data.size() / sizeof(::capnp::word))); auto reader = array_reader.getRoot(); - return load_array_schema_response_from_capnp(reader); + return load_array_schema_response_from_capnp(reader, memory_tracker); } default: { throw Status_SerializationError( @@ -2016,7 +2028,8 @@ Status array_schema_serialize( "Cannot serialize; serialization not enabled.")); } -ArraySchema array_schema_deserialize(SerializationType, const Buffer&) { +ArraySchema array_schema_deserialize( + SerializationType, const Buffer&, shared_ptr) { throw StatusException(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } @@ -2077,7 +2090,7 @@ void serialize_load_array_schema_response( } ArraySchema deserialize_load_array_schema_response( - SerializationType, const Buffer&) { + SerializationType, const Buffer&, shared_ptr) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index f0c2d7ee666..31df6b94a06 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -51,6 +51,7 @@ class Array; class Buffer; class ArraySchema; class Dimension; +class MemoryTracker; enum class SerializationType : uint8_t; namespace serialization { @@ -109,10 +110,13 @@ Status array_schema_to_capnp( * * @param schema_reader Cap'n proto object * @param uri A URI object + * @param memory_tracker The memory tracker to use. * @return a new ArraySchema */ ArraySchema array_schema_from_capnp( - const capnp::ArraySchema::Reader& schema_reader, const URI& uri); + const capnp::ArraySchema::Reader& schema_reader, + const URI& uri, + shared_ptr memory_tracker); /** * Serialize a dimension label to cap'n proto object @@ -130,10 +134,12 @@ void dimension_label_to_capnp( * Deserialize a dimension label from a cap'n proto object * * @param reader Cap'n proto reader object. +* @param memory_tracker The memory tracker to use. * @return A new DimensionLabel. */ shared_ptr dimension_label_from_capnp( - const capnp::DimensionLabel::Reader& reader); + const capnp::DimensionLabel::Reader& reader, + shared_ptr memory_tracker); #endif // TILEDB_SERIALIZATION @@ -153,7 +159,9 @@ Status array_schema_serialize( const bool client_side); ArraySchema array_schema_deserialize( - SerializationType serialize_type, const Buffer& serialized_buffer); + SerializationType serialize_type, + const Buffer& serialized_buffer, + shared_ptr memory_tracker); Status nonempty_domain_serialize( const Array* array, @@ -205,7 +213,9 @@ void serialize_load_array_schema_response( Buffer& data); ArraySchema deserialize_load_array_schema_response( - SerializationType serialization_type, const Buffer& data); + SerializationType serialization_type, + const Buffer& data, + shared_ptr memory_tracker); } // namespace serialization } // namespace sm diff --git a/tiledb/sm/serialization/fragment_info.cc b/tiledb/sm/serialization/fragment_info.cc index 5be515c7e91..1b01ae7855c 100644 --- a/tiledb/sm/serialization/fragment_info.cc +++ b/tiledb/sm/serialization/fragment_info.cc @@ -276,13 +276,14 @@ Status single_fragment_info_to_capnp( Status fragment_info_from_capnp( const capnp::FragmentInfo::Reader& fragment_info_reader, const URI& array_uri, - FragmentInfo* fragment_info) { + FragmentInfo* fragment_info, + shared_ptr memory_tracker) { // Get array_schema_latest from capnp if (fragment_info_reader.hasArraySchemaLatest()) { auto array_schema_latest_reader = fragment_info_reader.getArraySchemaLatest(); - auto array_schema_latest{ - array_schema_from_capnp(array_schema_latest_reader, array_uri)}; + auto array_schema_latest{array_schema_from_capnp( + array_schema_latest_reader, array_uri, memory_tracker)}; array_schema_latest.set_array_uri(array_uri); fragment_info->array_schema_latest() = make_shared(HERE(), array_schema_latest); @@ -295,8 +296,8 @@ Status fragment_info_from_capnp( if (all_schemas_reader.hasEntries()) { auto entries = fragment_info_reader.getArraySchemasAll().getEntries(); for (auto array_schema_build : entries) { - auto schema{ - array_schema_from_capnp(array_schema_build.getValue(), array_uri)}; + auto schema{array_schema_from_capnp( + array_schema_build.getValue(), array_uri, memory_tracker)}; schema.set_array_uri(array_uri); auto key = std::string_view{ array_schema_build.getKey().cStr(), @@ -453,7 +454,8 @@ Status fragment_info_deserialize( FragmentInfo* fragment_info, SerializationType serialize_type, const URI& uri, - const Buffer& serialized_buffer) { + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { if (fragment_info == nullptr) return LOG_STATUS( Status_SerializationError("Error deserializing fragment info; null " @@ -471,7 +473,8 @@ Status fragment_info_deserialize( auto reader = builder.asReader(); // Deserialize - RETURN_NOT_OK(fragment_info_from_capnp(reader, uri, fragment_info)); + RETURN_NOT_OK(fragment_info_from_capnp( + reader, uri, fragment_info, memory_tracker)); break; } case SerializationType::CAPNP: { @@ -493,7 +496,8 @@ Status fragment_info_deserialize( auto reader = msg_reader.getRoot(); // Deserialize - RETURN_NOT_OK(fragment_info_from_capnp(reader, uri, fragment_info)); + RETURN_NOT_OK(fragment_info_from_capnp( + reader, uri, fragment_info, memory_tracker)); break; } default: { @@ -524,7 +528,11 @@ Status fragment_info_serialize( } Status fragment_info_deserialize( - FragmentInfo*, SerializationType, const URI&, const Buffer&) { + FragmentInfo*, + SerializationType, + const URI&, + const Buffer&, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/fragment_info.h b/tiledb/sm/serialization/fragment_info.h index bd69c37c60f..968a13d38a8 100644 --- a/tiledb/sm/serialization/fragment_info.h +++ b/tiledb/sm/serialization/fragment_info.h @@ -59,12 +59,14 @@ namespace serialization { * @param fragment_info_reader cap'n proto class. * @param uri array uri that the fragment belongs to * @param fragment_info fragment info object to deserialize into. + * @param memory_tracker The memory tracker to use. * @return Status */ Status fragment_info_from_capnp( const capnp::FragmentInfo::Reader& fragment_info_reader, const URI& uri, - FragmentInfo* fragment_info); + FragmentInfo* fragment_info, + shared_ptr memory_tracker); /** * Convert Fragment Info to Cap'n Proto message. @@ -124,13 +126,15 @@ Status fragment_info_serialize( * @param serialize_type format the data is serialized in: Cap'n Proto of JSON. * @param uri array uri that the fragment belongs to * @param serialized_buffer buffer to read serialized bytes from. + * @param memory_tracker The memory tracker to use. * @return Status */ Status fragment_info_deserialize( FragmentInfo* fragment_info, SerializationType serialize_type, const URI& uri, - const Buffer& serialized_buffer); + const Buffer& serialized_buffer, + shared_ptr memory_tracker); /** * Serialize a fragment info request via Cap'n Proto. diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 92414cd51d5..01755a141d9 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -2284,7 +2284,8 @@ Status array_from_query_deserialize( const Buffer& serialized_buffer, SerializationType serialize_type, Array& array, - StorageManager* storage_manager) { + StorageManager* storage_manager, + shared_ptr memory_tracker) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -2299,7 +2300,11 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = query_builder.asReader(); // Deserialize array instance. RETURN_NOT_OK(array_from_capnp( - query_reader.getArray(), storage_manager, &array, false)); + query_reader.getArray(), + storage_manager, + &array, + false, + memory_tracker)); break; } case SerializationType::CAPNP: { @@ -2327,7 +2332,11 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = reader.getRoot(); // Deserialize array instance. RETURN_NOT_OK(array_from_capnp( - query_reader.getArray(), storage_manager, &array, false)); + query_reader.getArray(), + storage_manager, + &array, + false, + memory_tracker)); break; } default: @@ -3189,7 +3198,11 @@ Status query_deserialize( } Status array_from_query_deserialize( - const Buffer&, SerializationType, Array&, StorageManager*) { + const Buffer&, + SerializationType, + Array&, + StorageManager*, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index 1a711db6b7d..a5ad85c0348 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -137,12 +137,14 @@ using CopyState = * @param serialized_buffer Buffer containing serialized query * @param serialize_type Serialization type of serialized query * @param array Array object to deserialize into + * @param memory_tracker Memory tracker to use for allocations. */ Status array_from_query_deserialize( const Buffer& serialized_buffer, SerializationType serialize_type, Array& array, - StorageManager* storage_manager); + StorageManager* storage_manager, + shared_ptr memory_tracker); /** * Serialize a query diff --git a/tiledb/sm/serialization/test/CMakeLists.txt b/tiledb/sm/serialization/test/CMakeLists.txt index abd02d04503..71525cd96b6 100644 --- a/tiledb/sm/serialization/test/CMakeLists.txt +++ b/tiledb/sm/serialization/test/CMakeLists.txt @@ -36,7 +36,7 @@ conclude(unit_test) commence(unit_test capnp_array_schema) this_target_sources(main.cc unit_capnp_array_schema.cc) - this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB tiledb_test_support_lib) # Enable serialization target_compile_definitions(unit_capnp_array_schema PRIVATE -DTILEDB_SERIALIZATION) diff --git a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc index f532d950a62..dcfb5887b50 100644 --- a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc +++ b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc @@ -34,6 +34,7 @@ #include +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" @@ -97,6 +98,7 @@ TEST_CASE( "Serialize and deserialize dimension label", "[DimensionLabel][serialization]") { shared_ptr dim_label{nullptr}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); SECTION("Internal dimension label") { // Create dimension label array schema. @@ -108,7 +110,7 @@ TEST_CASE( st = dims[0]->set_domain(&domain1[0]); REQUIRE(st.ok()); st = schema->set_domain(make_shared( - HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR)); + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker)); REQUIRE(st.ok()); st = schema->add_attribute( make_shared(HERE(), "label", Datatype::FLOAT64)); @@ -153,8 +155,8 @@ TEST_CASE( message.initRoot(); tiledb::sm::serialization::dimension_label_to_capnp( *dim_label.get(), &builder, true); - auto dim_label_clone = - tiledb::sm::serialization::dimension_label_from_capnp(builder); + auto dim_label_clone = tiledb::sm::serialization::dimension_label_from_capnp( + builder, memory_tracker); // Check dimension label properties and components. CHECK(dim_label->has_schema() == dim_label_clone->has_schema()); diff --git a/tiledb/sm/storage_manager/context_resources.cc b/tiledb/sm/storage_manager/context_resources.cc index 999f0f7566b..bd82b3c319b 100644 --- a/tiledb/sm/storage_manager/context_resources.cc +++ b/tiledb/sm/storage_manager/context_resources.cc @@ -76,7 +76,7 @@ ContextResources::ContextResources( auto server_address = config_.get("rest.server_address"); if (server_address) { auto client = tdb::make_shared(HERE()); - auto st = client->init(&stats(), &config_, &compute_tp(), logger_); + auto st = client->init(&stats(), &config_, &compute_tp(), logger_, *this); throw_if_not_ok(st); rest_client_ = client; } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 570c0e0f537..3f9fd6fcc3a 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -41,6 +41,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" #include "tiledb/common/memory.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array/array_directory.h" @@ -382,7 +383,10 @@ Status StorageManager::array_evolve_schema( "' not exists")); } - auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); + auto memory_tracker = resources_.create_memory_tracker(); + memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); + auto&& array_schema = + array_dir.load_array_schema_latest(encryption_key, memory_tracker); // Load required enumerations before evolution. auto enmr_names = schema_evolution->enumeration_names_to_extend(); @@ -455,7 +459,10 @@ Status StorageManagerCanonical::array_upgrade_version( static_cast(encryption_key_from_cfg.size()))); } - auto&& array_schema = array_dir.load_array_schema_latest(encryption_key_cfg); + auto memory_tracker = resources_.create_memory_tracker(); + memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); + auto&& array_schema = + array_dir.load_array_schema_latest(encryption_key_cfg, memory_tracker); if (array_schema->version() < constants::format_version) { array_schema->generate_uri(); diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index c8168cc65ed..b5aebe73092 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -51,6 +51,7 @@ using namespace tiledb::type; TEST_CASE("Subarray::add_ranges_list", "[subarray]") { // Setup an Array needed to construct the Subarray for testing // add_ranges_list. + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::shared_ptr sp_dim1 = make_shared(HERE(), "d1", Datatype::INT64); std::shared_ptr sp_dim2 = @@ -58,7 +59,7 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { uint64_t tile_extents[] = {2, 2}; std::vector> dims{sp_dim1, sp_dim2}; std::shared_ptr sp_dom = make_shared( - HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker); uint64_t local_DIM_DOMAIN[4] = {1, 12, 1, 12}; CHECK(sp_dim1->set_domain(&local_DIM_DOMAIN[0]).ok()); CHECK(sp_dim2->set_domain(&local_DIM_DOMAIN[2]).ok()); @@ -66,7 +67,8 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { CHECK(sp_dim2->set_tile_extent(&tile_extents[1]).ok()); std::shared_ptr sp_attrib = make_shared(HERE(), "a1", Datatype::INT32); - tiledb::sm::Domain dom{Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR}; + tiledb::sm::Domain dom{ + Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker}; std::shared_ptr sp_as = make_shared(HERE()); CHECK(sp_as->set_domain(sp_dom).ok()); From 3ec8b405caabae6be7eebac38510d8dd4071f4d7 Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Tue, 20 Feb 2024 12:00:54 -0500 Subject: [PATCH 02/12] Disable copy and move --- tiledb/sm/array_schema/domain.cc | 26 ----- tiledb/sm/array_schema/domain.h | 13 +-- .../test/unit_fragment_consolidator.cc | 6 +- tiledb/sm/query/test/unit_query_condition.cc | 94 ++++++++----------- tiledb/sm/rtree/test/unit_rtree.cc | 39 +++----- tiledb/sm/serialization/array_schema.h | 2 +- 6 files changed, 58 insertions(+), 122 deletions(-) diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index b2251ea1f32..65144a8a330 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -96,32 +96,6 @@ Domain::Domain( set_tile_cell_order_cmp_funcs(); } -Domain::Domain(Domain&& rhs) - : cell_num_per_tile_(rhs.cell_num_per_tile_) - , cell_order_(rhs.cell_order_) - , dimensions_(move(rhs.dimensions_)) - , dimension_ptrs_(move(rhs.dimension_ptrs_)) - , dim_num_(rhs.dim_num_) - , tile_order_(rhs.tile_order_) - , cell_order_cmp_func_(move(rhs.cell_order_cmp_func_)) - , cell_order_cmp_func_2_(move(rhs.cell_order_cmp_func_2_)) - , tile_order_cmp_func_(move(rhs.tile_order_cmp_func_)) { -} - -Domain& Domain::operator=(Domain&& rhs) { - cell_num_per_tile_ = rhs.cell_num_per_tile_; - cell_order_ = rhs.cell_order_; - dim_num_ = rhs.dim_num_; - cell_order_cmp_func_ = move(rhs.cell_order_cmp_func_); - tile_order_cmp_func_ = move(rhs.tile_order_cmp_func_); - dimensions_ = move(rhs.dimensions_); - dimension_ptrs_ = move(rhs.dimension_ptrs_); - tile_order_ = rhs.tile_order_; - cell_order_cmp_func_2_ = move(rhs.cell_order_cmp_func_2_); - - return *this; -} - /* ********************************* */ /* API */ /* ********************************* */ diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 6c7b51fe88f..d64398f6e1f 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -86,12 +86,6 @@ class Domain { Layout tile_order, shared_ptr memory_tracker); - /** Copy constructor. */ - Domain(const Domain&) = default; - - /** Move constructor. */ - Domain(Domain&& rhs); - /** Destructor. */ ~Domain() = default; @@ -99,11 +93,8 @@ class Domain { /* OPERATORS */ /* ********************************* */ - /** Copy-assignment operator. */ - DISABLE_COPY_ASSIGN(Domain); - - /** Move-assignment operator. */ - Domain& operator=(Domain&& rhs); + DISABLE_COPY_AND_COPY_ASSIGN(Domain); + DISABLE_MOVE_AND_MOVE_ASSIGN(Domain); /* ********************************* */ /* API */ diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index e3a08c5486f..6987e6aee32 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -52,7 +52,7 @@ shared_ptr make_schema( // Create the domain/dimensions. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; for (uint64_t d = 0; d < dim_types.size(); d++) { auto dim{make_shared( HERE(), "d" + std::to_string(d + 1), dim_types[d])}; @@ -131,9 +131,9 @@ shared_ptr make_schema( } } - REQUIRE(domain.add_dimension(dim).ok()); + REQUIRE(domain->add_dimension(dim).ok()); } - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); // Create the attributes. for (uint64_t a = 0; a < attr_types.size(); a++) { diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index b8f95270b34..c9e1efa8c48 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1586,15 +1586,15 @@ void test_apply( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -1646,15 +1646,15 @@ void test_apply( attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -1755,7 +1755,7 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -1763,10 +1763,8 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); std::vector> frag_md(1); frag_md[0] = make_shared( @@ -2305,7 +2303,7 @@ void test_apply_dense( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2313,10 +2311,8 @@ void test_apply_dense( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -2365,7 +2361,7 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2373,10 +2369,8 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -2477,7 +2471,7 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -2485,10 +2479,8 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -3007,7 +2999,7 @@ void test_apply_sparse( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -3015,10 +3007,8 @@ void test_apply_sparse( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -3067,7 +3057,7 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -3075,10 +3065,8 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -3838,7 +3826,7 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -3846,10 +3834,8 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -4128,15 +4114,15 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -4483,15 +4469,15 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -4803,7 +4789,7 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{ make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -4811,10 +4797,8 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, @@ -4906,15 +4890,15 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto memory_tracker = tiledb::test::create_test_memory_tracker(); - Domain domain(memory_tracker); + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index 0f172e1d270..ccaf6fb1392 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -143,8 +143,7 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { Domain dom1 = create_domain( {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); - const Domain d1{dom1}; - RTree rtree1(&d1, 3, tracker); + RTree rtree1(&dom1, 3, tracker); CHECK(!rtree1.set_leaf(0, mbrs_1d[0]).ok()); CHECK(rtree1.set_leaf_num(mbrs_1d.size()).ok()); for (size_t m = 0; m < mbrs_1d.size(); ++m) @@ -205,8 +204,7 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { tracker); auto mbrs_2d = create_mbrs( {1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}, tracker); - const Domain d2{dom2}; - RTree rtree2(&d2, 5, tracker); + RTree rtree2(&dom2, 5, tracker); CHECK(rtree2.set_leaves(mbrs_2d).ok()); rtree2.build_tree(); CHECK(rtree2.height() == 2); @@ -287,8 +285,7 @@ TEST_CASE("RTree: Test 1D R-tree, height 2", "[rtree][1d][2h]") { Domain dom1 = create_domain( {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); - const Domain d1{dom1}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -405,8 +402,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { tracker); auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}, tracker); - const Domain d2{dom2}; - RTree rtree(&d2, 3, tracker); + RTree rtree(&dom2, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -462,8 +458,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15, 11, 15, 20, 22, 16, 16, 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}, tracker); - const Domain d2{dom2}; - RTree rtree(&d2, 3, tracker); + RTree rtree(&dom2, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -542,8 +537,7 @@ TEST_CASE( tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 5, tracker); + RTree rtree(&dom, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -600,8 +594,7 @@ TEST_CASE( tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 5, tracker); + RTree rtree(&dom, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -658,8 +651,7 @@ TEST_CASE( tracker); auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20}, {5, 6, 7, 9, 11, 30}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -729,8 +721,7 @@ TEST_CASE( tracker); auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20, 21, 26}, {5, 6, 7, 9, 11, 30, 31, 40}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 2, tracker); + RTree rtree(&dom, 2, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -867,8 +858,7 @@ TEST_CASE( auto mbrs = create_str_mbrs<1>({"aa", "b", "eee", "g", "gggg", "ii"}, tracker); - const Domain d1{dom1}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -958,8 +948,7 @@ TEST_CASE( "oop"}, tracker); - const Domain d1{dom1}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -1045,8 +1034,7 @@ TEST_CASE( {"aa", "b", "eee", "g", "gggg", "ii", "jj", "lll", "m", "n", "oo", "qqq"}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -1153,8 +1141,7 @@ TEST_CASE( auto mbrs = create_str_int32_mbrs( {"aa", "b", "eee", "g", "gggg", "ii"}, {1, 5, 7, 8, 10, 14}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index 31df6b94a06..bdb5d8a007c 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -134,7 +134,7 @@ void dimension_label_to_capnp( * Deserialize a dimension label from a cap'n proto object * * @param reader Cap'n proto reader object. -* @param memory_tracker The memory tracker to use. + * @param memory_tracker The memory tracker to use. * @return A new DimensionLabel. */ shared_ptr dimension_label_from_capnp( From 322d948516b92d13e64166ad4833a15e22374f25 Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Wed, 21 Feb 2024 11:28:11 -0500 Subject: [PATCH 03/12] Use tdb::pmr for vector members --- tiledb/common/memory_tracker.h | 4 +++- tiledb/sm/array_schema/dimension_label.cc | 1 + tiledb/sm/array_schema/domain.cc | 18 +++++++++++++++--- tiledb/sm/array_schema/domain.h | 10 +++++----- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 037d923dccc..2a08a5e2185 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -110,7 +110,9 @@ enum class MemoryType { TILE_MAX_VALS, TILE_SUMS, TILE_NULL_COUNTS, - ENUMERATION + ENUMERATION, + DOMAIN, + DIMENSIONS, }; /** The type of MemoryTracker. */ diff --git a/tiledb/sm/array_schema/dimension_label.cc b/tiledb/sm/array_schema/dimension_label.cc index 6d273faa453..e0287a377be 100644 --- a/tiledb/sm/array_schema/dimension_label.cc +++ b/tiledb/sm/array_schema/dimension_label.cc @@ -28,6 +28,7 @@ #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 65144a8a330..33841224c1d 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -59,7 +59,12 @@ namespace tiledb::sm { /* ********************************* */ Domain::Domain(shared_ptr memory_tracker) - : memory_tracker_(memory_tracker) { + : memory_tracker_(memory_tracker) + , dimensions_(memory_tracker_->get_resource(MemoryType::DOMAIN)) + , dimension_ptrs_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) + , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) + , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAIN)) + , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) { cell_order_ = Layout::ROW_MAJOR; tile_order_ = Layout::ROW_MAJOR; dim_num_ = 0; @@ -73,9 +78,16 @@ Domain::Domain( shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , cell_order_(cell_order) - , dimensions_(dimensions) + , dimensions_( + dimensions.begin(), + dimensions.end(), + memory_tracker_->get_resource(MemoryType::DOMAIN)) + , dimension_ptrs_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , dim_num_(static_cast(dimensions.size())) - , tile_order_(tile_order) { + , tile_order_(tile_order) + , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) + , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAIN)) + , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) { /* * Verify that the input vector has no non-null elements in order to meet the * class invariant. Initialize the dimensions mirror. diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index d64398f6e1f..bf4f7383470 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -523,7 +523,7 @@ class Domain { * * @invariant All pointers in the vector are non-null. */ - std::vector> dimensions_; + tdb::pmr::vector> dimensions_; /** * Non-allocating mirror of the dimensions vector. @@ -535,7 +535,7 @@ class Domain { * * @invariant All pointers in the vector are non-null. */ - std::vector dimension_ptrs_; + tdb::pmr::vector dimension_ptrs_; /** The number of dimensions. */ unsigned dim_num_; @@ -551,7 +551,7 @@ class Domain { * - buff: The buffer that stores all coorinates; * - a, b: The positions of the two coordinates in the buffer to compare. */ - std::vector cell_order_cmp_func_; @@ -561,7 +561,7 @@ class Domain { * * - coord_a, coord_b: The two coordinates to compare. */ - std::vector + tdb::pmr::vector cell_order_cmp_func_2_; /** @@ -571,7 +571,7 @@ class Domain { * - dim: The dimension to compare on. * - coord_a, coord_b: The two coordinates to compare. */ - std::vector tile_order_cmp_func_; From cb4c75d5eb52d7be6165a28668f70c43f4e4d7c4 Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Wed, 21 Feb 2024 12:57:04 -0500 Subject: [PATCH 04/12] Update methods to use pmr::vector Domain::overlap_ratio is passed Subarray::is_default_ as a parameter, so this method was not updated. Once Subarray::is_default_ is a pmr::vector this method can be updated to take a pmr::vector parameter. --- test/src/unit-Reader.cc | 3 +- tiledb/sm/array_schema/domain.cc | 10 +++-- tiledb/sm/array_schema/domain.h | 2 +- .../sm/array_schema/test/unit_tile_domain.cc | 37 ++++++++++++++----- tiledb/sm/array_schema/tile_domain.h | 7 ++-- tiledb/sm/filter/webp_filter.cc | 18 ++++----- tiledb/sm/filter/webp_filter.h | 3 +- 7 files changed, 52 insertions(+), 28 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 25a5d16d685..f88ee4d768b 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -186,7 +186,8 @@ TEST_CASE_METHOD( int32_t domain_vec[] = {1, 10, 1, 15}; NDRange domain = {Range(&domain_vec[0], size), Range(&domain_vec[2], size)}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + tdb::pmr::vector tile_extents( + 2, tracker_->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 33841224c1d..ea54ae6fa40 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -586,12 +586,14 @@ const ByteVecValue& Domain::tile_extent(unsigned i) const { return dimension_ptrs_[i]->tile_extent(); } -std::vector Domain::tile_extents() const { - std::vector ret(dim_num_); - for (unsigned d = 0; d < dim_num_; ++d) +tdb::pmr::vector Domain::tile_extents() const { + tdb::pmr::vector ret( + dim_num_, memory_tracker_->get_resource(MemoryType::DOMAIN)); + for (unsigned d = 0; d < dim_num_; ++d) { ret[d] = tile_extent(d); + } - return ret; + return {ret, memory_tracker_->get_resource(MemoryType::DOMAIN)}; } uint64_t Domain::tile_num(const NDRange& ndrange) const { diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index bf4f7383470..7c83cd6e204 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -441,7 +441,7 @@ class Domain { const ByteVecValue& tile_extent(unsigned i) const; /** Returns the tile extents. */ - std::vector tile_extents() const; + tdb::pmr::vector tile_extents() const; /** * Returns the number of tiles intersecting the input ND range. diff --git a/tiledb/sm/array_schema/test/unit_tile_domain.cc b/tiledb/sm/array_schema/test/unit_tile_domain.cc index 771c2772687..e27b3c00253 100644 --- a/tiledb/sm/array_schema/test/unit_tile_domain.cc +++ b/tiledb/sm/array_schema/test/unit_tile_domain.cc @@ -31,13 +31,16 @@ */ #include +#include "src/mem_helpers.h" #include "tiledb/sm/array_schema/tile_domain.h" using namespace tiledb::sm; TEST_CASE("TileDomain: Test 1D", "[TileDomain][1d]") { int32_t tile_extent_v = 10; - std::vector tile_extents(1); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 1, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extent_v); Layout layout = Layout::ROW_MAJOR; @@ -72,7 +75,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {1, 10, 1, 10}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; @@ -115,7 +120,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {4, 10, 2, 8}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; @@ -153,7 +160,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {1, 10, 1, 10}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -191,7 +200,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {4, 10, 2, 8}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -228,7 +239,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 11, 20}; std::vector domain_slice = {4, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -260,7 +273,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 11, 20}; std::vector domain_slice = {2, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -297,7 +312,9 @@ TEST_CASE( std::vector domain_vec = {1, 10, 11, 20}; std::vector domain_slice = {2, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -323,7 +340,9 @@ TEST_CASE("TileDomain: Test 2D, covers", "[TileDomain][2d][covers]") { std::vector domain_slice_1 = {2, 6, 2, 8}; std::vector domain_slice_2 = {3, 6, 1, 7}; std::vector tile_extents_vec = {2, 5}; - std::vector tile_extents(2); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + tdb::pmr::vector tile_extents( + 2, memory_tracker->get_resource(MemoryType::DOMAIN)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; diff --git a/tiledb/sm/array_schema/tile_domain.h b/tiledb/sm/array_schema/tile_domain.h index a89e498e9af..fcf6a778702 100644 --- a/tiledb/sm/array_schema/tile_domain.h +++ b/tiledb/sm/array_schema/tile_domain.h @@ -36,6 +36,7 @@ #include #include +#include "tiledb/common/pmr.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/misc/types.h" @@ -81,13 +82,13 @@ class TileDomain { unsigned id, const NDRange& domain, const NDRange& domain_slice, - const std::vector tile_extents, + const tdb::pmr::vector& tile_extents, Layout layout) : id_(id) , dim_num_((unsigned)domain.size()) , domain_(domain) , domain_slice_(domain_slice) - , tile_extents_(tile_extents) + , tile_extents_(tile_extents.begin(), tile_extents.end()) , layout_(layout) { assert(layout == Layout::ROW_MAJOR || layout == Layout::COL_MAJOR); compute_tile_domain(domain, domain_slice, tile_extents); @@ -304,7 +305,7 @@ class TileDomain { void compute_tile_domain( const NDRange& domain, const NDRange& domain_slice, - const std::vector& tile_extents) { + const tdb::pmr::vector& tile_extents) { tile_domain_.resize(2 * dim_num_); for (unsigned d = 0; d < dim_num_; ++d) { auto ds = (const T*)domain_slice[d].data(); diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index 961209f7ecb..31bc59ac0ed 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -367,7 +367,7 @@ void WebpFilter::serialize_impl(Serializer& serializer) const { } template -void WebpFilter::set_extents(const std::vector& extents) { +void WebpFilter::set_extents(const tdb::pmr::vector& extents) { extents_ = {extents[0].rvalue_as(), extents[1].rvalue_as()}; uint8_t pixel_depth = format_ < WebpInputFormat::WEBP_RGBA ? 3 : 4; // X should be divisible by pixel_depth or RGB values will skew. @@ -385,21 +385,21 @@ void WebpFilter::set_extents(const std::vector& extents) { } template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); template void WebpFilter::set_extents( - const std::vector& extents); + const tdb::pmr::vector& extents); } // namespace tiledb::sm diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index 5842f6351be..f1fa51828d8 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -40,6 +40,7 @@ constexpr bool webp_filter_exists = false; #endif // TILEDB_WEBP #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/enums/filter_option.h" #include "tiledb/sm/enums/filter_type.h" #include "tiledb/sm/filter/filter.h" @@ -266,7 +267,7 @@ class WebpFilter : public Filter { * @param extents Extents retrieved from array Domain object. */ template - void set_extents(const std::vector& extents); + void set_extents(const tdb::pmr::vector& extents); /** * Get tile extents currently in use by this WebpFilter From 99ff2f5b0ebeb54cfef0f4269121771e6245e90c Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Wed, 21 Feb 2024 14:17:52 -0500 Subject: [PATCH 05/12] Rename DOMAIN->DOMAINS Avoids conflict with math.h definitions seen in Windows, OSX, and manylinux CI https://github.com/shaunrd0/TileDB/actions/runs/7993508034/job/21829493275#step:12:158 --- test/src/unit-Reader.cc | 2 +- .../test/compile_capi_domain_stub_main.cc | 1 - tiledb/common/memory_tracker.h | 2 +- tiledb/sm/array_schema/domain.cc | 20 +++++++++---------- .../sm/array_schema/test/unit_tile_domain.cc | 18 ++++++++--------- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index f88ee4d768b..c2b7dd642f1 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -187,7 +187,7 @@ TEST_CASE_METHOD( NDRange domain = {Range(&domain_vec[0], size), Range(&domain_vec[2], size)}; std::vector tile_extents_vec = {2, 5}; tdb::pmr::vector tile_extents( - 2, tracker_->get_resource(MemoryType::DOMAIN)); + 2, tracker_->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; diff --git a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc index ba46e5cf2bb..a64d36b134d 100644 --- a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc +++ b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc @@ -29,7 +29,6 @@ #include "../domain_api_internal.h" int main() { - // Domain is easy to deal with because it has a default constructor. tiledb_domain_handle_t x{(shared_ptr)nullptr}; return 0; } diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 2a08a5e2185..4d91f8eb172 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -111,7 +111,7 @@ enum class MemoryType { TILE_SUMS, TILE_NULL_COUNTS, ENUMERATION, - DOMAIN, + DOMAINS, DIMENSIONS, }; diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index ea54ae6fa40..3b8b4ee768b 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -60,11 +60,11 @@ namespace tiledb::sm { Domain::Domain(shared_ptr memory_tracker) : memory_tracker_(memory_tracker) - , dimensions_(memory_tracker_->get_resource(MemoryType::DOMAIN)) + , dimensions_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , dimension_ptrs_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) - , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) - , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAIN)) - , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) { + , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) { cell_order_ = Layout::ROW_MAJOR; tile_order_ = Layout::ROW_MAJOR; dim_num_ = 0; @@ -81,13 +81,13 @@ Domain::Domain( , dimensions_( dimensions.begin(), dimensions.end(), - memory_tracker_->get_resource(MemoryType::DOMAIN)) + memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , dimension_ptrs_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , dim_num_(static_cast(dimensions.size())) , tile_order_(tile_order) - , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) - , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAIN)) - , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAIN)) { + , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) { /* * Verify that the input vector has no non-null elements in order to meet the * class invariant. Initialize the dimensions mirror. @@ -588,12 +588,12 @@ const ByteVecValue& Domain::tile_extent(unsigned i) const { tdb::pmr::vector Domain::tile_extents() const { tdb::pmr::vector ret( - dim_num_, memory_tracker_->get_resource(MemoryType::DOMAIN)); + dim_num_, memory_tracker_->get_resource(MemoryType::DOMAINS)); for (unsigned d = 0; d < dim_num_; ++d) { ret[d] = tile_extent(d); } - return {ret, memory_tracker_->get_resource(MemoryType::DOMAIN)}; + return {ret, memory_tracker_->get_resource(MemoryType::DOMAINS)}; } uint64_t Domain::tile_num(const NDRange& ndrange) const { diff --git a/tiledb/sm/array_schema/test/unit_tile_domain.cc b/tiledb/sm/array_schema/test/unit_tile_domain.cc index e27b3c00253..18adfd2ca7d 100644 --- a/tiledb/sm/array_schema/test/unit_tile_domain.cc +++ b/tiledb/sm/array_schema/test/unit_tile_domain.cc @@ -40,7 +40,7 @@ TEST_CASE("TileDomain: Test 1D", "[TileDomain][1d]") { int32_t tile_extent_v = 10; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 1, memory_tracker->get_resource(MemoryType::DOMAIN)); + 1, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extent_v); Layout layout = Layout::ROW_MAJOR; @@ -77,7 +77,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; @@ -122,7 +122,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; @@ -162,7 +162,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -202,7 +202,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -241,7 +241,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -275,7 +275,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -314,7 +314,7 @@ TEST_CASE( std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -342,7 +342,7 @@ TEST_CASE("TileDomain: Test 2D, covers", "[TileDomain][2d][covers]") { std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAIN)); + 2, memory_tracker->get_resource(MemoryType::DOMAINS)); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; From 464846f34a20132321e4c41c644a52fde9f7c18f Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Wed, 21 Feb 2024 16:42:33 -0500 Subject: [PATCH 06/12] Fix some tests in unit_rtree --- tiledb/sm/rtree/test/unit_rtree.cc | 60 ++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index ccaf6fb1392..3a1679d1067 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -142,8 +142,10 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { int32_t dim_extent = 10; Domain dom1 = create_domain( {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); + const Domain d1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); - RTree rtree1(&dom1, 3, tracker); + RTree rtree1(&d1, 3, tracker); CHECK(!rtree1.set_leaf(0, mbrs_1d[0]).ok()); CHECK(rtree1.set_leaf_num(mbrs_1d.size()).ok()); for (size_t m = 0; m < mbrs_1d.size(); ++m) @@ -202,9 +204,15 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { {dim_dom_2, dim_dom_2}, {&dim_extent_2, &dim_extent_2}, tracker); + const Domain d2 = create_domain( + {"d1", "d2"}, + {Datatype::INT64, Datatype::INT64}, + {dim_dom_2, dim_dom_2}, + {&dim_extent_2, &dim_extent_2}, + tracker); auto mbrs_2d = create_mbrs( {1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}, tracker); - RTree rtree2(&dom2, 5, tracker); + RTree rtree2(&d2, 5, tracker); CHECK(rtree2.set_leaves(mbrs_2d).ok()); rtree2.build_tree(); CHECK(rtree2.height() == 2); @@ -242,7 +250,9 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { create_mbrs({1.0f, 3.0f, 5.0f, 10.0f, 20.0f, 22.0f}, tracker); Domain dom2f = create_domain( {"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}, tracker); - RTree rtreef(&dom2f, 5, tracker); + const Domain d2f = create_domain( + {"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}, tracker); + RTree rtreef(&d2f, 5, tracker); CHECK(rtreef.set_leaves(mbrs_f).ok()); rtreef.build_tree(); @@ -394,7 +404,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { std::vector is_default(2, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom2 = create_domain( + Domain dom1 = create_domain( {"d1", "d2"}, {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, @@ -402,7 +412,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { tracker); auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}, tracker); - RTree rtree(&dom2, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -448,7 +458,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { std::vector is_default(2, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom2 = create_domain( + Domain dom1 = create_domain( {"d1", "d2"}, {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, @@ -458,7 +468,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15, 11, 15, 20, 22, 16, 16, 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}, tracker); - RTree rtree(&dom2, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -529,7 +539,13 @@ TEST_CASE( int32_t int32_dom[] = {5, 10}; uint8_t uint8_extent = 2; int32_t int32_extent = 2; - Domain dom = create_domain( + Domain dom1 = create_domain( + {"d1", "d2"}, + {Datatype::UINT8, Datatype::INT32}, + {uint8_dom, int32_dom}, + {&uint8_extent, &int32_extent}, + tracker); + const Domain d1 = create_domain( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, @@ -537,7 +553,7 @@ TEST_CASE( tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}, tracker); - RTree rtree(&dom, 5, tracker); + RTree rtree(&d1, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -552,7 +568,7 @@ TEST_CASE( int32_t int32_r_no[] = {1, 10}; range_no[0] = Range(uint8_r_no, sizeof(uint8_r_no)); range_no[1] = Range(int32_r_no, sizeof(int32_r_no)); - double ratio = dom.overlap_ratio(range_no, is_default, mbrs[0]); + double ratio = dom1.overlap_ratio(range_no, is_default, mbrs[0]); CHECK(ratio == 0.0); // Check full domain overlap @@ -561,9 +577,9 @@ TEST_CASE( int32_t int32_r_full[] = {1, 10}; range_full[0] = Range(uint8_r_full, sizeof(uint8_r_full)); range_full[1] = Range(int32_r_full, sizeof(int32_r_full)); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[0]); CHECK(ratio == 1.0); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[1]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[1]); CHECK(ratio == 1.0); // Check partial domain overlap @@ -572,7 +588,7 @@ TEST_CASE( int32_t int32_r_part[] = {5, 5}; range_part[0] = Range(uint8_r_part, sizeof(uint8_r_part)); range_part[1] = Range(int32_r_part, sizeof(int32_r_part)); - ratio = dom.overlap_ratio(range_part, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_part, is_default, mbrs[0]); CHECK(ratio == 0.25); } @@ -586,7 +602,13 @@ TEST_CASE( float float_dom[] = {0.1f, 0.9f}; uint64_t uint64_extent = 2; float float_extent = 0.1f; - Domain dom = create_domain( + Domain dom1 = create_domain( + {"d1", "d2"}, + {Datatype::UINT64, Datatype::FLOAT32}, + {uint64_dom, float_dom}, + {&uint64_extent, &float_extent}, + tracker); + const Domain d1 = create_domain( {"d1", "d2"}, {Datatype::UINT64, Datatype::FLOAT32}, {uint64_dom, float_dom}, @@ -594,7 +616,7 @@ TEST_CASE( tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}, tracker); - RTree rtree(&dom, 5, tracker); + RTree rtree(&d1, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -609,7 +631,7 @@ TEST_CASE( float float_r_no[] = {.1f, .9f}; range_no[0] = Range(uint64_r_no, sizeof(uint64_r_no)); range_no[1] = Range(float_r_no, sizeof(float_r_no)); - double ratio = dom.overlap_ratio(range_no, is_default, mbrs[0]); + double ratio = dom1.overlap_ratio(range_no, is_default, mbrs[0]); CHECK(ratio == 0.0); // Check full domain overlap @@ -618,9 +640,9 @@ TEST_CASE( float float_r_full[] = {.1f, 1.0f}; range_full[0] = Range(uint64_r_full, sizeof(uint64_r_full)); range_full[1] = Range(float_r_full, sizeof(float_r_full)); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[0]); CHECK(ratio == 1.0); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[1]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[1]); CHECK(ratio == 1.0); // Check partial domain overlap @@ -629,7 +651,7 @@ TEST_CASE( float float_r_part[] = {.5f, .55f}; range_part[0] = Range(uint64_r_part, sizeof(uint64_r_part)); range_part[1] = Range(float_r_part, sizeof(float_r_part)); - ratio = dom.overlap_ratio(range_part, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_part, is_default, mbrs[0]); CHECK(ratio == 0.25); } From d88b23680addc420038cbcdcfa6a6f00f4991abf Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Fri, 23 Feb 2024 11:50:52 -0500 Subject: [PATCH 07/12] Fixes after merge --- test/src/unit-enumerations.cc | 3 +- test/src/unit-request-handlers.cc | 12 ++--- tiledb/sm/array/array.cc | 5 +- tiledb/sm/array/array_directory.h | 14 ++--- .../test/array_schema_test_support.h | 1 - tiledb/sm/c_api/tiledb.cc | 6 +-- tiledb/sm/c_api/tiledb_filestore.cc | 1 - tiledb/sm/fragment/fragment_info.cc | 4 +- tiledb/sm/query/test/unit_query_condition.cc | 54 +++++++++---------- .../test/unit_capnp_array_schema.cc | 4 +- tiledb/sm/storage_manager/storage_manager.cc | 6 +-- 11 files changed, 48 insertions(+), 62 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index d9a00ae90ef..755e69a0407 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -146,7 +146,6 @@ struct EnumerationFx { Config cfg_; Context ctx_; EncryptionKey enc_key_; - shared_ptr memory_tracker_; }; template @@ -2743,7 +2742,7 @@ shared_ptr EnumerationFx::get_array_directory() { shared_ptr EnumerationFx::get_array_schema_latest() { auto array_dir = get_array_directory(); - return array_dir->load_array_schema_latest(enc_key_, memory_tracker_); + return array_dir->load_array_schema_latest(enc_key_); } #ifdef TILEDB_SERIALIZATION diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 6f654e01e3c..3ff7b4caa4a 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -97,8 +97,8 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override { - auto schema = make_shared( - HERE(), ArrayType::SPARSE, memory_tracker_); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto dim = make_shared(HERE(), "dim1", Datatype::INT32); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); @@ -399,8 +399,8 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { // Create a schema to serialize - auto schema = make_shared( - HERE(), ArrayType::SPARSE, memory_tracker_); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto dim = make_shared(HERE(), "dim1", Datatype::INT32); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); @@ -446,8 +446,8 @@ shared_ptr HandleLoadArraySchemaRequestFx::call_handler( } shared_ptr HandleQueryPlanRequestFx::create_schema() { - auto schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker_); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); schema->set_capacity(10000); throw_if_not_ok(schema->set_cell_order(Layout::ROW_MAJOR)); throw_if_not_ok(schema->set_tile_order(Layout::ROW_MAJOR)); diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index a76fb196161..dac76a6051e 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1265,8 +1265,7 @@ Array::open_for_reads_without_fragments() { "array_open_read_without_fragments_load_schemas"); // Load array schemas - auto result = - array_directory().load_array_schemas(*encryption_key(), memory_tracker_); + auto result = array_directory().load_array_schemas(*encryption_key()); auto version = std::get<0>(result)->version(); ensure_supported_schema_version_for_read(version); @@ -1291,7 +1290,7 @@ Array::open_for_writes() { // Load array schemas auto&& [array_schema_latest, array_schemas_all] = - array_directory().load_array_schemas(*encryption_key(), memory_tracker_); + array_directory().load_array_schemas(*encryption_key()); // If building experimentally, this library should not be able to // write to newer-versioned or older-versioned arrays diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index e58cdf5b552..68a3c837165 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -329,8 +329,7 @@ class ArrayDirectory { static shared_ptr load_array_schema_from_uri( ContextResources& resources, const URI& array_schema_uri, - const EncryptionKey& encryption_key, - shared_ptr memory_tracker); + const EncryptionKey& encryption_key); /** * Get the full vac uri using the base URI and a vac uri that might be @@ -353,8 +352,7 @@ class ArrayDirectory { * @return Status, a new ArraySchema */ shared_ptr load_array_schema_latest( - const EncryptionKey& encryption_key, - shared_ptr memory_tracker) const; + const EncryptionKey& encryption_key) const; /** * It loads and returns the latest schema and all the array schemas @@ -372,9 +370,7 @@ class ArrayDirectory { tuple< shared_ptr, std::unordered_map>> - load_array_schemas( - const EncryptionKey& encryption_key, - shared_ptr memory_tracker) const; + load_array_schemas(const EncryptionKey& encryption_key) const; /** * Loads all schemas of an array from persistent storage into memory. @@ -387,9 +383,7 @@ class ArrayDirectory { * ArraySchemaMap Map of all array schemas found keyed by name */ std::unordered_map> - load_all_array_schemas( - const EncryptionKey& encryption_key, - shared_ptr memory_tracker) const; + load_all_array_schemas(const EncryptionKey& encryption_key) const; /** * Load the enumerations from the provided list of paths. diff --git a/tiledb/sm/array_schema/test/array_schema_test_support.h b/tiledb/sm/array_schema/test/array_schema_test_support.h index 5d7ae467dc9..7b4746c34ad 100644 --- a/tiledb/sm/array_schema/test/array_schema_test_support.h +++ b/tiledb/sm/array_schema/test/array_schema_test_support.h @@ -293,7 +293,6 @@ class TestArraySchema { */ TestArraySchema() = delete; - // TODO: Use TestArraySchema::memory_tracker_? /** * The simplest array constructor has defaults for everything but the list of * dimensions and attributes. Note that the domain is not specified diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 3d7502da6de..06c7edf3ec6 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -518,8 +518,7 @@ int32_t tiledb_array_schema_load( // Load latest array schema auto memory_tracker = ctx->context().resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); - auto&& array_schema_latest = - array_dir->load_array_schema_latest(key, memory_tracker); + auto&& array_schema_latest = array_dir->load_array_schema_latest(key); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; @@ -612,8 +611,7 @@ int32_t tiledb_array_schema_load_with_key( // Load latest array schema auto memory_tracker = ctx->context().resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); - auto&& array_schema_latest = - array_dir->load_array_schema_latest(key, memory_tracker); + auto&& array_schema_latest = array_dir->load_array_schema_latest(key); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index e48c7e8ee84..72470516186 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -141,7 +141,6 @@ int32_t tiledb_filestore_schema_create( tiledb::sm::FilterPipeline{}, tiledb::sm::ByteVecValue(std::move(tile_extent_vec))); - auto memory_tracker = ctx->resources().create_memory_tracker(); auto domain = make_shared(HERE(), memory_tracker); throw_if_not_ok(domain->add_dimension(dim)); diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 819ddc323e6..e335da64de4 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -754,7 +754,7 @@ shared_ptr FragmentInfo::get_array_schema(uint32_t fid) { auto memory_tracker = resources_->create_memory_tracker(); memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); return ArrayDirectory::load_array_schema_from_uri( - *resources_, schema_uri, encryption_key, memory_tracker); + *resources_, schema_uri, encryption_key); } Status FragmentInfo::get_array_schema_name( @@ -1006,7 +1006,7 @@ FragmentInfo::load_array_schemas_and_fragment_metadata( std::unordered_map> array_schemas_all; std::tie(array_schema_latest, array_schemas_all) = - array_dir.load_array_schemas(enc_key, memory_tracker); + array_dir.load_array_schemas(enc_key); const auto filtered_fragment_uris = [&]() { auto timer_se = diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 00a3a97bc00..0e326c1f901 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1578,8 +1578,8 @@ void test_apply( const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -1644,8 +1644,8 @@ void test_apply( const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); @@ -1749,8 +1749,8 @@ TEST_CASE( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2298,8 +2298,8 @@ void test_apply_dense( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2362,8 +2362,8 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); @@ -2468,8 +2468,8 @@ TEST_CASE( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2997,8 +2997,8 @@ void test_apply_sparse( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -3061,8 +3061,8 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); @@ -3833,8 +3833,8 @@ TEST_CASE( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); @@ -4117,8 +4117,9 @@ TEST_CASE( const Datatype type = GENERATE(Datatype::STRING_ASCII, Datatype::STRING_UTF8); // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(false); attr.set_cell_val_num(constants::var_num); @@ -4126,7 +4127,6 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - auto memory_tracker = memory_tracker; auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; uint32_t bounds[2] = {1, cells}; @@ -4474,8 +4474,8 @@ TEST_CASE( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(false); attr.set_cell_val_num(constants::var_num); @@ -4798,8 +4798,8 @@ TEST_CASE( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(true); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) @@ -4894,8 +4894,8 @@ TEST_CASE( // Initialize the array schema. auto memory_tracker = tiledb::test::create_test_memory_tracker(); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); diff --git a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc index c5ddb9ccf90..df387a90054 100644 --- a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc +++ b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc @@ -102,8 +102,8 @@ TEST_CASE( SECTION("Internal dimension label") { // Create dimension label array schema. Status st; - auto schema = make_shared( - HERE(), ArrayType::DENSE, memory_tracker); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); std::vector> dims{ make_shared(HERE(), "index", Datatype::UINT32)}; uint32_t domain1[2]{1, 64}; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 3f9fd6fcc3a..6d42ca108bc 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -385,8 +385,7 @@ Status StorageManager::array_evolve_schema( auto memory_tracker = resources_.create_memory_tracker(); memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); - auto&& array_schema = - array_dir.load_array_schema_latest(encryption_key, memory_tracker); + auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); // Load required enumerations before evolution. auto enmr_names = schema_evolution->enumeration_names_to_extend(); @@ -461,8 +460,7 @@ Status StorageManagerCanonical::array_upgrade_version( auto memory_tracker = resources_.create_memory_tracker(); memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); - auto&& array_schema = - array_dir.load_array_schema_latest(encryption_key_cfg, memory_tracker); + auto&& array_schema = array_dir.load_array_schema_latest(encryption_key_cfg); if (array_schema->version() < constants::format_version) { array_schema->generate_uri(); From da20cff283daf2a28fe3189f6323d77ff5ce6bae Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Fri, 23 Feb 2024 11:52:38 -0500 Subject: [PATCH 08/12] Changes from review --- .../test/compile_capi_domain_stub_main.cc | 2 +- tiledb/sm/array_schema/domain.h | 3 + tiledb/sm/query/test/unit_query_condition.cc | 98 +++++++++---------- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc index a64d36b134d..6b6a726109e 100644 --- a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc +++ b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc @@ -29,6 +29,6 @@ #include "../domain_api_internal.h" int main() { - tiledb_domain_handle_t x{(shared_ptr)nullptr}; + tiledb_domain_handle_t x{shared_ptr()}; return 0; } diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 7c83cd6e204..ad6edca8c87 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -76,6 +76,9 @@ class Domain { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ + /** Deleted default constructor */ + Domain() = delete; + /** Constructor. */ Domain(shared_ptr memory_tracker); diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 0e326c1f901..254cfc189e9 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1557,20 +1557,13 @@ void test_apply_tile( */ template void test_apply( - shared_ptr memory_tracker, - const Datatype type, - bool var_size = false, - bool nullable = false); + const Datatype type, bool var_size = false, bool nullable = false); /** * C-string template-specialization for `test_apply`. */ template <> -void test_apply( - shared_ptr memory_tracker, - const Datatype type, - bool var_size, - bool nullable) { +void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE((type == Datatype::STRING_ASCII || type == Datatype::STRING_UTF8)); const std::string field_name = "foo"; @@ -1578,6 +1571,7 @@ void test_apply( const char* fill_value = "ac"; // Initialize the array schema. + auto memory_tracker = tiledb::test::get_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -1634,16 +1628,13 @@ void test_apply( * Non-specialized template type for `test_apply`. */ template -void test_apply( - shared_ptr memory_tracker, - const Datatype type, - bool var_size, - bool nullable) { +void test_apply(const Datatype type, bool var_size, bool nullable) { const std::string field_name = "foo"; const uint64_t cells = 10; const T fill_value = 3; // Initialize the array schema. + auto memory_tracker = tiledb::test::get_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -1690,46 +1681,45 @@ void test_apply( } TEST_CASE("QueryCondition: Test apply", "[QueryCondition][apply]") { - auto tracker = tiledb::test::create_test_memory_tracker(); - test_apply(tracker, Datatype::INT8); - test_apply(tracker, Datatype::UINT8); - test_apply(tracker, Datatype::INT16); - test_apply(tracker, Datatype::UINT16); - test_apply(tracker, Datatype::INT32); - test_apply(tracker, Datatype::UINT32); - test_apply(tracker, Datatype::INT64); - test_apply(tracker, Datatype::UINT64); - test_apply(tracker, Datatype::FLOAT32); - test_apply(tracker, Datatype::FLOAT64); - test_apply(tracker, Datatype::CHAR); - test_apply(tracker, Datatype::DATETIME_YEAR); - test_apply(tracker, Datatype::DATETIME_MONTH); - test_apply(tracker, Datatype::DATETIME_WEEK); - test_apply(tracker, Datatype::DATETIME_DAY); - test_apply(tracker, Datatype::DATETIME_HR); - test_apply(tracker, Datatype::DATETIME_MIN); - test_apply(tracker, Datatype::DATETIME_SEC); - test_apply(tracker, Datatype::DATETIME_MS); - test_apply(tracker, Datatype::DATETIME_US); - test_apply(tracker, Datatype::DATETIME_NS); - test_apply(tracker, Datatype::DATETIME_PS); - test_apply(tracker, Datatype::DATETIME_FS); - test_apply(tracker, Datatype::DATETIME_AS); - test_apply(tracker, Datatype::TIME_HR); - test_apply(tracker, Datatype::TIME_MIN); - test_apply(tracker, Datatype::TIME_SEC); - test_apply(tracker, Datatype::TIME_MS); - test_apply(tracker, Datatype::TIME_US); - test_apply(tracker, Datatype::TIME_NS); - test_apply(tracker, Datatype::TIME_PS); - test_apply(tracker, Datatype::TIME_FS); - test_apply(tracker, Datatype::TIME_AS); - test_apply(tracker, Datatype::STRING_ASCII); - test_apply(tracker, Datatype::STRING_ASCII, true); - test_apply(tracker, Datatype::STRING_ASCII, false, true); - test_apply(tracker, Datatype::STRING_UTF8); - test_apply(tracker, Datatype::STRING_UTF8, true); - test_apply(tracker, Datatype::STRING_UTF8, false, true); + test_apply(Datatype::INT8); + test_apply(Datatype::UINT8); + test_apply(Datatype::INT16); + test_apply(Datatype::UINT16); + test_apply(Datatype::INT32); + test_apply(Datatype::UINT32); + test_apply(Datatype::INT64); + test_apply(Datatype::UINT64); + test_apply(Datatype::FLOAT32); + test_apply(Datatype::FLOAT64); + test_apply(Datatype::CHAR); + test_apply(Datatype::DATETIME_YEAR); + test_apply(Datatype::DATETIME_MONTH); + test_apply(Datatype::DATETIME_WEEK); + test_apply(Datatype::DATETIME_DAY); + test_apply(Datatype::DATETIME_HR); + test_apply(Datatype::DATETIME_MIN); + test_apply(Datatype::DATETIME_SEC); + test_apply(Datatype::DATETIME_MS); + test_apply(Datatype::DATETIME_US); + test_apply(Datatype::DATETIME_NS); + test_apply(Datatype::DATETIME_PS); + test_apply(Datatype::DATETIME_FS); + test_apply(Datatype::DATETIME_AS); + test_apply(Datatype::TIME_HR); + test_apply(Datatype::TIME_MIN); + test_apply(Datatype::TIME_SEC); + test_apply(Datatype::TIME_MS); + test_apply(Datatype::TIME_US); + test_apply(Datatype::TIME_NS); + test_apply(Datatype::TIME_PS); + test_apply(Datatype::TIME_FS); + test_apply(Datatype::TIME_AS); + test_apply(Datatype::STRING_ASCII); + test_apply(Datatype::STRING_ASCII, true); + test_apply(Datatype::STRING_ASCII, false, true); + test_apply(Datatype::STRING_UTF8); + test_apply(Datatype::STRING_UTF8, true); + test_apply(Datatype::STRING_UTF8, false, true); } TEST_CASE( From ded935d35bfedf5f61c821904f2a7da21c9fe621 Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Fri, 23 Feb 2024 12:00:58 -0500 Subject: [PATCH 09/12] Remove some stale docs --- tiledb/sm/array/array_directory.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 68a3c837165..9f29563b981 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -323,7 +323,6 @@ class ArrayDirectory { * * @param array_schema_uri The URI path of the array schema. * @param encryption_key The encryption key to use. - * @param memory_tracker The memory tracker to use. * @return Status, the loaded array schema */ static shared_ptr load_array_schema_from_uri( @@ -348,7 +347,6 @@ class ArrayDirectory { * @param array_dir The ArrayDirectory object used to retrieve the * various URIs in the array directory. * @param encryption_key The encryption key to use. - * @param memory_tracker The memory tracker to use. * @return Status, a new ArraySchema */ shared_ptr load_array_schema_latest( @@ -361,7 +359,6 @@ class ArrayDirectory { * @param array_dir The ArrayDirectory object used to retrieve the * various URIs in the array directory. * @param encryption_key The encryption key to use. - * @param memory_tracker The memory tracker to use. * @return tuple of Status, latest array schema and all array schemas. * Status Ok on success, else error * ArraySchema The latest array schema. @@ -376,7 +373,6 @@ class ArrayDirectory { * Loads all schemas of an array from persistent storage into memory. * * @param encryption_key The encryption key to use. - * @param memory_tracker The memory tracker to use. * @return tuple of Status and optional unordered map. If Status is an error * the unordered_map will be nullopt * Status Ok on success, else error From 74f7c9b3dfa37302dd89dfbcaf7c58a10e443b2b Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Fri, 23 Feb 2024 12:46:24 -0500 Subject: [PATCH 10/12] Remove accessor --- tiledb/sm/array_schema/domain.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index ad6edca8c87..511a64af0df 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -103,10 +103,6 @@ class Domain { /* API */ /* ********************************* */ - inline shared_ptr memory_tracker() { - return memory_tracker_; - } - /** * Adds a dimension to the domain. * From 146b594e6aa8eccc712aab50ec62881a046fb4ce Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Fri, 1 Mar 2024 11:35:09 -0500 Subject: [PATCH 11/12] Changes from review --- test/src/unit-Reader.cc | 3 +-- test/src/unit-cppapi-schema-evolution.cc | 3 ++- tiledb/sm/array_schema/domain.cc | 7 +++-- tiledb/sm/array_schema/domain.h | 2 +- .../sm/array_schema/test/unit_tile_domain.cc | 27 +++++++------------ tiledb/sm/array_schema/tile_domain.h | 4 +-- tiledb/sm/filter/webp_filter.cc | 18 ++++++------- tiledb/sm/filter/webp_filter.h | 2 +- 8 files changed, 28 insertions(+), 38 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index fe0759bf3c5..7fca3e59c5a 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -188,8 +188,7 @@ TEST_CASE_METHOD( int32_t domain_vec[] = {1, 10, 1, 15}; NDRange domain = {Range(&domain_vec[0], size), Range(&domain_vec[2], size)}; std::vector tile_extents_vec = {2, 5}; - tdb::pmr::vector tile_extents( - 2, tracker_->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index adb6ac68cc9..340145c0519 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -818,7 +818,8 @@ TEST_CASE( int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE(), tiledb::test::get_test_memory_tracker()); + auto dom = make_shared( + HERE(), tiledb::test::get_test_memory_tracker()); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 3b8b4ee768b..fa9d4859181 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -586,14 +586,13 @@ const ByteVecValue& Domain::tile_extent(unsigned i) const { return dimension_ptrs_[i]->tile_extent(); } -tdb::pmr::vector Domain::tile_extents() const { - tdb::pmr::vector ret( - dim_num_, memory_tracker_->get_resource(MemoryType::DOMAINS)); +std::vector Domain::tile_extents() const { + std::vector ret(dim_num_); for (unsigned d = 0; d < dim_num_; ++d) { ret[d] = tile_extent(d); } - return {ret, memory_tracker_->get_resource(MemoryType::DOMAINS)}; + return ret; } uint64_t Domain::tile_num(const NDRange& ndrange) const { diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 511a64af0df..5544169cb44 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -440,7 +440,7 @@ class Domain { const ByteVecValue& tile_extent(unsigned i) const; /** Returns the tile extents. */ - tdb::pmr::vector tile_extents() const; + std::vector tile_extents() const; /** * Returns the number of tiles intersecting the input ND range. diff --git a/tiledb/sm/array_schema/test/unit_tile_domain.cc b/tiledb/sm/array_schema/test/unit_tile_domain.cc index 18adfd2ca7d..95676b33898 100644 --- a/tiledb/sm/array_schema/test/unit_tile_domain.cc +++ b/tiledb/sm/array_schema/test/unit_tile_domain.cc @@ -39,8 +39,7 @@ using namespace tiledb::sm; TEST_CASE("TileDomain: Test 1D", "[TileDomain][1d]") { int32_t tile_extent_v = 10; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 1, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(1); tile_extents[0].assign_as(tile_extent_v); Layout layout = Layout::ROW_MAJOR; @@ -76,8 +75,7 @@ TEST_CASE( std::vector domain_slice = {1, 10, 1, 10}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; @@ -121,8 +119,7 @@ TEST_CASE( std::vector domain_slice = {4, 10, 2, 8}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::ROW_MAJOR; @@ -161,8 +158,7 @@ TEST_CASE( std::vector domain_slice = {1, 10, 1, 10}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -201,8 +197,7 @@ TEST_CASE( std::vector domain_slice = {4, 10, 2, 8}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -240,8 +235,7 @@ TEST_CASE( std::vector domain_slice = {4, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -274,8 +268,7 @@ TEST_CASE( std::vector domain_slice = {2, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -313,8 +306,7 @@ TEST_CASE( std::vector domain_slice = {2, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; @@ -341,8 +333,7 @@ TEST_CASE("TileDomain: Test 2D, covers", "[TileDomain][2d][covers]") { std::vector domain_slice_2 = {3, 6, 1, 7}; std::vector tile_extents_vec = {2, 5}; auto memory_tracker = tiledb::test::create_test_memory_tracker(); - tdb::pmr::vector tile_extents( - 2, memory_tracker->get_resource(MemoryType::DOMAINS)); + std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); Layout layout = Layout::COL_MAJOR; diff --git a/tiledb/sm/array_schema/tile_domain.h b/tiledb/sm/array_schema/tile_domain.h index fcf6a778702..1cc96404a23 100644 --- a/tiledb/sm/array_schema/tile_domain.h +++ b/tiledb/sm/array_schema/tile_domain.h @@ -82,7 +82,7 @@ class TileDomain { unsigned id, const NDRange& domain, const NDRange& domain_slice, - const tdb::pmr::vector& tile_extents, + const std::vector& tile_extents, Layout layout) : id_(id) , dim_num_((unsigned)domain.size()) @@ -305,7 +305,7 @@ class TileDomain { void compute_tile_domain( const NDRange& domain, const NDRange& domain_slice, - const tdb::pmr::vector& tile_extents) { + const std::vector& tile_extents) { tile_domain_.resize(2 * dim_num_); for (unsigned d = 0; d < dim_num_; ++d) { auto ds = (const T*)domain_slice[d].data(); diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index 31bc59ac0ed..961209f7ecb 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -367,7 +367,7 @@ void WebpFilter::serialize_impl(Serializer& serializer) const { } template -void WebpFilter::set_extents(const tdb::pmr::vector& extents) { +void WebpFilter::set_extents(const std::vector& extents) { extents_ = {extents[0].rvalue_as(), extents[1].rvalue_as()}; uint8_t pixel_depth = format_ < WebpInputFormat::WEBP_RGBA ? 3 : 4; // X should be divisible by pixel_depth or RGB values will skew. @@ -385,21 +385,21 @@ void WebpFilter::set_extents(const tdb::pmr::vector& extents) { } template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); template void WebpFilter::set_extents( - const tdb::pmr::vector& extents); + const std::vector& extents); } // namespace tiledb::sm diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index f1fa51828d8..cd6648a06cb 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -267,7 +267,7 @@ class WebpFilter : public Filter { * @param extents Extents retrieved from array Domain object. */ template - void set_extents(const tdb::pmr::vector& extents); + void set_extents(const std::vector& extents); /** * Get tile extents currently in use by this WebpFilter From 5815a5971a165e5d55719169e4381cf61e0a14bd Mon Sep 17 00:00:00 2001 From: Shaun Reed Date: Mon, 4 Mar 2024 17:54:02 -0500 Subject: [PATCH 12/12] Domains memory_type_to_str --- tiledb/common/memory_tracker.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 6e81503d562..5a7d28c4928 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -76,6 +76,8 @@ std::string memory_type_to_str(MemoryType type) { return "DimensionLabels"; case MemoryType::DIMENSIONS: return "Dimensions"; + case MemoryType::DOMAINS: + return "Domains"; case MemoryType::TILE_SUMS: return "TileSums"; case MemoryType::TILE_WRITER_DATA: