Enumeration feature first draft

TileDB-Inc · Feb 16, 2024 · e29d57d · e29d57d
1 parent ee51025
commit e29d57d
Show file tree

Hide file tree

Showing 14 changed files with 148 additions and 33 deletions.
diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc
@@ -158,14 +158,23 @@ QueryCondition create_qc(
 TEST_CASE_METHOD(
     EnumerationFx, "Create Empty Enumeration", "[enumeration][empty]") {
   Enumeration::create(
-      default_enmr_name, Datatype::INT32, 1, false, nullptr, 0, nullptr, 0);
+      tiledb::test::create_test_memory_tracker(),
+      default_enmr_name,
+      Datatype::INT32,
+      1,
+      false,
+      nullptr,
+      0,
+      nullptr,
+      0);
 }
 
 TEST_CASE_METHOD(
     EnumerationFx,
     "Create Empty Var Sized Enumeration",
     "[enumeration][empty]") {
   Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -246,6 +255,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][invalid-offsets-args]") {
   uint64_t offsets = 0;
   auto enmr = Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -315,6 +325,7 @@ TEST_CASE_METHOD(
     "[enumeration][basic][fixed][multi-cell-val-num]") {
   std::vector<int> values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
   auto enmr = Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       2,
@@ -334,6 +345,7 @@ TEST_CASE_METHOD(
       "Invalid data buffer must not be nullptr for fixed sized data.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
+          tiledb::test::create_test_memory_tracker(),
           default_enmr_name,
           Datatype::INT32,
           1,
@@ -354,7 +366,15 @@ TEST_CASE_METHOD(
       "Invalid data size; must be non-zero for fixed size data.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
-          default_enmr_name, Datatype::INT32, 1, false, &val, 0, nullptr, 0),
+          tiledb::test::create_test_memory_tracker(),
+          default_enmr_name,
+          Datatype::INT32,
+          1,
+          false,
+          &val,
+          0,
+          nullptr,
+          0),
       matcher);
 }
 
@@ -367,6 +387,7 @@ TEST_CASE_METHOD(
       "Var sized enumeration values require a non-null offsets pointer.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
+          tiledb::test::create_test_memory_tracker(),
           default_enmr_name,
           Datatype::STRING_ASCII,
           constants::var_num,
@@ -388,6 +409,7 @@ TEST_CASE_METHOD(
       "Var sized enumeration values require a non-zero offsets size.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
+          tiledb::test::create_test_memory_tracker(),
           default_enmr_name,
           Datatype::STRING_ASCII,
           constants::var_num,
@@ -409,6 +431,7 @@ TEST_CASE_METHOD(
       "is non-zero.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
+          tiledb::test::create_test_memory_tracker(),
           default_enmr_name,
           Datatype::STRING_ASCII,
           constants::var_num,
@@ -430,6 +453,7 @@ TEST_CASE_METHOD(
       "require data.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
+          tiledb::test::create_test_memory_tracker(),
           default_enmr_name,
           Datatype::STRING_ASCII,
           constants::var_num,
@@ -452,6 +476,7 @@ TEST_CASE_METHOD(
       "offset.");
   REQUIRE_THROWS_WITH(
       Enumeration::create(
+          tiledb::test::create_test_memory_tracker(),
           default_enmr_name,
           Datatype::STRING_ASCII,
           constants::var_num,
@@ -469,6 +494,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][invalid-name]") {
   std::vector<int> values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       std::string(),
       Datatype::INT32,
       2,
@@ -485,6 +511,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][invalid-name]") {
   std::vector<int> values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       "",
       Datatype::INT32,
       2,
@@ -501,6 +528,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][invalid-name]") {
   std::vector<int> values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       "an/bad/path",
       Datatype::INT32,
@@ -518,6 +546,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][invalid-cell-val-num]") {
   std::vector<int> values = {1, 2, 3};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       0,
@@ -534,6 +563,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][data-nullptr]") {
   std::vector<int> values = {1, 2, 3};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       1,
@@ -550,6 +580,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][data-zero-size]") {
   std::vector<int> values = {1, 2, 3};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       1,
@@ -567,6 +598,7 @@ TEST_CASE_METHOD(
   auto data = "foobarbazbam";
   std::vector<uint64_t> offsets = {0, 3, 6, 9};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -584,6 +616,7 @@ TEST_CASE_METHOD(
   auto data = "foobarbazbam";
   std::vector<uint64_t> offsets = {0, 3, 6, 9};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -601,6 +634,7 @@ TEST_CASE_METHOD(
   std::vector<int> values = {0, 1, 2, 3, 4};
   std::vector<uint64_t> offsets = {0, 3, 6, 9};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       1,
@@ -617,6 +651,7 @@ TEST_CASE_METHOD(
     "[enumeration][error][offsets-not-required]") {
   std::vector<int> values = {0, 1, 2, 3, 4};
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       1,
@@ -636,6 +671,7 @@ TEST_CASE_METHOD(
   // Passing 3 for the offsets size is incorrect because the offsets size has
   // to be a multiple of `sizeof(uint64_t)`
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -654,6 +690,7 @@ TEST_CASE_METHOD(
   std::vector<uint64_t> offsets = {0, 3, 6, 100};
   // The last offset is larger than data_size
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -672,6 +709,7 @@ TEST_CASE_METHOD(
   // Passing 3 for the data size is invalid as its not a multiple of
   // sizeof(int)
   REQUIRE_THROWS(Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       1,
@@ -742,6 +780,7 @@ TEST_CASE_METHOD(
   std::vector<int> extend_values = {5, 6, 7, 8, 9, 10};
   std::vector<int> final_values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
   auto enmr1 = Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       default_enmr_name,
       Datatype::INT32,
       2,
@@ -946,7 +985,8 @@ TEST_CASE_METHOD(
   memset(data, 1, 4);
 
   Deserializer deserializer(tile.data(), tile.size());
-  REQUIRE_THROWS(Enumeration::deserialize(deserializer));
+  REQUIRE_THROWS(Enumeration::deserialize(
+      tiledb::test::create_test_memory_tracker(), deserializer));
 }
 
 TEST_CASE_METHOD(
@@ -1391,6 +1431,7 @@ TEST_CASE_METHOD(
   std::vector<uint8_t> data(1024 * 1024 * 10 + 1);
   std::vector<uint64_t> offsets = {0};
   auto enmr = Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       "enmr_name",
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -1420,6 +1461,7 @@ TEST_CASE_METHOD(
   // Create more than 50MiB of enumeration data
   for (size_t i = 0; i < 10; i++) {
     auto enmr = Enumeration::create(
+        tiledb::test::create_test_memory_tracker(),
         "enmr_name_" + std::to_string(i),
         Datatype::STRING_ASCII,
         constants::var_num,
@@ -1548,6 +1590,7 @@ TEST_CASE_METHOD(
 
   // We have to force this condition by hand
   auto enmr3 = tiledb::sm::Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       enmr2->name(),
       // Notice we're reusing the existing path name from enmr1
       enmr1->path_name(),
@@ -2146,8 +2189,17 @@ TEST_CASE_METHOD(
   auto schema1 = create_schema();
 
   auto enmr1 = Enumeration::create(
-      "empty_fixed", Datatype::INT32, 1, false, nullptr, 0, nullptr, 0);
+      tiledb::test::create_test_memory_tracker(),
+      "empty_fixed",
+      Datatype::INT32,
+      1,
+      false,
+      nullptr,
+      0,
+      nullptr,
+      0);
   auto enmr2 = Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       "empty_var",
       Datatype::STRING_ASCII,
       constants::var_num,
@@ -2442,6 +2494,7 @@ shared_ptr<const Enumeration> EnumerationFx::create_enumeration(
       raw_values[i] = values[i] ? 1 : 0;
     }
     return Enumeration::create(
+        tiledb::test::create_test_memory_tracker(),
         name,
         tp.type_,
         tp.cell_val_num_,
@@ -2452,6 +2505,7 @@ shared_ptr<const Enumeration> EnumerationFx::create_enumeration(
         0);
   } else if constexpr (std::is_pod_v<T>) {
     return Enumeration::create(
+        tiledb::test::create_test_memory_tracker(),
         name,
         tp.type_,
         tp.cell_val_num_,
@@ -2478,6 +2532,7 @@ shared_ptr<const Enumeration> EnumerationFx::create_enumeration(
     }
 
     return Enumeration::create(
+        tiledb::test::create_test_memory_tracker(),
         name,
         tp.type_,
         tp.cell_val_num_,
@@ -2492,7 +2547,15 @@ shared_ptr<const Enumeration> EnumerationFx::create_enumeration(
 shared_ptr<const Enumeration> EnumerationFx::create_empty_enumeration(
     Datatype type, uint32_t cell_val_num, bool ordered, std::string name) {
   return Enumeration::create(
-      name, type, cell_val_num, ordered, nullptr, 0, nullptr, 0);
+      tiledb::test::create_test_memory_tracker(),
+      name,
+      type,
+      cell_val_num,
+      ordered,
+      nullptr,
+      0,
+      nullptr,
+      0);
 }
 
 template <typename T>
@@ -2566,7 +2629,8 @@ void EnumerationFx::check_storage_deserialization(
   auto tile = serialize_to_tile(enmr);
 
   Deserializer deserializer(tile.data(), tile.size());
-  auto deserialized = Enumeration::deserialize(deserializer);
+  auto deserialized = Enumeration::deserialize(
+      tiledb::test::create_test_memory_tracker(), deserializer);
 
   REQUIRE(deserialized->name() == enmr->name());
   REQUIRE(deserialized->path_name().empty() == false);
@@ -2762,8 +2826,8 @@ shared_ptr<ArraySchemaEvolution> EnumerationFx::ser_des_array_schema_evolution(
       ase, stype, &buf, client_side));
 
   ArraySchemaEvolution* ret;
-  throw_if_not_ok(
-      serialization::array_schema_evolution_deserialize(&ret, stype, buf));
+  throw_if_not_ok(serialization::array_schema_evolution_deserialize(
+      &ret, stype, buf, tiledb::test::create_test_memory_tracker()));
 
   return shared_ptr<ArraySchemaEvolution>(ret);
 }

diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc
@@ -385,6 +385,7 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration(
   }
 
   return Enumeration::create(
+      tiledb::test::create_test_memory_tracker(),
       name,
       Datatype::STRING_ASCII,
       constants::var_num,

diff --git a/tiledb/api/c_api/enumeration/enumeration_api.cc b/tiledb/api/c_api/enumeration/enumeration_api.cc
@@ -34,10 +34,12 @@
 #include "../string/string_api_internal.h"
 #include "enumeration_api_experimental.h"
 #include "enumeration_api_internal.h"
+#include "tiledb/common/memory_tracker.h"
 
 namespace tiledb::api {
 
 capi_return_t tiledb_enumeration_alloc(
+    shared_ptr<tiledb::sm::MemoryTracker> memory_tracker,
     const char* name,
     tiledb_datatype_t type,
     uint32_t cell_val_num,
@@ -66,6 +68,7 @@ capi_return_t tiledb_enumeration_alloc(
 
   try {
     *enumeration = tiledb_enumeration_handle_t::make_handle(
+        memory_tracker,
         std::string(name),
         datatype,
         cell_val_num,
@@ -191,8 +194,11 @@ CAPI_INTERFACE(
     const void* offsets,
     uint64_t offsets_size,
     tiledb_enumeration_t** enumeration) {
+  auto memory_tracker = ctx->context().resources().create_memory_tracker();
+  memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ENUMERATION_CREATE);
   return api_entry_context<tiledb::api::tiledb_enumeration_alloc>(
       ctx,
+      memory_tracker,
       name,
       type,
       cell_val_num,

diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h
@@ -121,9 +121,11 @@ enum class MemoryType {
 enum class MemoryTrackerType {
   ANONYMOUS,
   ARRAY_CREATE,
+  ARRAY_EVOLVE,
   ARRAY_LOAD,
   ARRAY_READ,
   ARRAY_WRITE,
+  ENUMERATION_CREATE,
   FRAGMENT_INFO_LOAD,
   QUERY_READ,
   QUERY_WRITE,

diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc
@@ -1335,7 +1335,7 @@ shared_ptr<const Enumeration> ArrayDirectory::load_enumeration(
   }
 
   Deserializer deserializer(tile.data(), tile.size());
-  return Enumeration::deserialize(deserializer);
+  return Enumeration::deserialize(memory_tracker, deserializer);
 }
 
 }  // namespace tiledb::sm