From 263f57e612582534929c448d30201b4a47a25507 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Mon, 23 Oct 2023 10:43:46 -0500 Subject: [PATCH] Add ArraySchemaEvolution::extend_enumeration This commit adds the new ArraySchemaEvolution::extend_enumeration API for extending enumerations during array schema evolution. Enumerations passed to this API should come from the result of a call to Enumeration::extend. --- test/src/unit-cppapi-enumerations.cc | 29 ++ test/src/unit-enumerations.cc | 260 +++++++++++++++++- tiledb/sm/array_schema/array_schema.cc | 44 +++ tiledb/sm/array_schema/array_schema.h | 10 + .../sm/array_schema/array_schema_evolution.cc | 61 +++- .../sm/array_schema/array_schema_evolution.h | 26 ++ tiledb/sm/c_api/tiledb.cc | 25 ++ tiledb/sm/c_api/tiledb_experimental.h | 35 +++ tiledb/sm/cpp_api/array_schema_evolution.h | 25 +- .../serialization/array_schema_evolution.cc | 53 +++- tiledb/sm/serialization/tiledb-rest.capnp | 7 +- tiledb/sm/serialization/tiledb-rest.capnp.c++ | 67 +++-- tiledb/sm/serialization/tiledb-rest.capnp.h | 104 ++++++- tiledb/sm/storage_manager/storage_manager.cc | 21 ++ 14 files changed, 714 insertions(+), 53 deletions(-) diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc index 959411361adf..9414920a2805 100644 --- a/test/src/unit-cppapi-enumerations.cc +++ b/test/src/unit-cppapi-enumerations.cc @@ -345,6 +345,35 @@ TEST_CASE_METHOD( REQUIRE(rc != TILEDB_OK); } +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: ArraySchemaEvolution - Extend Enumeration", + "[enumeration][array-schema-evolution][extend-enumeration]") { + ArraySchemaEvolution ase(ctx_); + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + CHECK_NOTHROW(ase.extend_enumeration(enmr)); +} + + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: ArraySchemaEvolution - Extend Enumeration - Check nullptr", + "[enumeration][array-schema-evolution][drop-enumeration]") { + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + + auto rc = tiledb_array_schema_evolution_extend_enumeration( + ctx_.ptr().get(), nullptr, enmr.ptr().get()); + REQUIRE(rc != TILEDB_OK); + + ArraySchemaEvolution ase(ctx_); + rc = tiledb_array_schema_evolution_extend_enumeration( + ctx_.ptr().get(), ase.ptr().get(), nullptr); + REQUIRE(rc != TILEDB_OK); +} + TEST_CASE_METHOD( CPPEnumerationFx, "CPP: ArraySchemaEvolution - Drop Enumeration", diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index b3aecbeb7593..3ff4d751e183 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -75,6 +75,12 @@ struct EnumerationFx { Datatype type = static_cast(255), std::string enmr_name = default_enmr_name); + shared_ptr create_empty_enumeration( + Datatype type, + uint32_t cell_val_num, + bool ordered = false, + std::string enmr_name = default_enmr_name); + template shared_ptr extend_enumeration( shared_ptr enmr, const std::vector& values); @@ -712,6 +718,19 @@ TEST_CASE_METHOD( REQUIRE(enmr2->is_extension_of(enmr1)); } +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Empty Fixed Size", + "[enumeration][extension][fixed]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr1 = create_empty_enumeration(Datatype::INT32, 1); + auto enmr2 = extend_enumeration(enmr1, values); + check_enumeration( + enmr2, default_enmr_name, values, Datatype::INT32, 1, false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + TEST_CASE_METHOD( EnumerationFx, "Enumeration Extension Fixed Size Multi-Cell Value", @@ -755,6 +774,25 @@ TEST_CASE_METHOD( REQUIRE(enmr2->is_extension_of(enmr1)); } +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Empty Var Size", + "[enumeration][extension][var-sized]") { + std::vector values = {"fred", "wilma", "barney", "betty"}; + auto enmr1 = + create_empty_enumeration(Datatype::STRING_ASCII, constants::var_num); + auto enmr2 = extend_enumeration(enmr1, values); + check_enumeration( + enmr2, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + TEST_CASE_METHOD( EnumerationFx, "Enumeration Extension Invalid Data", @@ -1411,6 +1449,90 @@ TEST_CASE_METHOD( REQUIRE_THROWS(schema->drop_enumeration("not_an_enumeration")); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration is nullptr", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + auto matcher = Catch::Matchers::ContainsSubstring( + "Error adding enumeration. Enumeration must not be nullptr."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(nullptr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration Does Not Exist", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "foo"); + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration with name 'foo' does not exist in this ArraySchema."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration Not Loaded", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "test_enmr"); + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration with name 'test_enmr' is not loaded."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration Not An Extension", + "[enumeration][array-schema][error]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + + auto schema = make_shared(HERE(), array->array_schema_latest()); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "test_enmr"); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Provided enumeration is not an extension of the current state of " + "'test_enmr'"); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Duplicate Enumeration Path Name", + "[enumeration][array-schema][error]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + + auto schema = make_shared(HERE(), array->array_schema_latest()); + auto enmr1 = schema->get_enumeration("test_enmr"); + + std::vector extra_values = {"manatee", "narwhal", "oppossum"}; + auto enmr2 = extend_enumeration(enmr1, extra_values); + + // We have to force this condition by hand + auto enmr3 = tiledb::sm::Enumeration::create( + enmr2->name(), + // Notice we're reusing the existing path name from enmr1 + enmr1->path_name(), + enmr2->type(), + enmr2->cell_val_num(), + enmr2->ordered(), + enmr2->data().data(), + enmr2->data().size(), + enmr2->offsets().data(), + enmr2->offsets().size()); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration path name for 'test_enmr' already exists in this schema."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr3), matcher); +} + /* ********************************* */ /* Testing ArraySchemaEvolution */ /* ********************************* */ @@ -1509,10 +1631,37 @@ TEST_CASE_METHOD( CHECK_NOTHROW(ase->evolve_schema(orig_schema)); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Extend", + "[enumeration][array-schema-evolution][enmr-to-extend]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + auto orig_schema = array->array_schema_latest_ptr(); + + std::vector values_to_add = {"firefly", "gerbil", "hamster"}; + auto old_enmr = orig_schema->get_enumeration("test_enmr"); + REQUIRE(old_enmr != nullptr); + auto new_enmr = extend_enumeration(old_enmr, values_to_add); + + auto ase = make_shared(HERE()); + ase->extend_enumeration(new_enmr); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration", - "[enumeration][array-schema-evolution][enmr-to-add]") { + "[enumeration][array-schema-evolution][enmr-to-drop]") { + auto ase = make_shared(HERE()); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Add Then Drop Enumeration", + "[enumeration][array-schema-evolution][enmr-to-drop]") { create_array(); auto orig_schema = get_array_schema_latest(); auto ase1 = make_shared(HERE()); @@ -1529,14 +1678,6 @@ TEST_CASE_METHOD( CHECK_NOTHROW(ase2->evolve_schema(new_schema)); } -TEST_CASE_METHOD( - EnumerationFx, - "ArraySchemaEvolution - Drop Enumeration", - "[enumeration][array-schema-evolution][enmr-to-drop]") { - auto ase = make_shared(HERE()); - CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); -} - TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration Repeated", @@ -1664,6 +1805,30 @@ TEST_CASE_METHOD( REQUIRE_THROWS(ase->evolve_schema(orig_schema)); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Extend Enumeration nullptr", + "[enumeration][array-schema-evolution][extend][error]") { + auto ase = make_shared(HERE()); + auto matcher = Catch::Matchers::ContainsSubstring( + "Cannot extend enumeration; Input enumeration is null"); + REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Extend Enumeration Already Extended", + "[enumeration][array-schema-evolution][extend][error]") { + auto ase = make_shared(HERE()); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Cannot extend enumeration; Input enumeration name has already " + "been extended in this evolution."); + REQUIRE_NOTHROW(ase->extend_enumeration(enmr)); + REQUIRE_THROWS_WITH(ase->extend_enumeration(enmr), matcher); +} + /* ********************************* */ /* Testing QueryCondition */ /* ********************************* */ @@ -1708,6 +1873,45 @@ TEST_CASE_METHOD( REQUIRE(data2.rvalue_as() == 2); } +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Enumeration Value After Extension", + "[enumeration][query-condition][extend][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + + auto schema = array->array_schema_latest_ptr(); + + // Create two copies of the same query condition for assertions + auto qc1 = create_qc("attr1", std::string("gerbil"), QueryConditionOp::EQ); + auto qc2 = qc1; + + // Check that we fail the rewrite before extension. + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration value not found for field 'attr1'"); + REQUIRE_THROWS_WITH( + qc1.rewrite_enumeration_conditions(*(schema.get())), matcher); + + // Extend enumeration via schema evolution. + std::vector values_to_add = {"firefly", "gerbil", "hamster"}; + auto old_enmr = schema->get_enumeration("test_enmr"); + auto new_enmr = extend_enumeration(old_enmr, values_to_add); + + auto ase = make_shared(HERE()); + ase->extend_enumeration(new_enmr); + auto st = ctx_.storage_manager()->array_evolve_schema( + array->array_uri(), ase.get(), array->get_encryption_key()); + throw_if_not_ok(st); + + // Check that we can no rewrite the query condition. + array = get_array(QueryType::READ); + array->load_all_enumerations(); + schema = array->array_schema_latest_ptr(); + + REQUIRE_NOTHROW(qc2.rewrite_enumeration_conditions(*(schema.get()))); +} + TEST_CASE_METHOD( EnumerationFx, "QueryCondition - Skip enumeration rewrite", @@ -1926,6 +2130,7 @@ TEST_CASE_METHOD( auto enmrs_to_add1 = ase1.enumeration_names_to_add(); auto enmrs_to_add2 = ase2->enumeration_names_to_add(); + REQUIRE(enmrs_to_add1.size() == 2); REQUIRE(vec_cmp(enmrs_to_add1, enmrs_to_add2)); for (auto& name : enmrs_to_add1) { @@ -1935,6 +2140,37 @@ TEST_CASE_METHOD( } } +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - ArraySchemaEvolution Serialization With Extensions", + "[enumeration][capnp][basic][array-schema-evolution]") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + std::vector values1 = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values1, false, Datatype::INT32, "enmr1"); + + std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; + auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); + + ArraySchemaEvolution ase1; + ase1.extend_enumeration(enmr1); + ase1.extend_enumeration(enmr2); + + auto ase2 = ser_des_array_schema_evolution(&ase1, client_side, ser_type); + + auto enmrs_to_extend1 = ase1.enumeration_names_to_extend(); + auto enmrs_to_extend2 = ase2->enumeration_names_to_extend(); + REQUIRE(enmrs_to_extend2.size() == 2); + REQUIRE(vec_cmp(enmrs_to_extend1, enmrs_to_extend2)); + + for (auto& name : enmrs_to_extend1) { + REQUIRE(ase1.enumeration_to_extend(name) != nullptr); + REQUIRE(ase2->enumeration_to_extend(name) != nullptr); + REQUIRE(ase1.enumeration_to_extend(name) != ase2->enumeration_to_extend(name)); + } +} + TEST_CASE_METHOD( EnumerationFx, "Cap'N Proto - Basic Backwards Compatible Query Serialization", @@ -2188,6 +2424,12 @@ shared_ptr EnumerationFx::create_enumeration( } } +shared_ptr EnumerationFx::create_empty_enumeration( + Datatype type, uint32_t cell_val_num, bool ordered, std::string name) { + return Enumeration::create( + name, type, cell_val_num, ordered, nullptr, 0, nullptr, 0); +} + template shared_ptr EnumerationFx::extend_enumeration( shared_ptr enmr, const std::vector& values) { diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 68374d690dee..5198e5678ab4 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1070,6 +1070,50 @@ void ArraySchema::add_enumeration(shared_ptr enmr) { enumeration_path_map_[enmr->name()] = enmr->path_name(); } +void ArraySchema::extend_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaException( + "Error adding enumeration. Enumeration must not be nullptr."); + } + + auto it = enumeration_map_.find(enmr->name()); + if (it == enumeration_map_.end()) { + throw ArraySchemaException( + "Error extending enumeration. Enumeration with name '" + enmr->name() + + "' does not exist in this ArraySchema."); + } + + if (it->second == nullptr) { + throw ArraySchemaException( + "Error extending enumeration. Enumeration with name '" + enmr->name() + + "' is not loaded."); + } + + if (!enmr->is_extension_of(it->second)) { + throw ArraySchemaException( + "Error extending enumeration. Provided enumeration is not an extension " + "of the current state of '" + enmr->name() + "'"); + } + + if (enumeration_path_map_.find(enmr->name()) == enumeration_path_map_.end()) { + throw ArraySchemaException( + "Error extending enumeration. Invalid enumeration path map state for " + "enumeration '" + + enmr->name() + "'"); + } + + for (auto& enmr_path : enumeration_path_map_) { + if (enmr->path_name() == enmr_path.second) { + throw ArraySchemaException( + "Error extending enumeration. Enumeration path name for '" + + enmr->name() + "' already exists in this schema."); + } + } + + enumeration_map_[enmr->name()] = enmr; + enumeration_path_map_[enmr->name()] = enmr->path_name(); +} + void ArraySchema::store_enumeration(shared_ptr enmr) { if (enmr == nullptr) { throw ArraySchemaException( diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index 934389284b52..8a352df7550a 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -387,6 +387,16 @@ class ArraySchema { */ void add_enumeration(shared_ptr enmr); + /** + * Extend an Enumeration on this ArraySchema. + * + * N.B., this method is intended to be called via ArraySchemaEvolution. + * Calling it from anywhere else is likely incorrect. + * + * @param enmr The extended enumeration. + */ + void extend_enumeration(shared_ptr enmr); + /** * Check if an enumeration exists with the given name. * diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 6c6fb66809a1..7ef355d05a5f 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -79,11 +79,14 @@ ArraySchemaEvolution::ArraySchemaEvolution( std::unordered_map> attrs_to_add, std::unordered_set attrs_to_drop, std::unordered_map> enmrs_to_add, + std::unordered_map> + enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range) : attributes_to_add_map_(attrs_to_add) , attributes_to_drop_(attrs_to_drop) , enumerations_to_add_map_(enmrs_to_add) + , enumerations_to_extend_map_(enmrs_to_extend) , enumerations_to_drop_(enmrs_to_drop) , timestamp_range_(timestamp_range) { } @@ -112,6 +115,10 @@ shared_ptr ArraySchemaEvolution::evolve_schema( schema->add_enumeration(enmr.second); } + for (auto& enmr : enumerations_to_extend_map_) { + schema->extend_enumeration(enmr.second); + } + // Add attributes. for (auto& attr : attributes_to_add_map_) { throw_if_not_ok(schema->add_attribute(attr.second, false)); @@ -250,15 +257,63 @@ shared_ptr ArraySchemaEvolution::enumeration_to_add( return it->second; } +void ArraySchemaEvolution::extend_enumeration( + shared_ptr enmr) { + std::lock_guard lock(mtx_); + + if (enmr == nullptr) { + throw ArraySchemaEvolutionException( + "Cannot extend enumeration; Input enumeration is null"); + } + + auto it = enumerations_to_extend_map_.find(enmr->name()); + if (it != enumerations_to_extend_map_.end()) { + throw ArraySchemaEvolutionException( + "Cannot extend enumeration; Input enumeration name has already " + "been extended in this evolution."); + } + + enumerations_to_extend_map_[enmr->name()] = enmr; +} + +std::vector ArraySchemaEvolution::enumeration_names_to_extend() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_extend_map_.size()); + for (auto elem : enumerations_to_extend_map_) { + names.push_back(elem.first); + } + + return names; +} + +shared_ptr ArraySchemaEvolution::enumeration_to_extend( + const std::string& name) const { + std::lock_guard lock(mtx_); + auto it = enumerations_to_extend_map_.find(name); + + if (it == enumerations_to_extend_map_.end()) { + return nullptr; + } + + return it->second; +} + void ArraySchemaEvolution::drop_enumeration( const std::string& enumeration_name) { std::lock_guard lock(mtx_); enumerations_to_drop_.insert(enumeration_name); - auto it = enumerations_to_add_map_.find(enumeration_name); - if (it != enumerations_to_add_map_.end()) { + auto add_it = enumerations_to_add_map_.find(enumeration_name); + if (add_it != enumerations_to_add_map_.end()) { // Reset the pointer and erase it - enumerations_to_add_map_.erase(it); + enumerations_to_add_map_.erase(add_it); + } + + auto extend_it = enumerations_to_extend_map_.find(enumeration_name); + if (extend_it != enumerations_to_extend_map_.end()) { + enumerations_to_extend_map_.erase(extend_it); } } diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 14d98b98d0d7..be2f2bead0b0 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -83,6 +83,8 @@ class ArraySchemaEvolution { std::unordered_set attrs_to_drop, std::unordered_map> enmrs_to_add, + std::unordered_map> + enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range); @@ -142,6 +144,26 @@ class ArraySchemaEvolution { shared_ptr enumeration_to_add( const std::string& name) const; + /** + * Extend an enumeration. + * + * @param enmr The enumeration with its extension. + */ + void extend_enumeration(shared_ptr enmr); + + /** Returns the names of the enumerations to extend. */ + std::vector enumeration_names_to_extend() const; + + /** + * Returns a constant pointer to the selected enumeration or nullptr if it + * does not exist. + * + * @param name The name of the enumeration to extend. + * @return shared_ptr The enumeration to extend. + */ + shared_ptr enumeration_to_extend( + const std::string& name) const; + /** * Drops an enumeration * @@ -175,6 +197,10 @@ class ArraySchemaEvolution { std::unordered_map> enumerations_to_add_map_; + /** Enumerations to extend. */ + std::unordered_map> + enumerations_to_extend_map_; + /** The names of array enumerations to be dropped. */ std::unordered_set enumerations_to_drop_; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 111474dd9d41..10d55edc3c3c 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -917,6 +917,22 @@ capi_return_t tiledb_array_schema_evolution_add_enumeration( return TILEDB_OK; } +capi_return_t tiledb_array_schema_evolution_extend_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + auto enmr = enumeration->copy(); + array_schema_evolution->array_schema_evolution_->extend_enumeration(enmr); + + return TILEDB_OK; +} + capi_return_t tiledb_array_schema_evolution_drop_enumeration( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, @@ -5650,6 +5666,15 @@ int32_t tiledb_array_schema_evolution_add_enumeration( ctx, array_schema_evolution, enmr); } +capi_return_t tiledb_array_schema_evolution_extend_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enmr) noexcept { + return api_entry< + tiledb::api::tiledb_array_schema_evolution_extend_enumeration>( + ctx, array_schema_evolution, enmr); +} + capi_return_t tiledb_array_schema_evolution_drop_enumeration( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 69128ce599d8..a814a34043e3 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -212,6 +212,41 @@ TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_add_enumeration( tiledb_array_schema_evolution_t* array_schema_evolution, tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; +/** + * Extends an enumeration during array schema evolution. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* original_enmr = get_existing_enumeration(); + * const void* data = get_new_data(); + * uint64_t data_size = get_new_data_size(); + * tiledb_enumeration_t* new_enmr; + * tiledb_enumeration_extend( + * ctx, + * original_enmr, + * data, + * data_size, + * nullptr, + * 0, + * &new_enmr); + * tiledb_array_schema_evolution_extend_enumeration( + * ctx, + * array_schema_evolution, + * enmr); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param enumeration The enumeration to be extended. This should be the result + * of a call to tiledb_enumeration_extend. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_extend_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + /** * Drops an enumeration from an array schema evolution. * diff --git a/tiledb/sm/cpp_api/array_schema_evolution.h b/tiledb/sm/cpp_api/array_schema_evolution.h index a8397974e0a4..31cfa5523194 100644 --- a/tiledb/sm/cpp_api/array_schema_evolution.h +++ b/tiledb/sm/cpp_api/array_schema_evolution.h @@ -163,7 +163,7 @@ class ArraySchemaEvolution { * values)); * @endcode * - * @param attr The Attribute to add + * @param enmr The Enumeration to add. * @return Reference to this `ArraySchemaEvolution` instance. */ ArraySchemaEvolution& add_enumeration(const Enumeration& enmr) { @@ -173,6 +173,29 @@ class ArraySchemaEvolution { return *this; } + /** + * Extends an Enumeration during array schema evolution. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Enumeration old_enmr = array->get_enumeration("some_enumeration"); + * std::vector new_values = {"cyan", "magenta", "mauve"}; + * tiledb::Enumeration new_enmr = old_enmr->extend(new_values); + * tiledb::ArraySchemaEvolution schema_evolution(ctx); + * schema_evolution.extend_enumeration(new_enmr); + * @endcode + * + * @param enmr The Enumeration to extend. + * @return Reference to this `ArraySchemaEvolution` instance. + */ + ArraySchemaEvolution& extend_enumeration(const Enumeration& enmr) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_extend_enumeration( + ctx.ptr().get(), evolution_.get(), enmr.ptr().get())); + return *this; + } + /** * Drops an enumeration. * diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index ae993b384a4c..c60b9882a674 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -105,13 +105,13 @@ Status array_schema_evolution_to_capnp( attribute_to_capnp(attr_to_add, &attribute_builder); } + // Enumerations to add auto enmr_names_to_add = array_schema_evolution->enumeration_names_to_add(); - auto num_enmrs = enmr_names_to_add.size(); - - if (num_enmrs > 0) { + if (enmr_names_to_add.size() > 0) { auto enmrs_to_add_builder = - array_schema_evolution_builder->initEnumerationsToAdd(num_enmrs); - for (size_t i = 0; i < num_enmrs; i++) { + array_schema_evolution_builder->initEnumerationsToAdd( + enmr_names_to_add.size()); + for (size_t i = 0; i < enmr_names_to_add.size(); i++) { auto enmr = array_schema_evolution->enumeration_to_add(enmr_names_to_add[i]); auto builder = enmrs_to_add_builder[i]; @@ -119,16 +119,31 @@ Status array_schema_evolution_to_capnp( } } - // Enumerations to drop - std::vector enmr_names_to_drop = - array_schema_evolution->enumeration_names_to_drop(); - - auto enumerations_to_drop_builder = - array_schema_evolution_builder->initEnumerationsToDrop( - enmr_names_to_drop.size()); + // Enumerations to extend + auto enmr_names_to_extend = + array_schema_evolution->enumeration_names_to_extend(); + if (enmr_names_to_extend.size() > 0) { + auto enmrs_to_extend_builder = + array_schema_evolution_builder->initEnumerationsToExtend( + enmr_names_to_extend.size()); + for (size_t i = 0; i < enmr_names_to_extend.size(); i++) { + auto enmr = array_schema_evolution->enumeration_to_extend( + enmr_names_to_extend[i]); + auto builder = enmrs_to_extend_builder[i]; + enumeration_to_capnp(enmr, builder); + } + } - for (size_t i = 0; i < enmr_names_to_drop.size(); i++) { - enumerations_to_drop_builder.set(i, enmr_names_to_drop[i]); + // Enumerations to drop + auto enmr_names_to_drop = array_schema_evolution->enumeration_names_to_drop(); + if (enmr_names_to_drop.size() > 0) { + auto enumerations_to_drop_builder = + array_schema_evolution_builder->initEnumerationsToDrop( + enmr_names_to_drop.size()); + + for (size_t i = 0; i < enmr_names_to_drop.size(); i++) { + enumerations_to_drop_builder.set(i, enmr_names_to_drop[i]); + } } auto timestamp_builder = @@ -166,6 +181,15 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_add[enmr->name()] = enmr; } + // Create enumerations to extend + std::unordered_map> + enmrs_to_extend; + auto enmrs_to_extend_reader = evolution_reader.getEnumerationsToExtend(); + for (auto enmr_reader : enmrs_to_extend_reader) { + auto enmr = enumeration_from_capnp(enmr_reader); + enmrs_to_extend[enmr->name()] = enmr; + } + // Create enumerations to drop std::unordered_set enmrs_to_drop; auto enmrs_to_drop_reader = evolution_reader.getEnumerationsToDrop(); @@ -187,6 +211,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( attrs_to_add, attrs_to_drop, enmrs_to_add, + enmrs_to_extend, enmrs_to_drop, ts_range)); } diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 77fb19c6886b..6535efeb0c62 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -179,6 +179,9 @@ struct ArraySchemaEvolution { enumerationsToDrop @4 :List(Text); # Enumeration names to be dropped + + enumerationsToExtend @5 :List(Enumeration); + # Enumerations to be extended. } struct Attribute { @@ -1259,7 +1262,7 @@ struct LoadArraySchemaResponse { } struct QueryChannel { - # structure representing a query channel, that is a stream of data within + # structure representing a query channel, that is a stream of data within # a TileDB query. Such channels can be generated for the purpose of avoiding # processing result items multiple times in more complex queries such as e.g. # grouping queries. @@ -1281,6 +1284,6 @@ struct Aggregate { # name of the input field the aggregate is applied on name @2 :Text; - # the name of aggregate, e.g. COUNT, MEAN, SUM used for constructing the + # the name of aggregate, e.g. COUNT, MEAN, SUM used for constructing the # correct object during deserialization } diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index 5e6afe2644a1..d2d8f5821fe3 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -1157,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<143> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 31, 1, 0, 0, + 33, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1176,42 +1176,49 @@ static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 20, 0, 0, 0, 3, 0, 4, 0, + 24, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 0, 0, 0, 138, 0, 0, 0, + 153, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 0, 0, 0, 3, 0, 1, 0, - 156, 0, 0, 0, 2, 0, 1, 0, + 156, 0, 0, 0, 3, 0, 1, 0, + 184, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 130, 0, 0, 0, + 181, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 122, 0, 0, 0, + 205, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 146, 0, 0, 0, + 229, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 0, 0, 0, 3, 0, 1, 0, - 232, 0, 0, 0, 2, 0, 1, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 229, 0, 0, 0, 154, 0, 0, 0, + 1, 1, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 4, 1, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 32, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 60, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1277,6 +1284,20 @@ static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 69, 120, + 116, 101, 110, 100, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1287,11 +1308,11 @@ static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 5, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, - 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 143, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 6, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index f4f8b9694c14..dcd1f3aeb43f 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -246,7 +246,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 5) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 6) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -3150,6 +3150,12 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader getEnumerationsToDrop() const; + inline bool hasEnumerationsToExtend() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToExtend() const; + private: ::capnp::_::StructReader _reader; template @@ -3276,6 +3282,28 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> disownEnumerationsToDrop(); + inline bool hasEnumerationsToExtend(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToExtend(); + inline void setEnumerationsToExtend( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToExtend(unsigned int size); + inline void adoptEnumerationsToExtend( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToExtend(); + private: ::capnp::_::StructBuilder _builder; template @@ -17884,6 +17912,80 @@ ArraySchemaEvolution::Builder::disownEnumerationsToDrop() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToExtend() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToExtend() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToExtend() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<5>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToExtend() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<5>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToExtend( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToExtend(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToExtend( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToExtend() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<5>() * + ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index e7377c6658b6..6b5cfb32a34b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -755,6 +755,27 @@ Status StorageManager::array_evolve_schema( auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); + // Load required enumerations before evolution. + auto enmr_names = schema_evolution->enumeration_names_to_extend(); + if (enmr_names.size() > 0) { + std::unordered_set enmr_path_set; + for (auto name : enmr_names) { + enmr_path_set.insert(array_schema->get_enumeration_path_name(name)); + } + std::vector enmr_paths; + for (auto path : enmr_path_set) { + enmr_paths.emplace_back(path); + } + + MemoryTracker tracker; + auto loaded_enmrs = array_dir.load_enumerations_from_paths( + enmr_paths, encryption_key, tracker); + + for (auto enmr : loaded_enmrs) { + array_schema->store_enumeration(enmr); + } + } + // Evolve schema auto array_schema_evolved = schema_evolution->evolve_schema(array_schema);