From 84ec5e4ec8524a0e1865e27398b178ebfd69a362 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 31 Aug 2023 14:23:04 -0500 Subject: [PATCH] Add Enumeration support for Array v2 Serialization This adds the new Array::serialize_enumerations() method that indicates whether enumerations should be loaded and returned for v2 serialization. --- test/src/unit-enumerations.cc | 67 +++++++++++++++++++++++++++++++- tiledb/sm/array/array.cc | 10 +++++ tiledb/sm/array/array.h | 5 +++ tiledb/sm/serialization/array.cc | 4 ++ 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index e1a7cd8c640d..675826af3bd8 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -53,6 +53,7 @@ #ifdef TILEDB_SERIALIZATION #include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/array.h" #include "tiledb/sm/serialization/array_schema.h" #include "tiledb/sm/serialization/array_schema_evolution.h" #include "tiledb/sm/serialization/query.h" @@ -114,6 +115,8 @@ struct EnumerationFx { void ser_des_query( Query* q_in, Query* q_out, bool client_side, SerializationType stype); + void ser_des_array(Context& ctx, Array* in, Array* out, bool client_size, SerializationType stype); + template bool vec_cmp(std::vector v1, std::vector v2); @@ -1550,6 +1553,41 @@ TEST_CASE_METHOD( REQUIRE(node2->use_enumeration() == true); } +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Array v2 Serialization", + "[enumeration][capnp][serialization][v2][array]") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + auto do_load = GENERATE(std::string("true"), std::string("false")); + + create_array(); + + Config cfg; + throw_if_not_ok(cfg.set("rest.use_refactored_array_open", "true")); + throw_if_not_ok(cfg.set("rest.load_enumerations_on_array_open", do_load)); + Context ctx(cfg); + + auto a1 = make_shared(HERE(), uri_, ctx.storage_manager()); + throw_if_not_ok(a1->open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0)); + REQUIRE(a1->serialize_enumerations() == (do_load == "true" ? true : false)); + REQUIRE(a1->array_schema_latest_ptr()->get_loaded_enumeration_names().size() == 0); + + auto a2 = make_shared(HERE(), uri_, ctx.storage_manager()); + + ser_des_array(ctx, a1.get(), a2.get(), client_side, ser_type); + + auto schema = a2->array_schema_latest_ptr(); + auto names = schema->get_enumeration_names(); + auto loaded = schema->get_loaded_enumeration_names(); + + if (do_load == "true") { + REQUIRE(vec_cmp(loaded, names)); + } else { + REQUIRE(loaded.size() == 0); + } +} + #endif // ifdef TILEDB_SERIALIZATIONs /* ********************************* */ @@ -1839,9 +1877,9 @@ shared_ptr EnumerationFx::create_schema() { throw_if_not_ok(schema->set_domain(dom)); std::vector values = {"ant", "bat", "cat", "dog", "emu"}; - auto enmr = + auto enmr1 = create_enumeration(values, false, Datatype::STRING_ASCII, "test_enmr"); - schema->add_enumeration(enmr); + schema->add_enumeration(enmr1); auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); attr1->set_enumeration_name("test_enmr"); @@ -1850,6 +1888,14 @@ shared_ptr EnumerationFx::create_schema() { auto attr2 = make_shared(HERE(), "attr2", Datatype::STRING_ASCII); throw_if_not_ok(schema->add_attribute(attr2)); + std::vector names = {"fred", "wilma", "barney", "betty"}; + auto enmr2 = create_enumeration(names, false, Datatype::STRING_UTF8, "flintstones"); + schema->add_enumeration(enmr2); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); + attr3->set_enumeration_name("flintstones"); + throw_if_not_ok(schema->add_attribute(attr3)); + return schema; } @@ -1923,6 +1969,18 @@ void EnumerationFx::ser_des_query( &(ctx_.resources().compute_tp()))); } +void EnumerationFx::ser_des_array( + Context& ctx, + Array* in, + Array* out, + bool client_side, + SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_serialize(in, stype, &buf, client_side)); + throw_if_not_ok( + serialization::array_deserialize(out, stype, buf, ctx.storage_manager())); +} + #else // No TILEDB_SERIALIZATION ArraySchema EnumerationFx::ser_des_array_schema( @@ -1939,6 +1997,11 @@ void EnumerationFx::ser_des_query(Query*, Query*, bool, SerializationType) { throw std::logic_error("Serialization not enabled."); } +void EnumerataionFx::ser_des_array( + Context&, Array*, Array* bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + #endif // TILEDB_SERIALIZATION template diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f6832aba5712..e70444cc35c3 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1103,6 +1103,16 @@ bool Array::serialize_non_empty_domain() const { return serialize_ned_array_open; } +bool Array::serialize_enumerations() const { + auto serialize = config_.get("rest.load_enumerations_on_array_open"); + if (!serialize.has_value()) { + throw std::runtime_error( + "Cannot get rest.load_enumerations_on_array_open configuration option " + "from config"); + } + return serialize.value(); +} + bool Array::serialize_metadata() const { auto found = false; auto serialize_metadata_array_open = false; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 7ae09bd32ddf..ac232947d159 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -536,6 +536,11 @@ class Array { * open. */ bool serialize_non_empty_domain() const; + /** + * Checks the config to se if enumerations should be serialized on array open. + */ + bool serialize_enumerations() const; + /** * Checks the config to see if metadata should be serialized on array open. */ diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 1ca351286aec..f73fb212a867 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -134,6 +134,10 @@ Status array_to_capnp( array_builder->setQueryType(query_type_str(array->get_query_type())); + if (array->use_refactored_array_open() && array->serialize_enumerations()) { + array->load_all_enumerations(); + } + const auto& array_schema_latest = array->array_schema_latest(); auto array_schema_latest_builder = array_builder->initArraySchemaLatest(); RETURN_NOT_OK(array_schema_to_capnp(