diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 4ed163239ca..11c9ed1427a 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2879,7 +2879,7 @@ shared_ptr EnumerationFx::create_schema() { void EnumerationFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); + throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); } shared_ptr EnumerationFx::get_array(QueryType type) { diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index ff0c5965563..17c5287909f 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -357,7 +357,7 @@ RequestHandlerFx::~RequestHandlerFx() { void RequestHandlerFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); + throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); } void RequestHandlerFx::delete_array() { diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f2f228630db..0080ee73bba 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -39,6 +39,7 @@ #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/auxiliary.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/crypto/crypto.h" @@ -51,6 +52,8 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" +#include "tiledb/sm/object/object.h" +#include "tiledb/sm/object/object_mutex.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/query/update_value.h" #include "tiledb/sm/rest/rest_client.h" @@ -174,6 +177,102 @@ const EncryptionKey* Array::encryption_key() const { return opened_array_->encryption_key(); } +Status Array::create( + ContextResources& resources, + const URI& array_uri, + const shared_ptr& array_schema, + const EncryptionKey& encryption_key) { + // Check array schema + if (array_schema == nullptr) { + throw ArrayException("Cannot create array; Empty array schema"); + } + + // Check if array exists + if (is_array(resources, array_uri)) { + throw ArrayException( + "Cannot create array; Array '" + array_uri.to_string() + + "' already exists"); + } + + std::lock_guard lock{object_mtx}; + array_schema->set_array_uri(array_uri); + array_schema->generate_uri(); + array_schema->check(resources.config()); + + // Create array directory + throw_if_not_ok(resources.vfs().create_dir(array_uri)); + + // Create array schema directory + URI array_schema_dir_uri = + array_uri.join_path(constants::array_schema_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_schema_dir_uri)); + + // Create the enumerations directory inside the array schema directory + URI array_enumerations_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_enumerations_uri)); + + // Create commit directory + URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_commit_uri)); + + // Create fragments directory + URI array_fragments_uri = + array_uri.join_path(constants::array_fragments_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_fragments_uri)); + + // Create array metadata directory + URI array_metadata_uri = + array_uri.join_path(constants::array_metadata_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_metadata_uri)); + + // Create fragment metadata directory + URI array_fragment_metadata_uri = + array_uri.join_path(constants::array_fragment_meta_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_fragment_metadata_uri)); + + // Create dimension label directory + URI array_dimension_labels_uri = + array_uri.join_path(constants::array_dimension_labels_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_dimension_labels_uri)); + + // Get encryption key from config + Status st; + if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { + bool found = false; + std::string encryption_key_from_cfg = + resources.config().get("sm.encryption_key", &found); + assert(found); + std::string encryption_type_from_cfg = + resources.config().get("sm.encryption_type", &found); + assert(found); + auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st_enc); + EncryptionType encryption_type_cfg = etc.value(); + + EncryptionKey encryption_key_cfg; + if (encryption_key_from_cfg.empty()) { + throw_if_not_ok( + encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + } else { + throw_if_not_ok(encryption_key_cfg.set_key( + encryption_type_cfg, + (const void*)encryption_key_from_cfg.c_str(), + static_cast(encryption_key_from_cfg.size()))); + } + st = store_array_schema(resources, array_schema, encryption_key_cfg); + } else { + st = store_array_schema(resources, array_schema, encryption_key); + } + + if (!st.ok()) { + throw_if_not_ok(resources.vfs().remove_dir(array_uri)); + return st; + } + + return Status::Ok(); +} + // Used in Consolidator Status Array::open_without_fragments( EncryptionType encryption_type, diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 49d441cf0c0..08ed2754558 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -370,6 +370,21 @@ class Array { * serialization in pre TileDB 2.4 */ const URI& array_uri_serialized() const; + /** + * Creates a TileDB array, storing its schema. + * + * @param resources The context resources. + * @param array_uri The URI of the array to be created. + * @param array_schema The array schema. + * @param encryption_key The encryption key to use. + * @return Status + */ + static Status create( + ContextResources& resources, + const URI& array_uri, + const shared_ptr& array_schema, + const EncryptionKey& encryption_key); + /** * Opens the array for reading at a timestamp retrieved from the config * or for writing. diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index c0b56da2e64..69dc74761bf 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -124,11 +124,7 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - Status st = sm->array_create(uri, schema, key); - if (!st.ok()) { - throw std::runtime_error( - "[WhiteboxConsistencyController] Could not create array."); - } + throw_if_not_ok(Array::create(sm->resources(), uri, schema, key)); tdb_unique_ptr array(new Array{uri, sm, *this}); return array; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 5824c5dee92..f9e28b05b11 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2562,8 +2562,8 @@ int32_t tiledb_array_create( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the array - throw_if_not_ok(ctx->storage_manager()->array_create( - uri, array_schema->array_schema_, key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key)); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2581,8 +2581,11 @@ int32_t tiledb_array_create( } // Create the dimension label array with the same key. - throw_if_not_ok(ctx->storage_manager()->array_create( - dim_label_ref.uri(uri), dim_label_ref.schema(), key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), + dim_label_ref.uri(uri), + dim_label_ref.schema(), + key)); } } return TILEDB_OK; @@ -2630,8 +2633,8 @@ int32_t tiledb_array_create_with_key( key_length)); // Create the array - throw_if_not_ok(ctx->storage_manager()->array_create( - uri, array_schema->array_schema_, key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key)); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2649,8 +2652,11 @@ int32_t tiledb_array_create_with_key( } // Create the dimension label array with the same key. - throw_if_not_ok(ctx->storage_manager()->array_create( - dim_label_ref.uri(uri), dim_label_ref.schema(), key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), + dim_label_ref.uri(uri), + dim_label_ref.schema(), + key)); } } return TILEDB_OK; diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index a0168e896c4..6246d1c6f75 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -95,10 +95,7 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - Status st = sm_->array_create(uri, schema, key); - if (!st.ok()) { - throw std::runtime_error("Could not create array."); - } + throw_if_not_ok(Array::create(resources_, uri, schema, key)); tdb_unique_ptr array(new Array{uri, sm_.get()}); return array; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index b7dc00ca8e1..e8a58c3e27c 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -131,102 +131,6 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManagerCanonical::array_create( - const URI& array_uri, - const shared_ptr& array_schema, - const EncryptionKey& encryption_key) { - // Check array schema - if (array_schema == nullptr) { - throw StorageManagerException("Cannot create array; Empty array schema"); - } - - // Check if array exists - if (is_array(resources_, array_uri)) { - throw StorageManagerException( - "Cannot create array; Array '" + array_uri.to_string() + - "' already exists"); - } - - std::lock_guard lock{object_mtx}; - array_schema->set_array_uri(array_uri); - array_schema->generate_uri(); - array_schema->check(config_); - - // Create array directory - throw_if_not_ok(resources_.vfs().create_dir(array_uri)); - - // Create array schema directory - URI array_schema_dir_uri = - array_uri.join_path(constants::array_schema_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); - - // Create the enumerations directory inside the array schema directory - URI array_enumerations_uri = - array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_enumerations_uri)); - - // Create commit directory - URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_commit_uri)); - - // Create fragments directory - URI array_fragments_uri = - array_uri.join_path(constants::array_fragments_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_fragments_uri)); - - // Create array metadata directory - URI array_metadata_uri = - array_uri.join_path(constants::array_metadata_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_metadata_uri)); - - // Create fragment metadata directory - URI array_fragment_metadata_uri = - array_uri.join_path(constants::array_fragment_meta_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_fragment_metadata_uri)); - - // Create dimension label directory - URI array_dimension_labels_uri = - array_uri.join_path(constants::array_dimension_labels_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_dimension_labels_uri)); - - // Get encryption key from config - Status st; - if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { - bool found = false; - std::string encryption_key_from_cfg = - config_.get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - config_.get("sm.encryption_type", &found); - assert(found); - auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st_enc); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - RETURN_NOT_OK( - encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); - } else { - RETURN_NOT_OK(encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - static_cast(encryption_key_from_cfg.size()))); - } - st = store_array_schema(resources_, array_schema, encryption_key_cfg); - } else { - st = store_array_schema(resources_, array_schema, encryption_key); - } - - // Store array schema - if (!st.ok()) { - throw_if_not_ok(resources_.vfs().remove_dir(array_uri)); - return st; - } - - return Status::Ok(); -} - Status StorageManager::array_evolve_schema( const URI& array_uri, ArraySchemaEvolution* schema_evolution, diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 9c70fbf39b5..f2c7d55127d 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -135,19 +135,6 @@ class StorageManagerCanonical { /* API */ /* ********************************* */ - /** - * Creates a TileDB array storing its schema. - * - * @param array_uri The URI of the array to be created. - * @param array_schema The array schema. - * @param encryption_key The encryption key to use. - * @return Status - */ - Status array_create( - const URI& array_uri, - const shared_ptr& array_schema, - const EncryptionKey& encryption_key); - /** * Evolve a TileDB array schema and store its new schema. * diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 61ea4bda2c9..86e4ec1a724 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021 TileDB, Inc. + * @copyright Copyright (c) 2021-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -91,7 +91,8 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { tiledb::sm::Array a(tiledb::sm::URI{"mem://junk"}, ctx.storage_manager()); tiledb::sm::EncryptionKey ek; CHECK(ek.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0).ok()); - CHECK(ctx.storage_manager()->array_create(a.array_uri(), sp_as, ek).ok()); + CHECK(tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek) + .ok()); CHECK(a.open( tiledb::sm::QueryType::READ, tiledb::sm::EncryptionType::NO_ENCRYPTION,