diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..aadb977758a --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Flag all PR configuration changes for documentation follow-up. +/tiledb/sm/cpp_api/config.h @TileDB-Inc/tiledb-docs +/tiledb/api/c_api/config/config_api_external.h @TileDB-Inc/tiledb-docs diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index c4265f87c6b..7b26ecee8c7 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -61,7 +61,6 @@ jobs: TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} TILEDB_CMAKE_BUILD_TYPE: 'Release' VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' - CXXFLAGS: '/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR' # https://github.com/actions/runner-images/issues/10004 steps: - name: 'tiledb env prep' run: | diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 4efbb6798fa..2a32d021836 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -61,6 +61,11 @@ on: description: 'CMake generator' required: false type: string + build_only: + default: false + description: 'Whether to only build TileDB and not run tests' + required: false + type: boolean env: BACKWARDS_COMPATIBILITY_ARRAYS: OFF @@ -176,6 +181,7 @@ jobs: - name: 'Test libtiledb' id: test + if: ${{ !inputs.build_only }} shell: bash env: ASAN_OPTIONS: ${{ inputs.asan && 'detect_leaks=0' || '' }} @@ -266,6 +272,7 @@ jobs: /cores/ - name: 'Test status check' + if: ${{ !inputs.build_only }} run: | # tiledb_unit is configured to set a variable TILEDB_CI_SUCCESS=1 # following the test run. If this variable is not set, the build should fail. diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 79ebf4512bd..0e2403f4671 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -65,7 +65,7 @@ jobs: run: | cmake --build build --target check --config ${{ matrix.config || 'Release' }} - test_hdfs: + test_hdfs_build: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: HDFS @@ -73,6 +73,7 @@ jobs: matrix_compiler_cc: 'gcc-13' matrix_compiler_cxx: 'g++-13' timeout: 300 + build_only: true bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror' create_issue_on_fail: @@ -81,7 +82,7 @@ jobs: runs-on: ubuntu-latest needs: - test - - test_hdfs + - test_hdfs_build if: failure() || cancelled() steps: - name: Checkout TileDB `dev` diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index 4131f1802a5..b0f3f25bc08 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -10,8 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # Temporarily reverting to windows-2019 until https://github.com/actions/runner-images/issues/10004 gets fixed. - os: [macos-latest, ubuntu-latest, windows-2019] + os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false name: Build - ${{ matrix.os }} timeout-minutes: 90 @@ -54,11 +53,6 @@ jobs: shell: bash if: ${{ !startsWith(matrix.os, 'windows-') }} - - name: 'Print env' - run: set - shell: cmd - if: ${{ startsWith(matrix.os, 'windows-') }} - - name: 'Build standalone unit tests' run: | cmake -S . \ diff --git a/CMakeLists.txt b/CMakeLists.txt index db8d8068ac5..7c4d2db7bac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,6 +292,8 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer_list/buffer_list_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/config/config_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/context/context_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/current_domain/current_domain_api_enum.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/data_order/data_order_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/data_order/data_order_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/datatype/datatype_api_enum.h" @@ -307,6 +309,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filter/filter_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filter_list/filter_list_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/group/group_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/object/object_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/object/object_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query/query_api_enum.h" diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index fded8179437..c12905c0542 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -49,6 +49,10 @@ if (DEFINED TILEDB_VCPKG AND NOT TILEDB_VCPKG) message(FATAL_ERROR "Disabling TILEDB_VCPKG is not supported. To disable automatically downloading vcpkg, enable the TILEDB_DISABLE_AUTO_VCPKG option, or set ENV{TILEDB_DISABLE_AUTO_VCPKG} to any value.") endif() +if (TILEDB_HDFS) + message(DEPRECATION "The HDFS storage backend is deprecated and receiving build-only official validation. It will be removed in TileDB 2.28.") +endif() + # enable assertions by default for debug builds if (CMAKE_BUILD_TYPE STREQUAL "Debug") set(TILEDB_ASSERTIONS TRUE) diff --git a/examples/c_api/current_domain.c b/examples/c_api/current_domain.c new file mode 100644 index 00000000000..8a626389841 --- /dev/null +++ b/examples/c_api/current_domain.c @@ -0,0 +1,228 @@ +/** + * @file current_domain.c + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 1D sparse array with a current + * domain, print it, expand it with array schema evolution, and print it again. + */ + +#include +#include +#include +#include + +// Name of array. +const char* array_name = "current_domain_array"; + +void create_array() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // The array will be 1000x1 with dimension "d1", with domain [1,1000]. + int dim_domain[] = {1, 1000}; + int tile_extent = 50; + tiledb_dimension_t* d1; + tiledb_dimension_alloc( + ctx, "d1", TILEDB_INT32, &dim_domain[0], &tile_extent, &d1); + + // Create domain + tiledb_domain_t* domain; + tiledb_domain_alloc(ctx, &domain); + tiledb_domain_add_dimension(ctx, domain, d1); + + // Create current domain + tiledb_current_domain_t* current_domain; + tiledb_current_domain_create(ctx, ¤t_domain); + + // Create an n-dimensional rectangle + tiledb_ndrectangle_t* ndrect; + tiledb_ndrectangle_alloc(ctx, domain, &ndrect); + + // Assign the range [1, 100] to the rectangle's first dimension + int32_t expanded_current_domain[] = {1, 100}; + tiledb_range_t range = { + &expanded_current_domain[0], + sizeof(expanded_current_domain[0]), + &expanded_current_domain[1], + sizeof(expanded_current_domain[1])}; + tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range); + + // Assign the rectangle to the current domain + tiledb_current_domain_set_ndrectangle(current_domain, ndrect); + + // Create a single attribute "a" so each cell can store an integer + tiledb_attribute_t* a; + tiledb_attribute_alloc(ctx, "a", TILEDB_INT32, &a); + + // Create array schema + tiledb_array_schema_t* array_schema; + tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema); + tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); + tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR); + tiledb_array_schema_set_domain(ctx, array_schema, domain); + tiledb_array_schema_add_attribute(ctx, array_schema, a); + + // Assign the current domain to the array schema + tiledb_array_schema_set_current_domain(ctx, array_schema, current_domain); + + // Create array + tiledb_array_create(ctx, array_name, array_schema); + + // Clean up + tiledb_array_schema_free(&array_schema); + tiledb_attribute_free(&a); + tiledb_ndrectangle_free(&ndrect); + tiledb_current_domain_free(¤t_domain); + tiledb_domain_free(&domain); + tiledb_dimension_free(&d1); + tiledb_ctx_free(&ctx); +} + +void print_current_domain() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // Load array schema + tiledb_array_schema_t* array_schema; + tiledb_array_schema_load(ctx, array_name, &array_schema); + + // Get current domain + tiledb_current_domain_t* current_domain; + tiledb_array_schema_get_current_domain(ctx, array_schema, ¤t_domain); + + // Check if current domain is empty + uint32_t is_empty; + tiledb_current_domain_get_is_empty(current_domain, &is_empty); + + if (is_empty) { + printf("Current domain: empty\n"); + } else { + // Get current domain type + tiledb_current_domain_type_t current_domain_type; + tiledb_current_domain_get_type(current_domain, ¤t_domain_type); + + if (current_domain_type == TILEDB_NDRECTANGLE) { + printf("Current domain type: NDRECTANGLE\n"); + + // Get the ND rectangle + tiledb_ndrectangle_t* ndrect; + tiledb_current_domain_get_ndrectangle(current_domain, &ndrect); + + tiledb_range_t range; + tiledb_ndrectangle_get_range_from_name(ctx, ndrect, "d1", &range); + + printf( + "Current domain range: [%d, %d]\n", + *(int*)range.min, + *(int*)range.max); + + // Clean up + tiledb_ndrectangle_free(&ndrect); + } else { + printf("Current domain type: unknown\n"); + } + } + + // Clean up + tiledb_current_domain_free(¤t_domain); + tiledb_array_schema_free(&array_schema); + tiledb_ctx_free(&ctx); +} + +void expand_current_domain() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // Load array schema + tiledb_array_schema_t* array_schema; + tiledb_array_schema_load(ctx, array_name, &array_schema); + + // Get domain + tiledb_domain_t* domain; + tiledb_array_schema_get_domain(ctx, array_schema, &domain); + + // Create schema evolution + tiledb_array_schema_evolution_t* schema_evolution; + tiledb_array_schema_evolution_alloc(ctx, &schema_evolution); + + // Create the new current domain + tiledb_current_domain_t* new_current_domain; + tiledb_current_domain_create(ctx, &new_current_domain); + + // Create an n-dimensional rectangle + tiledb_ndrectangle_t* ndrect; + tiledb_ndrectangle_alloc(ctx, domain, &ndrect); + + // Assign the range [1, 200] to the rectangle's first dimension + int32_t expanded_current_domain[] = {1, 200}; + tiledb_range_t range = { + &expanded_current_domain[0], + sizeof(expanded_current_domain[0]), + &expanded_current_domain[1], + sizeof(expanded_current_domain[1])}; + tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range); + + // Set the rectangle to the current domain + tiledb_current_domain_set_ndrectangle(new_current_domain, ndrect); + + // Expand the current domain + tiledb_array_schema_evolution_expand_current_domain( + ctx, schema_evolution, new_current_domain); + + // Evolve the array + tiledb_array_evolve(ctx, array_name, schema_evolution); + + // Clean up + tiledb_ndrectangle_free(&ndrect); + tiledb_current_domain_free(&new_current_domain); + tiledb_array_schema_evolution_free(&schema_evolution); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&array_schema); + tiledb_ctx_free(&ctx); +} + +int main() { + // Get object type + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + tiledb_object_t type; + tiledb_object_type(ctx, array_name, &type); + tiledb_ctx_free(&ctx); + + if (type != TILEDB_ARRAY) { + create_array(); + print_current_domain(); + expand_current_domain(); + } + + print_current_domain(); + return 0; +} diff --git a/examples/cpp_api/current_domain.cc b/examples/cpp_api/current_domain.cc new file mode 100644 index 00000000000..0ba8f337310 --- /dev/null +++ b/examples/cpp_api/current_domain.cc @@ -0,0 +1,163 @@ +/** + * @file current_domain.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 1D sparse array with a current + * domain, print it, expand it with array schema evolution, and print it again. + */ + +#include +#include +#include +#include + +using namespace tiledb; + +// Name of array. +std::string array_name("current_domain_example_array"); + +void create_array(Context& ctx, const std::string& array_uri) { + // Create a TileDB domain + Domain domain(ctx); + + // Add a dimension to the domain + auto d1 = Dimension::create(ctx, "d1", {{1, 1000}}, 50); + domain.add_dimension(d1); + + // Create a CurrentDomain object + CurrentDomain current_domain(ctx); + + // Create an NDRectangle object + NDRectangle ndrect(ctx, domain); + + // Assign the range [1, 100] to the rectangle's first dimension + ndrect.set_range("d1", 1, 100); + + // Assign the NDRectangle to the CurrentDomain + current_domain.set_ndrectangle(ndrect); + + // Create a TileDB sparse array schema + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain) + .set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}) + .set_capacity(100) + .set_cell_order(TILEDB_ROW_MAJOR) + .set_tile_order(TILEDB_ROW_MAJOR); + + // Create a single attribute + schema.add_attribute(Attribute::create(ctx, "a")); + + // Assign the current domain to the array schema + ArraySchemaExperimental::set_current_domain(ctx, schema, current_domain); + + // Create the (empty) array on disk + Array::create(array_uri, schema); +} + +void print_current_domain(Context& ctx) { + // Get array schema + ArraySchema schema(ctx, array_name); + + // Get current domain + CurrentDomain current_domain = + ArraySchemaExperimental::current_domain(ctx, schema); + + // Check if the current domain is empty + if (current_domain.is_empty()) { + std::cout << "Current domain: empty" << std::endl; + return; + } + + // Get the current domain type + tiledb_current_domain_type_t current_domain_type = current_domain.type(); + + // Check the current domain type + if (current_domain_type != TILEDB_NDRECTANGLE) { + std::cout << "Current domain type: unknown" << std::endl; + return; + } + + std::cout << "Current domain type: NDRECTANGLE" << std::endl; + + // Get the current domain's NDRectangle + NDRectangle ndrect = current_domain.ndrectangle(); + + // Get the range of the rectangle's first dimension + std::array range = ndrect.range("d1"); + + // Print the range + std::cout << "Current domain range: [" << range[0] << ", " << range[1] << "]" + << std::endl; +} + +void expand_current_domain(Context& ctx) { + // Get the array schema + ArraySchema schema(ctx, array_name); + + // Get the domain + Domain domain = schema.domain(); + + // Create an ArraySchemaEvolution object + ArraySchemaEvolution schema_evolution(ctx); + + // Create the new CurrentDomain object + CurrentDomain new_current_domain(ctx); + + // Create an NDRectangle object + NDRectangle ndrect(ctx, domain); + + // Assign the range [1, 200] to the rectangle's first dimension + ndrect.set_range("d1", 1, 200); + + // Set the NDRectangle to the CurrentDomain + new_current_domain.set_ndrectangle(ndrect); + + // Set the current domain to the array schema evolution + schema_evolution.expand_current_domain(new_current_domain); + + // Evolve the array + schema_evolution.array_evolve(array_name); +} + +int main() { + Context ctx; + + // Create a new simple array + create_array(ctx, array_name); + + // Print the current domain + print_current_domain(ctx); + + // Expand the current domain + expand_current_domain(ctx); + + // Print the current domain again + print_current_domain(ctx); + + return 0; +} diff --git a/experimental/tiledb/common/dag/edge/test/unit_edge.cc b/experimental/tiledb/common/dag/edge/test/unit_edge.cc index fe2fdf46e4d..e8ea4c22381 100644 --- a/experimental/tiledb/common/dag/edge/test/unit_edge.cc +++ b/experimental/tiledb/common/dag/edge/test/unit_edge.cc @@ -54,7 +54,7 @@ using namespace tiledb::common; /** * Attach an `Edge` to a `Source` and a `Sink`, two stage */ -TEST_CASE("Edge: Attach a Source and Sink with a two stage Edge", "[edge") { +TEST_CASE("Edge: Attach a Source and Sink with a two stage Edge", "[edge]") { Source left; Sink right; Edge mid(left, right); @@ -63,7 +63,7 @@ TEST_CASE("Edge: Attach a Source and Sink with a two stage Edge", "[edge") { /** * Attach an `Edge` to a `Source` and a `Sink` */ -TEST_CASE("Edge: Attach a Source and Sink with an Edge", "[edge") { +TEST_CASE("Edge: Attach a Source and Sink with an Edge", "[edge]") { Source left; Sink right; Edge mid(left, right); @@ -72,7 +72,7 @@ TEST_CASE("Edge: Attach a Source and Sink with an Edge", "[edge") { /** * Attach an `Edge` to a `Source` and a `Sink, using CTAD` */ -TEST_CASE("Edge: Attach a Source and Sink with an Edge, using CTAD", "[edge") { +TEST_CASE("Edge: Attach a Source and Sink with an Edge, using CTAD", "[edge]") { Source left; Sink right; Edge mid(left, right); @@ -589,7 +589,7 @@ TEST_CASE("Edge: Async pass n integers", "[edge]") { /** * Attach an `Edge` to a `ProducerNode` and a `ConsumerNode` */ -TEST_CASE("Edge: Attach a Producer and Consumer with an Edge", "[edge") { +TEST_CASE("Edge: Attach a Producer and Consumer with an Edge", "[edge]") { ProducerNode left([]() { return 0UL; }); ConsumerNode right([](size_t) {}); @@ -600,7 +600,7 @@ TEST_CASE("Edge: Attach a Producer and Consumer with an Edge", "[edge") { * Attach an `Edge` to a `ProducerNode` and a `ConsumerNode, using CTAD` */ TEST_CASE( - "Edge: Attach a Producer and Consumer with an Edge, using CTAD", "[edge") { + "Edge: Attach a Producer and Consumer with an Edge, using CTAD", "[edge]") { ProducerNode left([]() { return 0UL; }); ConsumerNode right([](size_t) {}); diff --git a/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc b/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc index 4955d5f6a48..d227484a8ef 100644 --- a/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc +++ b/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc @@ -548,6 +548,9 @@ int main(int argc, char* argv[]) { size_t stages = 2; size_t trips = 2; + // This is unused until we migrate to catch2. + bool durations = false; + auto cli = Opt(block_size, "block_size")["-b"]["--block_size"]("Block size") | Opt(width, "width")["-w"]["--width"]("Width") | Opt(number, "number")["-n"]["--number"]("Number") | @@ -556,7 +559,8 @@ int main(int argc, char* argv[]) { Opt(scheduler, "scheduler")["-s"]["--scheduler"]( "Scheduler (bountiful, duffs, throw_catch, or frugal") | Opt(stages, "stages")["-t"]["--stages"]("Stages (2 or 3)") | - Opt(trips, "trips")["-p"]["--trips"]("Trips"); + Opt(trips, "trips")["-p"]["--trips"]("Trips") | + Opt(durations, "durations")["-d"]["--durations"]("Durations"); auto result = cli.parse(Args(argc, argv)); if (!result) { diff --git a/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc b/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc index b0e9f6cd5b2..78fc626126f 100644 --- a/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc +++ b/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc @@ -533,6 +533,9 @@ int main(int argc, char* argv[]) { size_t stages = 2; size_t trips = 2; + // This is unused until we migrate to catch2. + bool durations = false; + auto cli = Opt(block_size, "block_size")["-b"]["--block_size"]("Block size") | Opt(width, "width")["-w"]["--width"]("Width") | Opt(number, "number")["-n"]["--number"]("Number") | @@ -541,7 +544,8 @@ int main(int argc, char* argv[]) { Opt(scheduler, "scheduler")["-s"]["--scheduler"]( "Scheduler (bountiful, duffs, throw_catch, or frugal") | Opt(stages, "stages")["-t"]["--stages"]("Stages (2 or 3)") | - Opt(trips, "trips")["-p"]["--trips"]("Trips"); + Opt(trips, "trips")["-p"]["--trips"]("Trips") | + Opt(durations, "durations")["-d"]["--durations"]("Durations"); auto result = cli.parse(Args(argc, argv)); if (!result) { diff --git a/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt b/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt index 9ef986294bc..20f853da2f9 100644 --- a/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt +++ b/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt @@ -57,7 +57,7 @@ target_sources(unit_nodes_sieve PUBLIC # add_test( NAME "unit_nodes_sieve" - COMMAND $ --durations=yes + COMMAND $ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) add_dependencies(all_unit_tests "unit_nodes_sieve") diff --git a/experimental/tiledb/common/dag/ports/test/CMakeLists.txt b/experimental/tiledb/common/dag/ports/test/CMakeLists.txt index cae840acad1..104b98511d2 100644 --- a/experimental/tiledb/common/dag/ports/test/CMakeLists.txt +++ b/experimental/tiledb/common/dag/ports/test/CMakeLists.txt @@ -52,7 +52,7 @@ target_sources(unit_ports_sieve PUBLIC # add_test( NAME "unit_ports_sieve" - COMMAND $ --durations=yes + COMMAND $ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) add_dependencies(all_unit_tests "unit_ports_sieve") diff --git a/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc b/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc index 11fcb04ae58..0a8638907c4 100644 --- a/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc +++ b/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc @@ -1,5 +1,5 @@ /** - * @file unit_concurrent_set.cc + * @file unit_concurrent_map.cc * * @section LICENSE * diff --git a/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc b/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc index f93aae7b6ca..56abcd66c8d 100644 --- a/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc +++ b/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc @@ -39,10 +39,10 @@ namespace tiledb::common {} using namespace tiledb::common; -TEST_CASE("ConcurrentSet: Construct", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Construct", "[concurrent_set]") { } -TEST_CASE("ConcurrentSet: Test empty", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test empty", "[concurrent_set]") { ConcurrentSet numbers; CHECK(numbers.empty() == true); @@ -51,19 +51,19 @@ TEST_CASE("ConcurrentSet: Test empty", "[concurrent_set") { CHECK(numbers.empty() == false); } -TEST_CASE("ConcurrentSet: Test size", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test size", "[concurrent_set]") { ConcurrentSet numbers{1, 3, 5, 7, 11}; CHECK(numbers.size() == 5); } -TEST_CASE("ConcurrentSet: Test clear", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test clear", "[concurrent_set]") { ConcurrentSet numbers{1, 2, 3}; CHECK(numbers.size() == 3); numbers.clear(); CHECK(numbers.empty()); } -TEST_CASE("ConcurrentSet: Test insert", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test insert", "[concurrent_set]") { ConcurrentSet set; auto result_1 = set.insert(3); @@ -110,11 +110,11 @@ size_t set_emplace() { return set.size(); } -TEST_CASE("ConcurrentSet: Test emplace", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test emplace", "[concurrent_set]") { CHECK(set_emplace() == nof_operations * nof_operations * nof_operations); } -TEST_CASE("ConcurrentSet: Test erase", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test erase", "[concurrent_set]") { ConcurrentSet numbers = {1, 2, 3, 4, 1, 2, 3, 4}; CHECK(numbers.size() == 4); @@ -132,7 +132,7 @@ TEST_CASE("ConcurrentSet: Test erase", "[concurrent_set") { CHECK(numbers.erase(2) == 0); } -TEST_CASE("ConcurrentSet: Test swap", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test swap", "[concurrent_set]") { { ConcurrentSet a1{3, 1, 3, 2, 7}, a2{5, 4, 5}; @@ -178,7 +178,7 @@ TEST_CASE("ConcurrentSet: Test swap", "[concurrent_set") { } } -TEST_CASE("ConcurrentSet: Test extract", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test extract", "[concurrent_set]") { ConcurrentSet cont0{1, 2, 3}; ConcurrentSet cont1{1, 2, 3}; ConcurrentSet cont2{2, 3}; @@ -217,7 +217,7 @@ bool operator<(const FatKey& fk1, const FatKey& fk2) { return fk1.x < fk2.x; } -TEST_CASE("ConcurrentSet: Test find", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test find", "[concurrent_set]") { // simple comparison demo ConcurrentSet example = {1, 2, 3, 4}; diff --git a/format_spec/filter_pipeline.md b/format_spec/filter_pipeline.md index 96ef8786793..dc4260da2dd 100644 --- a/format_spec/filter_pipeline.md +++ b/format_spec/filter_pipeline.md @@ -42,13 +42,22 @@ The filter options are configuration parameters for the filters that do not chan ### Main Compressor Options -For the compression filters \(any of the filter types `TILEDB_FILTER_{GZIP,ZSTD,LZ4,RLE,BZIP2,DOUBLE_DELTA,DELTA,DICTIONARY}`\) the filter options have internal format: +For the main compression filters \(any of the filter types `TILEDB_FILTER_{GZIP,ZSTD,LZ4,RLE,BZIP2,DICTIONARY}`\) the filter options have internal format: | **Field** | **Type** | **Description** | | :--- | :--- | :--- | -| Compressor type | `uint8_t` | Type of compression \(e.g. `TILEDB_BZIP2`\) | +| Compressor type | `uint8_t` | Type of compression \(e.g. `TILEDB_FILTER_BZIP2`\) | | Compression level | `int32_t` | Compression level used \(ignored by some compressors\). | -| Reinterpret datatype | `uint8_t` | Type to reinterpret data prior to compression. Used for DOUBLE_DELTA and DELTA only. | + +### Delta Compressor Options + +For the `TILEDB_FILTER_DELTA` and `TILEDB_FILTER_DOUBLE_DELTA` compression filters the filter options have internal format: + +| **Field** | **Type** | **Description** | +| :--- | :--- | :--- | +| Compressor type | `uint8_t` | Type of compression \(e.g. `TILEDB_FILTER_DELTA`\) | +| Compression level | `int32_t` | Ignored | +| Reinterpret datatype | `uint8_t` | Type to reinterpret data prior to compression. | ### Bit-width Reduction Options @@ -78,4 +87,4 @@ The filter options for `TILEDB_FILTER_POSITIVE_DELTA` has internal format: ### Other Filter Options -The remaining filters \(`TILEDB_FILTER_{BITSHUFFLE,BYTESHUFFLE,CHECKSUM_MD5,CHECKSUM_256,XOR,DICTIONARY}`\) do not serialize any options. +The remaining filters \(`TILEDB_FILTER_{BITSHUFFLE,BYTESHUFFLE,CHECKSUM_MD5,CHECKSUM_256,XOR}`\) do not serialize any options. diff --git a/format_spec/filters/dictionary_encoding.md b/format_spec/filters/dictionary_encoding.md index 53ecbaea5e9..67f9599ca50 100644 --- a/format_spec/filters/dictionary_encoding.md +++ b/format_spec/filters/dictionary_encoding.md @@ -6,7 +6,7 @@ The Dictionary Encoding filter compresses losslessly string data by creating a s As an example in pseudocode: ``` - input_data = "HG543232", "HG543232", "HG543232", "HG54", "HG54", "A", "HG543232", "HG54"] + input_data = ["HG543232", "HG543232", "HG543232", "HG54", "HG54", "A", "HG543232", "HG54"] # apply dictionary encoding -> dictionary = ["HG543232", "HG54", "A"] output_data = [0, 0, 0, 1, 1, 2, 0, 1] diff --git a/format_spec/filters/double_delta.md b/format_spec/filters/double_delta.md new file mode 100644 index 00000000000..31a9db62cd1 --- /dev/null +++ b/format_spec/filters/double_delta.md @@ -0,0 +1,32 @@ +--- +title: Double Delta Filter +--- + +## Double Delta Filter + +The double delta filter compresses integer type data by first computing the delta between consecutive elements, then the delta between the deltas, and bit-packing the result. + +### Filter Enum Value + +The filter enum value for the double delta filter is `6` (`TILEDB_FILTER_DOUBLE_DELTA` enum). + +### Input and Output Layout + +The input data layout will be an array of integer numbers (each known as `in_{n}`, with `n` starting from 0). Their type (henceforth known as `input_t`) is inferred from the output type of the previous filter, or the tile's datatype if this is the first filter in the pipeline, but can be overriden by the [_Reinterpret datatype_ field](../filter_pipeline.md#delta-compressor-options) in the filter options. + +The output data layout consists of the following fields: + +|Field|Type|Description| +|:---|:---|:---| +|`bitsize`|`uint8_t`|Minimum number of bits required to represent any `dd_n` value.| +|`n`|`uint64_t`|Number of values in the input data.| +|`in_0`|`input_t`|First input value.| +|`in_1`|`input_t`|Second input value.| +|`sign_2`|`bit`|Sign of `(in_2 - in_1) - (in_1 - in_0)`.| +|`dd_2`|`bit[bitsize]`|Absolute value of `(in_2 - in_1) - (in_1 - in_0)`.| +|…|…|…| +|`sign_n`|`bit`|Sign of `(in_n - in_{n - 1}) - (in_{n - 1} - in_{n - 2})`.| +|`dd_n`|`bit[bitsize]`|Absolute value of `(in_n - in_{n - 1}) - (in_{n - 1} - in_{n - 2})`.| +|`pad`|`bit[((n - 2) * (bitsize + 1)) % 64]`|Padding to the next 64-bit boundary.| + +If `bitsize` was computed as equal to `sizeof(input_t) * 8 - 1` (i.e. double delta compression would not have yielded any size savings), double delta compression is not applied and the input data will be added to the output stream unchanged, after `bitsize` and `n`, which are always written. diff --git a/format_spec/tile.md b/format_spec/tile.md index 6677fa5fa77..23085be5f7a 100644 --- a/format_spec/tile.md +++ b/format_spec/tile.md @@ -193,38 +193,13 @@ The encryption filter metadata have the following on-disk format: | :--- | :--- | :--- | | Num metadata parts | `uint32_t` | Number of encrypted metadata parts | | Num data parts | `uint32_t` | Number of encrypted data parts | -| AES Metadata Part 1 | `AESPartMD` | Metadata part 1 | +| AES Metadata Part 1 | `AESPart` | Metadata part 1 | | … | … | … | -| AES Metadata Part N | `AESPartMD` | Metadata part N | +| AES Metadata Part N | `AESPart` | Metadata part N | | AES Data Part 1 | `AESPart` | Data part 1 | | … | … | … | | AES Data Part N | `AESPart` | Data part N | -The `AESPartMD` field has the following on-disk format: - -| **Field** | **Type** | **Description** | -| :--- | :--- | :--- | -| Num metadata parts | `uint32_t` | Number of metadata parts | -| Num data parts | `uint32_t` | Number of data parts | -| Plaintext length for metadata part 1 | `uint32_t` | Number of bytes of plaintext metadata part 1 | -| Ciphertext length for metadata part 1 | `uint32_t` | Number of bytes of ciphertext metadata part 1 | -| IV bytes for metadata part 1 | `uint32_t` | Number of bytes of AES-256-GCM IV bytes for metadata part 1 | -| Tag bytes for metadata part 1 | `uint32_t` | Number of bytes of AES-256-GCM tag for metadata part 1 | -| … | … | … | -| Plaintext length for metadata part N | `uint32_t` | Number of bytes of plaintext metadata part N | -| Ciphertext length for metadata part N | `uint32_t` | Number of bytes of ciphertext metadata part N | -| IV bytes for metadata part N | `uint32_t` | Number of bytes of AES-256-GCM IV bytes for metadata part N | -| Tag bytes for metadata part N | `uint32_t` | Number of bytes of AES-256-GCM tag for metadata part N | -| Plaintext length for data part 1 | `uint32_t` | Number of bytes of plaintext data part 1 | -| Ciphertext length for data part 1 | `uint32_t` | Number of bytes of ciphertext data part 1 | -| IV bytes for data part 1 | `uint32_t` | Number of bytes of AES-256-GCM IV bytes for data part 1 | -| Tag bytes for data part 1 | `uint32_t` | Number of bytes of AES-256-GCM tag for data part 1 | -| … | … | … | -| Plaintext length for data part N | `uint32_t` | Number of bytes of plaintext data part N | -| Ciphertext length for data part N | `uint32_t` | Number of bytes of ciphertext data part N | -| IV bytes for data part N | `uint32_t` | Number of bytes of AES-256-GCM IV bytes for data part N | -| Tag bytes for data part N | `uint32_t` | Number of bytes of AES-256-GCM tag for data part N | - The original metadata is **not** included in the metadata output. The `AESPart` field has the following on-disk format: diff --git a/ports/libmagic/CMakeLists.txt b/ports/libmagic/CMakeLists.txt index a6248bdf871..3d731affe3d 100644 --- a/ports/libmagic/CMakeLists.txt +++ b/ports/libmagic/CMakeLists.txt @@ -91,6 +91,10 @@ set(targets ${targets} libmagic) target_link_libraries(libmagic PRIVATE PCRE2::POSIX) +if(WIN32) + target_link_libraries(libmagic PRIVATE shlwapi) +endif() + target_include_directories(libmagic PUBLIC "$" diff --git a/ports/triplets/x64-windows-asan.cmake b/ports/triplets/x64-windows-asan.cmake index 2d1e356817c..d19486d2cfe 100644 --- a/ports/triplets/x64-windows-asan.cmake +++ b/ports/triplets/x64-windows-asan.cmake @@ -7,7 +7,3 @@ set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) # bigobj is needed for capnp. set(VCPKG_C_FLAGS "/fsanitize=address /bigobj") set(VCPKG_CXX_FLAGS "/fsanitize=address /bigobj") - -# https://github.com/actions/runner-images/issues/10004 -set(VCPKG_C_FLAGS "${VCPKG_C_FLAGS} /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") -set(VCPKG_CXX_FLAGS "${VCPKG_CXX_FLAGS} /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") diff --git a/ports/triplets/x64-windows.cmake b/ports/triplets/x64-windows.cmake index eb54e7f5a55..c87b4f5dd38 100644 --- a/ports/triplets/x64-windows.cmake +++ b/ports/triplets/x64-windows.cmake @@ -2,8 +2,4 @@ set(VCPKG_TARGET_ARCHITECTURE x64) set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_LIBRARY_LINKAGE static) -# https://github.com/actions/runner-images/issues/10004 -set(VCPKG_C_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") -set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") - set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5133fa933cc..851154c4bc3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -207,6 +207,8 @@ if (TILEDB_CPP_API) src/cpp-integration-filter-pipeline.cc src/test-cppapi-dense-array-dimension-label.cc src/test-cppapi-dimension-label.cc + src/test-cppapi-ndrectangle.cc + src/test-cppapi-current-domain.cc src/test-cppapi-subarray-labels.cc src/unit-cppapi-array.cc src/unit-cppapi-checksum.cc diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 23f6da83ee3..e7ecf827fe9 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -85,3 +85,8 @@ add_test( COMMAND $ --durations=yes WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + +if (BUILD_SHARED_LIBS) + # Add the location of the tiledb shared library to PATH, to help Windows find it. + set_property(TEST tiledb_regression PROPERTY ENVIRONMENT_MODIFICATION "PATH=path_list_append:$") +endif() diff --git a/test/src/test-cppapi-current-domain.cc b/test/src/test-cppapi-current-domain.cc new file mode 100644 index 00000000000..3441e1e3d30 --- /dev/null +++ b/test/src/test-cppapi-current-domain.cc @@ -0,0 +1,242 @@ +/** + * @file test-cppapi-current-domain.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the currentDomain C++ API + */ + +#include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +#include + +using namespace tiledb::test; + +struct CurrentDomainFx { + VFSTestSetup vfs_test_setup_; + + // Constructors/destructors. + CurrentDomainFx(); + + // TileDB context + tiledb_ctx_t* ctx_c_; + tiledb::Context ctx_; +}; + +CurrentDomainFx::CurrentDomainFx() { + tiledb::Config config; + ctx_c_ = vfs_test_setup_.ctx_c; + ctx_ = vfs_test_setup_.ctx(); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "C++ API: CurrentDomain - Integer dimensions", + "[cppapi][ArraySchema][currentDomain]") { + // Create domain + tiledb::Domain domain(ctx_); + auto d1 = tiledb::Dimension::create(ctx_, "x", {{0, 100}}, 10); + auto d2 = tiledb::Dimension::create(ctx_, "y", {{0, 100}}, 10); + domain.add_dimension(d1); + domain.add_dimension(d2); + + // Create an NDRectangle and set ranges + tiledb::NDRectangle ndrect(ctx_, domain); + int range_one[] = {10, 20}; + int range_two[] = {30, 40}; + ndrect.set_range(0, range_one[0], range_one[1]); + ndrect.set_range(1, range_two[0], range_two[1]); + + // Get and check ranges + auto range = ndrect.range(0); + CHECK(range[0] == 10); + CHECK(range[1] == 20); + range = ndrect.range(1); + CHECK(range[0] == 30); + CHECK(range[1] == 40); + + // Create a currentDomain and set the NDRectangle + tiledb::CurrentDomain current_domain(ctx_); + current_domain.set_ndrectangle(ndrect); + + CHECK_FALSE(current_domain.is_empty()); + + auto rect = current_domain.ndrectangle(); + + // Get and check ranges + range = rect.range(0); + CHECK(range[0] == 10); + CHECK(range[1] == 20); + range = rect.range(1); + CHECK(range[0] == 30); + CHECK(range[1] == 40); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "C++ API: CurrentDomain - String dimensions", + "[cppapi][ArraySchema][string-dims][currentDomain]") { + // Create domain + auto d1 = tiledb::Dimension::create( + ctx_, "d1", TILEDB_STRING_ASCII, nullptr, nullptr); + auto d2 = tiledb::Dimension::create( + ctx_, "d2", TILEDB_STRING_ASCII, nullptr, nullptr); + tiledb::Domain domain(ctx_); + domain.add_dimension(d1); + domain.add_dimension(d2); + + // Create an NDRectangle and set ranges + tiledb::NDRectangle ndrect(ctx_, domain); + ndrect.set_range(0, std::string("a"), std::string("c")); + ndrect.set_range(1, std::string("b"), std::string("db")); + + // Get and check ranges + std::array range = ndrect.range(0); + CHECK(range[0] == "a"); + CHECK(range[1] == "c"); + range = ndrect.range(1); + CHECK(range[0] == "b"); + CHECK(range[1] == "db"); + + // Create a currentDomain and set the NDRectangle + tiledb::CurrentDomain current_domain(ctx_); + current_domain.set_ndrectangle(ndrect); + + CHECK_FALSE(current_domain.is_empty()); + + auto rect = current_domain.ndrectangle(); + + // Get and check ranges + range = ndrect.range(0); + CHECK(range[0] == "a"); + CHECK(range[1] == "c"); + range = ndrect.range(1); + CHECK(range[0] == "b"); + CHECK(range[1] == "db"); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "C++ API: CurrentDomain - Add to ArraySchema", + "[cppapi][ArraySchema][currentDomain]") { + // Create domain. + tiledb::Domain domain(ctx_); + auto d = tiledb::Dimension::create(ctx_, "d", {{1, 999}}, 2); + domain.add_dimension(d); + + // Create array schema. + tiledb::ArraySchema schema(ctx_, TILEDB_SPARSE); + schema.set_domain(domain); + schema.add_attribute(tiledb::Attribute::create(ctx_, "a")); + + // Create NDRectangle and set ranges + tiledb::NDRectangle ndrect(ctx_, domain); + int range_one[] = {10, 20}; + ndrect.set_range(0, range_one[0], range_one[1]); + + // Create the currentDomain and set NDRectangle + tiledb::CurrentDomain current_domain(ctx_); + current_domain.set_ndrectangle(ndrect); + + CHECK_NOTHROW(tiledb::ArraySchemaExperimental::set_current_domain( + ctx_, schema, current_domain)); + + auto cd = tiledb::ArraySchemaExperimental::current_domain(ctx_, schema); + CHECK(!cd.is_empty()); + + // Check if ranges are the same + CHECK(cd.ndrectangle().range(0)[0] == ndrect.range(0)[0]); + CHECK(cd.ndrectangle().range(0)[1] == ndrect.range(0)[1]); + + CHECK_THROWS(cd.ndrectangle().range(1)); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "C++ API: CurrentDomain - Evolve", + "[cppapi][ArraySchema][currentDomain]") { + const std::string array_name = "test_current_domain_expansion"; + + tiledb::VFS vfs(ctx_); + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } + + // Create domain. + tiledb::Domain domain(ctx_); + auto d = tiledb::Dimension::create(ctx_, "d", {{1, 999}}, 2); + domain.add_dimension(d); + + // Create array schema. + tiledb::ArraySchema schema(ctx_, TILEDB_SPARSE); + schema.set_domain(domain); + schema.add_attribute(tiledb::Attribute::create(ctx_, "a")); + + // Create NDRectangle and set ranges + tiledb::NDRectangle ndrect(ctx_, domain); + int range_one[] = {10, 20}; + ndrect.set_range(0, range_one[0], range_one[1]); + + // Create the currentDomain and set NDRectangle + tiledb::CurrentDomain current_domain(ctx_); + current_domain.set_ndrectangle(ndrect); + + tiledb::ArraySchemaExperimental::set_current_domain( + ctx_, schema, current_domain); + + // Create array + tiledb::Array::create(array_name, schema); + + // Create new currentDomain to test evolution + tiledb::CurrentDomain new_current_domain(ctx_); + int range_two[] = {5, 30}; + tiledb::NDRectangle ndrect_two(ctx_, domain); + ndrect_two.set_range(0, range_two[0], range_two[1]); + new_current_domain.set_ndrectangle(ndrect_two); + + // Schema evolution + tiledb::ArraySchemaEvolution se(ctx_); + se.expand_current_domain(new_current_domain); + se.array_evolve(array_name); + + // Open array to check the ranges + tiledb::Array array(ctx_, array_name, TILEDB_READ); + auto s = array.schema(); + auto cd = tiledb::ArraySchemaExperimental::current_domain(ctx_, s); + auto n = cd.ndrectangle(); + CHECK(n.range(0)[0] == ndrect_two.range(0)[0]); + CHECK(n.range(0)[1] == ndrect_two.range(0)[1]); + + // Clean up. + array.close(); + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } +} diff --git a/test/src/test-cppapi-ndrectangle.cc b/test/src/test-cppapi-ndrectangle.cc new file mode 100644 index 00000000000..f29f9d576a6 --- /dev/null +++ b/test/src/test-cppapi-ndrectangle.cc @@ -0,0 +1,79 @@ +/** + * @file test-cppapi-ndrectangle.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the NDRectangle C++ API + */ + +#include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +#include + +using namespace tiledb::test; + +TEST_CASE_METHOD( + TemporaryDirectoryFixture, + "NDRectangle - Basic", + "[cppapi][ArraySchema][NDRectangle]") { + // Create the C++ context. + tiledb::Context ctx_{ctx, false}; + tiledb::Domain domain(ctx_); + auto d1 = tiledb::Dimension::create(ctx_, "x", {{0, 100}}, 10); + auto d2 = tiledb::Dimension::create(ctx_, "y", {{0, 100}}, 10); + domain.add_dimension(d1); + domain.add_dimension(d2); + + tiledb::NDRectangle ndrect(ctx_, domain); + + int range_one[] = {10, 20}; + ndrect.set_range(0, range_one[0], range_one[1]); + + int range_two[] = {30, 40}; + ndrect.set_range(1, range_two[0], range_two[1]); + + // Check setting range in non-existent dim + CHECK_THROWS(ndrect.set_range(2, range_two[0], range_two[1])); + + // Get range + auto range = ndrect.range(0); + CHECK(range[0] == 10); + CHECK(range[1] == 20); + range = ndrect.range(1); + CHECK(range[0] == 30); + CHECK(range[1] == 40); + + range = ndrect.range("x"); + CHECK(range[0] == 10); + CHECK(range[1] == 20); + range = ndrect.range("y"); + CHECK(range[0] == 30); + CHECK(range[1] == 40); +} diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index bbc3ba3a669..776278622a2 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -170,6 +170,7 @@ TEST_CASE_METHOD( Subarray subarray(&array, &g_helper_stats, g_helper_logger()); DefaultChannelAggregates default_channel_aggregates; auto params = StrategyParams( + context.resources(), array.memory_tracker(), tracker_, context.storage_manager(), diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 5166195b92a..6d9c7cfc14d 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -2444,3 +2444,332 @@ TEST_CASE_METHOD( remove_temp_dir( array_uri + "/" + tiledb::sm::constants::array_schema_dir_name); } + +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test CurrentDomain schema APIs", + "[capi][array-schema][current_domain][args]") { + tiledb_current_domain_t* crd = nullptr; + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + + CHECK( + tiledb_array_schema_set_current_domain(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_array_schema_set_current_domain(ctx_, nullptr, nullptr) == + TILEDB_ERR); + tiledb_array_schema_t* array_schema = nullptr; + REQUIRE( + tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &array_schema) == + TILEDB_OK); + CHECK( + tiledb_array_schema_set_current_domain(ctx_, array_schema, nullptr) == + TILEDB_ERR); + + CHECK( + tiledb_array_schema_get_current_domain(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_array_schema_get_current_domain(ctx_, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_array_schema_get_current_domain(ctx_, array_schema, nullptr) == + TILEDB_ERR); + + CHECK( + tiledb_array_schema_evolution_expand_current_domain( + nullptr, nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_array_schema_evolution_expand_current_domain( + ctx_, nullptr, nullptr) == TILEDB_ERR); + tiledb_array_schema_evolution_t* evo; + REQUIRE(tiledb_array_schema_evolution_alloc(ctx_, &evo) == TILEDB_OK); + CHECK( + tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, nullptr) == + TILEDB_ERR); + + tiledb_array_schema_evolution_free(&evo); + tiledb_array_schema_free(&array_schema); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); +} + +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test CurrentDomain schema APIs", + "[capi][array-schema][current_domain][get_set]") { + tiledb_array_schema_t* schema; + REQUIRE(tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &schema) == TILEDB_OK); + + tiledb_current_domain_t* crd = nullptr; + REQUIRE( + tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); + + uint32_t is_empty = 0; + REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + CHECK(is_empty == 1); + + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + + tiledb_dimension_t* d1; + REQUIRE( + tiledb_dimension_alloc( + ctx_, "d1", TILEDB_INT64, &DIM_DOMAIN[0], &TILE_EXTENTS[0], &d1) == + TILEDB_OK); + + tiledb_domain_t* domain; + REQUIRE(tiledb_domain_alloc(ctx_, &domain) == TILEDB_OK); + REQUIRE(tiledb_domain_add_dimension(ctx_, domain, d1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_set_domain(ctx_, schema, domain) == TILEDB_OK); + + tiledb_attribute_t* attr1; + REQUIRE( + tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT32, &attr1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_add_attribute(ctx_, schema, attr1) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_set_capacity(ctx_, schema, CAPACITY) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_cell_order(ctx_, schema, CELL_ORDER) == + TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_tile_order(ctx_, schema, TILE_ORDER) == + TILEDB_OK); + + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_set_current_domain(ctx_, schema, crd) == TILEDB_OK); + + SupportedFsLocal local_fs; + std::string array_name = + local_fs.file_prefix() + local_fs.temp_dir() + ARRAY_NAME; + create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + + // No range was set on the ndrectangle, can't create array + CHECK(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_ERR); + + tiledb_error_t* err; + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + const char* errmsg; + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals("TileDB internal: This current domain has no " + "range specified for dimension idx: 0")); + + tiledb_range_t range; + int64_t min = 2; + int64_t max = 100; + range.min = &min; + range.min_size = sizeof(int64_t); + range.max = &max; + range.max_size = sizeof(int64_t); + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + + // Range is out of schema domain bounds + CHECK(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_ERR); + + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals( + "TileDB internal: This array current domain has ranges past the " + "boundaries of the array schema domain")); + + max = 5; + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_OK); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_attribute_free(&attr1); + tiledb_dimension_free(&d1); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&schema); + + // Open array, read back current domain from schema and check + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); + REQUIRE(tiledb_array_get_schema(ctx_, array, &schema) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); + + REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + tiledb_range_t outrange; + REQUIRE( + tiledb_ndrectangle_get_range_from_name(ctx_, ndr, "d1", &outrange) == + TILEDB_OK); + CHECK(*(int64_t*)outrange.min == min); + CHECK(*(int64_t*)outrange.max == max); + CHECK(outrange.min_size == range.min_size); + CHECK(outrange.max_size == range.max_size); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_array_schema_free(&schema); + REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); + tiledb_array_free(&array); + + delete_array(array_name); + remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); +} + +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test CurrentDomain schema APIs", + "[capi][array-schema][current_domain][evolution]") { + tiledb_array_schema_t* schema; + REQUIRE(tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &schema) == TILEDB_OK); + + tiledb_dimension_t* d1; + REQUIRE( + tiledb_dimension_alloc( + ctx_, "d1", TILEDB_INT64, &DIM_DOMAIN[0], &TILE_EXTENTS[0], &d1) == + TILEDB_OK); + + tiledb_domain_t* domain; + REQUIRE(tiledb_domain_alloc(ctx_, &domain) == TILEDB_OK); + REQUIRE(tiledb_domain_add_dimension(ctx_, domain, d1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_set_domain(ctx_, schema, domain) == TILEDB_OK); + + tiledb_attribute_t* attr1; + REQUIRE( + tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT32, &attr1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_add_attribute(ctx_, schema, attr1) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_set_capacity(ctx_, schema, CAPACITY) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_cell_order(ctx_, schema, CELL_ORDER) == + TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_tile_order(ctx_, schema, TILE_ORDER) == + TILEDB_OK); + + tiledb_current_domain_t* crd = nullptr; + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); + + tiledb_range_t range; + int64_t min = 2; + int64_t max = 5; + range.min = &min; + range.min_size = sizeof(int64_t); + range.max = &max; + range.max_size = sizeof(int64_t); + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_current_domain(ctx_, schema, crd) == TILEDB_OK); + + SupportedFsLocal local_fs; + std::string array_name = + local_fs.file_prefix() + local_fs.temp_dir() + ARRAY_NAME; + create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + + REQUIRE(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_OK); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + + // Evolve the schema + tiledb_array_schema_evolution_t* evo; + REQUIRE(tiledb_array_schema_evolution_alloc(ctx_, &evo) == TILEDB_OK); + + // Expansion with empty domain is an error + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + CHECK( + tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, crd) == + TILEDB_ERR); + + tiledb_error_t* err; + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + const char* errmsg; + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals( + "ArraySchemaEvolution: Unable to expand the array current domain, " + "the new current domain specified is empty")); + + REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); + max = 3; + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, crd) == + TILEDB_OK); + + // The shape is smaller here so it should fail. + CHECK(tiledb_array_evolve(ctx_, array_name.c_str(), evo) == TILEDB_ERR); + + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals( + "ArraySchema: The current domain of an array can only be expanded, " + "please adjust your new current domain object.")); + + max = 7; + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_array_evolve(ctx_, array_name.c_str(), evo) == TILEDB_OK); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_array_schema_evolution_free(&evo); + tiledb_attribute_free(&attr1); + tiledb_dimension_free(&d1); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&schema); + + // Open array, read back current domain from schema and check + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); + REQUIRE(tiledb_array_get_schema(ctx_, array, &schema) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); + + REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + tiledb_range_t outrange; + REQUIRE( + tiledb_ndrectangle_get_range_from_name(ctx_, ndr, "d1", &outrange) == + TILEDB_OK); + CHECK(*(int64_t*)outrange.min == min); + CHECK(*(int64_t*)outrange.max == max); + CHECK(outrange.min_size == range.min_size); + CHECK(outrange.max_size == range.max_size); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_array_schema_free(&schema); + REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); + tiledb_array_free(&array); + + delete_array(array_name); + remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); +} diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 5f8abc0260d..b7835cb4e02 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -615,6 +615,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["rest.load_enumerations_on_array_open"] = "false"; all_param_values["rest.use_refactored_array_open"] = "true"; all_param_values["rest.use_refactored_array_open_and_query_submit"] = "true"; + all_param_values["rest.payer_namespace"] = ""; all_param_values["sm.allow_separate_attribute_writes"] = "false"; all_param_values["sm.allow_updates_experimental"] = "false"; all_param_values["sm.encryption_key"] = ""; diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index 4bff6dbb0be..5370a65cfb0 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -51,7 +51,7 @@ struct ConsolidationWithTimestampsFx { // TileDB context. Context ctx_; VFS vfs_; - sm::StorageManager* sm_; + sm::ContextResources* resources_; // Constructors/destructors. ConsolidationWithTimestampsFx(); @@ -99,7 +99,7 @@ ConsolidationWithTimestampsFx::ConsolidationWithTimestampsFx() Config config; config.set("sm.consolidation.buffer_size", "1000"); ctx_ = Context(config); - sm_ = ctx_.ptr().get()->storage_manager(); + resources_ = &ctx_.ptr().get()->resources(); vfs_ = VFS(ctx_); } @@ -113,7 +113,7 @@ void ConsolidationWithTimestampsFx::set_legacy() { config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); ctx_ = Context(config); - sm_ = ctx_.ptr().get()->storage_manager(); + resources_ = &ctx_.ptr().get()->resources(); vfs_ = VFS(ctx_); } @@ -419,7 +419,7 @@ TEST_CASE_METHOD( }; for (auto& ts : test_timestamps) { - sm::ArrayDirectory array_dir(sm_->resources(), array_uri, ts[0], ts[1]); + sm::ArrayDirectory array_dir(*resources_, array_uri, ts[0], ts[1]); // Check that only the consolidated fragment is visible. auto filtered_fragment_uris = array_dir.filtered_fragment_uris(false); @@ -437,7 +437,7 @@ TEST_CASE_METHOD( }; for (auto& ts : test_timestamps) { - sm::ArrayDirectory array_dir(sm_->resources(), array_uri, ts[0], ts[1]); + sm::ArrayDirectory array_dir(*resources_, array_uri, ts[0], ts[1]); // Check that no fragment is visible auto filtered_fragment_uris = array_dir.filtered_fragment_uris(false); @@ -525,7 +525,7 @@ TEST_CASE_METHOD( }; for (auto& ts : test_timestamps) { - sm::ArrayDirectory array_dir(sm_->resources(), array_uri, ts[0], ts[1]); + sm::ArrayDirectory array_dir(*resources_, array_uri, ts[0], ts[1]); // Check that only the first fragment is visible on an old array. auto filtered_fragment_uris = array_dir.filtered_fragment_uris(false); @@ -543,7 +543,7 @@ TEST_CASE_METHOD( }; for (auto& ts : test_timestamps) { - sm::ArrayDirectory array_dir(sm_->resources(), array_uri, ts[0], ts[1]); + sm::ArrayDirectory array_dir(*resources_, array_uri, ts[0], ts[1]); // Check that no fragment is visible auto filtered_fragment_uris = array_dir.filtered_fragment_uris(false); @@ -1274,7 +1274,7 @@ TEST_CASE_METHOD( cfg["sm.mem.consolidation.reader_weight"] = "5000"; cfg["sm.mem.consolidation.writer_weight"] = "5000"; ctx_ = Context(cfg); - sm_ = ctx_.ptr().get()->storage_manager(); + resources_ = &ctx_.ptr().get()->resources(); vfs_ = VFS(ctx_); // Write first fragment. diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 002ee1a53dd..8373b9f8023 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -59,7 +59,6 @@ struct DeletesFx { VFSTestSetup vfs_test_setup_; Context ctx_; VFS vfs_; - sm::StorageManager* sm_; const std::string array_name_; // to be used for direct VFS operations in [rest] testcases where array_name_ @@ -126,7 +125,6 @@ DeletesFx::DeletesFx() vfs_test_setup_.update_config(config.ptr().get()); ctx_ = vfs_test_setup_.ctx(); vfs_ = VFS(ctx_); - sm_ = ctx_.ptr().get()->storage_manager(); } void DeletesFx::set_purge_deleted_cells() { @@ -136,7 +134,6 @@ void DeletesFx::set_purge_deleted_cells() { vfs_test_setup_.update_config(config.ptr().get()); ctx_ = vfs_test_setup_.ctx(); vfs_ = VFS(ctx_); - sm_ = ctx_.ptr().get()->storage_manager(); } void DeletesFx::set_legacy() { @@ -147,7 +144,6 @@ void DeletesFx::set_legacy() { vfs_test_setup_.update_config(config.ptr().get()); ctx_ = vfs_test_setup_.ctx(); vfs_ = VFS(ctx_); - sm_ = ctx_.ptr().get()->storage_manager(); } void DeletesFx::create_dir(const std::string& path) { @@ -184,7 +180,6 @@ void DeletesFx::create_sparse_array(bool allows_dups, bool encrypt) { vfs_test_setup_.update_config(config.ptr().get()); ctx_ = vfs_test_setup_.ctx(); vfs_ = VFS(ctx_); - sm_ = ctx_.ptr().get()->storage_manager(); // Create dimensions. auto d1 = Dimension::create(ctx_, "d1", {{1, 4}}, 2); diff --git a/test/src/unit-cppapi-fill_values.cc b/test/src/unit-cppapi-fill_values.cc index 47c1cfe1cb3..b4f6e8ea7c9 100644 --- a/test/src/unit-cppapi-fill_values.cc +++ b/test/src/unit-cppapi-fill_values.cc @@ -98,7 +98,7 @@ void create_array_1d( schema.set_domain(domain); schema.add_attributes(a1, a2, a3); - CHECK_NOTHROW(Array::create(array_name, schema)); + Array::create(array_name, schema); } void write_array_1d_partial( diff --git a/test/src/unit-cppapi-update-queries.cc b/test/src/unit-cppapi-update-queries.cc index 897eb91eb11..3c1c33b8dc6 100644 --- a/test/src/unit-cppapi-update-queries.cc +++ b/test/src/unit-cppapi-update-queries.cc @@ -51,7 +51,6 @@ struct UpdatesFx { // TileDB context. Context ctx_; VFS vfs_; - sm::StorageManager* sm_; std::string key_ = "0123456789abcdeF0123456789abcdeF"; const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; @@ -84,7 +83,6 @@ UpdatesFx::UpdatesFx() config.set("sm.consolidation.buffer_size", "1000"); config["sm.allow_updates_experimental"] = "true"; ctx_ = Context(config); - sm_ = ctx_.ptr().get()->storage_manager(); vfs_ = VFS(ctx_); } @@ -217,7 +215,6 @@ void UpdatesFx::check_update_conditions( // Compare to negated condition. auto cmp = qcs[i].ptr()->query_condition_->negated_condition(); CHECK(tiledb::test::ast_equal(conditions->at(i).ast(), cmp.ast())); - auto& loaded_update_values = update_values->at(i); for (uint64_t j = 0; j < uvs[i].size(); j++) { CHECK(uvs[i][j].field_name() == loaded_update_values[j].field_name()); diff --git a/test/src/unit-cppapi-webp-filter.cc b/test/src/unit-cppapi-webp-filter.cc index 259e668da30..129d26035b1 100644 --- a/test/src/unit-cppapi-webp-filter.cc +++ b/test/src/unit-cppapi-webp-filter.cc @@ -314,7 +314,7 @@ TEMPLATE_LIST_TEST_CASE( "WebP filter can only be applied to dense arrays")); } - REQUIRE_NOTHROW(Array::create(webp_array_name, valid_schema)); + Array::create(webp_array_name, valid_schema); if (vfs.is_dir(webp_array_name)) { vfs.remove_dir(webp_array_name); diff --git a/test/src/unit-curl.cc b/test/src/unit-curl.cc index 9241941ad2f..e3e79017bc7 100644 --- a/test/src/unit-curl.cc +++ b/test/src/unit-curl.cc @@ -128,3 +128,29 @@ TEST_CASE( .ok()); CHECK(rest_client.rest_server() == "http://localhost:8080"); } + +TEST_CASE( + "RestClient: Ensure custom headers are set", + "[rest-client][custom-headers]") { + tiledb::sm::Config cfg; + REQUIRE(cfg.set("rest.custom_headers.abc", "def").ok()); + REQUIRE(cfg.set("rest.custom_headers.ghi", "jkl").ok()); + + ContextResources resources( + cfg, tiledb::test::g_helper_logger(), 1, 1, "test"); + const auto& extra_headers = resources.rest_client()->extra_headers(); + CHECK(extra_headers.at("abc") == "def"); + CHECK(extra_headers.at("ghi") == "jkl"); +} + +TEST_CASE( + "RestClient: Ensure payer namespace is set", + "[rest-client][payer-namespace]") { + tiledb::sm::Config cfg; + REQUIRE(cfg.set("rest.payer_namespace", "foo").ok()); + + ContextResources resources( + cfg, tiledb::test::g_helper_logger(), 1, 1, "test"); + const auto& extra_headers = resources.rest_client()->extra_headers(); + CHECK(extra_headers.at("X-Payer") == "foo"); +} diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index e8aee8b2df7..3e1a1dedaba 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2064,11 +2064,11 @@ TEST_CASE_METHOD( auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); - throw_if_not_ok(tiledb::sm::Array::evolve_array_schema( + tiledb::sm::Array::evolve_array_schema( ctx_.resources(), array->array_uri(), ase.get(), - array->get_encryption_key())); + array->get_encryption_key()); // Check that we can not rewrite the query condition. array = get_array(QueryType::READ); @@ -2398,10 +2398,10 @@ TEST_CASE_METHOD( auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); qc1.set_use_enumeration(false); - Query q1(ctx_.storage_manager(), array); + Query q1(ctx_.resources(), ctx_.storage_manager(), array); throw_if_not_ok(q1.set_condition(qc1)); - Query q2(ctx_.storage_manager(), array); + Query q2(ctx_.resources(), ctx_.storage_manager(), array); ser_des_query(&q1, &q2, client_side, ser_type); auto qc2 = q2.condition(); @@ -2434,10 +2434,10 @@ TEST_CASE_METHOD( throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3)); - Query q1(ctx_.storage_manager(), array); + Query q1(ctx_.resources(), ctx_.storage_manager(), array); throw_if_not_ok(q1.set_condition(qc3)); - Query q2(ctx_.storage_manager(), array); + Query q2(ctx_.resources(), ctx_.storage_manager(), array); ser_des_query(&q1, &q2, client_side, ser_type); auto qc4 = q2.condition(); @@ -2881,7 +2881,7 @@ shared_ptr EnumerationFx::create_schema() { void EnumerationFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); + Array::create(ctx_.resources(), uri_, schema, enc_key_); } shared_ptr EnumerationFx::get_array(QueryType type) { diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index b4702f87f30..b7368840d7a 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -357,7 +357,7 @@ RequestHandlerFx::~RequestHandlerFx() { void RequestHandlerFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); + Array::create(ctx_.resources(), uri_, schema, enc_key_); } void RequestHandlerFx::delete_array() { diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 1999bd8e5d5..b5e6094707b 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -709,37 +709,42 @@ TEST_CASE( #ifdef HAVE_AZURE TEST_CASE("VFS: Construct Azure Blob Storage endpoint URIs", "[azure][uri]") { + // Test the construction of Azure Blob Storage URIs from account name and SAS + // token. We are not actually connecting to Azure Blob Storage in this test. std::string sas_token, custom_endpoint, expected_endpoint; SECTION("No SAS token") { sas_token = ""; - expected_endpoint = "https://devstoreaccount1.blob.core.windows.net"; + expected_endpoint = "https://exampleaccount.blob.core.windows.net"; } SECTION("SAS token without leading question mark") { sas_token = "baz=qux&foo=bar"; expected_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + "https://exampleaccount.blob.core.windows.net?baz=qux&foo=bar"; } SECTION("SAS token with leading question mark") { sas_token = "?baz=qux&foo=bar"; expected_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + "https://exampleaccount.blob.core.windows.net?baz=qux&foo=bar"; } SECTION("SAS token in both endpoint and config option") { sas_token = "baz=qux&foo=bar"; custom_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + "https://exampleaccount.blob.core.windows.net?baz=qux&foo=bar"; expected_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; - } - SECTION("No SAS token") { - sas_token = ""; - expected_endpoint = "https://devstoreaccount1.blob.core.windows.net"; + "https://exampleaccount.blob.core.windows.net?baz=qux&foo=bar"; } Config config; require_tiledb_ok( - config.set("vfs.azure.storage_account_name", "devstoreaccount1")); + config.set("vfs.azure.storage_account_name", "exampleaccount")); require_tiledb_ok(config.set("vfs.azure.blob_endpoint", custom_endpoint)); require_tiledb_ok(config.set("vfs.azure.storage_sas_token", sas_token)); + if (sas_token.empty()) { + // If the SAS token is empty, the VFS will try to connect to Microsoft Entra + // ID to obtain credentials, which can take a long time because of retries. + // Set a dummy access key (which won't be used because we are not going to + // perform any requests) to prevent Entra ID from being chosen. + require_tiledb_ok(config.set("vfs.azure.storage_account_key", "foobar")); + } tiledb::sm::Azure azure; ThreadPool thread_pool(1); require_tiledb_ok(azure.init(config, &thread_pool)); diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index e8406640e12..e6a646dd883 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -119,6 +119,8 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/stats.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/capi_string.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/subarray.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/ndrectangle.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/current_domain.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/subarray_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/type.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/utils.h @@ -169,7 +171,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/gzip_compressor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/lz4_compressor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/rle_compressor.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/util/gzip_wrappers.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/zstd_compressor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/config/config.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/config/config_iter.cc @@ -245,6 +246,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object_iter.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object_mutex.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -437,36 +439,8 @@ target_link_libraries(TILEDB_CORE_OBJECTS target_link_libraries(TILEDB_CORE_OBJECTS INTERFACE object_store_definitions) -############################################################ -# provide actions/target for preparation of magic.mgc data for embedding/build - -set(MGC_GZIPPED_BIN_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/..") -set(MGC_GZIPPED_BIN_OUTPUT_FILE "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}/magic_mgc_gzipped.bin") -set(MGC_GZIPPED_BIN_INPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/sm/misc") -set(MGC_GZIPPED_BIN_INPUT_FILE "${MGC_GZIPPED_BIN_INPUT_DIRECTORY}/magic_mgc_gzipped.bin.tar.bz2") - -add_custom_command( - OUTPUT "${MGC_GZIPPED_BIN_OUTPUT_FILE}" - DEPENDS "${MGC_GZIPPED_BIN_INPUT_FILE}" - COMMAND ${CMAKE_COMMAND} -E tar x "${MGC_GZIPPED_BIN_INPUT_FILE}" - WORKING_DIRECTORY "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}" -) -add_custom_target(gen_mgc_unarch ALL - DEPENDS ${MGC_GZIPPED_BIN_OUTPUT_FILE} -) add_dependencies(TILEDB_CORE_OBJECTS gen_mgc_unarch) -add_custom_target( - update-embedded-magic-data - COMMAND "$" < "${libmagic_DICTIONARY}" "${MGC_GZIPPED_BIN_OUTPUT_FILE}" - # need to work in 'local' directory with no prefix paths so no paths are included in archive - WORKING_DIRECTORY "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}" - COMMAND ${CMAKE_COMMAND} -E tar cvj "magic_mgc_gzipped.bin.tar.bz2" "magic_mgc_gzipped.bin" - COMMAND ${CMAKE_COMMAND} -E copy "magic_mgc_gzipped.bin.tar.bz2" "${MGC_GZIPPED_BIN_INPUT_FILE}" - DEPENDS "${libmagic_DICTIONARY}" - COMMENT "Re-generate ${MGC_GZIPPED_BIN_INPUT_FILE} for embedded magic.mgc support" -) - target_include_directories(TILEDB_CORE_OBJECTS PRIVATE "${TILEDB_CORE_INCLUDE_DIR}" diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index 11d62110bcd..e511ee25a45 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -58,6 +58,9 @@ add_subdirectory(buffer) # `buffer_list`: depends on `buffer` add_subdirectory(buffer_list) +# `current_domain`: depends on `ndrectangle` +add_subdirectory(current_domain) + # `error`: no dependencies add_subdirectory(error) @@ -100,6 +103,9 @@ add_subdirectory(attribute) # `group`: depends on `config`, `context` add_subdirectory(group) +# `ndrectangle`: depends on `domain` +add_subdirectory(ndrectangle) + # `object`: no dependencies add_subdirectory(object) diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 161a5a75ae6..8cb17bdf53e 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -709,6 +709,13 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `rest.capnp_traversal_limit`
* CAPNP traversal limit used in the deserialization of messages(bytes)
* **Default**: 536870912 (512MB) + * - `rest.custom_headers.*`
+ * (Optional) Prefix for custom headers on REST requests. For each custom + * header, use "rest.custom_headers.header_key" = "header_value"
+ * **Optional. No Default** + * - `rest.payer_namespace`
+ * The namespace that should be charged for the request.
+ * **Default**: no default set * - `filestore.buffer_size`
* Specifies the size in bytes of the internal buffers used in the filestore * API. The size should be bigger than the minimum tile size filestore diff --git a/tiledb/api/c_api/current_domain/CMakeLists.txt b/tiledb/api/c_api/current_domain/CMakeLists.txt new file mode 100644 index 00000000000..22b56da1a59 --- /dev/null +++ b/tiledb/api/c_api/current_domain/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# tiledb/api/c_api/current_domain/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + current_domain_api.cc + ) +gather_sources(${SOURCES}) + +commence(object_library capi_current_domain) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(baseline current_domain capi_context_stub) +conclude(object_library) + +add_test_subdirectory() + diff --git a/tiledb/api/c_api/current_domain/current_domain_api.cc b/tiledb/api/c_api/current_domain/current_domain_api.cc new file mode 100644 index 00000000000..49b219a21c3 --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api.cc @@ -0,0 +1,155 @@ +/** + * @file tiledb/api/c_api/current_domain/current_domain_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the current domain C API of TileDB. + **/ + +#include "current_domain_api_external_experimental.h" +#include "current_domain_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/domain/domain_api_internal.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::api { + +capi_return_t tiledb_current_domain_create( + tiledb_ctx_t* ctx, tiledb_current_domain_t** current_domain) { + ensure_context_is_valid(ctx); + ensure_output_pointer_is_valid(current_domain); + + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_CREATE); + *current_domain = tiledb_current_domain_handle_t::make_handle( + memory_tracker, sm::constants::current_domain_version); + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_free( + tiledb_current_domain_t** current_domain) { + ensure_output_pointer_is_valid(current_domain); + ensure_handle_is_valid(*current_domain); + tiledb_current_domain_handle_t::break_handle(*current_domain); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_set_ndrectangle( + tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t* ndr) { + ensure_handle_is_valid(current_domain); + ensure_handle_is_valid(ndr); + + current_domain->current_domain()->set_ndrectangle(ndr->ndrectangle()); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_get_ndrectangle( + tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t** ndr) { + ensure_handle_is_valid(current_domain); + ensure_output_pointer_is_valid(ndr); + + *ndr = tiledb_ndrectangle_handle_t::make_handle( + current_domain->current_domain()->ndrectangle()); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_get_is_empty( + tiledb_current_domain_t* current_domain, uint32_t* is_empty) { + ensure_handle_is_valid(current_domain); + ensure_output_pointer_is_valid(is_empty); + + *is_empty = current_domain->current_domain()->empty(); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_get_type( + tiledb_current_domain_t* current_domain, + tiledb_current_domain_type_t* type) { + ensure_handle_is_valid(current_domain); + ensure_output_pointer_is_valid(type); + + *type = static_cast( + current_domain->current_domain()->type()); + + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_plain; +using tiledb::api::api_entry_with_context; + +CAPI_INTERFACE( + current_domain_create, + tiledb_ctx_t* ctx, + tiledb_current_domain_t** current_domain) { + return api_entry_with_context( + ctx, current_domain); +} + +CAPI_INTERFACE(current_domain_free, tiledb_current_domain_t** current_domain) { + return api_entry_plain( + current_domain); +} + +CAPI_INTERFACE( + current_domain_set_ndrectangle, + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t* ndr) { + return api_entry_plain( + current_domain, ndr); +} + +CAPI_INTERFACE( + current_domain_get_ndrectangle, + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t** ndr) { + return api_entry_plain( + current_domain, ndr); +} + +CAPI_INTERFACE( + current_domain_get_is_empty, + tiledb_current_domain_t* current_domain, + uint32_t* is_empty) { + return api_entry_plain( + current_domain, is_empty); +} + +CAPI_INTERFACE( + current_domain_get_type, + tiledb_current_domain_t* current_domain, + tiledb_current_domain_type_t* type) { + return api_entry_plain( + current_domain, type); +} diff --git a/tiledb/api/c_api/current_domain/current_domain_api_enum.h b/tiledb/api/c_api/current_domain/current_domain_api_enum.h new file mode 100644 index 00000000000..1d3154ed68c --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api_enum.h @@ -0,0 +1,34 @@ +/* + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * NOTE: The values of these enums are serialized to the array schema and/or + * fragment metadata. Therefore, the values below should never change, + * otherwise backwards compatibility breaks. + */ + +// clang-format off +#ifdef TILEDB_CURRENT_DOMAIN_TYPE_ENUM + TILEDB_CURRENT_DOMAIN_TYPE_ENUM(NDRECTANGLE) = 0, +#endif diff --git a/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h b/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h new file mode 100644 index 00000000000..cc3db794f1a --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h @@ -0,0 +1,185 @@ +/** + * @file + * tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the current_domain experimental C API for TileDB. + */ + +#ifndef TILEDB_CAPI_CURRENT_DOMAIN_API_EXTERNAL_EXPERIMENTAL_H +#define TILEDB_CAPI_CURRENT_DOMAIN_API_EXTERNAL_EXPERIMENTAL_H + +#include "tiledb/api/c_api/api_external_common.h" +#include "tiledb/api/c_api/context/context_api_external.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** The current domain type */ +typedef enum { +#define TILEDB_CURRENT_DOMAIN_TYPE_ENUM(id) TILEDB_##id +#include "tiledb/api/c_api/current_domain/current_domain_api_enum.h" +#undef TILEDB_CURRENT_DOMAIN_TYPE_ENUM +} tiledb_current_domain_type_t; + +typedef struct tiledb_current_domain_handle_t tiledb_current_domain_t; + +/** + * Create a current domain object + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param ctx The TileDB context + * @param current_domain The current domain to be allocated + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_create( + tiledb_ctx_t* ctx, + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + +/** + * Free the resources associated with a current domain object + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param current_domain The current domain to be freed + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_free( + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + +/** + * Set a N-dimensional rectangle representation on a current domain. + * Error if the current domain passed is not empty. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * + * tiledb_range_t range; + * range.min = &min; + * range.min_size = sizeof(min); + * range.max = &max; + * range.max_size = sizeof(max); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim", &range); + * + * tiledb_current_domain_set_ndrectangle(current_domain, ndr); + * + * tiledb_ndrectangle_free(&ndr); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param current_domain The current domain to modify + * @param ndr The N-dimensional rectangle to be set + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_set_ndrectangle( + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t* ndr) TILEDB_NOEXCEPT; + +/** + * Get the N-dimensional rectangle associated with the current domain object. + * Error if the current domain is empty or a different representation is set. + * + * It is the responsability of the caller to free the resources associated + * with the ndrectangle when the handle isn't needed anymore. + * + * **Example:** + * + * @code{.c} + * tiledb_ndrectangle_t *ndr; + * tiledb_current_domain_get_ndrectangle(current_domain, &ndr); + * @endcode + * + * @param current_domain The current domain to query + * @param ndr The N-dimensional rectangle of the current domain + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_get_ndrectangle( + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t** ndr) TILEDB_NOEXCEPT; + +/** + * Query if the current domain object is empty or not. + * + * **Example:** + * + * @code{.c} + * uint32_t empty = 0; + * tiledb_current_domain_get_is_empty(current_domain, &empty); + * @endcode + * + * @param current_domain The current domain to query + * @param is_empty True if nothing is set on the current domain + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_get_is_empty( + tiledb_current_domain_t* current_domain, + uint32_t* is_empty) TILEDB_NOEXCEPT; + +/** + * Query the type of current domain set on the object + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_type_t type; + * tiledb_current_domain_get_type(current_domain, &type); + * @endcode + * + * @param current_domain The current domain to query + * @param type The type of representation set on the current domain + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_get_type( + tiledb_current_domain_t* current_domain, + tiledb_current_domain_type_t* type) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_CURRENT_DOMAIN_API_EXTERNAL_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/current_domain/current_domain_api_internal.h b/tiledb/api/c_api/current_domain/current_domain_api_internal.h new file mode 100644 index 00000000000..18d14574547 --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api_internal.h @@ -0,0 +1,83 @@ +/** + * @file tiledb/api/c_api/current_domain/current_domain_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internal C API implementation of the Current Domain + */ + +#ifndef TILEDB_CAPI_CURRENT_DOMAIN_INTERNAL_H +#define TILEDB_CAPI_CURRENT_DOMAIN_INTERNAL_H + +#include "tiledb/api/c_api_support/argument_validation.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/sm/array_schema/current_domain.h" + +struct tiledb_current_domain_handle_t + : public tiledb::api::CAPIHandle { + /** + * Type name + */ + static constexpr std::string_view object_type_name{"tiledb_current_domain_t"}; + + private: + std::shared_ptr current_domain_; + + public: + /** + * Default constructor doesn't make sense + */ + tiledb_current_domain_handle_t() = delete; + + /** + * Constructs a handle with an empty CurrentDomain instance. + * @param memory_tracker The tracker to use in the internal CurrentDomain + * @param version The on-disk format version of the CurrentDomain + */ + explicit tiledb_current_domain_handle_t( + shared_ptr memory_tracker, + format_version_t version) + : current_domain_{make_shared( + HERE(), memory_tracker, version)} { + } + + /** + * Constructor. + * @param current_domain A CurrentDomain instance to assign to this handle + */ + explicit tiledb_current_domain_handle_t( + std::shared_ptr current_domain) + : current_domain_(current_domain) { + } + + [[nodiscard]] inline shared_ptr current_domain() + const { + return current_domain_; + } +}; + +#endif // TILEDB_CAPI_CURRENT_DOMAIN_INTERNAL_H diff --git a/tiledb/api/c_api/current_domain/test/CMakeLists.txt b/tiledb/api/c_api/current_domain/test/CMakeLists.txt new file mode 100644 index 00000000000..a9af423b2dc --- /dev/null +++ b/tiledb/api/c_api/current_domain/test/CMakeLists.txt @@ -0,0 +1,36 @@ +# +# tiledb/api/c_api/current_domain/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_current_domain) + this_target_sources(unit_capi_current_domain.cc) + if (NOT MSVC) + target_compile_options(unit_capi_current_domain PRIVATE -Wno-deprecated-declarations) + endif() + this_target_object_libraries(capi_current_domain) + this_target_link_libraries(tiledb_test_support_lib) +conclude(unit_test) diff --git a/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc b/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc new file mode 100644 index 00000000000..f2f52fd19df --- /dev/null +++ b/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc @@ -0,0 +1,41 @@ +/** + * @file + * tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../current_domain_api_external_experimental.h" + +int main() { + tiledb_current_domain_create(nullptr, nullptr); + tiledb_current_domain_free(nullptr); + tiledb_current_domain_set_ndrectangle(nullptr, nullptr); + tiledb_current_domain_get_ndrectangle(nullptr, nullptr); + tiledb_current_domain_get_is_empty(nullptr, nullptr); + tiledb_current_domain_get_type(nullptr, nullptr); + + return 0; +} diff --git a/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc b/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc new file mode 100644 index 00000000000..d34db5fc529 --- /dev/null +++ b/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc @@ -0,0 +1,153 @@ +/** + * @file tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the current_domain C API. + */ + +#include +#include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/config/config_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_internal.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" + +using namespace tiledb::test; + +struct CAPICurrentDomainFx : TemporaryDirectoryFixture { + CAPICurrentDomainFx() + : array_name_(temp_dir_ + "curent_domain_array") { + create_domain(); + } + ~CAPICurrentDomainFx() { + free_domain(); + } + void create_domain(); + void free_domain(); + + std::string array_name_; + tiledb_dimension_t *d1_, *d2_; + tiledb_domain_t* domain_; +}; + +void CAPICurrentDomainFx::create_domain() { + // Create dimensions + uint64_t tile_extents[] = {2, 2}; + uint64_t dim_domain[] = {1, 10, 1, 10}; + + int rc = tiledb_dimension_alloc( + ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_dimension_alloc( + ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2_); + CHECK(rc == TILEDB_OK); + + // Create domain + rc = tiledb_domain_alloc(ctx, &domain_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d2_); + CHECK(rc == TILEDB_OK); +} + +void CAPICurrentDomainFx::free_domain() { + tiledb_dimension_free(&d1_); + tiledb_dimension_free(&d2_); + tiledb_domain_free(&domain_); +} + +TEST_CASE_METHOD( + CAPICurrentDomainFx, + "C API: argument validation", + "[capi][current_domain][args]") { + CHECK( + tiledb_current_domain_create(nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK(tiledb_current_domain_create(ctx, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_free(nullptr) == TILEDB_ERR); + tiledb_current_domain_t* crd = nullptr; + CHECK(tiledb_current_domain_free(&crd) == TILEDB_ERR); + + crd = nullptr; + CHECK(tiledb_current_domain_create(ctx, &crd) == TILEDB_OK); + + CHECK(tiledb_current_domain_set_ndrectangle(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_set_ndrectangle(crd, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_get_ndrectangle(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_ndrectangle(crd, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_get_is_empty(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_is_empty(crd, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_get_type(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_type(crd, nullptr) == TILEDB_ERR); + + tiledb_current_domain_free(&crd); +} + +TEST_CASE_METHOD( + CAPICurrentDomainFx, + "C API: Setting ND rectangles works", + "[capi][current_domain][ndr]") { + tiledb_current_domain_t* crd = nullptr; + REQUIRE(tiledb_current_domain_create(ctx, &crd) == TILEDB_OK); + + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); + + uint32_t is_empty = 0; + REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + CHECK(is_empty == 1); + + tiledb_current_domain_type_t type; + CHECK(tiledb_current_domain_get_type(crd, &type) == TILEDB_ERR); + + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + + REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + CHECK(is_empty == 0); + + REQUIRE(tiledb_current_domain_get_type(crd, &type) == TILEDB_OK); + CHECK(type == TILEDB_NDRECTANGLE); + + tiledb_ndrectangle_t* out_ndr = nullptr; + REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &out_ndr) == TILEDB_OK); + CHECK(out_ndr != nullptr); + + // Verify that they point to the same tiledb::sm::NDRectangle instance. + CHECK(ndr->ndrectangle().get() == out_ndr->ndrectangle().get()); + + tiledb_ndrectangle_free(&out_ndr); + tiledb_ndrectangle_free(&ndr); + tiledb_current_domain_free(&crd); +} diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index ba4b15371c2..03ad16cc12d 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -81,8 +81,7 @@ capi_return_t tiledb_group_create( tiledb_ctx_handle_t* ctx, const char* group_uri) { ensure_group_uri_argument_is_valid(group_uri); - throw_if_not_ok( - tiledb::sm::Group::create(ctx->resources(), tiledb::sm::URI(group_uri))); + tiledb::sm::Group::create(ctx->resources(), tiledb::sm::URI(group_uri)); return TILEDB_OK; } @@ -576,8 +575,7 @@ capi_return_t tiledb_group_consolidate_metadata( ensure_group_uri_argument_is_valid(group_uri); auto cfg = (config == nullptr) ? ctx->config() : config->config(); - throw_if_not_ok(tiledb::sm::Group::consolidate_metadata( - ctx->resources(), group_uri, cfg)); + tiledb::sm::Group::consolidate_metadata(ctx->resources(), group_uri, cfg); return TILEDB_OK; } diff --git a/tiledb/api/c_api/ndrectangle/CMakeLists.txt b/tiledb/api/c_api/ndrectangle/CMakeLists.txt new file mode 100644 index 00000000000..6e22000eebb --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# tiledb/api/c_api/ndrectangle/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + ndrectangle_api.cc + ) +gather_sources(${SOURCES}) + +commence(object_library capi_ndrectangle) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(baseline ndrectangle capi_context_stub) +conclude(object_library) + +add_test_subdirectory() + diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc b/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc new file mode 100644 index 00000000000..5acd46226e5 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc @@ -0,0 +1,238 @@ +/** + * @file tiledb/api/c_api/ndrectangle/ndrectangle_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the NDRectangle C API of TileDB. + **/ + +#include "ndrectangle_api_external_experimental.h" +#include "ndrectangle_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/dimension/dimension_api_internal.h" +#include "tiledb/api/c_api/domain/domain_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::api { + +/** + * Ensure the argument is a valid range pointer + * + * @param range A range struct + */ +inline void ensure_range_ptr_is_valid(tiledb_range_t* range) { + ensure_output_pointer_is_valid(range); +} + +/** + * Ensure the argument is a valid char pointer + * + * @param name A char pointer + */ +inline void ensure_dim_name_is_valid(const char* name) { + if (!name) { + throw CAPIStatusException("argument `name` may not be nullptr"); + } +} + +/** + * Lays out an internal Range into a tiledb_range_t + * + * @param r The internal sm Range + * @param range The C API range + */ +inline void internal_range_to_range(const Range& r, tiledb_range_t* range) { + if (r.var_size()) { + range->min_size = r.start_size(); + range->max_size = r.end_size(); + range->min = static_cast(r.start_str().data()); + range->max = static_cast(r.end_str().data()); + } else { + range->min_size = r.size() / 2; + range->max_size = r.size() / 2; + range->min = static_cast(r.start_fixed()); + range->max = static_cast(r.end_fixed()); + } +} + +capi_return_t tiledb_ndrectangle_alloc( + tiledb_ctx_t* ctx, tiledb_domain_t* domain, tiledb_ndrectangle_t** ndr) { + ensure_context_is_valid(ctx); + ensure_domain_is_valid(domain); + ensure_output_pointer_is_valid(ndr); + + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_CREATE); + *ndr = tiledb_ndrectangle_handle_t::make_handle( + memory_tracker, domain->copy_domain()); + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_free(tiledb_ndrectangle_t** ndr) { + ensure_output_pointer_is_valid(ndr); + ensure_handle_is_valid(*ndr); + tiledb_ndrectangle_handle_t::break_handle(*ndr); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_get_range_from_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_dim_name_is_valid(name); + ensure_range_ptr_is_valid(range); + + auto& r = ndr->ndrectangle()->get_range_for_name(name); + + internal_range_to_range(r, range); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_get_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_range_ptr_is_valid(range); + + auto& r = ndr->ndrectangle()->get_range(idx); + internal_range_to_range(r, range); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_set_range_for_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_dim_name_is_valid(name); + ensure_range_ptr_is_valid(range); + + Range r; + auto idx = ndr->ndrectangle()->domain()->get_dimension_index(name); + if (ndr->ndrectangle()->domain()->dimension_ptr(idx)->var_size()) { + r.set_range_var(range->min, range->min_size, range->max, range->max_size); + } else { + r.set_range_fixed(range->min, range->max, range->min_size); + } + + ndr->ndrectangle()->set_range_for_name(r, name); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_set_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_range_ptr_is_valid(range); + + Range r; + if (ndr->ndrectangle()->domain()->dimension_ptr(idx)->var_size()) { + r.set_range_var(range->min, range->min_size, range->max, range->max_size); + } else { + r.set_range_fixed(range->min, range->max, range->min_size); + } + + ndr->ndrectangle()->set_range(r, idx); + + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_plain; +using tiledb::api::api_entry_with_context; + +CAPI_INTERFACE( + ndrectangle_alloc, + tiledb_ctx_t* ctx, + tiledb_domain_t* domain, + tiledb_ndrectangle_t** ndr) { + return api_entry_with_context( + ctx, domain, ndr); +} + +CAPI_INTERFACE(ndrectangle_free, tiledb_ndrectangle_t** ndr) { + return api_entry_plain(ndr); +} + +CAPI_INTERFACE( + ndrectangle_get_range_from_name, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + return api_entry_with_context< + tiledb::api::tiledb_ndrectangle_get_range_from_name>( + ctx, ndr, name, range); +} + +CAPI_INTERFACE( + ndrectangle_get_range, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + return api_entry_with_context( + ctx, ndr, idx, range); +} + +CAPI_INTERFACE( + ndrectangle_set_range_for_name, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + return api_entry_with_context< + tiledb::api::tiledb_ndrectangle_set_range_for_name>( + ctx, ndr, name, range); +} + +CAPI_INTERFACE( + ndrectangle_set_range, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + return api_entry_with_context( + ctx, ndr, idx, range); +} diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h b/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h new file mode 100644 index 00000000000..e0f80ea22b8 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h @@ -0,0 +1,205 @@ +/** + * @file + * tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the NDRectangle experimental C API for TileDB. + */ + +#ifndef TILEDB_CAPI_NDRECTANGLE_API_EXTERNAL_EXPERIMENTAL_H +#define TILEDB_CAPI_NDRECTANGLE_API_EXTERNAL_EXPERIMENTAL_H + +#include "tiledb/api/c_api/api_external_common.h" +#include "tiledb/api/c_api/context/context_api_external.h" +#include "tiledb/api/c_api/domain/domain_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** C API struct to specify the limits of a dimension for an ND rectangle. */ +typedef struct { + const void* min; + uint64_t min_size; + const void* max; + uint64_t max_size; +} tiledb_range_t; + +typedef struct tiledb_ndrectangle_handle_t tiledb_ndrectangle_t; + +/** + * Allocate an N-dimensional rectangle given a TileDB array schema domain. + * The resulted rectangle will maintain the same number of dimensions as the + * array schema domain. + * + * **Example:** + * + * @code{.c} + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * tiledb_ndrectangle_free(&ndr); + * @endcode + * + * @param ctx The TileDB context + * @param domain The TileDB array schema domain + * @param ndr The n-dimensional rectangle to be allocated + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_alloc( + tiledb_ctx_t* ctx, + tiledb_domain_t* domain, + tiledb_ndrectangle_t** ndr) TILEDB_NOEXCEPT; + +/** + * Free the resources associated with the N-dimensional rectangle arg + * + * **Example:** + * + * @code{.c} + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * tiledb_ndrectangle_free(&ndr); + * @endcode + * + * @param ndr The n-dimensional rectangle to be freed + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_free(tiledb_ndrectangle_t** ndr) + TILEDB_NOEXCEPT; + +/** + * Get the range set on an N-dimensional rectangle for a dimension name + * + * The pointers within the returned range struct point to resources tied to + * the lifetime of the `tiledb_ndrectangle_t` object, it is not the + * responsibility of the caller to free those resources, attempting + * to do so results in undefined behavior. + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * tiledb_ndrectangle_get_range_from_name(ctx, ndr, "dim", &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param name The name of the dimension for which range should be returned + * @param range The range returned as output argument + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_range_from_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +/** + * Get the range set on an N-dimensional rectangle for a dimension index. + * + * The pointers within the returned range struct point to resources tied to + * the lifetime of the `tiledb_ndrectangle_t` object, it is not the + * responsibility of the caller to free those resources, attempting + * to do so results in undefined behavior. + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * tiledb_ndrectangle_get_range_from_name(ctx, ndr, 1, &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param idx The index of the dimension for which range should be returned + * @param range The range returned as output argument + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +/** + * Set the range on an N-dimensional rectangle for a dimension name + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * range.min = &min; + * range.min_size = sizeof(min); + * range.max = &max; + * range.max_size = sizeof(max); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim", &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param name The name of the dimension for which range should be set + * @param range The range to be set on the ND rectangle + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_set_range_for_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +/** + * Set the range on an N-dimensional rectangle for dimension at idx. + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * range.min = &min; + * range.min_size = sizeof(min); + * range.max = &max; + * range.max_size = sizeof(max); + * tiledb_ndrectangle_set_range(ctx, ndr, 1, &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param idx The index of the dimension for which range should be set + * @param range The range to be set on the ND rectangle + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_set_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_NDRECTANGLE_API_EXTERNAL_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h b/tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h new file mode 100644 index 00000000000..a418003a7be --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h @@ -0,0 +1,94 @@ +/** + * @file tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internal C API implementation of a ND rectangle + */ + +#ifndef TILEDB_CAPI_NDRECTANGLE_INTERNAL_H +#define TILEDB_CAPI_NDRECTANGLE_INTERNAL_H + +#include "tiledb/api/c_api_support/argument_validation.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/sm/array_schema/ndrectangle.h" + +struct tiledb_ndrectangle_handle_t + : public tiledb::api::CAPIHandle { + /** + * Type name + */ + static constexpr std::string_view object_type_name{"tiledb_ndrectangle_t"}; + + private: + std::shared_ptr ndrectangle_; + + public: + /** + * Default constructor doesn't make sense + */ + tiledb_ndrectangle_handle_t() = delete; + + /** + * Constructs a handle with a NDRectangle instance. + * @param memory_tracker The memory tracker to use in the internal NDRectangle + * @param domain The ArraySchema domain used for internal validations + */ + explicit tiledb_ndrectangle_handle_t( + shared_ptr memory_tracker, + shared_ptr domain) + : ndrectangle_{make_shared( + HERE(), memory_tracker, domain)} { + } + + /** + * Ordinary constructor. + * @param ndrectangle An internal tiledb NDRectangle instance + */ + explicit tiledb_ndrectangle_handle_t( + shared_ptr ndrectangle) + : ndrectangle_(ndrectangle) { + } + + /** + * Get the internal instance of NDRectangle + * @returns The internal NDRectangle + */ + [[nodiscard]] inline shared_ptr ndrectangle() const { + return ndrectangle_; + } + + /** + * Set the internal NDRectangle instance to be managed by this handle + * @param ndr The NDRectangle to manage + */ + inline void set_ndrectangle(shared_ptr ndr) { + ndrectangle_ = ndr; + } +}; + +#endif // TILEDB_CAPI_NDRECTANGLE_INTERNAL_H diff --git a/tiledb/api/c_api/ndrectangle/test/CMakeLists.txt b/tiledb/api/c_api/ndrectangle/test/CMakeLists.txt new file mode 100644 index 00000000000..d656225aff9 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/test/CMakeLists.txt @@ -0,0 +1,36 @@ +# +# tiledb/api/c_api/ndrectangle/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_ndrectangle) + this_target_sources(unit_capi_ndrectangle.cc) + if (NOT MSVC) + target_compile_options(unit_capi_ndrectangle PRIVATE -Wno-deprecated-declarations) + endif() + this_target_object_libraries(capi_ndrectangle) + this_target_link_libraries(tiledb_test_support_lib) +conclude(unit_test) diff --git a/tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc b/tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc new file mode 100644 index 00000000000..ad3c4013519 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc @@ -0,0 +1,40 @@ +/** + * @file tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../ndrectangle_api_external_experimental.h" + +int main() { + tiledb_ndrectangle_alloc(nullptr, nullptr, nullptr); + tiledb_ndrectangle_free(nullptr); + tiledb_ndrectangle_get_range_from_name(nullptr, nullptr, nullptr, nullptr); + tiledb_ndrectangle_get_range(nullptr, nullptr, 0, nullptr); + tiledb_ndrectangle_set_range_for_name(nullptr, nullptr, nullptr, nullptr); + tiledb_ndrectangle_set_range(nullptr, nullptr, 0, nullptr); + + return 0; +} diff --git a/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc b/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc new file mode 100644 index 00000000000..49221b0b3c9 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc @@ -0,0 +1,190 @@ +/** + * @file tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the NDRectangle C API. + */ + +#include +#include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/dimension/dimension_api_external.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" + +using namespace tiledb::test; + +struct CAPINDRectangleFx : TemporaryDirectoryFixture { + CAPINDRectangleFx() { + create_domain(); + } + ~CAPINDRectangleFx() { + free_domain(); + } + void create_domain(); + void free_domain(); + + tiledb_dimension_t *d1_, *d2_; + tiledb_domain_t* domain_; +}; + +void CAPINDRectangleFx::create_domain() { + // Create dimensions + uint64_t tile_extents[] = {2, 2}; + uint64_t dim_domain[] = {1, 10, 1, 10}; + + int rc = tiledb_dimension_alloc( + ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_dimension_alloc( + ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2_); + CHECK(rc == TILEDB_OK); + + // Create domain + rc = tiledb_domain_alloc(ctx, &domain_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d2_); + CHECK(rc == TILEDB_OK); +} + +void CAPINDRectangleFx::free_domain() { + tiledb_dimension_free(&d1_); + tiledb_dimension_free(&d2_); + tiledb_domain_free(&domain_); +} + +TEST_CASE_METHOD( + CAPINDRectangleFx, + "C API: argument validation", + "[capi][ndrectangle][args]") { + CHECK( + tiledb_ndrectangle_alloc(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_alloc(ctx, nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_alloc(ctx, domain_, nullptr) == TILEDB_ERR); + + CHECK(tiledb_ndrectangle_free(nullptr) == TILEDB_ERR); + tiledb_ndrectangle_t* ndr = nullptr; + CHECK(tiledb_ndrectangle_free(&ndr) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_get_range_from_name( + nullptr, nullptr, nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, nullptr, nullptr, nullptr) == + TILEDB_ERR); + + REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, "dim1", nullptr) == + TILEDB_ERR); + tiledb_range_t range; + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, "doesntexist", &range) == + TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_get_range(nullptr, nullptr, 0, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_get_range(ctx, nullptr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_get_range(ctx, ndr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_get_range(ctx, ndr, 2, &range) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_set_range(nullptr, nullptr, 0, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_set_range(ctx, nullptr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_set_range(ctx, ndr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_set_range(ctx, ndr, 2, &range) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_set_range_for_name( + nullptr, nullptr, nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, nullptr, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim1", nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, "doesntexist", &range) == + TILEDB_ERR); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); +} + +TEST_CASE_METHOD( + CAPINDRectangleFx, + "C API: setting and getting ranges works", + "[capi][ndrectangle][range]") { + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); + + tiledb_range_t range; + uint64_t min = 2; + uint64_t max = 5; + range.min = &min; + range.min_size = sizeof(uint64_t); + range.max = &max; + range.max_size = sizeof(uint64_t); + + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, "d1", &range) == + TILEDB_OK); + + CHECK(tiledb_ndrectangle_set_range(ctx, ndr, 1, &range) == TILEDB_OK); + + tiledb_range_t out_range_d1; + + REQUIRE( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, "d1", &out_range_d1) == + TILEDB_OK); + CHECK(range.min_size == out_range_d1.min_size); + CHECK(range.min_size == out_range_d1.min_size); + CHECK(std::memcmp(range.min, out_range_d1.min, range.min_size) == 0); + CHECK(std::memcmp(range.max, out_range_d1.max, range.max_size) == 0); + + tiledb_range_t out_range_d2; + REQUIRE( + tiledb_ndrectangle_get_range(ctx, ndr, 1, &out_range_d2) == TILEDB_OK); + CHECK(range.min_size == out_range_d2.min_size); + CHECK(range.min_size == out_range_d2.min_size); + CHECK(std::memcmp(range.min, out_range_d2.min, range.min_size) == 0); + CHECK(std::memcmp(range.max, out_range_d2.max, range.max_size) == 0); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); +} diff --git a/tiledb/api/cpp_api_support/test/CMakeLists.txt b/tiledb/api/cpp_api_support/test/CMakeLists.txt index 30acaadf619..4b0097f0a5e 100644 --- a/tiledb/api/cpp_api_support/test/CMakeLists.txt +++ b/tiledb/api/cpp_api_support/test/CMakeLists.txt @@ -30,3 +30,7 @@ commence(unit_test cppapi_string) this_target_sources(unit_cppapi_string.cc) this_target_object_libraries(capi_string) conclude(unit_test) + +if(TILEDB_SANITIZER) + target_compile_definitions(unit_cppapi_string PRIVATE -DHAVE_SANITIZER) +endif() diff --git a/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc b/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc index ffddb3f5012..0157f99f68d 100644 --- a/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc +++ b/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc @@ -71,6 +71,7 @@ TEST_CASE( REQUIRE(result == test_string); } +#ifndef HAVE_SANITIZER TEST_CASE( "CAPIString: Test that accessing freed handle fails", "[capi_string][freed_handle]") { @@ -82,6 +83,7 @@ TEST_CASE( size_t length = 0; REQUIRE(tiledb_string_view(handle_copy, &chars, &length) == TILEDB_ERR); } +#endif TEST_CASE( "CAPIString: Test convert_to_string with null handle", "[capi_string]") { diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h index a5d15a6f0bb..dccbd15a650 100644 --- a/tiledb/common/pmr.h +++ b/tiledb/common/pmr.h @@ -187,77 +187,7 @@ class list : public pmr_list { /* ********************************* */ template -using pmr_vector = std::vector>; - -template -class vector : public pmr_vector { - public: - // This class exists to ensure that all uses of it are provided with a - // valid std::pmr based allocator. This is so that as we switch from - // std::vector to using this class we don't forget to provide the allocator - // which is quite easy to do. - // - // If these constructors look confusing, just know that all we're doing is - // copying the current definitions from cppreference and then adjusting types - // to require the PMR based allocator. - - // I have absolutely no idea if all of these aliases are required. The - // allocator_type is the important one. I've copied the others just in - // case since I do know that PMR aware containers at least require - // allocator_type. - using value_type = typename pmr_vector::value_type; - using allocator_type = typename pmr_vector::allocator_type; - using size_type = typename pmr_vector::size_type; - using difference_type = typename pmr_vector::difference_type; - using reference = typename pmr_vector::reference; - using const_reference = typename pmr_vector::const_reference; - using pointer = typename pmr_vector::pointer; - using const_pointer = typename pmr_vector::const_pointer; - using iterator = typename pmr_vector::iterator; - using const_iterator = typename pmr_vector::const_iterator; - using reverse_iterator = typename pmr_vector::reverse_iterator; - using const_reverse_iterator = - typename pmr_vector::const_reverse_iterator; - - // Delete all default constructors because they don't require an allocator - constexpr vector() noexcept(noexcept(allocator_type())) = delete; - constexpr vector(const vector& other) = delete; - constexpr vector(vector&& other) noexcept = delete; - - // Delete non-allocator aware copy and move assign. - constexpr vector& operator=(const vector& other) = delete; - constexpr vector& operator=(vector&& other) noexcept = delete; - - constexpr explicit vector(const allocator_type& alloc) noexcept - : pmr_vector(alloc) { - } - - constexpr vector( - size_type count, const Tp& value, const allocator_type& alloc) - : pmr_vector(count, value, alloc) { - } - - constexpr explicit vector(size_type count, const allocator_type& alloc) - : pmr_vector(count, alloc) { - } - - template - constexpr vector(InputIt first, InputIt last, const allocator_type& alloc) - : pmr_vector(first, last, alloc) { - } - - constexpr vector(const vector& other, const allocator_type& alloc) - : pmr_vector(other, alloc) { - } - - constexpr vector(vector&& other, const allocator_type& alloc) - : pmr_vector(other, alloc) { - } - - constexpr vector(std::initializer_list init, const allocator_type& alloc) - : pmr_vector(init, alloc) { - } -}; +using vector = std::vector>; /* ********************************* */ /* PMR UNORDERED MAP DECLARATION */ diff --git a/tiledb/common/util/alt_var_length_view.h b/tiledb/common/util/alt_var_length_view.h index b80a80f957f..c61e7378902 100644 --- a/tiledb/common/util/alt_var_length_view.h +++ b/tiledb/common/util/alt_var_length_view.h @@ -351,4 +351,39 @@ alt_var_length_view(R, R, J, I, I, K) template alt_var_length_view(R, R, J, I, I, K, L) -> alt_var_length_view, std::ranges::subrange>; + +/** + * Actually reorder the data underlying an alt_var_length_view (un-virtualize + * the permutation). Upon return, data will have the sorted data, offsets will + * have the SIZES of each subrange, and the subranges will be "in order" + * @tparam S Type of the subranges + * @tparam R Type of the data range + * @tparam I Type of the index / offset / size range + * @tparam B Type of the scratch buffer + * @param subranges The alt_var_length_view + * @param data The data range underlying the alt_var_length_view + * @param offsets The offsets range to be written to + * @param buffer Scratch space used for reordering, must have enough space to + * hold all of the data + * @return + * + * @todo Make this a member of alt_var_length_view + */ +template < + std::ranges::forward_range S, + std::ranges::forward_range R, + std::ranges::forward_range I, + std::ranges::random_access_range B> +auto actualize(S& subranges, R& data, I& offsets, B& buffer) { + auto x = buffer.begin(); + auto o = offsets.begin(); + for (auto& s : subranges) { + std::ranges::copy(s.begin(), s.end(), x); + auto n = s.size(); + s = std::ranges::subrange(x, x + n); + x += n; + *o++ = n; // x - buffer.begin(); + } + std::ranges::copy(buffer.begin(), buffer.begin() + data.size(), data.begin()); +} #endif // TILEDB_ALT_VAR_LENGTH_VIEW_H diff --git a/tiledb/common/util/permute.h b/tiledb/common/util/permute.h new file mode 100644 index 00000000000..3ef151930d1 --- /dev/null +++ b/tiledb/common/util/permute.h @@ -0,0 +1,89 @@ +/** + * @file tiledb/common/util/permute.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#ifndef TILEDB_PERMUTE_H +#define TILEDB_PERMUTE_H + +#include +#include +#include +#include + +/** + * Permute an array in-place according to a given permutation array. + * @tparam Vector + * @tparam Perm + * @tparam Done Should generally be `std::vector` + * @param b Array to be sorted + * @param perm Array specifying the permutation + * @param done Array of flags indicating whether an element has been processed + * or not. All elements must be cleared (set to false) on entry. Note that it + * will be modified while the function is executing but on return, all the + * entries will be cleared. + */ +template < + std::ranges::random_access_range Vector, + std::ranges::random_access_range Perm, + std::ranges::random_access_range Done> +void permute(Vector& b, const Perm& perm, Done& done) { + using flag_type = std::remove_cvref_t; + constexpr flag_type zero_flag{0}; + constexpr flag_type one_flag{1}; + + // @todo Is there a better way to track done-ness? Hi bit of perm? + for (size_t i = 0; i < perm.size(); ++i) { + if (zero_flag != done[i]) { + done[i] = zero_flag; + continue; + } + + size_t ix = i; + size_t px = perm[ix]; + while (!done[px]) { + // @todo Find with ADL??? + std::swap(b[ix], b[px]); + + done[ix] = one_flag; + ix = px; + px = perm[ix]; + } + } +} + +template < + std::ranges::random_access_range Vector, + std::ranges::random_access_range Perm> +void permute(Vector& b, const Perm& perm) { + std::vector done(perm.size(), false); + permute(b, perm, done); +} + +#endif diff --git a/tiledb/common/util/test/CMakeLists.txt b/tiledb/common/util/test/CMakeLists.txt index 60b71e92dc8..0d6787bbf13 100644 --- a/tiledb/common/util/test/CMakeLists.txt +++ b/tiledb/common/util/test/CMakeLists.txt @@ -50,6 +50,12 @@ commence(unit_test proxy_sort) ) conclude(unit_test) +commence(unit_test sort_chunk) + this_target_sources( + unit_sort_chunk.cc +) +conclude(unit_test) + commence(unit_test sort_zip) this_target_sources( unit_sort_zip.cc diff --git a/tiledb/common/util/test/unit_alt_var_length_view.cc b/tiledb/common/util/test/unit_alt_var_length_view.cc index 50164071e2e..b76e6a0e99d 100644 --- a/tiledb/common/util/test/unit_alt_var_length_view.cc +++ b/tiledb/common/util/test/unit_alt_var_length_view.cc @@ -603,3 +603,51 @@ TEST_CASE("alt_var_length_view: Sort", "[alt_var_length_view]") { } } } + +TEST_CASE("alt_var_length_view: Sort and actualize", "[alt_var_length_view]") { + std::vector r = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}; + std::vector s(r); + std::vector o = {0, 3, 6, 10, 12}; + alt_var_length_view, std::vector> v(r, o); + + SECTION("Sort by size, ascending") { + std::vector> expected = { + {11.0, 12.0}, + {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0}, + {7.0, 8.0, 9.0, 10.0}, + }; + + std::sort(v.begin(), v.end(), [](auto& a, auto& b) { + return a.size() < b.size(); + }); + + CHECK(v.begin()->size() == 2); + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 3); + CHECK((v.begin() + 3)->size() == 4); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + + // Check the underlying data has not changed even though sorted + CHECK(std::ranges::equal(r, s)); + std::vector scratch(size(r)); + + actualize(v, r, o, scratch); + + // Check the underlying data has changed to the expected sorted order + CHECK(std::ranges::equal( + r, + std::vector{ + 11.0, 12.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0})); + + // The alt_var_length_view should still "look" the same + CHECK(std::ranges::equal(o, std::vector{2, 3, 3, 4, 12})); + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } +} diff --git a/tiledb/common/util/test/unit_permutation_sort.cc b/tiledb/common/util/test/unit_permutation_sort.cc index 23bf78abf15..a4d2c87169b 100644 --- a/tiledb/common/util/test/unit_permutation_sort.cc +++ b/tiledb/common/util/test/unit_permutation_sort.cc @@ -38,10 +38,13 @@ #include #include "tiledb/common/util/alt_var_length_view.h" #include "tiledb/common/util/permutation_view.h" + #include "tiledb/common/util/proxy_sort.h" #include "tiledb/common/util/var_length_view.h" #include "tiledb/stdx/__ranges/zip_view.h" +#include "tiledb/common/util/permute.h" + TEST_CASE("permutation_sort: Null test", "[permutation_sort][null_test]") { REQUIRE(true); } @@ -339,3 +342,124 @@ TEST_CASE( } } } + +/** + * This function is set up to time various ways of sorting a set of arrays + * according to how one of them is ordered. It compares the time of sorting + * with a proxy sort and in-place permutation vs in-place sort. + * + * Doing in-place sort with the zip view is the most efficient (especially when + * parallelized. + * + * To do the timings, access a version of timer.h (e.g., from the experimental + * dag hierarchy and uncomment the timer related lines. To do the timing of + * parallel zip sort you will also need to use a parallel sort algorithm. You + * may also want to increase the size of N to 20'000'000. + */ +TEST_CASE("permutation_sort: time", "[permutation_sort]") { + uint32_t seed = Catch::rngSeed(); + size_t N = 2'000'000; + std::mt19937 g(seed); + + std::vector perm(N); + std::iota(begin(perm), end(perm), 0); + + auto init_19 = std::vector(N); + std::iota(begin(init_19), end(init_19), 19); + std::vector shuffled(init_19); + + std::shuffle(shuffled.begin(), shuffled.end(), g); + std::vector sorted0(shuffled); + std::vector sorted1(shuffled); + std::vector sorted2(shuffled); + std::vector sorted3(shuffled); + std::vector sorted4(shuffled); + std::vector sorted5(shuffled); + std::vector sorted6(shuffled); + std::vector sorted7(shuffled); + std::vector sorted8(shuffled); + std::vector sorted9(shuffled); + + CHECK(!std::ranges::equal(shuffled, init_19)); + + // Different numbers of vectors have different performance characteristics + // auto z = zip(shuffled, sorted0, sorted1, sorted2); + // auto z = zip(shuffled, sorted0, sorted1, sorted2, sorted3, sorted4); + // auto z = zip(shuffled, sorted0, sorted1, sorted2, sorted3, sorted4, + // sorted5, sorted6); + auto z = + zip(shuffled, + sorted0, + sorted1, + sorted2, + sorted3, + sorted4, + sorted5, + sorted6, + sorted7, + sorted8, + sorted9); + + std::vector done(N, 0); + + SECTION("separate sort") { + std::ranges::sort(shuffled); + std::ranges::sort(sorted0); + std::ranges::sort(sorted1); + std::ranges::sort(sorted2); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + SECTION("proxy sort permute") { + proxy_sort(shuffled, perm); + + permute(shuffled, perm); + permute(sorted0, perm); + permute(sorted1, perm); + permute(sorted2, perm); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + SECTION("proxy sort permute zip") { + proxy_sort(shuffled, perm); + + permute(z, perm, done); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + + SECTION("zip sort") { + std::sort(z.begin(), z.end(), [](auto&& a, auto&& b) { + return std::get<0>(a) < std::get<0>(b); + }); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + +// Enable if pmergesort is available +#if 0 + SECTION("zip par sort") { + + pmergesort(z.begin(), z.end(), [](auto&& a, auto&& b) { + return std::get<0>(a) < std::get<0>(b); + }); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } +#endif +} diff --git a/tiledb/common/util/test/unit_sort_chunk.cc b/tiledb/common/util/test/unit_sort_chunk.cc new file mode 100644 index 00000000000..309ba2562e9 --- /dev/null +++ b/tiledb/common/util/test/unit_sort_chunk.cc @@ -0,0 +1,158 @@ +/** + * @file tiledb/common/util/test/unit_sort_chunk.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements tests for sorting chunks in a chunk_view over a vector. + */ + +#include +#include +#include +#include +#include +#include "oberon.h" +#include "tiledb/common/util/alt_var_length_view.h" +#include "tiledb/stdx/ranges" + +#include "tiledb/common/util/print_types.h" + +TEST_CASE("sort_chunk: Null test", "[sort_chunk][null_test]") { +} + +/* + * Test that sorting each chunk in a chunk view will sort blocks of elements in + * the underlying vector. + */ +TEST_CASE("sort_chunk: std::vector", "[sort_chunk]") { + size_t N = 1024; + size_t chunk_size = 16; + size_t num_chunks = N / chunk_size; + + std::vector v(N); + std::iota(v.begin(), v.end(), 17); + std::ranges::reverse(v); + std::vector expected(v); + + /* Sort blocks of underlying vector */ + for (size_t i = 0; i < num_chunks; ++i) { + for (size_t j = 0; j < chunk_size; ++j) { + std::sort( + expected.begin() + i * chunk_size, + expected.begin() + (i + 1) * chunk_size); + } + } + + /* Create chunk view of v */ + auto cv = stdx::views::chunk(v, (long)chunk_size); + CHECK(std::ranges::size(cv) == (long)num_chunks); + + /* Sort each chunk */ + for (auto&& cu : cv) { + std::sort(cu.begin(), cu.end()); + } + + CHECK(std::ranges::equal(v, expected)); +} + +TEST_CASE("sort_chunk: zip_view of std::vector", "sort_chunk") { + size_t N = 1024; + size_t chunk_size = 16; + size_t num_chunks = N / chunk_size; + + std::vector v(N), w(N + 5); + std::iota(v.begin(), v.end(), 17); + std::iota(w.begin(), w.end(), -13); + + auto z = stdx::views::zip(v, w); + + std::ranges::reverse(v); + std::ranges::reverse(w); + std::vector xv(v), xw(w); + + /* Sort blocks of underlying vectors */ + for (size_t i = 0; i < num_chunks; ++i) { + for (size_t j = 0; j < chunk_size; ++j) { + std::sort(xv.begin() + i * chunk_size, xv.begin() + (i + 1) * chunk_size); + std::sort(xw.begin() + i * chunk_size, xw.begin() + (i + 1) * chunk_size); + } + } + + /* Create chunk view of v */ + auto cz = stdx::ranges::chunk_view(z, chunk_size); + CHECK(std::ranges::size(cz) == (long)num_chunks); + + /* Sort each chunk */ + for (auto&& cy : cz) { + std::sort(cy.begin(), cy.end()); + } + + CHECK(std::ranges::equal(z, stdx::views::zip(xv, xw))); + CHECK(!std::ranges::is_sorted(z)); + CHECK(!std::ranges::is_sorted(xv)); + CHECK(!std::ranges::is_sorted(xw)); +} + +TEST_CASE("sort_chunk: zip_view of vector and alt_var_length_view") { + auto ob1 = alt_var_length_view(oberon::cs1, oberon::ps1); + REQUIRE(size(ob1) == 10); + std::vector v(10); + std::iota(v.begin(), v.end(), 17); + std::ranges::reverse(v); + + auto z = stdx::views::zip(v, ob1); + auto cz = stdx::views::chunk(z, 2); + CHECK(size(cz) == 5); + CHECK(size(cz[0]) == 2); + + for (auto&& cy : cz) { + std::sort(cy.begin(), cy.end(), [](auto&& a, auto&& b) { + return std::get<0>(a) < std::get<0>(b); + }); + } + + // 26, 25, 24, 23, 22, 21, 20, 19, 18, 17 + // -> 25, 26, 23, 24, 21, 22, 19, 20, 17, 18 + // Where ox lips and the nod ding vio let grows + // -> ox Where and lips nod the vio ding grows let + CHECK(std::get<0>(cz[0][0]) == 25); + CHECK(std::get<0>(cz[0][1]) == 26); + CHECK(std::ranges::equal(get<1>(cz[0][0]), std::vector{'o', 'x'})); + CHECK(std::ranges::equal( + get<1>(cz[0][1]), std::vector{'W', 'h', 'e', 'r', 'e'})); + CHECK(std::ranges::equal(get<1>(cz[1][0]), std::vector{'a', 'n', 'd'})); + CHECK(std::ranges::equal( + get<1>(cz[1][1]), std::vector{'l', 'i', 'p', 's'})); + CHECK(std::ranges::equal(get<1>(cz[2][0]), std::vector{'n', 'o', 'd'})); + CHECK(std::ranges::equal(get<1>(cz[2][1]), std::vector{'t', 'h', 'e'})); + CHECK(std::ranges::equal(get<1>(cz[3][0]), std::vector{'v', 'i', 'o'})); + CHECK(std::ranges::equal( + get<1>(cz[3][1]), std::vector{'d', 'i', 'n', 'g'})); + CHECK(std::ranges::equal( + get<1>(cz[4][0]), std::vector{'g', 'r', 'o', 'w', 's'})); + CHECK(std::ranges::equal(get<1>(cz[4][1]), std::vector{'l', 'e', 't'})); +} diff --git a/tiledb/common/util/test/unit_sort_zip.cc b/tiledb/common/util/test/unit_sort_zip.cc index 2f1c829be7a..6ef60e671c8 100644 --- a/tiledb/common/util/test/unit_sort_zip.cc +++ b/tiledb/common/util/test/unit_sort_zip.cc @@ -1,5 +1,5 @@ /** - * @file unit_sort_zip.cc + * @file tiledb/common/util/test/unit_sort_zip.cc * * @section LICENSE * diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 59091561a59..6e9d7722ef2 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -159,12 +159,12 @@ OpenedArray::load_delete_and_update_conditions() { throw_if_not_ok(conditions[i].check(array_schema_latest())); return Status::Ok(); }); - RETURN_NOT_OK_TUPLE(status, nullopt, nullopt); + throw_if_not_ok(status); return {Status::Ok(), conditions, update_values}; } -Status Array::evolve_array_schema( +void Array::evolve_array_schema( ContextResources& resources, const URI& array_uri, ArraySchemaEvolution* schema_evolution, @@ -175,8 +175,10 @@ Status Array::evolve_array_schema( } if (array_uri.is_tiledb()) { - return resources.rest_client()->post_array_schema_evolution_to_rest( - array_uri, schema_evolution); + throw_if_not_ok( + resources.rest_client()->post_array_schema_evolution_to_rest( + array_uri, schema_evolution)); + return; } // Load URIs from the array directory @@ -218,15 +220,12 @@ Status Array::evolve_array_schema( // Evolve schema auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); - - Status st = - store_array_schema(resources, array_schema_evolved, encryption_key); - if (!st.ok()) { - throw ArrayException( - "Cannot evolve schema; Not able to store evolved array schema."); + try { + store_array_schema(resources, array_schema_evolved, encryption_key); + } catch (...) { + std::throw_with_nested(ArrayException( + "Cannot evolve schema; Not able to store evolved array schema.")); } - - return Status::Ok(); } const URI& Array::array_uri() const { @@ -241,7 +240,7 @@ const EncryptionKey* Array::encryption_key() const { return opened_array_->encryption_key(); } -Status Array::create( +void Array::create( ContextResources& resources, const URI& array_uri, const shared_ptr& array_schema, @@ -306,41 +305,39 @@ Status Array::create( array_uri.join_path(constants::array_dimension_labels_dir_name); throw_if_not_ok(resources.vfs().create_dir(array_dimension_labels_uri)); - // Get encryption key from config - Status st; - if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { - bool found = false; - std::string encryption_key_from_cfg = - resources.config().get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - resources.config().get("sm.encryption_type", &found); - assert(found); - auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); - throw_if_not_ok(st_enc); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - throw_if_not_ok( - encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + // Store the array schema + try { + // Get encryption key from config + if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { + bool found = false; + std::string encryption_key_from_cfg = + resources.config().get("sm.encryption_key", &found); + assert(found); + std::string encryption_type_from_cfg = + resources.config().get("sm.encryption_type", &found); + assert(found); + auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st_enc); + EncryptionType encryption_type_cfg = etc.value(); + + EncryptionKey encryption_key_cfg; + if (encryption_key_from_cfg.empty()) { + throw_if_not_ok( + encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + } else { + throw_if_not_ok(encryption_key_cfg.set_key( + encryption_type_cfg, + (const void*)encryption_key_from_cfg.c_str(), + static_cast(encryption_key_from_cfg.size()))); + } + store_array_schema(resources, array_schema, encryption_key_cfg); } else { - throw_if_not_ok(encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - static_cast(encryption_key_from_cfg.size()))); + store_array_schema(resources, array_schema, encryption_key); } - st = store_array_schema(resources, array_schema, encryption_key_cfg); - } else { - st = store_array_schema(resources, array_schema, encryption_key); - } - - if (!st.ok()) { + } catch (...) { throw_if_not_ok(resources.vfs().remove_dir(array_uri)); - return st; + throw; } - - return Status::Ok(); } // Used in Consolidator @@ -386,7 +383,7 @@ Status Array::open_without_fragments( if (remote_) { auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { - throw Status_ArrayError( + throw ArrayException( "Cannot open array; remote array with no REST client."); } /* #TODO Change get_array_schema_from_rest function signature to @@ -675,7 +672,7 @@ Status Array::close() { set_metadata_loaded(true); auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { - throw Status_ArrayError( + throw ArrayException( "Error closing array; remote array with no REST client."); } throw_if_not_ok(rest_client->post_array_metadata_to_rest( @@ -699,7 +696,7 @@ Status Array::close() { opened_array_.reset(); } catch (std::exception& e) { is_opening_or_closing_ = false; - throw Status_ArrayError(e.what()); + throw ArrayException(e.what()); } is_opening_or_closing_ = false; @@ -833,7 +830,7 @@ void Array::delete_fragments_list(const std::vector& fragment_uris) { } } -Status Array::encryption_type( +void Array::encryption_type( ContextResources& resources, const URI& uri, EncryptionType* encryption_type) { @@ -859,8 +856,6 @@ Status Array::encryption_type( auto&& header = GenericTileIO::read_generic_tile_header( resources, array_dir->latest_array_schema_uri(), 0); *encryption_type = static_cast(header.encryption_type); - - return Status::Ok(); } shared_ptr Array::get_enumeration( @@ -1159,7 +1154,7 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { set_array_closed(); } catch (std::exception& e) { is_opening_or_closing_ = false; - throw Status_ArrayError(e.what()); + throw ArrayException(e.what()); } is_opening_or_closing_ = false; @@ -2113,7 +2108,7 @@ const ArrayDirectory& Array::load_array_directory() { return array_directory(); } -Status Array::upgrade_version( +void Array::upgrade_version( ContextResources& resources, const URI& array_uri, const Config& override_config) { @@ -2168,8 +2163,7 @@ Status Array::upgrade_version( throw_if_not_ok(resources.vfs().create_dir(array_schema_dir_uri)); // Store array schema - throw_if_not_ok( - store_array_schema(resources, array_schema, encryption_key_cfg)); + store_array_schema(resources, array_schema, encryption_key_cfg); // Create commit directory if necessary URI array_commit_uri = @@ -2186,8 +2180,6 @@ Status Array::upgrade_version( array_uri.join_path(constants::array_fragment_meta_dir_name); throw_if_not_ok(resources.vfs().create_dir(array_fragment_metadata_uri)); } - - return Status::Ok(); } Status Array::compute_non_empty_domain() { diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index c1e6ce1a9b3..3394939b42e 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -375,9 +375,8 @@ class Array { * @param array_uri The uri of the array whose schema is to be evolved. * @param schema_evolution The schema evolution. * @param encryption_key The encryption key to use. - * @return Status */ - static Status evolve_array_schema( + static void evolve_array_schema( ContextResources& resources, const URI& array_uri, ArraySchemaEvolution* array_schema, @@ -397,9 +396,8 @@ class Array { * @param array_uri The URI of the array to be created. * @param array_schema The array schema. * @param encryption_key The encryption key to use. - * @return Status */ - static Status create( + static void create( ContextResources& resources, const URI& array_uri, const shared_ptr& array_schema, @@ -578,9 +576,8 @@ class Array { * @param resources The context resources. * @param uri The URI of the array. * @param encryption_type Set to the encryption type. - * @return Status */ - static Status encryption_type( + static void encryption_type( ContextResources& resources, const URI& uri, EncryptionType* encryption_type); @@ -1058,9 +1055,8 @@ class Array { * @param config Configuration parameters for the upgrade * (`nullptr` means default, which will use the config associated with * this instance). - * @return Status */ - static Status upgrade_version( + static void upgrade_version( ContextResources& resources, const URI& uri, const Config& config); private: diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 189bdf20dfd..32f966b4606 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -1229,7 +1229,6 @@ URI ArrayDirectory::select_latest_array_schema_uri() { } optional latest_uri = nullopt; - uint64_t latest_ts = 0; for (auto& uri : array_schema_uris_) { FragmentID fragment_id{uri}; @@ -1239,9 +1238,8 @@ URI ArrayDirectory::select_latest_array_schema_uri() { } auto ts_range{fragment_id.timestamp_range()}; - if (ts_range.second > latest_ts && ts_range.second <= timestamp_end_) { + if (ts_range.second <= timestamp_end_) { latest_uri = uri; - latest_ts = ts_range.second; } } diff --git a/tiledb/sm/array/test/unit_consistency.cc b/tiledb/sm/array/test/unit_consistency.cc index 1aeb1522eb9..d494e43c3b3 100644 --- a/tiledb/sm/array/test/unit_consistency.cc +++ b/tiledb/sm/array/test/unit_consistency.cc @@ -168,7 +168,7 @@ TEST_CASE( WhiteboxConsistencyController x; const URI uri = URI("whitebox_single_array"); - // Create a StorageManager + // Create a ContextResources Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); @@ -193,7 +193,7 @@ TEST_CASE( "[ConsistencyController][array][vector]") { WhiteboxConsistencyController x; - // Create a StorageManager + // Create a ContextResources Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); @@ -235,7 +235,7 @@ TEST_CASE( WhiteboxConsistencyController x; const URI uri = URI("whitebox_modify_exclusive"); - // Create a StorageManager + // Create a ContextResources Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 426fcfabb53..38a0e621c18 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -1,5 +1,5 @@ /** - * @file tiledb/sm/array/test/unit_consistency.cc + * @file tiledb/sm/array/test/unit_consistency.h * * @section LICENSE * @@ -124,7 +124,7 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - throw_if_not_ok(Array::create(resources, uri, schema, key)); + Array::create(resources, uri, schema, key); tdb_unique_ptr array(new Array{resources, uri, *this}); return array; diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 4b006374087..19c7c8b20c7 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -144,7 +144,7 @@ ArraySchema::ArraySchema( FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , uri_(uri) @@ -1437,7 +1437,7 @@ shared_ptr ArraySchema::deserialize( // Load the array current domain, if this is an older array, it'll get by // default an empty current domain object - auto current_domain = make_shared( + auto current_domain = make_shared( memory_tracker, constants::current_domain_version); if (version >= constants::current_domain_min_format_version) { current_domain = @@ -1815,7 +1815,7 @@ void ArraySchema::generate_uri( } void ArraySchema::expand_current_domain( - shared_ptr new_current_domain) { + shared_ptr new_current_domain) { if (new_current_domain == nullptr) { throw ArraySchemaException( "The argument specified for current domain expansion is nullptr."); @@ -1835,12 +1835,11 @@ void ArraySchema::expand_current_domain( current_domain_ = new_current_domain; } -shared_ptr ArraySchema::get_current_domain() const { +shared_ptr ArraySchema::get_current_domain() const { return current_domain_; } -void ArraySchema::set_current_domain( - shared_ptr current_domain) { +void ArraySchema::set_current_domain(shared_ptr current_domain) { if (current_domain == nullptr) { throw ArraySchemaException( "The argument specified for setting the current domain on the " diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index a07101cf19a..5edc323d5fd 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -143,7 +143,7 @@ class ArraySchema { FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker); /** @@ -593,18 +593,17 @@ class ArraySchema { * * @param new_current_domain The new array current domain we want to expand to */ - void expand_current_domain( - shared_ptr new_current_domain); + void expand_current_domain(shared_ptr new_current_domain); /** * Set the array current domain on the schema * * @param current_domain The array current domain we want to set on the schema */ - void set_current_domain(shared_ptr current_domain); + void set_current_domain(shared_ptr current_domain); /** Array current domain accessor */ - shared_ptr get_current_domain() const; + shared_ptr get_current_domain() const; private: /* ********************************* */ @@ -722,7 +721,7 @@ class ArraySchema { FilterPipeline coords_filters_; /** The array current domain */ - shared_ptr current_domain_; + shared_ptr current_domain_; /** Mutex for thread-safety. */ mutable std::mutex mtx_; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 2cafe3180a0..09aab9ff329 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -92,7 +92,7 @@ ArraySchemaEvolution::ArraySchemaEvolution( enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , attributes_to_add_map_( @@ -379,7 +379,7 @@ std::pair ArraySchemaEvolution::timestamp_range() const { } void ArraySchemaEvolution::expand_current_domain( - shared_ptr current_domain) { + shared_ptr current_domain) { if (current_domain == nullptr) { throw ArraySchemaEvolutionException( "Cannot expand the array current domain; Input current domain is null"); @@ -395,7 +395,7 @@ void ArraySchemaEvolution::expand_current_domain( current_domain_to_expand_ = current_domain; } -shared_ptr ArraySchemaEvolution::current_domain_to_expand() +shared_ptr ArraySchemaEvolution::current_domain_to_expand() const { std::lock_guard lock(mtx_); diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index bdec153e483..8959178128e 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -92,7 +92,7 @@ class ArraySchemaEvolution { enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker); DISABLE_COPY_AND_COPY_ASSIGN(ArraySchemaEvolution); @@ -196,12 +196,12 @@ class ArraySchemaEvolution { * * @param current_domain The new current domain to expand to */ - void expand_current_domain(shared_ptr current_domain); + void expand_current_domain(shared_ptr current_domain); /** * Accessor for the current domain we want to expand to */ - shared_ptr current_domain_to_expand() const; + shared_ptr current_domain_to_expand() const; private: /* ********************************* */ @@ -241,7 +241,7 @@ class ArraySchemaEvolution { std::pair timestamp_range_; /** The array current domain to expand */ - shared_ptr current_domain_to_expand_; + shared_ptr current_domain_to_expand_; /** Mutex for thread-safety. */ mutable std::mutex mtx_; diff --git a/tiledb/sm/array_schema/auxiliary.cc b/tiledb/sm/array_schema/auxiliary.cc index 0d05fc878b1..af595d5f6b4 100644 --- a/tiledb/sm/array_schema/auxiliary.cc +++ b/tiledb/sm/array_schema/auxiliary.cc @@ -51,7 +51,7 @@ namespace tiledb::sm { /* API */ /* ********************************* */ -Status store_array_schema( +void store_array_schema( ContextResources& resources, const shared_ptr& array_schema, const EncryptionKey& encryption_key) { @@ -122,8 +122,6 @@ Status store_array_schema( auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); GenericTileIO::store_data(resources, abs_enmr_uri, tile, encryption_key); } - - return Status::Ok(); } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/auxiliary.h b/tiledb/sm/array_schema/auxiliary.h index 85919a9e870..e2e76335eaf 100644 --- a/tiledb/sm/array_schema/auxiliary.h +++ b/tiledb/sm/array_schema/auxiliary.h @@ -60,9 +60,8 @@ class EncryptionKey; * @param resources The context resources. * @param array_schema The array schema to be stored. * @param encryption_key The encryption key to use. - * @return Status */ -Status store_array_schema( +void store_array_schema( ContextResources& resources, const shared_ptr& array_schema, const EncryptionKey& encryption_key); diff --git a/tiledb/sm/array_schema/current_domain.cc b/tiledb/sm/array_schema/current_domain.cc index 9aad114d5a1..489d0977557 100644 --- a/tiledb/sm/array_schema/current_domain.cc +++ b/tiledb/sm/array_schema/current_domain.cc @@ -54,12 +54,12 @@ CurrentDomain::CurrentDomain( CurrentDomain::CurrentDomain( shared_ptr memory_tracker, format_version_t version, - shared_ptr ndr) + shared_ptr ndr) : CurrentDomain(memory_tracker, version) { set_ndrectangle(ndr); } -shared_ptr CurrentDomain::deserialize( +shared_ptr CurrentDomain::deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain) { @@ -149,7 +149,7 @@ void CurrentDomain::dump(FILE* out) const { } } -void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { +void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { if (!empty_) { throw std::logic_error( "Setting a rectangle on a non-empty CurrentDomain object is not " @@ -160,7 +160,7 @@ void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { empty_ = false; } -shared_ptr CurrentDomain::ndrectangle() const { +shared_ptr CurrentDomain::ndrectangle() const { if (empty_ || type_ != CurrentDomainType::NDRECTANGLE) { throw std::logic_error( "It's not possible to get the ndrectangle from this current domain if " diff --git a/tiledb/sm/array_schema/current_domain.h b/tiledb/sm/array_schema/current_domain.h index b73f3d2314d..90e1c6084ca 100644 --- a/tiledb/sm/array_schema/current_domain.h +++ b/tiledb/sm/array_schema/current_domain.h @@ -77,7 +77,7 @@ class CurrentDomain { CurrentDomain( shared_ptr memory_tracker, format_version_t version, - shared_ptr ndr); + shared_ptr ndr); /** Destructor. */ ~CurrentDomain() = default; @@ -101,7 +101,7 @@ class CurrentDomain { * @param domain The array schema domain. * @return A new CurrentDomain. */ - static shared_ptr deserialize( + static shared_ptr deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain); @@ -117,6 +117,11 @@ class CurrentDomain { * @return Returns the type of current domain stored in this instance */ CurrentDomainType type() const { + if (empty_) { + throw std::logic_error( + "It's not possible to read the type, this CurrentDomain instance is " + "empty."); + } return type_; } @@ -147,14 +152,14 @@ class CurrentDomain { * * @param ndr A NDRectangle to be set on this CurrentDomain object. */ - void set_ndrectangle(std::shared_ptr ndr); + void set_ndrectangle(std::shared_ptr ndr); /** * Throws if the current domain doesn't have a NDRectangle set * * @return Returns the ndrectangle set on a current domain. */ - shared_ptr ndrectangle() const; + shared_ptr ndrectangle() const; /** * Checks if the argument fully contains this current domain. @@ -198,7 +203,7 @@ class CurrentDomain { bool empty_; /** The ndrectangle current domain */ - shared_ptr ndrectangle_; + shared_ptr ndrectangle_; /** The format version of this CurrentDomain */ format_version_t version_; diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 40ecc0e31aa..397cbdf0d83 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -227,7 +227,7 @@ class Domain { * @return non-null pointer to the dimension */ inline const Dimension* dimension_ptr(dimension_size_type i) const { - if (i > dim_num_) { + if (i >= dim_num_) { throw std::invalid_argument("invalid dimension index"); } return dimension_ptrs_[i]; diff --git a/tiledb/sm/array_schema/ndrectangle.cc b/tiledb/sm/array_schema/ndrectangle.cc index 009f844b28b..bc0df55855c 100644 --- a/tiledb/sm/array_schema/ndrectangle.cc +++ b/tiledb/sm/array_schema/ndrectangle.cc @@ -77,7 +77,7 @@ NDRange NDRectangle::deserialize_ndranges( return nd; } -shared_ptr NDRectangle::deserialize( +shared_ptr NDRectangle::deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain) { diff --git a/tiledb/sm/array_schema/ndrectangle.h b/tiledb/sm/array_schema/ndrectangle.h index 84a4c1a2a81..4ea35c1a3d1 100644 --- a/tiledb/sm/array_schema/ndrectangle.h +++ b/tiledb/sm/array_schema/ndrectangle.h @@ -99,7 +99,7 @@ class NDRectangle { * @param domain The domain from array schema * @return A new ndrectangle. */ - static shared_ptr deserialize( + static shared_ptr deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain); diff --git a/tiledb/sm/array_schema/test/CMakeLists.txt b/tiledb/sm/array_schema/test/CMakeLists.txt index 98726420175..9ed957e082b 100644 --- a/tiledb/sm/array_schema/test/CMakeLists.txt +++ b/tiledb/sm/array_schema/test/CMakeLists.txt @@ -36,6 +36,7 @@ commence(unit_test array_schema) unit_attribute.cc unit_dimension.cc unit_dimension_label.cc + unit_domain.cc unit_domain_data.cc unit_tile_domain.cc unit_current_domain.cc diff --git a/tiledb/sm/array_schema/test/unit_current_domain.cc b/tiledb/sm/array_schema/test/unit_current_domain.cc index 077ddc0a0c6..54dfe96b074 100644 --- a/tiledb/sm/array_schema/test/unit_current_domain.cc +++ b/tiledb/sm/array_schema/test/unit_current_domain.cc @@ -1,5 +1,5 @@ /** - * @file unit-current_domain.cc + * @file unit_current_domain.cc * * @section LICENSE * @@ -58,7 +58,7 @@ class CurrentDomainFx { CurrentDomainFx(); ~CurrentDomainFx(); - shared_ptr create_current_domain( + shared_ptr create_current_domain( const NDRange& ranges, shared_ptr schema, shared_ptr ndrectangle = nullptr, @@ -68,9 +68,9 @@ class CurrentDomainFx { shared_ptr schema, const NDRange& ranges); storage_size_t calculate_serialized_size( - shared_ptr current_domain); + shared_ptr current_domain); shared_ptr serialize_to_tile( - shared_ptr current_domain); + shared_ptr current_domain); shared_ptr create_schema(); @@ -81,7 +81,7 @@ class CurrentDomainFx { shared_ptr get_array_directory(); shared_ptr get_array_schema_latest(); void check_current_domains_equal( - shared_ptr s1, shared_ptr s2); + shared_ptr s1, shared_ptr s2); void rm_array(); @@ -117,7 +117,7 @@ void CurrentDomainFx::rm_array() { } template -shared_ptr CurrentDomainFx::create_current_domain( +shared_ptr CurrentDomainFx::create_current_domain( const NDRange& ranges, shared_ptr schema, shared_ptr ndrectangle, @@ -158,7 +158,7 @@ void CurrentDomainFx::check_storage_serialization( template void CurrentDomainFx::check_current_domains_equal( - shared_ptr s1, shared_ptr s2) { + shared_ptr s1, shared_ptr s2) { REQUIRE(s1->empty() == s2->empty()); REQUIRE(s1->type() == s2->type()); REQUIRE(s1->version() == s2->version()); @@ -168,7 +168,7 @@ void CurrentDomainFx::check_current_domains_equal( template storage_size_t CurrentDomainFx::calculate_serialized_size( - shared_ptr current_domain) { + shared_ptr current_domain) { storage_size_t num_bytes = 0; // uint32_t - version @@ -200,7 +200,7 @@ storage_size_t CurrentDomainFx::calculate_serialized_size( template shared_ptr CurrentDomainFx::serialize_to_tile( - shared_ptr current_domain) { + shared_ptr current_domain) { SizeComputationSerializer size_serializer; current_domain->serialize(size_serializer); @@ -261,7 +261,7 @@ shared_ptr CurrentDomainFx::create_schema_var() { template void CurrentDomainFx::create_array(shared_ptr schema) { - throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); + Array::create(ctx_.resources(), uri_, schema, enc_key_); } template @@ -524,8 +524,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( // Persist evolved schema and read it back, check it shows the correct // rectangle - throw_if_not_ok(Array::evolve_array_schema( - this->ctx_.resources(), this->uri_, ase.get(), this->enc_key_)); + Array::evolve_array_schema( + this->ctx_.resources(), this->uri_, ase.get(), this->enc_key_); // Read it back auto new_schema = this->get_array_schema_latest(); diff --git a/tiledb/sm/array_schema/test/unit_domain.cc b/tiledb/sm/array_schema/test/unit_domain.cc index 27ea16fb0cf..2d2f7ac0048 100644 --- a/tiledb/sm/array_schema/test/unit_domain.cc +++ b/tiledb/sm/array_schema/test/unit_domain.cc @@ -38,9 +38,12 @@ #include +#include "src/mem_helpers.h" + using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm; +using namespace tiledb::test; template inline T& dom_buffer_offset(void* p) { @@ -112,8 +115,14 @@ TEST_CASE("Domain: Test deserialization", "[domain][deserialize]") { dom_buffer_offset(p) = null_tile_extent2; Deserializer deserializer(&serialized_buffer, sizeof(serialized_buffer)); - auto dom{Domain::deserialize( - deserializer, 10, Layout::ROW_MAJOR, Layout::ROW_MAJOR)}; + FilterPipeline fp; + auto dom{tiledb::sm::Domain::deserialize( + deserializer, + 10, + Layout::ROW_MAJOR, + Layout::ROW_MAJOR, + fp, + tiledb::test::get_test_memory_tracker())}; CHECK(dom->dim_num() == dim_num); auto dim1{dom->dimension_ptr("d1")}; @@ -128,3 +137,8 @@ TEST_CASE("Domain: Test deserialization", "[domain][deserialize]") { CHECK(dim2->cell_val_num() == cell_val_num2); CHECK(dim2->filters().size() == num_filters2); } + +TEST_CASE("Domain: Test dimension_ptr is not oob", "[domain][oob]") { + auto d = tiledb::sm::Domain(tiledb::test::get_test_memory_tracker()); + REQUIRE_THROWS(d.dimension_ptr(0)); +} diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 7b10b8f755e..b5129672691 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -40,6 +40,7 @@ #include "tiledb/api/c_api/buffer/buffer_api_internal.h" #include "tiledb/api/c_api/buffer_list/buffer_list_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" #include "tiledb/api/c_api/domain/domain_api_internal.h" #include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" @@ -73,6 +74,7 @@ #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/object/object.h" +#include "tiledb/sm/object/object_iter.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/rest/rest_client.h" @@ -784,6 +786,40 @@ int32_t tiledb_array_schema_has_attribute( return TILEDB_OK; } +int32_t tiledb_array_schema_set_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_handle_is_valid(current_domain); + + array_schema->array_schema_->set_current_domain( + current_domain->current_domain()); + + return TILEDB_OK; +} + +int32_t tiledb_array_schema_get_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_output_pointer_is_valid(current_domain); + + // There is always a current domain on an ArraySchema instance, + // when none was set explicitly, there is an empty current domain. + *current_domain = tiledb_current_domain_handle_t::make_handle( + array_schema->array_schema_->get_current_domain()); + + return TILEDB_OK; +} + /* ********************************* */ /* SCHEMA EVOLUTION */ /* ********************************* */ @@ -906,6 +942,22 @@ capi_return_t tiledb_array_schema_evolution_drop_enumeration( return TILEDB_OK; } +capi_return_t tiledb_array_schema_evolution_expand_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_current_domain_t* expanded_domain) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_handle_is_valid(expanded_domain); + + array_schema_evolution->array_schema_evolution_->expand_current_domain( + expanded_domain->current_domain()); + + return TILEDB_OK; +} + int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, @@ -975,8 +1027,8 @@ int32_t tiledb_query_alloc( } // Create query - (*query)->query_ = new (std::nothrow) - tiledb::sm::Query(ctx->storage_manager(), array->array_); + (*query)->query_ = new (std::nothrow) tiledb::sm::Query( + ctx->resources(), ctx->storage_manager(), array->array_); if ((*query)->query_ == nullptr) { auto st = Status_Error( "Failed to allocate TileDB query object; Memory allocation failed"); @@ -2561,8 +2613,8 @@ int32_t tiledb_array_create( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the array - throw_if_not_ok(tiledb::sm::Array::create( - ctx->resources(), uri, array_schema->array_schema_, key)); + tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2580,11 +2632,11 @@ int32_t tiledb_array_create( } // Create the dimension label array with the same key. - throw_if_not_ok(tiledb::sm::Array::create( + tiledb::sm::Array::create( ctx->resources(), dim_label_ref.uri(uri), dim_label_ref.schema(), - key)); + key); } } return TILEDB_OK; @@ -2632,8 +2684,8 @@ int32_t tiledb_array_create_with_key( key_length)); // Create the array - throw_if_not_ok(tiledb::sm::Array::create( - ctx->resources(), uri, array_schema->array_schema_, key)); + tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2651,11 +2703,11 @@ int32_t tiledb_array_create_with_key( } // Create the dimension label array with the same key. - throw_if_not_ok(tiledb::sm::Array::create( + tiledb::sm::Array::create( ctx->resources(), dim_label_ref.uri(uri), dim_label_ref.schema(), - key)); + key); } } return TILEDB_OK; @@ -2665,6 +2717,7 @@ int32_t tiledb_array_consolidate( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { api::ensure_config_is_valid_if_present(config); tiledb::sm::Consolidator::array_consolidate( + ctx->resources(), array_uri, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, @@ -2684,6 +2737,7 @@ int32_t tiledb_array_consolidate_with_key( // Sanity checks tiledb::sm::Consolidator::array_consolidate( + ctx->resources(), array_uri, static_cast(encryption_type), encryption_key, @@ -2710,6 +2764,7 @@ int32_t tiledb_array_consolidate_fragments( } tiledb::sm::Consolidator::fragments_consolidate( + ctx->resources(), array_uri, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, @@ -2724,6 +2779,7 @@ int32_t tiledb_array_consolidate_fragments( int32_t tiledb_array_vacuum( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { tiledb::sm::Consolidator::array_vacuum( + ctx->resources(), array_uri, (config == nullptr) ? ctx->config() : config->config(), ctx->storage_manager()); @@ -2888,8 +2944,8 @@ int32_t tiledb_array_encryption_type( // Get encryption type tiledb::sm::EncryptionType enc; - throw_if_not_ok(sm::Array::encryption_type( - ctx->resources(), tiledb::sm::URI(array_uri), &enc)); + sm::Array::encryption_type( + ctx->resources(), tiledb::sm::URI(array_uri), &enc); *encryption_type = static_cast(enc); return TILEDB_OK; @@ -3015,11 +3071,11 @@ int32_t tiledb_array_evolve( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Evolve schema - throw_if_not_ok(tiledb::sm::Array::evolve_array_schema( + tiledb::sm::Array::evolve_array_schema( ctx->resources(), uri, array_schema_evolution->array_schema_evolution_, - key)); + key); // Success return TILEDB_OK; @@ -3070,10 +3126,10 @@ int32_t tiledb_array_upgrade_version( } // Upgrade version - throw_if_not_ok(tiledb::sm::Array::upgrade_version( + tiledb::sm::Array::upgrade_version( ctx->resources(), uri, - (config == nullptr) ? ctx->config() : config->config())); + (config == nullptr) ? ctx->config() : config->config()); return TILEDB_OK; } @@ -3084,22 +3140,20 @@ int32_t tiledb_array_upgrade_version( int32_t tiledb_object_type( tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) { - auto uri = tiledb::sm::URI(path); - tiledb::sm::ObjectType object_type; - throw_if_not_ok(tiledb::sm::object_type(ctx->resources(), uri, &object_type)); - - *type = static_cast(object_type); + ensure_output_pointer_is_valid(type); + *type = static_cast( + tiledb::sm::object_type(ctx->resources(), tiledb::sm::URI(path))); return TILEDB_OK; } int32_t tiledb_object_remove(tiledb_ctx_t* ctx, const char* path) { - throw_if_not_ok(object_remove(ctx->resources(), path)); + object_remove(ctx->resources(), path); return TILEDB_OK; } int32_t tiledb_object_move( tiledb_ctx_t* ctx, const char* old_path, const char* new_path) { - throw_if_not_ok(object_move(ctx->resources(), old_path, new_path)); + object_move(ctx->resources(), old_path, new_path); return TILEDB_OK; } @@ -3119,9 +3173,8 @@ int32_t tiledb_object_walk( } // Create an object iterator - tiledb::sm::StorageManager::ObjectIter* obj_iter; - throw_if_not_ok(ctx->storage_manager()->object_iter_begin( - &obj_iter, path, static_cast(order))); + tiledb::sm::ObjectIter* obj_iter = tiledb::sm::object_iter_begin( + ctx->resources(), path, static_cast(order)); // For as long as there is another object and the callback indicates to // continue, walk over the TileDB objects in the path @@ -3132,9 +3185,9 @@ int32_t tiledb_object_walk( do { if (SAVE_ERROR_CATCH( ctx, - ctx->storage_manager()->object_iter_next( - obj_iter, &obj_name, &obj_type, &has_next))) { - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_next( + ctx->resources(), obj_iter, &obj_name, &obj_type, &has_next))) { + tiledb::sm::object_iter_free(obj_iter); return TILEDB_ERR; } if (!has_next) @@ -3143,7 +3196,7 @@ int32_t tiledb_object_walk( } while (rc == 1); // Clean up - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_free(obj_iter); if (rc == -1) return TILEDB_ERR; @@ -3166,8 +3219,8 @@ int32_t tiledb_object_ls( } // Create an object iterator - tiledb::sm::StorageManager::ObjectIter* obj_iter; - throw_if_not_ok(ctx->storage_manager()->object_iter_begin(&obj_iter, path)); + tiledb::sm::ObjectIter* obj_iter = + tiledb::sm::object_iter_begin(ctx->resources(), path); // For as long as there is another object and the callback indicates to // continue, walk over the TileDB objects in the path @@ -3178,9 +3231,9 @@ int32_t tiledb_object_ls( do { if (SAVE_ERROR_CATCH( ctx, - ctx->storage_manager()->object_iter_next( - obj_iter, &obj_name, &obj_type, &has_next))) { - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_next( + ctx->resources(), obj_iter, &obj_name, &obj_type, &has_next))) { + tiledb::sm::object_iter_free(obj_iter); return TILEDB_ERR; } if (!has_next) @@ -3189,7 +3242,7 @@ int32_t tiledb_object_ls( } while (rc == 1); // Clean up - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_free(obj_iter); if (rc == -1) return TILEDB_ERR; @@ -3724,7 +3777,7 @@ int32_t tiledb_deserialize_query_and_array( buffer->buffer(), (tiledb::sm::SerializationType)serialize_type, *(*array)->array_, - ctx->storage_manager(), + ctx->resources(), memory_tracker)); // Create query struct @@ -3738,8 +3791,8 @@ int32_t tiledb_deserialize_query_and_array( } // Create query - (*query)->query_ = new (std::nothrow) - tiledb::sm::Query(ctx->storage_manager(), (*array)->array_); + (*query)->query_ = new (std::nothrow) tiledb::sm::Query( + ctx->resources(), ctx->storage_manager(), (*array)->array_); if ((*query)->query_ == nullptr) { auto st = Status_Error( "Failed to allocate TileDB query object; Memory allocation failed"); @@ -4317,7 +4370,8 @@ capi_return_t tiledb_handle_query_plan_request( api::ensure_buffer_is_valid(request); api::ensure_buffer_is_valid(response); - tiledb::sm::Query query(ctx->storage_manager(), array->array_); + tiledb::sm::Query query( + ctx->resources(), ctx->storage_manager(), array->array_); tiledb::sm::serialization::deserialize_query_plan_request( static_cast(serialization_type), request->buffer(), @@ -5540,6 +5594,24 @@ CAPI_INTERFACE( ctx, array_schema, name, has_attr); } +CAPI_INTERFACE( + array_schema_set_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) { + return api_entry( + ctx, array_schema, current_domain); +} + +CAPI_INTERFACE( + array_schema_get_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) { + return api_entry( + ctx, array_schema, current_domain); +} + /* ********************************* */ /* SCHEMA EVOLUTION */ /* ********************************* */ @@ -5605,6 +5677,16 @@ CAPI_INTERFACE( ctx, array_schema_evolution, enumeration_name); } +CAPI_INTERFACE( + array_schema_evolution_expand_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_current_domain_t* expanded_domain) { + return api_entry< + tiledb::api::tiledb_array_schema_evolution_expand_current_domain>( + ctx, array_schema_evolution, expanded_domain); +} + TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, diff --git a/tiledb/sm/c_api/tiledb_enum.h b/tiledb/sm/c_api/tiledb_enum.h index 016809bc5d9..9fb10866c03 100644 --- a/tiledb/sm/c_api/tiledb_enum.h +++ b/tiledb/sm/c_api/tiledb_enum.h @@ -132,7 +132,3 @@ /** application/pdf*/ TILEDB_MIME_TYPE_ENUM(MIME_PDF) = 2, #endif - -#ifdef TILEDB_CURRENT_DOMAIN_TYPE_ENUM - TILEDB_CURRENT_DOMAIN_TYPE_ENUM(NDRECTANGLE) = 0, -#endif diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 56bd9290185..691ce1f5586 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -42,6 +42,7 @@ * API sections */ #include "tiledb/api/c_api/attribute/attribute_api_external_experimental.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" #include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" #include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" @@ -295,6 +296,49 @@ TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( uint64_t lo, uint64_t hi) TILEDB_NOEXCEPT; +/** + * Expands the current domain during array schema evolution. + * TileDB will enforce that the new current domain is expanding + * on the current one and not contracting during `tiledb_array_evolve`. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *new_domain; + * tiledb_current_domain_create(ctx, &new_domain); + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * + * tiledb_range_t range; + * range.min = &expanded_min; + * range.min_size = sizeof(expanded_min); + * range.max = &expanded_max; + * range.max_size = sizeof(expanded_max); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim1", &range); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim2", &range); + * + * tiledb_current_domain_set_ndrectangle(new_domain, ndr); + * + * tiledb_array_schema_evolution_expand_current_domain(ctx, + * array_schema_evolution, new_domain); + * + * tiledb_array_evolve(ctx, array_uri, array_schema_evolution); + * + * tiledb_ndrectangle_free(&ndr); + * tiledb_current_domain_free(&new_domain); + * + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param expanded_domain The current domain we want to expand the schema to. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_expand_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_current_domain_t* expanded_domain) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY SCHEMA */ /* ********************************* */ @@ -354,6 +398,51 @@ TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( tiledb_array_schema_t* array_schema, tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; +/** + * Sets the current domain on the array schema + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_array_schema_set_current_domain(ctx, array_schema, current_domain); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param current_domain The current domain to set on the schema + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_set_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) TILEDB_NOEXCEPT; + +/** + * Gets the current domain set on the array schema or + * creates an empty current domain if none was set. + * It is the responsability of the caller to free the resources associated + * with the current domain when the handle isn't needed anymore. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_array_schema_get_current_domain(ctx, array_schema, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param current_domain The current domain set on the schema + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_get_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY */ /* ********************************* */ diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index cc2d4d1ce03..d0541879c2a 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -274,7 +274,8 @@ int32_t tiledb_filestore_uri_import( auto buffer_size = get_buffer_size_from_config(context.resources().config(), tile_extent); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::GLOBAL_ORDER)); std::vector buffer(buffer_size); @@ -296,7 +297,8 @@ int32_t tiledb_filestore_uri_import( query.set_subarray(subarray); auto tiledb_cloud_fix = [&](uint64_t start, uint64_t end) { - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); tiledb::sm::Subarray subarray_cloud_fix( array.get(), nullptr, context.resources().logger(), true); @@ -429,7 +431,8 @@ int32_t tiledb_filestore_uri_export( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); subarray.add_range(0, std::move(subarray_range)); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); query.set_subarray(subarray); @@ -532,7 +535,8 @@ int32_t tiledb_filestore_buffer_import( static_cast(0), ""); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); tiledb::sm::Subarray subarray( @@ -601,7 +605,8 @@ int32_t tiledb_filestore_buffer_export( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); subarray.add_range(0, std::move(subarray_range)); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); query.set_subarray(subarray); uint64_t size_tmp = size; diff --git a/tiledb/sm/compressors/CMakeLists.txt b/tiledb/sm/compressors/CMakeLists.txt index f513591f84f..ca42d431f42 100644 --- a/tiledb/sm/compressors/CMakeLists.txt +++ b/tiledb/sm/compressors/CMakeLists.txt @@ -4,7 +4,7 @@ # # The MIT License # -# Copyright (c) 2021-2022 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -34,7 +34,7 @@ commence(object_library compressors) this_target_sources( bzip_compressor.cc dd_compressor.cc delta_compressor.cc dict_compressor.cc gzip_compressor.cc lz4_compressor.cc - rle_compressor.cc zstd_compressor.cc util/gzip_wrappers.cc) + rle_compressor.cc zstd_compressor.cc) this_target_object_libraries(baseline buffer) find_package(BZip2 REQUIRED) find_package(LZ4 MODULE REQUIRED) @@ -43,33 +43,4 @@ commence(object_library compressors) this_target_link_libraries(BZip2::BZip2 lz4::lz4 ZLIB::ZLIB ${ZSTD_TARGET}) conclude(object_library) -# -# tiledb crude gzip archiver initially for embedding magic.mgc -# -add_executable(tdb_gzip_embedded_data) -target_sources(tdb_gzip_embedded_data PRIVATE - util/tdb_gzip_embedded_data.cc -) - -target_link_libraries( - tdb_gzip_embedded_data PUBLIC - filter - compressors - ) - -target_include_directories(tdb_gzip_embedded_data PRIVATE - $ -) - -# Link to Threads::Threads if library or flag needed to enable threading. -find_package(Threads REQUIRED) - -if (CMAKE_THREAD_LIBS_INIT) - target_link_libraries(tdb_gzip_embedded_data PRIVATE Threads::Threads) -endif() -# On Linux, must explicitly link -ldl for static linking to libzstd. -if (NOT WIN32) - target_link_libraries(tdb_gzip_embedded_data PRIVATE dl) -endif() - add_test_subdirectory() diff --git a/tiledb/sm/compressors/util/gzip_wrappers.cc b/tiledb/sm/compressors/util/gzip_wrappers.cc deleted file mode 100644 index cb8912e3407..00000000000 --- a/tiledb/sm/compressors/util/gzip_wrappers.cc +++ /dev/null @@ -1,100 +0,0 @@ -/** - * @file gzip_wrappers.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines gzip_compress and gzip_uncompress, routines - * that slightly simplify use of the gzip compression routines. - */ - -#include "gzip_wrappers.h" - -#include - -void gzip_compress( - shared_ptr& out_gzipped_buf, - const void* in_bytes, - uint64_t nbytes) { - assert(out_gzipped_buf.get() != nullptr); - /** - * buffer format: - * uint64_t uncompressed_size - * uint64_t compressed_size - * uint8_t [compressed_size] - compressed data - */ - uint64_t overhead_size = 2 * sizeof(uint64_t); - tiledb::sm::ConstBuffer const_in_buf(in_bytes, nbytes); - // Ensure space in output buffer for worst acceptable case. - if (!out_gzipped_buf->realloc(nbytes + overhead_size).ok()) { - throw std::logic_error("gzip output buffer allocation error"); - } - out_gzipped_buf->reset_size(); - out_gzipped_buf->advance_size(overhead_size); - out_gzipped_buf->advance_offset(overhead_size); - tiledb::sm::GZip::compress(9, &const_in_buf, out_gzipped_buf.get()); - - // TODO: Handle possibility that 'error' is just 'not enuf buffer', i.e. - // unable to compress into <= space of in_buf (currently that is not - // returned as a distinct error from Gzip::compress()) - - // adjust to size actually used - out_gzipped_buf->set_size(out_gzipped_buf->offset()); - uint64_t compressed_size = out_gzipped_buf->offset() - overhead_size; - uint64_t uncompressed_size = nbytes; - // return next 'write()' position to beginning of buffer - out_gzipped_buf->reset_offset(); - // write sizes to beginning of buffer - throw_if_not_ok( - out_gzipped_buf->write(&uncompressed_size, sizeof(uncompressed_size))); - throw_if_not_ok( - out_gzipped_buf->write(&compressed_size, sizeof(compressed_size))); -} - -void gzip_decompress( - shared_ptr& out_buf, const uint8_t* comp_buf) { - /** - * Expected compressed buffer format: - * uint64_t uncompressed_size - * uint64_t compressed_size - * uint8_t [compressed_size] - compressed data - */ - uint64_t expanded_size = *reinterpret_cast(comp_buf); - uint64_t compressed_size = - *reinterpret_cast(comp_buf + sizeof(uint64_t)); - - const uint8_t* comp_data = comp_buf + 2 * sizeof(uint64_t); - - out_buf->resize(expanded_size); - - tiledb::sm::PreallocatedBuffer pa_gunzip_out_buf( - out_buf->data(), - expanded_size); // the expected uncompressed size - - tiledb::sm::ConstBuffer gzipped_input_buffer(comp_data, compressed_size); - - tiledb::sm::GZip::decompress(&gzipped_input_buffer, &pa_gunzip_out_buf); -} diff --git a/tiledb/sm/compressors/util/gzip_wrappers.h b/tiledb/sm/compressors/util/gzip_wrappers.h deleted file mode 100644 index bafacfc7cf3..00000000000 --- a/tiledb/sm/compressors/util/gzip_wrappers.h +++ /dev/null @@ -1,67 +0,0 @@ -/** - * @file gzip_wrappers.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declare gzip_compress and gzip_uncompress, routines - * that slightly simplify use of the gzip compression routines. - */ - -#ifndef TILEDB_GZIP_WRAPPER_H -#define TILEDB_GZIP_WRAPPER_H - -#include "../gzip_compressor.h" - -#include "tiledb/common/common.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/filter/filter_buffer.h" -#include "tiledb/sm/filter/filter_storage.h" -#include "tiledb/sm/misc/types.h" - -/** - * gzip compress data from in_buf into a simple format in out_gzipped_buf - * that can be passed to gzip_uncompress to retrieve the uncompressed data. - * - * @param out_gzipped_buf The returned buffer with the compressed data. - * @param in_buf The input buffer containing data to be compressed - */ -void gzip_compress( - shared_ptr& out_gzipped_buf, - const void* in_bytes, - uint64_t nbytes); - -/** - * gzip decompress data from inbuf into linear series of bytesw in outbuf - * that can be passed to gzip_uncompress to retrieve the uncompressed data. - * - * @param out_buf The input buffer containing data to be compressed - * @param comp_buf The buffer with the compressed data to be decompressed - */ -void gzip_decompress( - shared_ptr& out_buf, const uint8_t* comp_buf); - -#endif // TILEDB_GZIP_WRAPPER_H diff --git a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc deleted file mode 100644 index a3384443fb2..00000000000 --- a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc +++ /dev/null @@ -1,214 +0,0 @@ -/** - * @file compile_filter_main.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "../bzip_compressor.h" -#include "../dd_compressor.h" -#include "../dict_compressor.h" -#include "../gzip_compressor.h" -#include "../lz4_compressor.h" -#include "../rle_compressor.h" -#include "../zstd_compressor.h" - -#include "tiledb/common/common.h" -#include "tiledb/common/scoped_executor.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/filter/filter_buffer.h" -#include "tiledb/sm/filter/filter_storage.h" -#include "tiledb/sm/misc/types.h" - -#include "gzip_wrappers.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#endif - -int main(int argc, char* argv[]) { - (void)sizeof(tiledb::sm::GZip); - - const int seg_sz = 4096; - char fbuf[seg_sz]; - auto filter_stg = make_shared(HERE()); - auto inbuf = make_shared(HERE()); - auto zipped_buf = make_shared(HERE(), &*filter_stg); - uint64_t nread; - uint64_t cntread = 0; - - auto infile = stdin; - auto outfile = stdout; - - // note: If stdout used ( '>' ), compressed data subject to being - // intermixed with any application output, corrupting the compressed - // output stream. - - if (argc > 1) { - outfile = fopen(argv[1], "w+b"); - if (!outfile) { - fprintf(stderr, "Unable to create file %s\n", argv[1]); - exit(-2); - } - } - auto closefile = [&]() { fclose(outfile); }; - tiledb::common::ScopedExecutor onexit1(closefile); - -#ifdef _WIN32 - // need to be sure in/out are in binay mode, windows default won't be!!! - if ((-1 == _setmode(fileno(stdin), _O_BINARY)) || - (-1 == _setmode(fileno(stdout), _O_BINARY))) { - fprintf(stderr, "failure setting stdin/stdout to binary mode!"); - exit(-1); - } -#endif - - do { - nread = fread(fbuf, 1, sizeof(fbuf), infile); - if (nread) { - throw_if_not_ok(inbuf->write(fbuf, nread)); - cntread += nread; - } - } while (nread == sizeof(fbuf)); - - tiledb::sm::ConstBuffer const_inbuf(&*inbuf); - // Ensure space in output buffer for worst case. - if (!zipped_buf->prepend_buffer(inbuf->size()).ok()) { - printf("output buffer allocation error!\n"); - exit(-3); - } - tiledb::sm::Buffer* out_buffer_ptr = zipped_buf->buffer_ptr(0); - assert(out_buffer_ptr != nullptr); - out_buffer_ptr->reset_offset(); - tiledb::sm::GZip::compress(9, &const_inbuf, out_buffer_ptr); - std::cerr << "sizes input " << inbuf->size() << ", compressed " - << out_buffer_ptr->size() << std::endl; - if (0) // leave available for future possible diagnostic need - { - // save compressed data to allow examination from filesystem - auto outgzip = fopen("magic-mgc.tdbgzip", "wb"); - if (!outgzip) { - fprintf(stderr, "Unable to create magic-mgc.tdbgzip!\n"); - exit(-21); - } - if (fwrite(out_buffer_ptr->data(0), 1, out_buffer_ptr->size(), outgzip) != - out_buffer_ptr->size()) { - fprintf(stderr, "write failure magic-mgc.tdbgzip!\n"); - exit(-22); - } - fclose(outgzip); - } - auto tdb_gzip_buf = make_shared(HERE()); - - unsigned maxnperline = 128; - auto addbytevals = [&](void* _pbytes, uint64_t nbytes) { - uint8_t* pbytes = reinterpret_cast(_pbytes); - static unsigned cntout = 0; - for (auto i = 0u; i < nbytes; ++i) { - fprintf(outfile, "'\\x%02x',", pbytes[i]); - if (++cntout > maxnperline) { - cntout = 0; - fprintf(outfile, "\n"); - } - } - }; - addbytevals(&cntread, sizeof(cntread)); - throw_if_not_ok(tdb_gzip_buf->write(&cntread, sizeof(cntread))); - uint64_t compressed_size = zipped_buf->size(); - addbytevals(&compressed_size, sizeof(compressed_size)); - throw_if_not_ok( - tdb_gzip_buf->write(&compressed_size, sizeof(compressed_size))); - auto nremaining = zipped_buf->size(); - // vs19 complained... - // A) could not find raw string literal terminator - // for almost identical error, though not quite same cause, can see - // https://developercommunity.visualstudio.com/t/bug-in-raw-string-implementation-converning-eof-02/254730) - // vs19 16.11.3 was in use with tiledb failed #include of raw literal file - // B) that (all data as) one string was to large a string - // C) many shorter strings expecting concatention were causing - // compiler heap memory errors (100+GB) - // D) caused warnings trying to do direct 0xXX numeric values - // end result format is comma delimited '\xXX' characters with periodic line - // splits - while (nremaining) { - auto ntowrite = nremaining > seg_sz ? seg_sz : nremaining; - // TBD: error from ->read()? - if (!zipped_buf->read(fbuf, ntowrite).ok()) { - printf("ERROR reading from compressed data.\n"); - exit(-7); - } - if (!tdb_gzip_buf->write(fbuf, ntowrite).ok()) { - printf("ERROR writing compressed format buffer."); - exit(-11); - } - addbytevals(fbuf, ntowrite); - nremaining -= ntowrite; - } - - // brief sanity check that wrapper compression matches unwrapped compression - shared_ptr out_gzipped_buf = - make_shared(HERE()); - gzip_compress(out_gzipped_buf, inbuf->data(0), inbuf->size()); - if (out_gzipped_buf->size() != tdb_gzip_buf->size()) { - printf( - "Error, compressed data sizes mismatch! %" PRIu64 ", %" PRIu64 "u\n", - out_gzipped_buf->size(), - tdb_gzip_buf->size()); - exit(-13); - } - if (memcmp( - out_gzipped_buf->data(0), - tdb_gzip_buf->data(0), - out_gzipped_buf->size()) // tdb_gzip_buf->size()) - ) { - printf("Error, compressed data mismatch!\n"); - exit(-17); - } - - shared_ptr expanded_buffer = - make_shared(HERE()); - - // brief sanity check the decompressed()d data matches original - tdb_gzip_buf->set_offset(0); - gzip_decompress(expanded_buffer, static_cast(tdb_gzip_buf->data())); - if (expanded_buffer->size() != inbuf->size()) { - fprintf(stderr, "re-expanded size different from original size!\n"); - exit(-29); - } - if (memcmp(expanded_buffer->data(), inbuf->data(), inbuf->size())) { - printf("Error uncompress data != original data!\n"); - exit(-21); - } - - return 0; -} diff --git a/tiledb/sm/compressors/zstd_compressor.cc b/tiledb/sm/compressors/zstd_compressor.cc index 203d1c0bac3..2ac2158b7ff 100644 --- a/tiledb/sm/compressors/zstd_compressor.cc +++ b/tiledb/sm/compressors/zstd_compressor.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -92,10 +92,6 @@ void ZStd::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check - if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - throw ZStdException( - "Failed decompressing with ZStd; invalid buffer format"); - if (decompress_ctx_pool == nullptr) { throw ZStdException( "Failed decompressing with ZStd; Resource pool not initialized"); @@ -104,13 +100,25 @@ void ZStd::decompress( ResourceGuard context_guard(*decompress_ctx_pool); auto& context = context_guard.get(); + decompress(context, *input_buffer, *output_buffer); +} + +void ZStd::decompress( + ZSTD_Decompress_Context& decompress_ctx, + ConstBuffer& input_buffer, + PreallocatedBuffer& output_buffer) { + // Sanity check + if (input_buffer.data() == nullptr || output_buffer.data() == nullptr) + throw ZStdException( + "Failed decompressing with ZStd; invalid buffer format"); + // Decompress uint64_t zstd_ret = ZSTD_decompressDCtx( - context.ptr(), - output_buffer->cur_data(), - output_buffer->free_space(), - input_buffer->data(), - input_buffer->size()); + decompress_ctx.ptr(), + output_buffer.cur_data(), + output_buffer.free_space(), + input_buffer.data(), + input_buffer.size()); // Check error if (ZSTD_isError(zstd_ret) != 0) { @@ -119,7 +127,7 @@ void ZStd::decompress( } // Set size decompressed data - output_buffer->advance_offset(zstd_ret); + output_buffer.advance_offset(zstd_ret); } uint64_t ZStd::overhead(uint64_t nbytes) { diff --git a/tiledb/sm/compressors/zstd_compressor.h b/tiledb/sm/compressors/zstd_compressor.h index 9db1c63cf74..23a33bf4c80 100644 --- a/tiledb/sm/compressors/zstd_compressor.h +++ b/tiledb/sm/compressors/zstd_compressor.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -119,6 +119,18 @@ class ZStd { ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); + /** + * Decompression function. + * + * @param decompress_ctx_pool decompression context + * @param input_buffer Input buffer to read from. + * @param output_buffer Output buffer to write the decompressed data to. + */ + static void decompress( + ZSTD_Decompress_Context& decompress_ctx, + ConstBuffer& input_buffer, + PreallocatedBuffer& output_buffer); + /** Returns the default compression level. */ static int default_level() { return default_level_; diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 4a5e677295c..325140d2125 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -96,6 +96,7 @@ const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "false"; const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "false"; +const std::string Config::REST_PAYER_NAMESPACE = ""; const std::string Config::SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES = "false"; const std::string Config::SM_ALLOW_UPDATES_EXPERIMENTAL = "false"; const std::string Config::SM_ENCRYPTION_KEY = ""; @@ -270,6 +271,7 @@ const std::map default_config_values = { std::make_pair( "rest.use_refactored_array_open_and_query_submit", Config::REST_USE_REFACTORED_QUERY_SUBMIT), + std::make_pair("rest.payer_namespace", Config::REST_PAYER_NAMESPACE), std::make_pair( "config.env_var_prefix", Config::CONFIG_ENVIRONMENT_VARIABLE_PREFIX), std::make_pair("config.logging_level", Config::CONFIG_LOGGING_LEVEL), diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 3dd6fa760f1..0ed2e5e050d 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -125,6 +125,9 @@ class Config { /** Refactored query submit is disabled by default */ static const std::string REST_USE_REFACTORED_QUERY_SUBMIT; + /** The namespace that should be charged for the request. */ + static const std::string REST_PAYER_NAMESPACE; + /** The prefix to use for checking for parameter environmental variables. */ static const std::string CONFIG_ENVIRONMENT_VARIABLE_PREFIX; diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 52c8356f3af..0608227adcc 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -47,8 +47,10 @@ namespace tiledb::sm { /* ****************************** */ ArrayMetaConsolidator::ArrayMetaConsolidator( - const Config& config, StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, + const Config& config, + StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { auto st = set_config(config); if (!st.ok()) { throw std::logic_error(st.message()); @@ -69,8 +71,8 @@ Status ArrayMetaConsolidator::consolidate( // Open array for reading auto array_uri = URI(array_name); - Array array_for_reads(storage_manager_->resources(), array_uri); - RETURN_NOT_OK(array_for_reads.open( + Array array_for_reads(resources_, array_uri); + throw_if_not_ok(array_for_reads.open( QueryType::READ, config_.timestamp_start_, config_.timestamp_end_, @@ -79,7 +81,7 @@ Status ArrayMetaConsolidator::consolidate( key_length)); // Open array for writing - Array array_for_writes(storage_manager_->resources(), array_uri); + Array array_for_writes(resources_, array_uri); RETURN_NOT_OK_ELSE( array_for_writes.open( QueryType::WRITE, encryption_type, encryption_key, key_length), @@ -144,10 +146,10 @@ Status ArrayMetaConsolidator::set_config(const Config& config) { Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_start", &config_.timestamp_start_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_end", &config_.timestamp_end_, &found)); assert(found); diff --git a/tiledb/sm/consolidator/array_meta_consolidator.h b/tiledb/sm/consolidator/array_meta_consolidator.h index 0393db98e70..a8ad9d1a731 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.h +++ b/tiledb/sm/consolidator/array_meta_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -54,11 +55,21 @@ class ArrayMetaConsolidator : public Consolidator { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * ArrayMetaConsolidator(ContextResources&, const Config&). + * + * @param resources The context resources. * @param config Config. - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ explicit ArrayMetaConsolidator( - const Config& config, StorageManager* storage_manager); + ContextResources& resources, + const Config& config, + StorageManager* storage_manager); /** Destructor. */ ~ArrayMetaConsolidator() = default; diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index d0859aa9582..e5ef4d42ec1 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -49,8 +49,9 @@ namespace tiledb::sm { /* CONSTRUCTOR */ /* ****************************** */ -CommitsConsolidator::CommitsConsolidator(StorageManager* storage_manager) - : Consolidator(storage_manager) { +CommitsConsolidator::CommitsConsolidator( + ContextResources& resources, StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { } /* ****************************** */ @@ -68,16 +69,16 @@ Status CommitsConsolidator::consolidate( // Open array for writing auto array_uri = URI(array_name); - Array array_for_writes(storage_manager_->resources(), array_uri); - RETURN_NOT_OK(array_for_writes.open( + Array array_for_writes(resources_, array_uri); + throw_if_not_ok(array_for_writes.open( QueryType::WRITE, encryption_type, encryption_key, key_length)); // Ensure write version is at least 12. auto write_version = array_for_writes.array_schema_latest().write_version(); - RETURN_NOT_OK(array_for_writes.close()); + throw_if_not_ok(array_for_writes.close()); if (write_version < 12) { - return logger_->status(Status_ConsolidatorError( - "Array version should be at least 12 to consolidate commits.")); + throw ConsolidatorException( + "Array version should be at least 12 to consolidate commits."); } // Get the array uri to consolidate from the array directory. diff --git a/tiledb/sm/consolidator/commits_consolidator.h b/tiledb/sm/consolidator/commits_consolidator.h index b5c1100b4ef..ba1bf226864 100644 --- a/tiledb/sm/consolidator/commits_consolidator.h +++ b/tiledb/sm/consolidator/commits_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -54,9 +55,18 @@ class CommitsConsolidator : public Consolidator { /** * Constructor. * - * @param storage_manager Storage manager. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * CommitsConsolidator(ContextResources&). + * + * @param resources The context resources. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ - explicit CommitsConsolidator(StorageManager* storage_manager); + explicit CommitsConsolidator( + ContextResources& resources, StorageManager* storage_manager); /** Destructor. */ ~CommitsConsolidator() = default; diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index b0e711b2b2b..a003a20fc5b 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -55,22 +55,26 @@ namespace tiledb::sm { /** Factory function to create the consolidator depending on mode. */ shared_ptr Consolidator::create( + ContextResources& resources, const ConsolidationMode mode, const Config& config, StorageManager* storage_manager) { switch (mode) { case ConsolidationMode::FRAGMENT_META: - return make_shared(HERE(), storage_manager); + return make_shared( + HERE(), resources, storage_manager); case ConsolidationMode::FRAGMENT: - return make_shared(HERE(), config, storage_manager); + return make_shared( + HERE(), resources, config, storage_manager); case ConsolidationMode::ARRAY_META: return make_shared( - HERE(), config, storage_manager); + HERE(), resources, config, storage_manager); case ConsolidationMode::COMMITS: - return make_shared(HERE(), storage_manager); + return make_shared( + HERE(), resources, storage_manager); case ConsolidationMode::GROUP_META: return make_shared( - HERE(), config, storage_manager); + HERE(), resources, config, storage_manager); default: return nullptr; } @@ -105,8 +109,9 @@ ConsolidationMode Consolidator::mode_from_config( /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -Consolidator::Consolidator(StorageManager* storage_manager) - : resources_(storage_manager->resources()) +Consolidator::Consolidator( + ContextResources& resources, StorageManager* storage_manager) + : resources_(resources) , storage_manager_(storage_manager) , consolidator_memory_tracker_(resources_.create_memory_tracker()) , stats_(resources_.stats().create_child("Consolidator")) @@ -125,8 +130,7 @@ Status Consolidator::consolidate( [[maybe_unused]] EncryptionType encryption_type, [[maybe_unused]] const void* encryption_key, [[maybe_unused]] uint32_t key_length) { - return logger_->status( - Status_ConsolidatorError("Cannot consolidate; Invalid object")); + throw ConsolidatorException("Cannot consolidate; Invalid object"); } void Consolidator::vacuum([[maybe_unused]] const char* array_name) { @@ -134,6 +138,7 @@ void Consolidator::vacuum([[maybe_unused]] const char* array_name) { } void Consolidator::array_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -147,19 +152,14 @@ void Consolidator::array_consolidate( } // Check if array exists - ObjectType obj_type; - throw_if_not_ok( - object_type(storage_manager->resources(), array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { + if (object_type(resources, array_uri) != ObjectType::ARRAY) { throw ConsolidatorException( "Cannot consolidate array; Array does not exist"); } if (array_uri.is_tiledb()) { throw_if_not_ok( - storage_manager->resources().rest_client()->post_consolidation_to_rest( - array_uri, config)); + resources.rest_client()->post_consolidation_to_rest(array_uri, config)); } else { // Get encryption key from config std::string encryption_key_from_cfg; @@ -190,13 +190,15 @@ void Consolidator::array_consolidate( // Consolidate auto mode = Consolidator::mode_from_config(config); - auto consolidator = Consolidator::create(mode, config, storage_manager); + auto consolidator = + Consolidator::create(resources, mode, config, storage_manager); throw_if_not_ok(consolidator->consolidate( array_name, encryption_type, encryption_key, key_length)); } } void Consolidator::fragments_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -211,11 +213,7 @@ void Consolidator::fragments_consolidate( } // Check if array exists - ObjectType obj_type; - throw_if_not_ok( - object_type(storage_manager->resources(), array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { + if (object_type(resources, array_uri) != ObjectType::ARRAY) { throw ConsolidatorException( "Cannot consolidate array; Array does not exist"); } @@ -249,7 +247,7 @@ void Consolidator::fragments_consolidate( // Consolidate auto consolidator = Consolidator::create( - ConsolidationMode::FRAGMENT, config, storage_manager); + resources, ConsolidationMode::FRAGMENT, config, storage_manager); auto fragment_consolidator = dynamic_cast(consolidator.get()); throw_if_not_ok(fragment_consolidator->consolidate_fragments( @@ -315,19 +313,20 @@ void Consolidator::write_consolidated_commits_file( } void Consolidator::array_vacuum( + ContextResources& resources, const char* array_name, const Config& config, StorageManager* storage_manager) { URI array_uri(array_name); if (array_uri.is_tiledb()) { throw_if_not_ok( - storage_manager->resources().rest_client()->post_vacuum_to_rest( - array_uri, config)); + resources.rest_client()->post_vacuum_to_rest(array_uri, config)); return; } auto mode = Consolidator::mode_from_config(config, true); - auto consolidator = Consolidator::create(mode, config, storage_manager); + auto consolidator = + Consolidator::create(resources, mode, config, storage_manager); consolidator->vacuum(array_name); } diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index 1c7d10ae238..dcbdc37ad2a 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -37,6 +37,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include @@ -76,13 +77,22 @@ class Consolidator { /** * Factory method to make a new Consolidator instance given the config mode. * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be create(ContextResources&, ...). + * + * @param resources The context resources. * @param mode Consolidation mode. * @param config Configuration parameters for the consolidation * (`nullptr` means default). - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) * @return New Consolidator instance or nullptr on error. */ static shared_ptr create( + ContextResources& resources, const ConsolidationMode mode, const Config& config, StorageManager* storage_manager); @@ -134,6 +144,14 @@ class Consolidator { /** * Consolidates the fragments of an array into a single one. * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * array_consolidate(ContextResources&, ...). + * + * @param resources The context resources. * @param array_name The name of the array to be consolidated. * @param encryption_type The encryption type of the array * @param encryption_key If the array is encrypted, the private encryption @@ -142,9 +160,11 @@ class Consolidator { * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config associated with * this instance). - * @param storage_manager The storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ static void array_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -155,6 +175,14 @@ class Consolidator { /** * Consolidates the fragments of an array into a single one. * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * fragments_consolidate(ContextResources&, ...). + * + * @param resources The context resources. * @param array_name The name of the array to be consolidated. * @param encryption_type The encryption type of the array * @param encryption_key If the array is encrypted, the private encryption @@ -164,9 +192,11 @@ class Consolidator { * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config associated with * this instance). - * @param storage_manager The storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ static void fragments_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -194,11 +224,21 @@ class Consolidator { * metadata. Note that this will coarsen the granularity of time traveling * (see docs for more information). * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * array_vacuum(ContextResources&, ...). + * + * @param resources The context resources. * @param array_name The name of the array to be vacuumed. * @param config Configuration parameters for vacuuming. - * @param storage_manager The storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ static void array_vacuum( + ContextResources& resources, const char* array_name, const Config& config, StorageManager* storage_manager); @@ -223,9 +263,18 @@ class Consolidator { /** * Constructor. * - * @param storage_manager Storage manager. + * Constructs a Consolidator object given a ContextResources reference. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Consolidator class on StorageManager. For + * now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be Consolidator(ContextResources&). + * + * @param resources The context resources. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ - explicit Consolidator(StorageManager* storage_manager); + explicit Consolidator( + ContextResources& resources, StorageManager* storage_manager); /** * Checks if the array is remote. diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 93d224f6ed3..2b8941f2a1d 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -187,8 +187,10 @@ void FragmentConsolidationWorkspace::resize_buffers( /* ****************************** */ FragmentConsolidator::FragmentConsolidator( - const Config& config, StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, + const Config& config, + StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { auto st = set_config(config); if (!st.ok()) { throw FragmentConsolidatorException(st.message()); @@ -209,15 +211,14 @@ Status FragmentConsolidator::consolidate( check_array_uri(array_name); // Open array for reading - auto array_for_reads{make_shared( - HERE(), storage_manager_->resources(), URI(array_name))}; - RETURN_NOT_OK(array_for_reads->open_without_fragments( + auto array_for_reads{make_shared(HERE(), resources_, URI(array_name))}; + throw_if_not_ok(array_for_reads->open_without_fragments( encryption_type, encryption_key, key_length)); // Open array for writing - auto array_for_writes{make_shared( - HERE(), storage_manager_->resources(), array_for_reads->array_uri())}; - RETURN_NOT_OK(array_for_writes->open( + auto array_for_writes{ + make_shared(HERE(), resources_, array_for_reads->array_uri())}; + throw_if_not_ok(array_for_writes->open( QueryType::WRITE, encryption_type, encryption_key, key_length)); // Disable consolidation with timestamps on older arrays. @@ -232,7 +233,7 @@ Status FragmentConsolidator::consolidate( // must be fetched (even before `config_.timestamp_start_`), // to compute the anterior ND range that can help determine // which dense fragments are consolidatable. - FragmentInfo fragment_info(URI(array_name), storage_manager_->resources()); + FragmentInfo fragment_info(URI(array_name), resources_); auto st = fragment_info.load( array_for_reads->array_directory(), config_.timestamp_start_, @@ -305,7 +306,7 @@ Status FragmentConsolidator::consolidate( RETURN_NOT_OK_ELSE( array_for_reads->close(), throw_if_not_ok(array_for_writes->close())); - RETURN_NOT_OK(array_for_writes->close()); + throw_if_not_ok(array_for_writes->close()); stats_->add_counter("consolidate_step_num", step); @@ -321,15 +322,14 @@ Status FragmentConsolidator::consolidate_fragments( auto timer_se = stats_->start_timer("consolidate_frags"); // Open array for reading - auto array_for_reads{make_shared( - HERE(), storage_manager_->resources(), URI(array_name))}; - RETURN_NOT_OK(array_for_reads->open_without_fragments( + auto array_for_reads{make_shared(HERE(), resources_, URI(array_name))}; + throw_if_not_ok(array_for_reads->open_without_fragments( encryption_type, encryption_key, key_length)); // Open array for writing - auto array_for_writes{make_shared( - HERE(), storage_manager_->resources(), array_for_reads->array_uri())}; - RETURN_NOT_OK(array_for_writes->open( + auto array_for_writes{ + make_shared(HERE(), resources_, array_for_reads->array_uri())}; + throw_if_not_ok(array_for_writes->open( QueryType::WRITE, encryption_type, encryption_key, key_length)); // Disable consolidation with timestamps on older arrays. @@ -344,7 +344,7 @@ Status FragmentConsolidator::consolidate_fragments( } // Get all fragment info - FragmentInfo fragment_info(URI(array_name), storage_manager_->resources()); + FragmentInfo fragment_info(URI(array_name), resources_); auto st = fragment_info.load( array_for_reads->array_directory(), 0, @@ -383,8 +383,8 @@ Status FragmentConsolidator::consolidate_fragments( } if (count != fragment_uris.size()) { - return logger_->status(Status_ConsolidatorError( - "Cannot consolidate; Not all fragments could be found")); + throw FragmentConsolidatorException( + "Cannot consolidate; Not all fragments could be found"); } FragmentConsolidationWorkspace cw(consolidator_memory_tracker_); @@ -429,7 +429,7 @@ void FragmentConsolidator::vacuum(const char* array_name) { // Get the fragment URIs and vacuum file URIs to be vacuumed ArrayDirectory array_dir( - storage_manager_->resources(), + resources_, URI(array_name), 0, std::numeric_limits::max(), @@ -583,8 +583,8 @@ Status FragmentConsolidator::consolidate_internal( vac_uri = array_for_reads->array_directory().get_vacuum_uri(*new_fragment_uri); } catch (std::exception& e) { - std::throw_with_nested( - std::logic_error("[FragmentConsolidator::consolidate_internal] ")); + FragmentConsolidatorException( + "Internal consolidation failed with exception" + std::string(e.what())); } // Read from one array and write to the other @@ -668,15 +668,20 @@ Status FragmentConsolidator::create_queries( // Create read query query_r = tdb_unique_ptr(tdb_new( - Query, storage_manager_, array_for_reads, nullopt, read_memory_budget)); - RETURN_NOT_OK(query_r->set_layout(Layout::GLOBAL_ORDER)); + Query, + resources_, + storage_manager_, + array_for_reads, + nullopt, + read_memory_budget)); + throw_if_not_ok(query_r->set_layout(Layout::GLOBAL_ORDER)); // Dense consolidation will do a tile aligned read. if (dense) { NDRange read_subarray = subarray; auto& domain{array_for_reads->array_schema_latest().domain()}; domain.expand_to_tiles(&read_subarray); - RETURN_NOT_OK(query_r->set_subarray_unsafe(read_subarray)); + throw_if_not_ok(query_r->set_subarray_unsafe(read_subarray)); } // Enable consolidation with timestamps on the reader, if applicable. @@ -695,15 +700,16 @@ Status FragmentConsolidator::create_queries( // Create write query query_w = tdb_unique_ptr(tdb_new( Query, + resources_, storage_manager_, array_for_writes, fragment_name, write_memory_budget)); - RETURN_NOT_OK(query_w->set_layout(Layout::GLOBAL_ORDER)); - RETURN_NOT_OK(query_w->disable_checks_consolidation()); + throw_if_not_ok(query_w->set_layout(Layout::GLOBAL_ORDER)); + throw_if_not_ok(query_w->disable_checks_consolidation()); query_w->set_fragment_size(config_.max_fragment_size_); if (array_for_reads->array_schema_latest().dense()) { - RETURN_NOT_OK(query_w->set_subarray_unsafe(subarray)); + throw_if_not_ok(query_w->set_subarray_unsafe(subarray)); } // Set the processed conditions on new fragment. @@ -931,11 +937,11 @@ Status FragmentConsolidator::set_config(const Config& config) { merged_config.inherit(config); bool found = false; config_.amplification_ = 0.0f; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.amplification", &config_.amplification_, &found)); assert(found); config_.steps_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.steps", &config_.steps_, &found)); assert(found); config_.buffer_size_ = 0; @@ -946,54 +952,54 @@ Status FragmentConsolidator::set_config(const Config& config) { "The `sm.consolidation.buffer_size configuration setting has been " "deprecated. Set consolidation buffer sizes using the newer " "`sm.mem.consolidation.buffers_weight` setting."); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.buffer_size", &config_.buffer_size_, &found)); assert(found); } config_.total_budget_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.total_budget", &config_.total_budget_, &found)); assert(found); config_.buffers_weight_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.consolidation.buffers_weight", &config_.buffers_weight_, &found)); assert(found); config_.reader_weight_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.consolidation.reader_weight", &config_.reader_weight_, &found)); assert(found); config_.writer_weight_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.consolidation.writer_weight", &config_.writer_weight_, &found)); assert(found); config_.max_fragment_size_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.max_fragment_size", &config_.max_fragment_size_, &found)); assert(found); config_.size_ratio_ = 0.0f; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.step_size_ratio", &config_.size_ratio_, &found)); assert(found); config_.purge_deleted_cells_ = false; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.purge_deleted_cells", &config_.purge_deleted_cells_, &found)); assert(found); config_.min_frags_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.step_min_frags", &config_.min_frags_, &found)); assert(found); config_.max_frags_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.step_max_frags", &config_.max_frags_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_start", &config_.timestamp_start_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_end", &config_.timestamp_end_, &found)); assert(found); std::string reader = @@ -1005,17 +1011,17 @@ Status FragmentConsolidator::set_config(const Config& config) { // Sanity checks if (config_.min_frags_ > config_.max_frags_) - return logger_->status(Status_ConsolidatorError( + throw FragmentConsolidatorException( "Invalid configuration; Minimum fragments config parameter is larger " - "than the maximum")); + "than the maximum"); if (config_.size_ratio_ > 1.0f || config_.size_ratio_ < 0.0f) - return logger_->status(Status_ConsolidatorError( + throw FragmentConsolidatorException( "Invalid configuration; Step size ratio config parameter must be in " - "[0.0, 1.0]")); + "[0.0, 1.0]"); if (config_.amplification_ < 0) - return logger_->status( - Status_ConsolidatorError("Invalid configuration; Amplification config " - "parameter must be non-negative")); + throw FragmentConsolidatorException( + "Invalid configuration; Amplification config parameter must be " + "non-negative"); return Status::Ok(); } diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index 5b828b40714..3fcae353d7e 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -40,6 +40,7 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/misc/types.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include @@ -173,11 +174,21 @@ class FragmentConsolidator : public Consolidator { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * FragmentConsolidator(ContextResources&, const Config&). + * + * @param resources The context resources. * @param config Config. - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ explicit FragmentConsolidator( - const Config& config, StorageManager* storage_manager); + ContextResources& resources, + const Config& config, + StorageManager* storage_manager); /** Destructor. */ ~FragmentConsolidator() = default; diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index b67262b3494..2df652f9fcd 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -52,8 +52,8 @@ namespace tiledb::sm { /* ****************************** */ FragmentMetaConsolidator::FragmentMetaConsolidator( - StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { } /* ****************************** */ @@ -70,8 +70,8 @@ Status FragmentMetaConsolidator::consolidate( check_array_uri(array_name); // Open array for reading - Array array(storage_manager_->resources(), URI(array_name)); - RETURN_NOT_OK( + Array array(resources_, URI(array_name)); + throw_if_not_ok( array.open(QueryType::READ, encryption_type, encryption_key, key_length)); // Include only fragments with footers / separate basic metadata @@ -129,7 +129,7 @@ Status FragmentMetaConsolidator::consolidate( return Status::Ok(); }); - RETURN_NOT_OK(status); + throw_if_not_ok(status); auto serialize_data = [&](Serializer& serializer, uint64_t offset) { // Write number of fragments @@ -166,10 +166,10 @@ Status FragmentMetaConsolidator::consolidate( serialize_data(serializer, offset); // Close array - RETURN_NOT_OK(array.close()); + throw_if_not_ok(array.close()); EncryptionKey enc_key; - RETURN_NOT_OK(enc_key.set_key(encryption_type, encryption_key, key_length)); + throw_if_not_ok(enc_key.set_key(encryption_type, encryption_key, key_length)); GenericTileIO tile_io(resources_, uri); [[maybe_unused]] uint64_t nbytes = 0; diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.h b/tiledb/sm/consolidator/fragment_meta_consolidator.h index eb5027b0c17..01c1ef574d9 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.h +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -54,9 +55,18 @@ class FragmentMetaConsolidator : public Consolidator { /** * Constructor. * - * @param storage_manager Storage manager. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * FragmentMetaConsolidator(ContextResources&). + * + * @param resources The context resources. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ - explicit FragmentMetaConsolidator(StorageManager* storage_manager); + explicit FragmentMetaConsolidator( + ContextResources& resources, StorageManager* storage_manager); /** Destructor. */ ~FragmentMetaConsolidator() = default; diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index ccb6099bb4a..d534e1e89fe 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -49,8 +49,10 @@ namespace tiledb::sm { /* ****************************** */ GroupMetaConsolidator::GroupMetaConsolidator( - const Config& config, StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, + const Config& config, + StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { auto st = set_config(config); if (!st.ok()) { throw std::logic_error(st.message()); @@ -68,12 +70,12 @@ Status GroupMetaConsolidator::consolidate( // Open group for reading auto group_uri = URI(group_name); - Group group_for_reads(storage_manager_->resources(), group_uri); + Group group_for_reads(resources_, group_uri); group_for_reads.open( QueryType::READ, config_.timestamp_start_, config_.timestamp_end_); // Open group for writing - Group group_for_writes(storage_manager_->resources(), group_uri); + Group group_for_writes(resources_, group_uri); try { group_for_writes.open(QueryType::WRITE); @@ -136,10 +138,10 @@ Status GroupMetaConsolidator::set_config(const Config& config) { Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_start", &config_.timestamp_start_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_end", &config_.timestamp_end_, &found)); assert(found); diff --git a/tiledb/sm/consolidator/group_meta_consolidator.h b/tiledb/sm/consolidator/group_meta_consolidator.h index e83618e925b..5137584ba9f 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.h +++ b/tiledb/sm/consolidator/group_meta_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,6 +37,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/status.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -53,11 +54,21 @@ class GroupMetaConsolidator : public Consolidator { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * GroupMetaConsolidator(ContextResources&, const Config&). + * + * @param resources The context resources. * @param config Config. - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ explicit GroupMetaConsolidator( - const Config& config, StorageManager* storage_manager); + ContextResources& resources, + const Config& config, + StorageManager* storage_manager); /** Destructor. */ ~GroupMetaConsolidator() = default; diff --git a/tiledb/sm/cpp_api/array_schema_evolution.h b/tiledb/sm/cpp_api/array_schema_evolution.h index 31cfa552319..4c5fb8882dd 100644 --- a/tiledb/sm/cpp_api/array_schema_evolution.h +++ b/tiledb/sm/cpp_api/array_schema_evolution.h @@ -36,6 +36,7 @@ #include "array_schema.h" #include "attribute.h" #include "context.h" +#include "current_domain.h" #include "deleter.h" #include "domain.h" #include "object.h" @@ -216,6 +217,21 @@ class ArraySchemaEvolution { return *this; } + /** + * Expands the current domain during array schema evolution. + * TileDB will enforce that the new current domain is expanding + * on the current one and not contracting during `tiledb_array_evolve`. + * + * @param expanded_domain The current domain we want to expand the schema to. + */ + ArraySchemaEvolution& expand_current_domain( + const CurrentDomain& expanded_domain) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_expand_current_domain( + ctx.ptr().get(), evolution_.get(), expanded_domain.ptr().get())); + return *this; + } + /** * Sets timestamp range. * diff --git a/tiledb/sm/cpp_api/array_schema_experimental.h b/tiledb/sm/cpp_api/array_schema_experimental.h index 4f6a746bd3f..8da27be3c94 100644 --- a/tiledb/sm/cpp_api/array_schema_experimental.h +++ b/tiledb/sm/cpp_api/array_schema_experimental.h @@ -34,6 +34,7 @@ #define TILEDB_CPP_API_ARRAY_SCHEMA_EXPERIMENTAL_H #include "array_schema.h" +#include "current_domain.h" #include "dimension_label_experimental.h" #include "enumeration_experimental.h" #include "filter_list.h" @@ -156,6 +157,38 @@ class ArraySchemaExperimental { return DimensionLabel(ctx, dim_label); } + /** + * Returns a copy of the schema's array currentDomain. To change the + * currentDomain, use `set_current_domain()`. + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @return Copy of the schema's currentDomain. + */ + static CurrentDomain current_domain( + const Context& ctx, const ArraySchema& array_schema) { + tiledb_current_domain_t* current_domain; + ctx.handle_error(tiledb_array_schema_get_current_domain( + ctx.ptr().get(), array_schema.ptr().get(), ¤t_domain)); + + return CurrentDomain(ctx, current_domain); + } + + /** + * Sets the currentDomain. + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param current_domain The currentDomain to use. + */ + static void set_current_domain( + const Context& ctx, + const ArraySchema& array_schema, + const CurrentDomain& current_domain) { + ctx.handle_error(tiledb_array_schema_set_current_domain( + ctx.ptr().get(), array_schema.ptr().get(), current_domain.ptr().get())); + } + /** * Add an enumeration to the array schema. * diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index cd1c2110ad9..e447683267b 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -889,6 +889,13 @@ class Config { * CAPNP traversal limit used in the deserialization of messages(bytes) *
* **Default**: 536870912 (512MB) + * - `rest.custom_headers.*`
+ * (Optional) Prefix for custom headers on REST requests. For each custom + * header, use "rest.custom_headers.header_key" = "header_value"
+ * **Optional. No Default** + * - `rest.payer_namespace`
+ * The namespace that should be charged for the request.
+ * **Default**: no default set * - `filestore.buffer_size`
* Specifies the size in bytes of the internal buffers used in the * filestore API. The size should be bigger than the minimum tile size diff --git a/tiledb/sm/cpp_api/current_domain.h b/tiledb/sm/cpp_api/current_domain.h new file mode 100644 index 00000000000..f8456dce276 --- /dev/null +++ b/tiledb/sm/cpp_api/current_domain.h @@ -0,0 +1,178 @@ +/** + * @file current_domain.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB current_domain object. + */ + +#ifndef TILEDB_CPP_API_CURRENT_DOMAIN_H +#define TILEDB_CPP_API_CURRENT_DOMAIN_H + +#include "context.h" +#include "core_interface.h" +#include "deleter.h" +#include "exception.h" +#include "ndrectangle.h" +#include "tiledb.h" +#include "tiledb_experimental.h" +#include "type.h" +#include "utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tiledb { +class CurrentDomain { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Creates a TileDB current_domain object. + * + * @param ctx TileDB context + * @param type The TileDB currentDomain type + */ + explicit CurrentDomain(const Context& ctx) + : ctx_(ctx) { + tiledb_current_domain_t* cd; + ctx.handle_error(tiledb_current_domain_create(ctx.ptr().get(), &cd)); + current_domain_ = std::shared_ptr(cd, deleter_); + } + + /** + * Creates a TileDB current_domain object. + * + * @param ctx TileDB context + * @param cd The TileDB currentDomain C api pointer + */ + CurrentDomain(const tiledb::Context& ctx, tiledb_current_domain_t* cd) + : ctx_(ctx) { + current_domain_ = std::shared_ptr(cd, deleter_); + } + + CurrentDomain(const CurrentDomain&) = default; + CurrentDomain(CurrentDomain&&) = default; + CurrentDomain& operator=(const CurrentDomain&) = default; + CurrentDomain& operator=(CurrentDomain&&) = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Returns a shared pointer to the C TileDB currentDomain object. + * + * @return The C API pointer + */ + std::shared_ptr ptr() const { + return current_domain_; + } + + /** + * Returns the currentDomain type. + * + * @return The currentDomain type. + */ + tiledb_current_domain_type_t type() const { + tiledb_current_domain_type_t type; + auto& ctx = ctx_.get(); + ctx.handle_error( + tiledb_current_domain_get_type(current_domain_.get(), &type)); + return type; + } + + /** + * Set a N-dimensional rectangle representation on a current domain. + * Error if the current domain passed is not empty. + * + * @param ndrect The N-dimensional rectangle to be used. + * @return Reference to this `currentDomain` instance. + */ + CurrentDomain& set_ndrectangle(const NDRectangle& ndrect) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_current_domain_set_ndrectangle( + current_domain_.get(), ndrect.ptr().get())); + + return *this; + } + + /** + * Get the N-dimensional rectangle associated with the current domain object, + * error if the current domain is empty or a different representation is set. + * + * @return The N-dimensional rectangle of the current domain + */ + NDRectangle ndrectangle() const { + tiledb_ndrectangle_t* nd; + auto& ctx = ctx_.get(); + ctx.handle_error( + tiledb_current_domain_get_ndrectangle(current_domain_.get(), &nd)); + return NDRectangle(ctx, nd); + } + + /** + * Check if the current_domain is empty + * + * @return True if the currentDomain is empty + */ + bool is_empty() const { + uint32_t ret; + auto& ctx = ctx_.get(); + ctx.handle_error( + tiledb_current_domain_get_is_empty(current_domain_.get(), &ret)); + return static_cast(ret); + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB context. */ + std::reference_wrapper ctx_; + + /** Pointer to the TileDB C current_domain object. */ + std::shared_ptr current_domain_; + + /** An auxiliary deleter. */ + impl::Deleter deleter_; +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_CURRENT_DOMAIN_H diff --git a/tiledb/sm/cpp_api/deleter.h b/tiledb/sm/cpp_api/deleter.h index 90d466c4403..0629c3383b6 100644 --- a/tiledb/sm/cpp_api/deleter.h +++ b/tiledb/sm/cpp_api/deleter.h @@ -117,6 +117,14 @@ class Deleter { tiledb_domain_free(&p); } + void operator()(tiledb_current_domain_t* p) const { + tiledb_current_domain_free(&p); + } + + void operator()(tiledb_ndrectangle_t* p) const { + tiledb_ndrectangle_free(&p); + } + void operator()(tiledb_enumeration_t* p) const { tiledb_enumeration_free(&p); } diff --git a/tiledb/sm/cpp_api/ndrectangle.h b/tiledb/sm/cpp_api/ndrectangle.h new file mode 100644 index 00000000000..11a94128f86 --- /dev/null +++ b/tiledb/sm/cpp_api/ndrectangle.h @@ -0,0 +1,321 @@ +/** + * @file ndrectangle.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB NDRectangle. + */ + +#ifndef TILEDB_CPP_API_NDRECTANGLE_H +#define TILEDB_CPP_API_NDRECTANGLE_H + +#include "context.h" +#include "deleter.h" +#include "dimension.h" +#include "domain.h" +#include "tiledb.h" +#include "type.h" + +#include +#include +#include + +namespace tiledb { + +class NDRectangle { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Constructor + * + * @param ctx The TileDB context + * @param domain The input Domain + */ + explicit NDRectangle(const tiledb::Context& ctx, const tiledb::Domain& domain) + : ctx_(ctx) { + tiledb_ndrectangle_t* capi_ndrect; + ctx.handle_error(tiledb_ndrectangle_alloc( + ctx.ptr().get(), domain.ptr().get(), &capi_ndrect)); + ndrect_ = std::shared_ptr(capi_ndrect, deleter_); + } + + /** + * Constructor + * + * @param ctx The TileDB context + * @param ndrect The NDRectangle C API pointer + * + * @note The object will manage the lifetime of tiledb_ndrectangle_t* + */ + NDRectangle(const tiledb::Context& ctx, tiledb_ndrectangle_t* ndrect) + : ctx_(ctx) { + ndrect_ = std::shared_ptr(ndrect, deleter_); + } + + NDRectangle(const NDRectangle&) = default; + NDRectangle(NDRectangle&&) = default; + NDRectangle& operator=(const NDRectangle&) = default; + NDRectangle& operator=(NDRectangle&&) = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Adds an 1D range along a dimension name, in the form (start, end). The + * datatype of the range must be the same as the dimension datatype. + * + * **Example:** + * + * @code{.cpp} + * // Set an 1D range on dimension 0, assuming the domain type is int64. + * int64_t start = 10; + * int64_t end = 20; + * ndrect.set_range(0, start, end); + * @endcode + * + * @tparam T The dimension datatype. + * @param dim_name The name of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @return Reference to this NDRectangle. + */ + template + NDRectangle& set_range(const std::string& dim_name, T start, T end) { + auto& ctx = ctx_.get(); + + // Create the tiledb_range_t struct and fill it + tiledb_range_t range; + range.min = static_cast(&start); + range.min_size = sizeof(T); + range.max = static_cast(&end); + range.max_size = sizeof(T); + + // Pass the struct to tiledb_ndrectangle_set_range_for_name + ctx.handle_error(tiledb_ndrectangle_set_range_for_name( + ctx.ptr().get(), ndrect_.get(), dim_name.c_str(), &range)); + + return *this; + } + + /** + * Adds an 1D range along a dimension index, in the form (start, end). The + * datatype of the range must be the same as the dimension datatype. + * + * **Example:** + * + * @code{.cpp} + * // Set an 1D range on dimension 0, assuming the domain type is int64. + * int64_t start = 10; + * int64_t end = 20; + * ndrect.set_range(0, start, end); + * @endcode + * + * @tparam T The dimension datatype. + * @param dim_idx The index of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @return Reference to this NDRectangle. + */ + template + NDRectangle& set_range(uint32_t dim_idx, T start, T end) { + auto& ctx = ctx_.get(); + // Create the tiledb_range_t struct and fill it + tiledb_range_t range; + range.min = static_cast(&start); + range.min_size = sizeof(T); + range.max = static_cast(&end); + range.max_size = sizeof(T); + + // Pass the struct to tiledb_ndrectangle_set_range + ctx.handle_error(tiledb_ndrectangle_set_range( + ctx.ptr().get(), ndrect_.get(), dim_idx, &range)); + + return *this; + } + + /** + * Adds a 1D string range along a dimension index, in the form (start, end). + * Applicable only to variable-sized dimensions + * + * @param dim_idx The index of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @return Reference to this NDRectangle. + */ + NDRectangle& set_range( + uint32_t dim_idx, const std::string& start, const std::string& end) { + auto& ctx = ctx_.get(); + // Create the tiledb_range_t struct and fill it + tiledb_range_t range; + range.min = static_cast(start.c_str()); + range.min_size = start.size(); + range.max = static_cast(end.c_str()); + range.max_size = end.size(); + + // Pass the struct to tiledb_ndrectangle_set_range + ctx.handle_error(tiledb_ndrectangle_set_range( + ctx.ptr().get(), ndrect_.get(), dim_idx, &range)); + + return *this; + } + + /** + * Adds a 1D string range along a dimension index, in the form (start, end). + * Applicable only to variable-sized dimensions + * + * @param dim_name The name of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @return Reference to this NDRectangle. + */ + NDRectangle& set_range( + const std::string& dim_name, + const std::string& start, + const std::string& end) { + auto& ctx = ctx_.get(); + // Create the tiledb_range_t struct and fill it + tiledb_range_t range; + range.min = static_cast(start.c_str()); + range.min_size = start.size(); + range.max = static_cast(end.c_str()); + range.max_size = end.size(); + + // Pass the struct to tiledb_ndrectangle_set_range_for_name + ctx.handle_error(tiledb_ndrectangle_set_range_for_name( + ctx.ptr().get(), ndrect_.get(), dim_name.c_str(), &range)); + + return *this; + } + + /** + * Retrieves a range for a given variable length string dimension index and + * range id. + * + * @param dim_idx The dimension index. + * @param range_idx The range index. + * @return A pair of the form (start, end). + */ + std::array range(unsigned dim_idx) { + auto& ctx = ctx_.get(); + tiledb_range_t range; + + ctx.handle_error(tiledb_ndrectangle_get_range( + ctx.ptr().get(), ndrect_.get(), dim_idx, &range)); + + std::string start_str(static_cast(range.min), range.min_size); + std::string end_str(static_cast(range.max), range.max_size); + + return {std::move(start_str), std::move(end_str)}; + } + + /** + * Retrieves a range for a given variable length string dimension index and + * range id. + * + * @param dim_name The dimension name. + * @param range_idx The range index. + * @return A pair of the form (start, end). + */ + std::array range(const std::string& dim_name) { + auto& ctx = ctx_.get(); + tiledb_range_t range; + ctx.handle_error(tiledb_ndrectangle_get_range_from_name( + ctx.ptr().get(), ndrect_.get(), dim_name.c_str(), &range)); + + std::string start_str(static_cast(range.min), range.min_size); + std::string end_str(static_cast(range.max), range.max_size); + + return {std::move(start_str), std::move(end_str)}; + } + + /** + * Retrieves a range for a given dimension name. The template datatype must be + * the same as that of the underlying array. + * + * @tparam T The dimension datatype. + * @param dim_name The dimension name. + * @return A duplex of the form (start, end). + */ + template + std::array range(const std::string& dim_name) { + auto& ctx = ctx_.get(); + tiledb_range_t range; + ctx.handle_error(tiledb_ndrectangle_get_range_from_name( + ctx.ptr().get(), ndrect_.get(), dim_name.c_str(), &range)); + + return {*(const T*)range.min, *(const T*)range.max}; + } + + /** + * Retrieves a range for a given dimension index. The template datatype must + * be the same as that of the underlying array. + * + * @tparam T The dimension datatype. + * @param dim_idx The dimension index. + * @return A duplex of the form (start, end). + */ + template + std::array range(unsigned dim_idx) { + auto& ctx = ctx_.get(); + tiledb_range_t range; + ctx.handle_error(tiledb_ndrectangle_get_range( + ctx.ptr().get(), ndrect_.get(), dim_idx, &range)); + + return {*(const T*)range.min, *(const T*)range.max}; + } + + /** + * Returns the C TileDB ndrect object. + * + * @return The C pointer to the NDRectangle object + */ + std::shared_ptr ptr() const { + return ndrect_; + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB context. */ + std::reference_wrapper ctx_; + + /** Pointer to the C TileDB domain object. */ + std::shared_ptr ndrect_; + + /** An auxiliary deleter. */ + impl::Deleter deleter_; +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_NDRECTANGLE_H diff --git a/tiledb/sm/cpp_api/tiledb_experimental b/tiledb/sm/cpp_api/tiledb_experimental index 63f208ff005..5ffb41d049f 100644 --- a/tiledb/sm/cpp_api/tiledb_experimental +++ b/tiledb/sm/cpp_api/tiledb_experimental @@ -45,5 +45,7 @@ #include "query_experimental.h" #include "subarray_experimental.h" #include "vfs_experimental.h" +#include "ndrectangle.h" +#include "current_domain.h" #endif // TILEDB_EXPERIMENTAL_CPP_H diff --git a/tiledb/sm/enums/current_domain_type.h b/tiledb/sm/enums/current_domain_type.h index 38c9f8492b9..0c35f6a98f4 100644 --- a/tiledb/sm/enums/current_domain_type.h +++ b/tiledb/sm/enums/current_domain_type.h @@ -46,7 +46,7 @@ namespace sm { /** A current domain type. */ enum class CurrentDomainType : uint8_t { #define TILEDB_CURRENT_DOMAIN_TYPE_ENUM(id) id -#include "tiledb/sm/c_api/tiledb_enum.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_enum.h" #undef TILEDB_CURRENT_DOMAIN_TYPE_ENUM }; diff --git a/tiledb/sm/filesystem/posix.cc b/tiledb/sm/filesystem/posix.cc index 6e0b678941b..a4419338be2 100644 --- a/tiledb/sm/filesystem/posix.cc +++ b/tiledb/sm/filesystem/posix.cc @@ -47,6 +47,7 @@ #include #include +#include #include #include #include diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index df47c6f05fd..22012bcac19 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -105,7 +105,7 @@ bool BitWidthReductionFilter::accepts_input_datatype(Datatype datatype) const { return false; } -Status BitWidthReductionFilter::run_forward( +void BitWidthReductionFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -155,9 +155,9 @@ Status BitWidthReductionFilter::run_forward( case Datatype::TIME_AS: if (tile.format_version() < 20) { // Return data as-is for backwards compatibility - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } return run_forward( tile, offsets_tile, input_metadata, input, output_metadata, output); @@ -169,14 +169,14 @@ Status BitWidthReductionFilter::run_forward( case Datatype::UINT8: default: // If bit width compression can't work, just return the input unmodified. - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } } template -Status BitWidthReductionFilter::run_forward( +void BitWidthReductionFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -207,24 +207,22 @@ Status BitWidthReductionFilter::run_forward( } // Allocate space in output buffer for the upper bound. - RETURN_NOT_OK(output->prepend_buffer(output_size_ub)); + throw_if_not_ok(output->prepend_buffer(output_size_ub)); Buffer* buffer_ptr = output->buffer_ptr(0); buffer_ptr->reset_offset(); assert(buffer_ptr != nullptr); // Forward the existing metadata - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); // Allocate a buffer for this filter's metadata and write the header. - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&input_size, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&total_num_windows, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&input_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&total_num_windows, sizeof(uint32_t))); // Compress all parts. for (unsigned i = 0; i < num_parts; i++) { - RETURN_NOT_OK(compress_part(&parts[i], output, output_metadata)); + throw_if_not_ok(compress_part(&parts[i], output, output_metadata)); } - - return Status::Ok(); } template diff --git a/tiledb/sm/filter/bit_width_reduction_filter.h b/tiledb/sm/filter/bit_width_reduction_filter.h index ebcab1ec93d..094f4e19c51 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.h +++ b/tiledb/sm/filter/bit_width_reduction_filter.h @@ -113,7 +113,7 @@ class BitWidthReductionFilter : public Filter { /** * Reduce the bit size of the given input into the given output. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -192,7 +192,7 @@ class BitWidthReductionFilter : public Filter { /** Run_forward method templated on the tile cell datatype. */ template - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const tile_offsets, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/bitshuffle_filter.cc b/tiledb/sm/filter/bitshuffle_filter.cc index ae3f23fb274..067e008b68f 100644 --- a/tiledb/sm/filter/bitshuffle_filter.cc +++ b/tiledb/sm/filter/bitshuffle_filter.cc @@ -58,7 +58,7 @@ void BitshuffleFilter::dump(FILE* out) const { fprintf(out, "BitShuffle"); } -Status BitshuffleFilter::run_forward( +void BitshuffleFilter::run_forward( const WriterTile& tile, WriterTile* const, FilterBuffer* input_metadata, @@ -68,39 +68,37 @@ Status BitshuffleFilter::run_forward( auto tile_type_size = static_cast(datatype_size(filter_data_type_)); // Output size does not change with this filter. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); // Compute the list of parts to shuffle std::vector parts; - RETURN_NOT_OK(compute_parts(input, &parts)); + throw_if_not_ok(compute_parts(input, &parts)); // Write the metadata auto num_parts = (uint32_t)parts.size(); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // Shuffle all parts for (const auto& part : parts) { auto part_size = (uint32_t)part.size(); - RETURN_NOT_OK(output_metadata->write(&part_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&part_size, sizeof(uint32_t))); if (part_size % tile_type_size != 0 || part_size % 8 != 0) { // Can't shuffle: just copy. std::memcpy(output_buf->cur_data(), part.data(), part_size); } else { - RETURN_NOT_OK(shuffle_part(tile, &part, output_buf)); + throw_if_not_ok(shuffle_part(tile, &part, output_buf)); } if (output_buf->owns_data()) output_buf->advance_size(part.size()); output_buf->advance_offset(part.size()); } - - return Status::Ok(); } Status BitshuffleFilter::compute_parts( diff --git a/tiledb/sm/filter/bitshuffle_filter.h b/tiledb/sm/filter/bitshuffle_filter.h index c1f9ca354cc..da663c28585 100644 --- a/tiledb/sm/filter/bitshuffle_filter.h +++ b/tiledb/sm/filter/bitshuffle_filter.h @@ -87,7 +87,7 @@ class BitshuffleFilter : public Filter { /** * Shuffle the bits of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/byteshuffle_filter.cc b/tiledb/sm/filter/byteshuffle_filter.cc index f92f46a9b6e..cd07c218c9e 100644 --- a/tiledb/sm/filter/byteshuffle_filter.cc +++ b/tiledb/sm/filter/byteshuffle_filter.cc @@ -58,7 +58,7 @@ void ByteshuffleFilter::dump(FILE* out) const { fprintf(out, "ByteShuffle"); } -Status ByteshuffleFilter::run_forward( +void ByteshuffleFilter::run_forward( const WriterTile& tile, WriterTile* const, FilterBuffer* input_metadata, @@ -66,7 +66,7 @@ Status ByteshuffleFilter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { // Output size does not change with this filter. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); @@ -74,23 +74,21 @@ Status ByteshuffleFilter::run_forward( auto parts = input->buffers(); auto num_parts = (uint32_t)parts.size(); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // Shuffle all parts for (const auto& part : parts) { auto part_size = (uint32_t)part.size(); - RETURN_NOT_OK(output_metadata->write(&part_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&part_size, sizeof(uint32_t))); - RETURN_NOT_OK(shuffle_part(tile, &part, output_buf)); + throw_if_not_ok(shuffle_part(tile, &part, output_buf)); if (output_buf->owns_data()) output_buf->advance_size(part.size()); output_buf->advance_offset(part.size()); } - - return Status::Ok(); } Status ByteshuffleFilter::shuffle_part( diff --git a/tiledb/sm/filter/byteshuffle_filter.h b/tiledb/sm/filter/byteshuffle_filter.h index ef25bf97214..88bb03c432e 100644 --- a/tiledb/sm/filter/byteshuffle_filter.h +++ b/tiledb/sm/filter/byteshuffle_filter.h @@ -79,7 +79,7 @@ class ByteshuffleFilter : public Filter { /** * Shuffle the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/checksum_md5_filter.cc b/tiledb/sm/filter/checksum_md5_filter.cc index 9fa157c0af2..ce1b7d416dd 100644 --- a/tiledb/sm/filter/checksum_md5_filter.cc +++ b/tiledb/sm/filter/checksum_md5_filter.cc @@ -60,7 +60,7 @@ void ChecksumMD5Filter::dump(FILE* out) const { fprintf(out, "ChecksumMD5"); } -Status ChecksumMD5Filter::run_forward( +void ChecksumMD5Filter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -68,7 +68,7 @@ Status ChecksumMD5Filter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { // Set output buffer to input buffer - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); // Add original input metadata as a view to the output metadata throw_if_not_ok(output_metadata->append_view(input_metadata)); @@ -82,17 +82,16 @@ Status ChecksumMD5Filter::run_forward( uint32_t part_md_size = Crypto::MD5_DIGEST_BYTES + sizeof(uint64_t); uint32_t metadata_size = (total_num_parts * part_md_size) + (2 * sizeof(uint32_t)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Checksum all parts for (auto& part : metadata_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); + throw_if_not_ok(checksum_part(&part, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(checksum_part(&part, output_metadata)); } Status ChecksumMD5Filter::run_reverse( diff --git a/tiledb/sm/filter/checksum_md5_filter.h b/tiledb/sm/filter/checksum_md5_filter.h index 54462895511..4faf0817300 100644 --- a/tiledb/sm/filter/checksum_md5_filter.h +++ b/tiledb/sm/filter/checksum_md5_filter.h @@ -84,7 +84,7 @@ class ChecksumMD5Filter : public Filter { /** * Encrypt the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/checksum_sha256_filter.cc b/tiledb/sm/filter/checksum_sha256_filter.cc index 2a6fbdf89c1..e91185e688a 100644 --- a/tiledb/sm/filter/checksum_sha256_filter.cc +++ b/tiledb/sm/filter/checksum_sha256_filter.cc @@ -60,7 +60,7 @@ void ChecksumSHA256Filter::dump(FILE* out) const { fprintf(out, "ChecksumSHA256"); } -Status ChecksumSHA256Filter::run_forward( +void ChecksumSHA256Filter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -68,7 +68,7 @@ Status ChecksumSHA256Filter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { // Set output buffer to input buffer - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); // Add original input metadata as a view to the output metadata throw_if_not_ok(output_metadata->append_view(input_metadata)); @@ -82,17 +82,16 @@ Status ChecksumSHA256Filter::run_forward( uint32_t part_md_size = Crypto::SHA256_DIGEST_BYTES + sizeof(uint64_t); uint32_t metadata_size = (total_num_parts * part_md_size) + (2 * sizeof(uint32_t)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Checksum all parts for (auto& part : metadata_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); + throw_if_not_ok(checksum_part(&part, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(checksum_part(&part, output_metadata)); } Status ChecksumSHA256Filter::run_reverse( diff --git a/tiledb/sm/filter/checksum_sha256_filter.h b/tiledb/sm/filter/checksum_sha256_filter.h index fdc79cb1547..2b47d04c600 100644 --- a/tiledb/sm/filter/checksum_sha256_filter.h +++ b/tiledb/sm/filter/checksum_sha256_filter.h @@ -85,7 +85,7 @@ class ChecksumSHA256Filter : public Filter { /** * Encrypt the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/compression_filter.cc b/tiledb/sm/filter/compression_filter.cc index 55c28759b8e..458437fe469 100644 --- a/tiledb/sm/filter/compression_filter.cc +++ b/tiledb/sm/filter/compression_filter.cc @@ -247,7 +247,7 @@ Status CompressionFilter::get_option_impl( return Status::Ok(); } -Status CompressionFilter::run_forward( +void CompressionFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -256,22 +256,24 @@ Status CompressionFilter::run_forward( FilterBuffer* output) const { // Easy case: no compression if (compressor_ == Compressor::NO_COMPRESSION) { - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } - if (input->size() > std::numeric_limits::max()) - return LOG_STATUS( - Status_FilterError("Input is too large to be compressed.")); + if (input->size() > std::numeric_limits::max()) { + throw FilterStatusException("Input is too large to be compressed."); + } if ((filter_data_type_ == Datatype::STRING_ASCII || filter_data_type_ == Datatype::STRING_UTF8) && offsets_tile) { if (compressor_ == Compressor::RLE || - compressor_ == Compressor::DICTIONARY_ENCODING) - return compress_var_string_coords( - *input, offsets_tile, *output, *output_metadata); + compressor_ == Compressor::DICTIONARY_ENCODING) { + throw_if_not_ok(compress_var_string_coords( + *input, offsets_tile, *output, *output_metadata)); + return; + } } std::vector data_parts = @@ -283,9 +285,10 @@ Status CompressionFilter::run_forward( 2 * sizeof(uint32_t) + total_num_parts * 2 * sizeof(uint32_t); auto num_metadata_parts = static_cast(metadata_parts.size()); auto num_data_parts = static_cast(data_parts.size()); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Allocate output data uint64_t output_size_ub = 0; @@ -295,18 +298,16 @@ Status CompressionFilter::run_forward( output_size_ub += part.size() + overhead(tile, part.size()); // Ensure space in output buffer for worst case. - RETURN_NOT_OK(output->prepend_buffer(output_size_ub)); + throw_if_not_ok(output->prepend_buffer(output_size_ub)); Buffer* buffer_ptr = output->buffer_ptr(0); assert(buffer_ptr != nullptr); buffer_ptr->reset_offset(); // Compress all parts. for (auto& part : metadata_parts) - RETURN_NOT_OK(compress_part(tile, &part, buffer_ptr, output_metadata)); + throw_if_not_ok(compress_part(tile, &part, buffer_ptr, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(compress_part(tile, &part, buffer_ptr, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(compress_part(tile, &part, buffer_ptr, output_metadata)); } Status CompressionFilter::run_reverse( diff --git a/tiledb/sm/filter/compression_filter.h b/tiledb/sm/filter/compression_filter.h index 4925e79610e..88d30e8dbb2 100644 --- a/tiledb/sm/filter/compression_filter.h +++ b/tiledb/sm/filter/compression_filter.h @@ -126,7 +126,7 @@ class CompressionFilter : public Filter { /** * Compress the given input into the given output. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.cc b/tiledb/sm/filter/encryption_aes256gcm_filter.cc index fd9ffc4483b..a485e187027 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.cc +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.cc @@ -70,18 +70,19 @@ void EncryptionAES256GCMFilter::dump(FILE* out) const { fprintf(out, "EncryptionAES256GCM"); } -Status EncryptionAES256GCMFilter::run_forward( +void EncryptionAES256GCMFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, FilterBuffer* output) const { - if (key_bytes_ == nullptr) - return LOG_STATUS(Status_FilterError("Encryption error; bad key.")); + if (key_bytes_ == nullptr) { + throw FilterStatusException("Encryption error; bad key."); + } // Allocate an initial output buffer. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); @@ -95,17 +96,16 @@ Status EncryptionAES256GCMFilter::run_forward( Crypto::AES256GCM_IV_BYTES; uint32_t metadata_size = 2 * sizeof(uint32_t) + total_num_parts * part_md_size; - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Encrypt all parts for (auto& part : metadata_parts) - RETURN_NOT_OK(encrypt_part(&part, output_buf, output_metadata)); + throw_if_not_ok(encrypt_part(&part, output_buf, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(encrypt_part(&part, output_buf, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(encrypt_part(&part, output_buf, output_metadata)); } Status EncryptionAES256GCMFilter::encrypt_part( diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.h b/tiledb/sm/filter/encryption_aes256gcm_filter.h index 31c8bdb252f..54872656405 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.h +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.h @@ -100,7 +100,7 @@ class EncryptionAES256GCMFilter : public Filter { /** * Encrypt the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/filter.cc b/tiledb/sm/filter/filter.cc index bbba192c075..81d39a70312 100644 --- a/tiledb/sm/filter/filter.cc +++ b/tiledb/sm/filter/filter.cc @@ -41,12 +41,6 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -class FilterStatusException : public StatusException { - public: - explicit FilterStatusException(const std::string& msg) - : StatusException("Filter", msg) { - } -}; Filter::Filter(FilterType type, Datatype filter_data_type) { type_ = type; diff --git a/tiledb/sm/filter/filter.h b/tiledb/sm/filter/filter.h index 0ceecc737a3..910b6ab11bb 100644 --- a/tiledb/sm/filter/filter.h +++ b/tiledb/sm/filter/filter.h @@ -53,6 +53,13 @@ enum class FilterOption : uint8_t; enum class FilterType : uint8_t; enum class Datatype : uint8_t; +class FilterStatusException : public StatusException { + public: + explicit FilterStatusException(const std::string& msg) + : StatusException("Filter", msg) { + } +}; + /** * A Filter processes or modifies a byte region, modifying it in place, or * producing output in new buffers. @@ -137,9 +144,8 @@ class Filter { * @param input Buffer with data to be filtered. * @param output_metadata Buffer with metadata for filtered data * @param output Buffer with filtered data (unused by in-place filters). - * @return */ - virtual Status run_forward( + virtual void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/filter_pipeline.cc b/tiledb/sm/filter/filter_pipeline.cc index d14d80e9c9a..1f5e84e75c1 100644 --- a/tiledb/sm/filter/filter_pipeline.cc +++ b/tiledb/sm/filter/filter_pipeline.cc @@ -269,13 +269,13 @@ Status FilterPipeline::filter_chunks_forward( f->init_compression_resource_pool(compute_tp->concurrency_level()); - RETURN_NOT_OK(f->run_forward( + f->run_forward( tile, offsets_tile, &input_metadata, &input_data, &output_metadata, - &output_data)); + &output_data); input_data.set_read_only(false); throw_if_not_ok(input_data.swap(output_data)); @@ -382,13 +382,13 @@ uint32_t FilterPipeline::max_chunk_size() const { return max_chunk_size_; } -Status FilterPipeline::run_forward( +void FilterPipeline::run_forward( stats::Stats* const writer_stats, WriterTile* const tile, WriterTile* const offsets_tile, ThreadPool* const compute_tp, bool use_chunking) const { - RETURN_NOT_OK( + throw_if_not_ok( tile ? Status::Ok() : Status_Error("invalid argument: null Tile*")); writer_stats->add_counter("write_filtered_byte_num", tile->size()); @@ -404,25 +404,28 @@ Status FilterPipeline::run_forward( // Get the chunk sizes for var size attributes. auto&& [st, chunk_offsets] = get_var_chunk_sizes(chunk_size, tile, offsets_tile); - RETURN_NOT_OK_ELSE(st, tile->filtered_buffer().clear()); + if (!st.ok()) { + tile->filtered_buffer().clear(); + throw_if_not_ok(st); + } // Run the filters over all the chunks and store the result in // 'filtered_buffer'. - RETURN_NOT_OK_ELSE( - filter_chunks_forward( - *tile, - offsets_tile, - chunk_size, - *chunk_offsets, - tile->filtered_buffer(), - compute_tp), - tile->filtered_buffer().clear()); + st = filter_chunks_forward( + *tile, + offsets_tile, + chunk_size, + *chunk_offsets, + tile->filtered_buffer(), + compute_tp); + if (!st.ok()) { + tile->filtered_buffer().clear(); + throw_if_not_ok(st); + } // The contents of 'buffer' have been filtered and stored // in 'filtered_buffer'. We can safely free 'buffer'. tile->clear_data(); - - return Status::Ok(); } void FilterPipeline::run_reverse_generic_tile( diff --git a/tiledb/sm/filter/filter_pipeline.h b/tiledb/sm/filter/filter_pipeline.h index 66c6093803f..b1233978479 100644 --- a/tiledb/sm/filter/filter_pipeline.h +++ b/tiledb/sm/filter/filter_pipeline.h @@ -221,9 +221,8 @@ class FilterPipeline { * @param compute_tp The thread pool for compute-bound tasks. * @param chunking True if the tile should be cut into chunks before * filtering, false if not. - * @return Status */ - Status run_forward( + void run_forward( stats::Stats* writer_stats, WriterTile* tile, WriterTile* offsets_tile, diff --git a/tiledb/sm/filter/float_scaling_filter.cc b/tiledb/sm/filter/float_scaling_filter.cc index f11d4703f52..c2287c4a60b 100644 --- a/tiledb/sm/filter/float_scaling_filter.cc +++ b/tiledb/sm/filter/float_scaling_filter.cc @@ -62,7 +62,7 @@ void FloatScalingFilter::serialize_impl(Serializer& serializer) const { } template -Status FloatScalingFilter::run_forward( +void FloatScalingFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -70,9 +70,9 @@ Status FloatScalingFilter::run_forward( auto input_parts = input->buffers(); uint32_t num_parts = static_cast(input_parts.size()); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // Iterate through all the input buffers. for (auto& i : input_parts) { @@ -80,8 +80,8 @@ Status FloatScalingFilter::run_forward( assert(s % sizeof(T) == 0); uint32_t num_elems_in_part = (s / sizeof(T)); uint32_t new_size = num_elems_in_part * sizeof(W); - RETURN_NOT_OK(output_metadata->write(&new_size, sizeof(uint32_t))); - RETURN_NOT_OK(output->prepend_buffer(new_size)); + throw_if_not_ok(output_metadata->write(&new_size, sizeof(uint32_t))); + throw_if_not_ok(output->prepend_buffer(new_size)); // Iterate through each input buffer, storing each raw float as // an integer with the value round((raw_float - offset) / scale). @@ -90,18 +90,16 @@ Status FloatScalingFilter::run_forward( T elem = part_data[j]; W converted_elem = static_cast( round((elem - static_cast(offset_)) / static_cast(scale_))); - RETURN_NOT_OK(output->write(&converted_elem, sizeof(W))); + throw_if_not_ok(output->write(&converted_elem, sizeof(W))); if (j != num_elems_in_part - 1) { output->advance_offset(sizeof(W)); } } } - - return Status::Ok(); } template -Status FloatScalingFilter::run_forward( +void FloatScalingFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -126,7 +124,7 @@ Status FloatScalingFilter::run_forward( } } -Status FloatScalingFilter::run_forward( +void FloatScalingFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/float_scaling_filter.h b/tiledb/sm/filter/float_scaling_filter.h index 518fe0ed38b..c50a2663a90 100644 --- a/tiledb/sm/filter/float_scaling_filter.h +++ b/tiledb/sm/filter/float_scaling_filter.h @@ -106,7 +106,7 @@ class FloatScalingFilter : public Filter { * stores it as integers with the value round((raw_float - offset) / scale) * with the pre-specified byte width. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -167,7 +167,7 @@ class FloatScalingFilter : public Filter { * Run forward, templated on the size of the input type. */ template - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -177,7 +177,7 @@ class FloatScalingFilter : public Filter { * Run forward, templated on the size of the input type and byte width. */ template - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, diff --git a/tiledb/sm/filter/noop_filter.cc b/tiledb/sm/filter/noop_filter.cc index 26a36abe6e0..b5beab218fd 100644 --- a/tiledb/sm/filter/noop_filter.cc +++ b/tiledb/sm/filter/noop_filter.cc @@ -55,16 +55,15 @@ void NoopFilter::dump(FILE* out) const { fprintf(out, "NoOp"); } -Status NoopFilter::run_forward( +void NoopFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, FilterBuffer* output) const { - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status NoopFilter::run_reverse( diff --git a/tiledb/sm/filter/noop_filter.h b/tiledb/sm/filter/noop_filter.h index 3e5bbc87f55..e2546dea669 100644 --- a/tiledb/sm/filter/noop_filter.h +++ b/tiledb/sm/filter/noop_filter.h @@ -59,7 +59,7 @@ class NoopFilter : public Filter { /** * Run forward. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/positive_delta_filter.cc b/tiledb/sm/filter/positive_delta_filter.cc index 9b469efe16c..f308cc788c2 100644 --- a/tiledb/sm/filter/positive_delta_filter.cc +++ b/tiledb/sm/filter/positive_delta_filter.cc @@ -68,7 +68,7 @@ bool PositiveDeltaFilter::accepts_input_datatype(Datatype datatype) const { return false; } -Status PositiveDeltaFilter::run_forward( +void PositiveDeltaFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -131,22 +131,22 @@ Status PositiveDeltaFilter::run_forward( case Datatype::TIME_AS: if (tile.format_version() < 20) { // Return data as-is for backwards compatibility. - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } return run_forward( tile, offsets_tile, input_metadata, input, output_metadata, output); default: // If encoding can't work, just return the input unmodified. - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } } template -Status PositiveDeltaFilter::run_forward( +void PositiveDeltaFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -174,23 +174,21 @@ Status PositiveDeltaFilter::run_forward( } // Allocate space in output buffer for the upper bound. - RETURN_NOT_OK(output->prepend_buffer(output_size_ub)); + throw_if_not_ok(output->prepend_buffer(output_size_ub)); Buffer* buffer_ptr = output->buffer_ptr(0); buffer_ptr->reset_offset(); assert(buffer_ptr != nullptr); // Forward the existing metadata - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); // Allocate a buffer for this filter's metadata and write the header. - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&total_num_windows, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&total_num_windows, sizeof(uint32_t))); // Compress all parts. for (unsigned i = 0; i < num_parts; i++) { - RETURN_NOT_OK(encode_part(&parts[i], output, output_metadata)); + throw_if_not_ok(encode_part(&parts[i], output, output_metadata)); } - - return Status::Ok(); } template diff --git a/tiledb/sm/filter/positive_delta_filter.h b/tiledb/sm/filter/positive_delta_filter.h index 01fcb3e51d8..57445b2f3a6 100644 --- a/tiledb/sm/filter/positive_delta_filter.h +++ b/tiledb/sm/filter/positive_delta_filter.h @@ -104,7 +104,7 @@ class PositiveDeltaFilter : public Filter { /** * Perform positive-delta encoding of the given input into the given output. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const, FilterBuffer* input_metadata, @@ -154,7 +154,7 @@ class PositiveDeltaFilter : public Filter { /** Run_forward method templated on the tile cell datatype. */ template - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const tile_offsets, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_1_in_place_filter.cc b/tiledb/sm/filter/test/add_1_in_place_filter.cc index f4c9425fade..d5683a123d4 100644 --- a/tiledb/sm/filter/test/add_1_in_place_filter.cc +++ b/tiledb/sm/filter/test/add_1_in_place_filter.cc @@ -44,7 +44,7 @@ void Add1InPlace::dump(FILE* out) const { (void)out; } -Status Add1InPlace::run_forward( +void Add1InPlace::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -52,7 +52,7 @@ Status Add1InPlace::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); output->reset_offset(); uint64_t nelts = input_size / sizeof(uint64_t); @@ -63,9 +63,7 @@ Status Add1InPlace::run_forward( } // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status Add1InPlace::run_reverse( diff --git a/tiledb/sm/filter/test/add_1_in_place_filter.h b/tiledb/sm/filter/test/add_1_in_place_filter.h index 67b33f33de6..ec64e9801cd 100644 --- a/tiledb/sm/filter/test/add_1_in_place_filter.h +++ b/tiledb/sm/filter/test/add_1_in_place_filter.h @@ -52,7 +52,7 @@ class Add1InPlace : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_1_including_metadata_filter.cc b/tiledb/sm/filter/test/add_1_including_metadata_filter.cc index dfc34493307..cd74e5b636b 100644 --- a/tiledb/sm/filter/test/add_1_including_metadata_filter.cc +++ b/tiledb/sm/filter/test/add_1_including_metadata_filter.cc @@ -44,7 +44,7 @@ void Add1IncludingMetadataFilter::dump(FILE* out) const { (void)out; } -Status Add1IncludingMetadataFilter::run_forward( +void Add1IncludingMetadataFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -57,46 +57,44 @@ Status Add1IncludingMetadataFilter::run_forward( md_nelts = input_md_size / sizeof(uint64_t); // Add another output buffer. - RETURN_NOT_OK(output->prepend_buffer(input_size + input_md_size)); + throw_if_not_ok(output->prepend_buffer(input_size + input_md_size)); output->reset_offset(); // Filter input data for (uint64_t i = 0; i < nelts; i++) { uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + throw_if_not_ok(input->read(&inc, sizeof(uint64_t))); inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + throw_if_not_ok(output->write(&inc, sizeof(uint64_t))); } // Finish any remaining bytes to ensure no data loss. auto rem = input_size % sizeof(uint64_t); for (unsigned i = 0; i < rem; i++) { char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); + throw_if_not_ok(input->read(&byte, sizeof(char))); + throw_if_not_ok(output->write(&byte, sizeof(char))); } // Now filter input metadata. for (uint64_t i = 0; i < md_nelts; i++) { uint64_t inc; - RETURN_NOT_OK(input_metadata->read(&inc, sizeof(uint64_t))); + throw_if_not_ok(input_metadata->read(&inc, sizeof(uint64_t))); inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + throw_if_not_ok(output->write(&inc, sizeof(uint64_t))); } rem = input_md_size % sizeof(uint64_t); for (unsigned i = 0; i < rem; i++) { char byte; - RETURN_NOT_OK(input_metadata->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); + throw_if_not_ok(input_metadata->read(&byte, sizeof(char))); + throw_if_not_ok(output->write(&byte, sizeof(char))); } // Because this filter modifies the input metadata, we need output metadata // that allows the original metadata to be reconstructed on reverse. Also // note that contrary to most filters, we don't forward the input metadata. - RETURN_NOT_OK(output_metadata->prepend_buffer(2 * sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&input_size, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&input_md_size, sizeof(uint32_t))); - - return Status::Ok(); + throw_if_not_ok(output_metadata->prepend_buffer(2 * sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&input_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&input_md_size, sizeof(uint32_t))); } Status Add1IncludingMetadataFilter::run_reverse( diff --git a/tiledb/sm/filter/test/add_1_including_metadata_filter.h b/tiledb/sm/filter/test/add_1_including_metadata_filter.h index c508caff401..93b62d919eb 100644 --- a/tiledb/sm/filter/test/add_1_including_metadata_filter.h +++ b/tiledb/sm/filter/test/add_1_including_metadata_filter.h @@ -54,7 +54,7 @@ class Add1IncludingMetadataFilter : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_1_out_of_place_filter.cc b/tiledb/sm/filter/test/add_1_out_of_place_filter.cc index aa7c7f47539..78cc89d89ba 100644 --- a/tiledb/sm/filter/test/add_1_out_of_place_filter.cc +++ b/tiledb/sm/filter/test/add_1_out_of_place_filter.cc @@ -43,7 +43,7 @@ void Add1OutOfPlace::dump(FILE* out) const { (void)out; } -Status Add1OutOfPlace::run_forward( +void Add1OutOfPlace::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -54,28 +54,26 @@ Status Add1OutOfPlace::run_forward( auto nelts = input_size / sizeof(uint64_t); // Add another output buffer. - RETURN_NOT_OK(output->prepend_buffer(input_size)); + throw_if_not_ok(output->prepend_buffer(input_size)); output->reset_offset(); for (uint64_t i = 0; i < nelts; i++) { uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + throw_if_not_ok(input->read(&inc, sizeof(uint64_t))); inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + throw_if_not_ok(output->write(&inc, sizeof(uint64_t))); } // Finish any remaining bytes to ensure no data loss. auto rem = input_size % sizeof(uint64_t); for (unsigned i = 0; i < rem; i++) { char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); + throw_if_not_ok(input->read(&byte, sizeof(char))); + throw_if_not_ok(output->write(&byte, sizeof(char))); } // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status Add1OutOfPlace::run_reverse( diff --git a/tiledb/sm/filter/test/add_1_out_of_place_filter.h b/tiledb/sm/filter/test/add_1_out_of_place_filter.h index 21065a002f6..c39d70c9d08 100644 --- a/tiledb/sm/filter/test/add_1_out_of_place_filter.h +++ b/tiledb/sm/filter/test/add_1_out_of_place_filter.h @@ -52,7 +52,7 @@ class Add1OutOfPlace : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_n_in_place_filter.cc b/tiledb/sm/filter/test/add_n_in_place_filter.cc index 15fa7df2f85..6452ae19f9b 100644 --- a/tiledb/sm/filter/test/add_n_in_place_filter.cc +++ b/tiledb/sm/filter/test/add_n_in_place_filter.cc @@ -45,7 +45,7 @@ void AddNInPlace::dump(FILE* out) const { (void)out; } -Status AddNInPlace::run_forward( +void AddNInPlace::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -53,7 +53,7 @@ Status AddNInPlace::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); output->reset_offset(); uint64_t nelts = input_size / sizeof(uint64_t); @@ -64,8 +64,7 @@ Status AddNInPlace::run_forward( } // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status AddNInPlace::run_reverse( diff --git a/tiledb/sm/filter/test/add_n_in_place_filter.h b/tiledb/sm/filter/test/add_n_in_place_filter.h index 97cba2c3c18..b5c5e3069c5 100644 --- a/tiledb/sm/filter/test/add_n_in_place_filter.h +++ b/tiledb/sm/filter/test/add_n_in_place_filter.h @@ -52,7 +52,7 @@ class AddNInPlace : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/filter_test_support.cc b/tiledb/sm/filter/test/filter_test_support.cc index f517c5b8650..907940fd013 100644 --- a/tiledb/sm/filter/test/filter_test_support.cc +++ b/tiledb/sm/filter/test/filter_test_support.cc @@ -68,13 +68,11 @@ void check_run_pipeline_full( const FilteredTileChecker& filtered_buffer_checker, shared_ptr memory_tracker) { // Run the pipeline forward. - CHECK(pipeline - .run_forward( - &dummy_stats, - tile.get(), - offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, - &tp) - .ok()); + pipeline.run_forward( + &dummy_stats, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, + &tp); // Check the original unfiltered data was removed. CHECK(tile->size() == 0); @@ -120,13 +118,11 @@ void check_run_pipeline_roundtrip( const TileDataGenerator* test_data, shared_ptr memory_tracker) { // Run the pipeline forward. - CHECK(pipeline - .run_forward( - &dummy_stats, - tile.get(), - offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, - &tp) - .ok()); + pipeline.run_forward( + &dummy_stats, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, + &tp); // Check the original unfiltered data was removed. CHECK(tile->size() == 0); diff --git a/tiledb/sm/filter/test/pseudo_checksum_filter.cc b/tiledb/sm/filter/test/pseudo_checksum_filter.cc index c65a7c7308e..0b12a02093c 100644 --- a/tiledb/sm/filter/test/pseudo_checksum_filter.cc +++ b/tiledb/sm/filter/test/pseudo_checksum_filter.cc @@ -43,7 +43,7 @@ void PseudoChecksumFilter::dump(FILE* out) const { (void)out; } -Status PseudoChecksumFilter::run_forward( +void PseudoChecksumFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -54,24 +54,22 @@ Status PseudoChecksumFilter::run_forward( auto nelts = input_size / sizeof(uint64_t); // The input is unmodified by this filter. - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); // Forward the existing metadata and prepend a metadata buffer for the // checksum. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(sizeof(uint64_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(sizeof(uint64_t))); output_metadata->reset_offset(); uint64_t sum = 0; for (uint64_t i = 0; i < nelts; i++) { uint64_t val; - RETURN_NOT_OK(input->read(&val, sizeof(uint64_t))); + throw_if_not_ok(input->read(&val, sizeof(uint64_t))); sum += val; } - RETURN_NOT_OK(output_metadata->write(&sum, sizeof(uint64_t))); - - return Status::Ok(); + throw_if_not_ok(output_metadata->write(&sum, sizeof(uint64_t))); } Status PseudoChecksumFilter::run_reverse( diff --git a/tiledb/sm/filter/test/pseudo_checksum_filter.h b/tiledb/sm/filter/test/pseudo_checksum_filter.h index 96197b7ad4c..2fee1dffe1c 100644 --- a/tiledb/sm/filter/test/pseudo_checksum_filter.h +++ b/tiledb/sm/filter/test/pseudo_checksum_filter.h @@ -52,7 +52,7 @@ class PseudoChecksumFilter : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc index e74eca6ca0a..9d271754844 100644 --- a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc @@ -54,7 +54,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { SECTION("- Single stage") { auto tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -103,7 +103,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { pipeline.get_filter()->set_max_window_size( window_size); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -139,7 +139,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -176,7 +176,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -205,7 +205,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -270,10 +270,7 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -343,10 +340,7 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); pipeline.get_filter()->set_max_window_size( window_size); - - CHECK(pipeline - .run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -384,9 +378,7 @@ TEST_CASE( CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -444,8 +436,7 @@ TEST_CASE( constants::cell_var_offset_size)); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), &offsets_tile32, &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), &offsets_tile32, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -476,10 +467,7 @@ TEST_CASE( } auto offsets_tile = make_offsets_tile(offsets, tracker); - - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc index 665fc405bef..507e7c0967a 100644 --- a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc @@ -53,7 +53,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -84,7 +84,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); @@ -144,9 +144,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { SECTION("- Single stage") { WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -178,9 +176,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc index 7bd576815e4..735430f4759 100644 --- a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc +++ b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc @@ -51,7 +51,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -82,7 +82,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); @@ -142,9 +142,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { SECTION("- Single stage") { WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -176,9 +174,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_checksum_pipeline.cc b/tiledb/sm/filter/test/unit_checksum_pipeline.cc index c7e667abf8d..f9877eab22c 100644 --- a/tiledb/sm/filter/test/unit_checksum_pipeline.cc +++ b/tiledb/sm/filter/test/unit_checksum_pipeline.cc @@ -56,7 +56,7 @@ TEST_CASE( ThreadPool tp(4); ChecksumMD5Filter md5_filter(Datatype::UINT64); md5_pipeline.add_filter(md5_filter); - CHECK(md5_pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + md5_pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -76,8 +76,7 @@ TEST_CASE( FilterPipeline sha_256_pipeline; ChecksumMD5Filter sha_256_filter(Datatype::UINT64); sha_256_pipeline.add_filter(sha_256_filter); - CHECK(sha_256_pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp) - .ok()); + sha_256_pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_encryption_pipeline.cc b/tiledb/sm/filter/test/unit_encryption_pipeline.cc index ed0ed9a1ed9..84c32c9b379 100644 --- a/tiledb/sm/filter/test/unit_encryption_pipeline.cc +++ b/tiledb/sm/filter/test/unit_encryption_pipeline.cc @@ -54,7 +54,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { pipeline.add_filter(EncryptionAES256GCMFilter(Datatype::UINT64)); // No key set - CHECK(!pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK_THROWS(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp)); // Create and set a key char key[32]; @@ -64,7 +64,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { filter->set_key(key); // Check success - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -79,7 +79,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { // Check error decrypting with wrong key. tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); key[0]++; filter->set_key(key); diff --git a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc index 5242b882fd2..4b02fdff0c0 100644 --- a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc +++ b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc @@ -53,7 +53,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { SECTION("- Single stage") { auto tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -101,7 +101,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { pipeline.get_filter()->set_max_window_size( window_size); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -123,7 +123,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(!pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK_THROWS(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp)); } } @@ -174,10 +174,7 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -249,10 +246,7 @@ TEST_CASE( pipeline.get_filter()->set_max_window_size( window_size); - - CHECK(pipeline - .run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -277,9 +271,8 @@ TEST_CASE( CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - !pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + CHECK_THROWS(pipeline.run_forward( + &dummy_stats, tile.get(), offsets_tile.get(), &tp)); } WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); diff --git a/tiledb/sm/filter/test/unit_webp_pipeline.cc b/tiledb/sm/filter/test/unit_webp_pipeline.cc index 27a74a7b975..c8013e5d4a6 100644 --- a/tiledb/sm/filter/test/unit_webp_pipeline.cc +++ b/tiledb/sm/filter/test/unit_webp_pipeline.cc @@ -85,9 +85,7 @@ TEST_CASE("Filter: Round trip WebpFilter RGB data", "[filter][webp]") { width * 3, Datatype::UINT8)); bool use_chunking = true; - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp, use_chunking) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp, use_chunking); // Check the original unfiltered data was removed. CHECK(tile->size() == 0); diff --git a/tiledb/sm/filter/test/unit_xor_pipeline.cc b/tiledb/sm/filter/test/unit_xor_pipeline.cc index 51ea0c0e6df..3f4e18a17ea 100644 --- a/tiledb/sm/filter/test/unit_xor_pipeline.cc +++ b/tiledb/sm/filter/test/unit_xor_pipeline.cc @@ -113,7 +113,7 @@ void testing_float_scaling_filter() { ->set_option(FilterOption::SCALE_FLOAT_OFFSET, &foffset) .ok()); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); // Check new size and number of chunks CHECK(tile->size() == 0); @@ -177,7 +177,7 @@ void testing_xor_filter(Datatype t) { ThreadPool tp(4); pipeline.add_filter(XORFilter(t)); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); // Check new size and number of chunks CHECK(tile->size() == 0); @@ -293,7 +293,7 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { } ThreadPool tp(4); - REQUIRE(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index fe4ea38d9c1..58b1e41a46e 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -54,7 +54,7 @@ namespace tiledb::sm { * a program crash. */ -Status WebpFilter::run_forward( +void WebpFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer*, @@ -101,7 +101,7 @@ using namespace tiledb::common; namespace tiledb::sm { -Status WebpFilter::run_forward( +void WebpFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -111,7 +111,7 @@ Status WebpFilter::run_forward( return run_forward(input_metadata, input, output_metadata, output); } -Status WebpFilter::run_forward( +void WebpFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -212,8 +212,6 @@ Status WebpFilter::run_forward( throw_if_not_ok(output->prepend_buffer(enc_size)); throw_if_not_ok(output->write(result, enc_size)); } - - return Status::Ok(); } Status WebpFilter::run_reverse( diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index cd6648a06cb..7e9a8b31a9d 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -171,9 +171,8 @@ class WebpFilter : public Filter { * @param input Buffer with data to be filtered. * @param output_metadata Buffer with metadata for filtered data. * @param output Buffer with filtered data (unused by in-place filters). - * @return Status::Ok() on success. Throws on failure. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -189,9 +188,8 @@ class WebpFilter : public Filter { * @param input Buffer with data to be filtered. * @param output_metadata Buffer with metadata for filtered data. * @param output Buffer with filtered data (unused by in-place filters). - * @return Status::Ok() on success. Throws on failure. */ - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, diff --git a/tiledb/sm/filter/xor_filter.cc b/tiledb/sm/filter/xor_filter.cc index 488ffc7fe46..b6ecdc6c36d 100644 --- a/tiledb/sm/filter/xor_filter.cc +++ b/tiledb/sm/filter/xor_filter.cc @@ -77,7 +77,7 @@ Datatype XORFilter::output_datatype(tiledb::sm::Datatype input_type) const { } } -Status XORFilter::run_forward( +void XORFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -105,25 +105,25 @@ Status XORFilter::run_forward( input_metadata, input, output_metadata, output); } default: { - return Status_FilterError( + throw FilterStatusException( "XORFilter::run_forward: datatype size cannot be converted to " "integer type."); } } - return Status_FilterError("XORFilter::run_forward: invalid datatype."); + throw FilterStatusException("XORFilter::run_forward: invalid datatype."); } template < typename T, typename std::enable_if::value>::type*> -Status XORFilter::run_forward( +void XORFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, FilterBuffer* output) const { // Output size does not change with this filter. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); @@ -131,18 +131,16 @@ Status XORFilter::run_forward( auto parts = input->buffers(); auto num_parts = (uint32_t)parts.size(); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // XOR all parts for (const auto& part : parts) { auto part_size = (uint32_t)part.size(); - RETURN_NOT_OK(output_metadata->write(&part_size, sizeof(uint32_t))); - RETURN_NOT_OK(xor_part(&part, output_buf)); + throw_if_not_ok(output_metadata->write(&part_size, sizeof(uint32_t))); + throw_if_not_ok(xor_part(&part, output_buf)); } - - return Status::Ok(); } template < diff --git a/tiledb/sm/filter/xor_filter.h b/tiledb/sm/filter/xor_filter.h index e6decfe7153..be118f7c601 100644 --- a/tiledb/sm/filter/xor_filter.h +++ b/tiledb/sm/filter/xor_filter.h @@ -90,7 +90,7 @@ class XORFilter : public Filter { * element in the part, and then the differences of each consecutive pair * of elements. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -119,7 +119,7 @@ class XORFilter : public Filter { template < typename T, typename std::enable_if::value>::type* = nullptr> - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 17ba2a5e5f7..6cfee3597b4 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -48,6 +48,13 @@ namespace tiledb::sm { +class FragmentInfoException : public StatusException { + public: + explicit FragmentInfoException(const std::string& message) + : StatusException("FragmentInfo", message) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -182,7 +189,7 @@ Status FragmentInfo::get_total_cell_num(uint64_t* cell_num) const { const std::string& FragmentInfo::fragment_name(uint32_t fid) const { ensure_loaded(); if (fid >= fragment_num()) - throw Status_FragmentInfoError( + throw FragmentInfoException( "Cannot get fragment URI; Invalid fragment index"); return single_fragment_info_vec_[fid].name(); @@ -936,7 +943,7 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { void FragmentInfo::ensure_loaded() const { if (!loaded_) { - throw Status_FragmentInfoError("Fragment info has not been loaded."); + throw FragmentInfoException("Fragment info has not been loaded."); } } diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 8afe38d9eb9..0db20232ec5 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -52,7 +52,6 @@ #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/query/readers/aggregators/tile_metadata.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" #include "tiledb/sm/tile/tile_metadata_generator.h" diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 7701b7b6009..bcc3f40f39f 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -82,16 +82,14 @@ Group::Group(ContextResources& resources, const URI& group_uri) memory_tracker_->set_type(MemoryTrackerType::GROUP); } -Status Group::create(ContextResources& resources, const URI& uri) { +void Group::create(ContextResources& resources, const URI& uri) { // Create group URI if (uri.is_invalid()) throw GroupException( "Cannot create group '" + uri.to_string() + "'; Invalid group URI"); // Check if group exists - bool exists; - throw_if_not_ok(is_group(resources, uri, &exists)); - if (exists) { + if (is_group(resources, uri)) { throw GroupException( "Cannot create group; Group '" + uri.to_string() + "' already exists"); } @@ -101,7 +99,7 @@ Status Group::create(ContextResources& resources, const URI& uri) { Group group(resources, uri); throw_if_not_ok( resources.rest_client()->post_group_create_to_rest(uri, &group)); - return Status::Ok(); + return; } // Create group directory @@ -118,7 +116,6 @@ Status Group::create(ContextResources& resources, const URI& uri) { // Create group detail folder throw_if_not_ok(resources.vfs().create_dir( uri.join_path(constants::group_detail_dir_name))); - return Status::Ok(); } void Group::open( @@ -245,11 +242,11 @@ void Group::close_for_writes() { if (group_details()->is_modified()) { const URI& group_detail_folder_uri = group_detail_uri(); auto group_detail_uri = generate_detail_uri(); - throw_if_not_ok(group_details()->store( + group_details()->store( resources_, group_detail_folder_uri, group_detail_uri, - *encryption_key())); + *encryption_key()); } } @@ -568,7 +565,7 @@ void Group::set_metadata_loaded(const bool metadata_loaded) { metadata_loaded_ = metadata_loaded; } -Status Group::consolidate_metadata( +void Group::consolidate_metadata( ContextResources& resources, const char* group_name, const Config& config) { // Check group URI URI group_uri(group_name); @@ -576,10 +573,7 @@ Status Group::consolidate_metadata( throw GroupException("Cannot consolidate group metadata; Invalid URI"); } // Check if group exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources, group_uri, &obj_type)); - - if (obj_type != ObjectType::GROUP) { + if (object_type(resources, group_uri) != ObjectType::GROUP) { throw GroupException( "Cannot consolidate group metadata; Group does not exist"); } @@ -587,10 +581,10 @@ Status Group::consolidate_metadata( // Consolidate // Encryption credentials are loaded by Group from config StorageManager sm(resources, resources.logger(), config); - auto consolidator = - Consolidator::create(ConsolidationMode::GROUP_META, config, &sm); - return consolidator->consolidate( - group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0); + auto consolidator = Consolidator::create( + resources, ConsolidationMode::GROUP_META, config, &sm); + throw_if_not_ok(consolidator->consolidate( + group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0)); } void Group::vacuum_metadata( @@ -602,16 +596,13 @@ void Group::vacuum_metadata( } // Check if group exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources, group_uri, &obj_type)); - - if (obj_type != ObjectType::GROUP) { + if (object_type(resources, group_uri) != ObjectType::GROUP) { throw GroupException("Cannot vacuum group metadata; Group does not exist"); } StorageManager sm(resources, resources.logger(), config); - auto consolidator = - Consolidator::create(ConsolidationMode::GROUP_META, config, &sm); + auto consolidator = Consolidator::create( + resources, ConsolidationMode::GROUP_META, config, &sm); consolidator->vacuum(group_name); } diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 2445d876fa3..b2ebb2f022b 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -74,9 +74,8 @@ class Group { * * @param resources The context resources. * @param uri The URI of the group to be created. - * @return Status */ - static Status create(ContextResources& resources, const URI& uri); + static void create(ContextResources& resources, const URI& uri); /** Returns the group directory object. */ const shared_ptr group_directory() const; @@ -236,9 +235,8 @@ class Group { * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config associated with * this instance). - * @return Status */ - static Status consolidate_metadata( + static void consolidate_metadata( ContextResources& resources, const char* group_name, const Config& config); diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index 3c5e9174bd5..cff44effcc5 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -92,8 +92,7 @@ void GroupDetails::mark_member_for_addition( absolute_group_member_uri = group_uri_.join_path(group_member_uri.to_string()); } - ObjectType type = ObjectType::INVALID; - throw_if_not_ok(object_type(resources, absolute_group_member_uri, &type)); + ObjectType type = object_type(resources, absolute_group_member_uri); if (type == ObjectType::INVALID) { throw GroupDetailsException( "Cannot add group member " + absolute_group_member_uri.to_string() + @@ -203,7 +202,7 @@ GroupDetails::members() const { return members_; } -Status GroupDetails::store( +void GroupDetails::store( ContextResources& resources, const URI& group_detail_folder_uri, const URI& group_detail_uri, @@ -231,8 +230,6 @@ Status GroupDetails::store( throw_if_not_ok(vfs.create_dir(group_detail_folder_uri)); } GenericTileIO::store_data(resources, group_detail_uri, tile, encryption_key); - - return Status::Ok(); } std::optional> GroupDetails::deserialize( diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index cf95bbd0838..71c91cbdb40 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -139,9 +139,8 @@ class GroupDetails { * @param group_detail_folder_uri group details folder * @param group_detail_uri uri for detail file to write * @param encryption_key encryption key for at-rest encryption - * @return status */ - Status store( + void store( ContextResources& resources, const URI& group_detail_folder_uri, const URI& group_detail_uri, diff --git a/tiledb/sm/group/group_details_v1.h b/tiledb/sm/group/group_details_v1.h index b04ed932dc0..84555ab0117 100644 --- a/tiledb/sm/group/group_details_v1.h +++ b/tiledb/sm/group/group_details_v1.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,13 +41,11 @@ #include "tiledb/sm/group/group_details.h" #include "tiledb/sm/group/group_member.h" #include "tiledb/sm/metadata/metadata.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/serialization/serializers.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class GroupDetails; @@ -79,7 +77,6 @@ class GroupDetailsV1 : public GroupDetails { /* Format version for class. */ inline static const format_version_t format_version_ = 1; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_GROUP_DETAILS_V1_H diff --git a/tiledb/sm/misc/CMakeLists.txt b/tiledb/sm/misc/CMakeLists.txt index 34e27a53e5f..92714ecfb66 100644 --- a/tiledb/sm/misc/CMakeLists.txt +++ b/tiledb/sm/misc/CMakeLists.txt @@ -26,6 +26,8 @@ include(common NO_POLICY_SCOPE) include(object_library) +find_package(LibMagic MODULE REQUIRED) + # # `cancelable_tasks` object library # @@ -41,6 +43,33 @@ commence(object_library constants) this_target_sources(constants.cc) conclude(object_library) +############################################################ +# provide actions/target for preparation of magic.mgc data for embedding/build + +# +# `mgc_dict` object library +# +# The file is used as a header but we don't name it .h to avoid it being picked up by IDEs +set(MGC_GZIPPED_H_OUTPUT_FILE "${PROJECT_BINARY_DIR}/magic_mgc.zst.h_") + +add_custom_command( + OUTPUT "${MGC_GZIPPED_H_OUTPUT_FILE}" + DEPENDS "${libmagic_DICTIONARY}" "${CMAKE_CURRENT_LIST_DIR}/generate_embedded_data_header.script.cmake" + COMMAND ${CMAKE_COMMAND} "-DINPUT_FILE=${libmagic_DICTIONARY}" "-DOUTPUT_FILE=${MGC_GZIPPED_H_OUTPUT_FILE}" -P "${CMAKE_CURRENT_LIST_DIR}/generate_embedded_data_header.script.cmake" +) +add_custom_target(gen_mgc_unarch + DEPENDS ${MGC_GZIPPED_H_OUTPUT_FILE} +) + +commence(object_library mgc_dict) + add_dependencies(mgc_dict gen_mgc_unarch) + # change to `this_target_include_directories` when available + target_include_directories(mgc_dict PRIVATE "${PROJECT_BINARY_DIR}") + this_target_object_libraries(buffer compressors) + this_target_link_libraries(libmagic) + this_target_sources(mgc_dict.cc) +conclude(object_library) + # # `math` object library # @@ -72,34 +101,3 @@ commence(object_library time) conclude(object_library) add_test_subdirectory() - -# -# `mgc_dict.*` tests are declared in this directory for the moment. -# -# See also `/tiledb/CMakeLists.txt` for related targets `gen_mgc_unarch` and -# `update-embedded-magic-data` -# -if (TILEDB_TESTS) - # simple unit test of magic.mgc embedded data vs external data - find_package(LibMagic MODULE REQUIRED) - if(NOT EXISTS ${libmagic_DICTIONARY}) - message(FATAL_ERROR "Failed to find libmagic 'magic.mgc' file.") - endif() - add_executable(unit_mgc_dict EXCLUDE_FROM_ALL) - target_sources(unit_mgc_dict PRIVATE - test/unit_mgc_dict.cc - mgc_dict.cc - ) - target_link_libraries(unit_mgc_dict PRIVATE - compressors - filter - libmagic - ) - - target_compile_options(unit_mgc_dict PRIVATE -DTILEDB_PATH_TO_MAGIC_MGC=\"${libmagic_DICTIONARY}\") - target_include_directories(unit_mgc_dict - PRIVATE - ${CMAKE_CURRENT_BINARY_DIR} - "${libmagic_INCLUDE_DIR}" - ) -endif() diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 8feb0542c06..438442a2b63 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -773,6 +773,9 @@ const unsigned concurrent_attr_reads = 2; /** The redirection header key in REST response. */ extern const std::string redirection_header_key = "location"; +/** The config key prefix for REST custom headers. */ +const std::string rest_header_prefix = "rest.custom_headers."; + /** String describing MIME_AUTODETECT. */ const std::string mime_autodetect_str = "AUTODETECT"; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index c675b8de4d3..5a854d404b8 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -760,6 +760,9 @@ const void* fill_value(Datatype type); /** The redirection header key in REST response. */ extern const std::string redirection_header_key; +/** The REST custom headers config key prefix. */ +extern const std::string rest_header_prefix; + /** Delimiter for lists passed as config parameter */ extern const std::string config_delimiter; diff --git a/tiledb/sm/misc/generate_embedded_data_header.script.cmake b/tiledb/sm/misc/generate_embedded_data_header.script.cmake new file mode 100755 index 00000000000..a5e4d78c4ec --- /dev/null +++ b/tiledb/sm/misc/generate_embedded_data_header.script.cmake @@ -0,0 +1,72 @@ +# +# tiledb/sm/misc/generate_embedded_data_header.script.cmake +# +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +if (NOT INPUT_FILE) + message(FATAL_ERROR "Must specify INPUT_FILE") +endif() + +if (NOT OUTPUT_FILE) + message(FATAL_ERROR "Must specify OUTPUT_FILE") +endif() + +cmake_path(GET INPUT_FILE FILENAME INPUT_FILENAME) +cmake_path(REPLACE_FILENAME OUTPUT_FILE "${INPUT_FILENAME}" OUTPUT_VARIABLE compressed_file) +set(compressed_file "${compressed_file}.zst") + +string(MAKE_C_IDENTIFIER ${INPUT_FILENAME} INPUT_VARIABLE) + +message(DEBUG "Compressing ${INPUT_FILE} to ${compressed_file}") +file(ARCHIVE_CREATE OUTPUT "${compressed_file}" PATHS ${INPUT_FILE} FORMAT raw COMPRESSION Zstd + # Level 12 was found to have the best balance between compression ratio and speed + # but is available in CMake 3.26+. + COMPRESSION_LEVEL 9 +) +file(SIZE ${compressed_file} COMPRESSED_SIZE) +message(DEBUG "Compressed size: ${COMPRESSED_SIZE} bytes") + +message(DEBUG "Reading compressed data from ${compressed_file}") +file(READ "${compressed_file}" compressed_data HEX) +file(REMOVE "${compressed_file}") + +message(DEBUG "Writing embedded data to ${OUTPUT_FILE}") + +# Split the data into lines of 128 byte literals. +set(MAX_CHARACTERS_PER_LINE 128) +string(REPEAT "[0-9a-f][0-9a-f]" ${MAX_CHARACTERS_PER_LINE} characters_per_line) +string(REGEX REPLACE "(${characters_per_line})" "\\1\n" compressed_data "${compressed_data}") + +# Convert the hex characters to C-style hex numbers. +string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1, " compressed_data "${compressed_data}") + +file(SIZE ${INPUT_FILE} DECOMPRESSED_SIZE) + +file(WRITE ${OUTPUT_FILE} + "static const unsigned char ${INPUT_VARIABLE}_compressed_bytes[] = {\n" + ${compressed_data} "\n" + "};\n" + "constexpr size_t ${INPUT_VARIABLE}_compressed_size = sizeof(${INPUT_VARIABLE}_compressed_bytes);\n" + "constexpr size_t ${INPUT_VARIABLE}_decompressed_size = ${DECOMPRESSED_SIZE};\n") diff --git a/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 b/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 deleted file mode 100644 index d23e06bc82a..00000000000 Binary files a/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 and /dev/null differ diff --git a/tiledb/sm/misc/mgc_dict.cc b/tiledb/sm/misc/mgc_dict.cc index 5a42f253e85..1bafe99fabb 100644 --- a/tiledb/sm/misc/mgc_dict.cc +++ b/tiledb/sm/misc/mgc_dict.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,55 +28,38 @@ #include "mgc_dict.h" -#include "tiledb/sm/compressors/util/gzip_wrappers.h" +#include "magic_mgc.zst.h_" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/compressors/zstd_compressor.h" #include #include #include -namespace tiledb { -namespace sm { +namespace tiledb::sm::magic_dict { -static const char magic_mgc_compressed_bytes[] = { -#include "magic_mgc_gzipped.bin" -}; - -shared_ptr magic_dict::expanded_buffer_; - -void* magic_dict::uncompressed_magic_dict_ = nullptr; - -static magic_dict magic_dict_object; - -magic_dict::magic_dict() { -} - -void magic_dict::prepare_data() { - if (uncompressed_magic_dict_) - return; - - expanded_buffer_ = make_shared(HERE()); - gzip_decompress( - expanded_buffer_, - reinterpret_cast(&magic_mgc_compressed_bytes[0])); - - uncompressed_magic_dict_ = expanded_buffer_.get()->data(); +void prepare_data(span expanded_buffer) { + ConstBuffer input(magic_mgc_compressed_bytes, magic_mgc_compressed_size); + PreallocatedBuffer output(expanded_buffer.data(), expanded_buffer.size()); + ZStd::ZSTD_Decompress_Context ctx; + ZStd::decompress(ctx, input, output); } -int magic_dict::magic_mgc_embedded_load(magic_t magic) { - if (!uncompressed_magic_dict_) - prepare_data(); +int magic_mgc_embedded_load(magic_t magic) { + auto buffer = expanded_buffer(); - void* data[1] = {uncompressed_magic_dict_}; - size_t sizes[1] = {expanded_buffer_->size()}; + void* data = const_cast(buffer.data()); + size_t size = buffer.size(); // zero ok, non-zero error - return magic_load_buffers(magic, &data[0], &sizes[0], 1); + return magic_load_buffers(magic, &data, &size, 1); } -const shared_ptr magic_dict::expanded_buffer() { - if (!uncompressed_magic_dict_) - prepare_data(); - return expanded_buffer_; +span expanded_buffer() { + static std::vector expanded_buffer(magic_mgc_decompressed_size); + static std::once_flag once_flag; + // Thread-safe initialization of the expanded data. + std::call_once(once_flag, [&]() { prepare_data(expanded_buffer); }); + return expanded_buffer; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::magic_dict diff --git a/tiledb/sm/misc/mgc_dict.h b/tiledb/sm/misc/mgc_dict.h index aa2b53fea15..65bb020ff3b 100644 --- a/tiledb/sm/misc/mgc_dict.h +++ b/tiledb/sm/misc/mgc_dict.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,49 +29,26 @@ #include "magic.h" #include "tiledb/common/common.h" -#include "tiledb/sm/misc/types.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm::magic_dict { -using tiledb::common::Status; - -class magic_dict { - public: - /** Default constructor */ - magic_dict(); - - /** - * Have libmagic load data from our embedded version. - * - * @param magic - libmagic object obtained from magic_open() - * @return the value libmgaic returns from magic_load_buffers(). - */ - static int magic_mgc_embedded_load(magic_t magic); - - /** - * Provides access to the internally expanded data. - * - * @return a shared pointer to the internal ByteVecValue holding - * the expanded data. - */ - static const shared_ptr expanded_buffer(); - - private: - /** - * decompress and init data items - */ - static void prepare_data(); - - /** - * raw pointer to expanded bytes for use with magic_load_buffers(), - * points to the data held within expanded_buffer_ - */ - static void* uncompressed_magic_dict_; +/** + * Have libmagic load data from our embedded version. + * + * @param magic - libmagic object obtained from magic_open() + * @return the value libmgaic returns from magic_load_buffers(). + */ +int magic_mgc_embedded_load(magic_t magic); - /** holds the expanded data until application exits. */ - static shared_ptr expanded_buffer_; -}; +/** + * Provides access to the internally expanded data. + * + * Data is stored in the library in a compressed form (approx. 270KB) and gets + * decompressed (approx. 7MB) on the first call to this function. Subsequent + * calls to this function will reuse the decompressed buffer. + * + * @return a span to the internal buffer holding the expanded data. + */ +span expanded_buffer(); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::magic_dict diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index a0dac49341c..8064459cd00 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -38,3 +38,11 @@ commence(unit_test misc) unit_math.cc ) conclude(unit_test) + +commence(unit_test mgc_dict) + this_target_object_libraries(mgc_dict) + this_target_compile_definitions(TILEDB_PATH_TO_MAGIC_MGC=\"${libmagic_DICTIONARY}\") + this_target_sources( + unit_mgc_dict.cc + ) +conclude(unit_test) diff --git a/tiledb/sm/misc/test/compile_mgc_dict_main.cc b/tiledb/sm/misc/test/compile_mgc_dict_main.cc new file mode 100644 index 00000000000..977a8c9b167 --- /dev/null +++ b/tiledb/sm/misc/test/compile_mgc_dict_main.cc @@ -0,0 +1,34 @@ +/** + * @file compile_mgc_dict_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../mgc_dict.h" + +int main() { + (void)tiledb::sm::magic_dict::expanded_buffer(); + return 0; +} diff --git a/tiledb/sm/misc/test/unit_mgc_dict.cc b/tiledb/sm/misc/test/unit_mgc_dict.cc index 579cf5a780a..c9ec25298e3 100644 --- a/tiledb/sm/misc/test/unit_mgc_dict.cc +++ b/tiledb/sm/misc/test/unit_mgc_dict.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,66 +32,36 @@ * magic checks return same values using both embedded and external data. */ -#include "tiledb/common/common.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/filter/filter_buffer.h" -#include "tiledb/sm/filter/filter_storage.h" +#include #include "tiledb/sm/misc/mgc_dict.h" -#include "tiledb/sm/misc/types.h" -#include #include -#include +#include #include #include -#ifdef _WIN32 -#include -#include -#endif - #ifndef TILEDB_PATH_TO_MAGIC_MGC #error "TILEDB_PATH_TO_MAGIC_MGC not defined!" #endif -using tiledb::sm::magic_dict; - -int check_embedded_data_validity() { - FILE* infile = nullptr; - infile = fopen(TILEDB_PATH_TO_MAGIC_MGC, "rb"); - if (!infile) { - fprintf(stderr, "ERROR: Unable to open %s\n", TILEDB_PATH_TO_MAGIC_MGC); - return 1; - } - - fseek(infile, 0L, SEEK_END); - uint64_t magic_mgc_len = ftell(infile); - fseek(infile, 0L, SEEK_SET); +using namespace tiledb::sm; - char* magic_mgc_data = tdb_new_array(char, magic_mgc_len); - if (fread(magic_mgc_data, 1, magic_mgc_len, infile) != magic_mgc_len) { - fprintf(stderr, "ERROR reading data from %s\n", TILEDB_PATH_TO_MAGIC_MGC); - return 4; +TEST_CASE("Test embedded data validity", "[mgc_dict][embedded_validity]") { + std::vector magic_mgc_data; + { + std::ifstream infile(TILEDB_PATH_TO_MAGIC_MGC, std::ios::binary); + magic_mgc_data.assign( + std::istreambuf_iterator(infile), + std::istreambuf_iterator()); } - fclose(infile); auto magic_mgc_embedded_data = tiledb::sm::magic_dict::expanded_buffer(); - if (magic_mgc_embedded_data->size() != magic_mgc_len) { - fprintf( - stderr, - "ERROR magic.mgc data len (%" PRIu64 - ") does not match embedded data length (%" PRIu64 ")\n", - magic_mgc_len, - static_cast(magic_mgc_embedded_data->size())); - return 7; - } - - if (memcmp(magic_mgc_data, magic_mgc_embedded_data->data(), magic_mgc_len)) { - fprintf(stderr, "ERROR magic.mgc data different from embedded data\n"); - return 10; - } - - return 0; + REQUIRE(magic_mgc_data.size() == magic_mgc_embedded_data.size()); + REQUIRE( + memcmp( + magic_mgc_data.data(), + magic_mgc_embedded_data.data(), + magic_mgc_data.size()) == 0); } char empty_txt[] = {""}; // further below, treated differently from others here @@ -224,282 +194,182 @@ char text_txt[] = { '\x6c', '\x69', '\x6e', '\x65', '\x73', '\x2e', '\x0a', }; -struct file_data_sizes_s { - const char* file_name; - char* file_data; - uint64_t file_data_len; -} file_data_sizes1[] = - {{"empty_text", empty_txt, strlen(empty_txt)}, - {"fileapi0_csv", fileapi0_csv, sizeof(fileapi0_csv)}, - {"fileapi1_csv", fileapi1_csv, sizeof(fileapi1_csv)}, - {"fileapi2_csv", fileapi2_csv, sizeof(fileapi2_csv)}, - {"fileapi3_csv", fileapi3_csv, sizeof(fileapi3_csv)}, - {"fileapi4_csv", fileapi4_csv, sizeof(fileapi4_csv)}, - {"fileapi5_csv", fileapi5_csv, sizeof(fileapi5_csv)}, - {"fileapi6_csv", fileapi6_csv, sizeof(fileapi6_csv)}, - {"fileapi7_csv", fileapi7_csv, sizeof(fileapi7_csv)}, - {"fileapi8_csv", fileapi8_csv, sizeof(fileapi8_csv)}, - {"fileapi9_csv", fileapi9_csv, sizeof(fileapi9_csv)}, - {"quickstart_dense_csv", - quickstart_dense_csv, - sizeof(quickstart_dense_csv)}, - {"quickstart_dense_csv_gz", - quickstart_dense_csv_gz, - sizeof(quickstart_dense_csv_gz)}, - {"text_txt", text_txt, sizeof(text_txt)}, +struct FileData { + const std::string_view file_name; + span file_data; - {0, 0, 0}}, - file_data_sizes2[] = { - {"text_txt", text_txt, sizeof(text_txt)}, - {"quickstart_dense_csv_gz", - quickstart_dense_csv_gz, - sizeof(quickstart_dense_csv_gz)}, - {"quickstart_dense_csv", - quickstart_dense_csv, - sizeof(quickstart_dense_csv)}, - {"fileapi9_csv", fileapi9_csv, sizeof(fileapi9_csv)}, - {"fileapi8_csv", fileapi8_csv, sizeof(fileapi8_csv)}, - {"fileapi7_csv", fileapi7_csv, sizeof(fileapi7_csv)}, - {"fileapi6_csv", fileapi6_csv, sizeof(fileapi6_csv)}, - {"fileapi5_csv", fileapi5_csv, sizeof(fileapi5_csv)}, - {"fileapi4_csv", fileapi4_csv, sizeof(fileapi4_csv)}, - {"fileapi3_csv", fileapi3_csv, sizeof(fileapi3_csv)}, - {"fileapi2_csv", fileapi2_csv, sizeof(fileapi2_csv)}, - {"fileapi1_csv", fileapi1_csv, sizeof(fileapi1_csv)}, - {"fileapi0_csv", fileapi0_csv, sizeof(fileapi0_csv)}, - {"empty_text", empty_txt, strlen(empty_txt)}, - - {0, 0, 0}}; - -int embedded_vs_external_identifications() { - int errcnt = 0; + constexpr FileData( + std::string_view file_name, const char* file_data, size_t file_data_len) + : file_name(file_name) + , file_data(file_data, file_data_len) { + } +}; +std::vector file_data = { + {"empty_text", empty_txt, strlen(empty_txt)}, + {"fileapi0_csv", fileapi0_csv, sizeof(fileapi0_csv)}, + {"fileapi1_csv", fileapi1_csv, sizeof(fileapi1_csv)}, + {"fileapi2_csv", fileapi2_csv, sizeof(fileapi2_csv)}, + {"fileapi3_csv", fileapi3_csv, sizeof(fileapi3_csv)}, + {"fileapi4_csv", fileapi4_csv, sizeof(fileapi4_csv)}, + {"fileapi5_csv", fileapi5_csv, sizeof(fileapi5_csv)}, + {"fileapi6_csv", fileapi6_csv, sizeof(fileapi6_csv)}, + {"fileapi7_csv", fileapi7_csv, sizeof(fileapi7_csv)}, + {"fileapi8_csv", fileapi8_csv, sizeof(fileapi8_csv)}, + {"fileapi9_csv", fileapi9_csv, sizeof(fileapi9_csv)}, + {"quickstart_dense_csv", + quickstart_dense_csv, + sizeof(quickstart_dense_csv)}, + {"quickstart_dense_csv_gz", + quickstart_dense_csv_gz, + sizeof(quickstart_dense_csv_gz)}, + {"text_txt", text_txt, sizeof(text_txt)}}; + +TEST_CASE("Test embedded data validity", "[mgc_dict][embedded_vs_external]") { magic_t magic_mimeenc_embedded; magic_t magic_mimeenc_external; magic_t magic_mimetyp_embedded; magic_t magic_mimetyp_external; - auto magic_opens = [&]() { - auto magic_debug_options = 0; // MAGIC_DEBUG | MAGIC_CHECK | MAGIC_ERROR; - magic_mimeenc_embedded = - magic_open(MAGIC_MIME_ENCODING | magic_debug_options); - if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimeenc_embedded); - rval != 0) { - auto str_rval = std::to_string(rval); - auto err = magic_error(magic_mimeenc_embedded); - if (!err) { - err = - "(magic_error() returned 0, unexpected error loading embedded " - "data "; - } - magic_close(magic_mimeenc_embedded); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 1; + auto magic_debug_options = 0; // MAGIC_DEBUG | MAGIC_CHECK | MAGIC_ERROR; + magic_mimeenc_embedded = + magic_open(MAGIC_MIME_ENCODING | magic_debug_options); + if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimeenc_embedded); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimeenc_embedded); + if (!err) { + err = + "(magic_error() returned 0, unexpected error loading embedded " + "data "; } - magic_mimeenc_external = - magic_open(MAGIC_MIME_ENCODING | magic_debug_options); - if (auto rval = - magic_load(magic_mimeenc_external, TILEDB_PATH_TO_MAGIC_MGC); - rval != 0) { - auto str_rval = std::to_string(rval); - auto err = magic_error(magic_mimeenc_external); - if (!err) { - err = - "(magic_error() returned 0, try setting env var 'MAGIC' to " - "location " - "of magic.mgc or " - "alternate!)"; - } - magic_close(magic_mimeenc_external); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 2; + magic_close(magic_mimeenc_embedded); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + magic_mimeenc_external = + magic_open(MAGIC_MIME_ENCODING | magic_debug_options); + if (auto rval = magic_load(magic_mimeenc_external, TILEDB_PATH_TO_MAGIC_MGC); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimeenc_external); + if (!err) { + err = + "(magic_error() returned 0, try setting env var 'MAGIC' to " + "location " + "of magic.mgc or " + "alternate!)"; + } + magic_close(magic_mimeenc_external); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + magic_mimetyp_embedded = magic_open(MAGIC_MIME_TYPE | magic_debug_options); + if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimetyp_embedded); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimetyp_embedded); + if (!err) { + err = + "(magic_error() returned 0, unexpected error loading embedded " + "data "; + } + magic_close(magic_mimetyp_embedded); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + magic_mimetyp_external = magic_open(MAGIC_MIME_TYPE | magic_debug_options); + if (auto rval = magic_load(magic_mimetyp_external, TILEDB_PATH_TO_MAGIC_MGC); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimetyp_external); + if (!err) { + err = + "(magic_error() returned 0, try setting env var 'MAGIC' to " + "location " + "of magic.mgc or " + "alternate!)"; } - magic_mimetyp_embedded = magic_open(MAGIC_MIME_TYPE | magic_debug_options); - if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimetyp_embedded); - rval != 0) { - auto str_rval = std::to_string(rval); - auto err = magic_error(magic_mimetyp_embedded); + magic_close(magic_mimetyp_external); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + + for (auto& file_item : file_data) { + const char* mime_type_embedded = nullptr; + const char* mime_type_external = nullptr; + const char* mime_enc_embedded = nullptr; + const char* mime_enc_external = nullptr; + + mime_type_embedded = magic_buffer( + magic_mimetyp_embedded, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_type_embedded) { + auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); + auto err = magic_error(magic_mimeenc_embedded); if (!err) { - err = - "(magic_error() returned 0, unexpected error loading embedded " - "data "; + err = "(magic_buffer(..._embedded) returned 0!)"; } - magic_close(magic_mimetyp_embedded); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 1; + magic_close(magic_mimeenc_embedded); + throw std::runtime_error( + std::string("cannot access mime_type - ") + str_rval + err); } - magic_mimetyp_external = magic_open(MAGIC_MIME_TYPE | magic_debug_options); - if (auto rval = - magic_load(magic_mimetyp_external, TILEDB_PATH_TO_MAGIC_MGC); - rval != 0) { - auto str_rval = std::to_string(rval); + mime_type_external = magic_buffer( + magic_mimetyp_external, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_type_external) { + auto str_rval = std::to_string(magic_errno(magic_mimetyp_external)); auto err = magic_error(magic_mimetyp_external); if (!err) { - err = - "(magic_error() returned 0, try setting env var 'MAGIC' to " - "location " - "of magic.mgc or " - "alternate!)"; + err = "(magic_buffer(..._external) returned 0!)"; } magic_close(magic_mimetyp_external); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 2; + throw std::runtime_error( + std::string("cannot access mime_type - ") + str_rval + err); } - return 0; - }; - auto magic_closes = [&]() { - magic_close(magic_mimeenc_embedded); - magic_close(magic_mimeenc_external); - magic_close(magic_mimetyp_embedded); - magic_close(magic_mimetyp_external); - }; + CHECK( + std::string_view(mime_type_embedded) == + std::string_view(mime_type_external)); - auto proc_list = [&](file_data_sizes_s file_data_sizes[], - bool global_open_close = true) { - if (global_open_close) { - if (auto rval = magic_opens(); rval != 0) { - return rval; + mime_enc_embedded = magic_buffer( + magic_mimeenc_embedded, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_enc_embedded) { + auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); + auto err = magic_error(magic_mimeenc_embedded); + if (!err) { + err = "(magic_buffer(..._embedded) returned 0!)"; } + magic_close(magic_mimeenc_embedded); + throw std::runtime_error( + std::string("cannot access mime_encoding - ") + str_rval + err); } - - auto file_item = &file_data_sizes[0]; - while (file_item->file_data) { - if (!global_open_close) { - if (auto rval = magic_opens(); rval != 0) { - return rval; - } - } - const char* mime_type_embedded = nullptr; - const char* mime_type_external = nullptr; - const char* mime_enc_embedded = nullptr; - const char* mime_enc_external = nullptr; - - mime_type_embedded = magic_buffer( - magic_mimetyp_embedded, - file_item->file_data, - file_item->file_data_len); - if (!mime_type_embedded) { - auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); - auto err = magic_error(magic_mimeenc_embedded); - if (!err) { - err = "(magic_buffer(..._embedded) returned 0!)"; - } - magic_close(magic_mimeenc_embedded); - std::cerr << std::string("cannot access mime_type - ") + str_rval + err; - return 2; - } - mime_type_external = magic_buffer( - magic_mimetyp_external, - file_item->file_data, - file_item->file_data_len); - if (!mime_type_external) { - auto str_rval = std::to_string(magic_errno(magic_mimetyp_external)); - auto err = magic_error(magic_mimetyp_external); - if (!err) { - err = "(magic_buffer(..._external) returned 0!)"; - } - magic_close(magic_mimetyp_external); - std::cerr << std::string("cannot access mime_type - ") + str_rval + err; - return 2; - } - - if (strcmp(mime_type_embedded, mime_type_external)) { - fprintf( - stderr, - "ERROR mismatch (%s) mime type embedded (%s) vs external (%s)\n", - file_item->file_name, - mime_type_embedded, - mime_type_external); - ++errcnt; - } - - mime_enc_embedded = magic_buffer( - magic_mimeenc_embedded, - file_item->file_data, - file_item->file_data_len); - if (!mime_enc_embedded) { - auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); - auto err = magic_error(magic_mimeenc_embedded); - if (!err) { - err = "(magic_buffer(..._embedded) returned 0!)"; - } - magic_close(magic_mimeenc_embedded); - std::cerr << std::string("cannot access mime_encoding - ") + str_rval + - err; - return 2; - } - mime_enc_external = magic_buffer( - magic_mimeenc_external, - file_item->file_data, - file_item->file_data_len); - if (!mime_enc_embedded) { - auto str_rval = std::to_string(magic_errno(magic_mimeenc_external)); - auto err = magic_error(magic_mimeenc_external); - if (!err) { - err = "(magic_buffer(..._embedded) returned 0!)"; - } - magic_close(magic_mimeenc_external); - std::cerr << std::string("cannot access mime_encoding - ") + str_rval + - err; - return 2; - } - - if (strcmp(mime_enc_embedded, mime_enc_external)) { - fprintf( - stderr, - "ERROR mismatch (%s) mime encoding embedded (%s) vs external " - "(%s)\n", - file_item->file_name, - mime_enc_embedded, - mime_enc_external); - ++errcnt; - } - file_item++; - if (!global_open_close) { - magic_closes(); + mime_enc_external = magic_buffer( + magic_mimeenc_external, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_enc_embedded) { + auto str_rval = std::to_string(magic_errno(magic_mimeenc_external)); + auto err = magic_error(magic_mimeenc_external); + if (!err) { + err = "(magic_buffer(..._embedded) returned 0!)"; } - } - if (global_open_close) { - magic_closes(); + magic_close(magic_mimeenc_external); + throw std::runtime_error( + std::string("cannot access mime_encoding - ") + str_rval + err); } - return 0; - }; - - if (auto rval = proc_list(file_data_sizes1, true); rval != 0) - return rval; - if (auto rval = proc_list(file_data_sizes2, true); rval != 0) - return rval; - - if (auto rval = proc_list(file_data_sizes1, false); rval != 0) - return rval; - if (auto rval = proc_list(file_data_sizes2, false); rval != 0) - return rval; - - if (errcnt) { - fprintf( - stderr, "%d mismatch errors magic embedded vs external data\n", errcnt); + CHECK( + std::string_view(mime_enc_embedded) == + std::string_view(mime_enc_external)); } - return errcnt ? 99 : 0; -} - -int main() { - int failures = 0; - - failures |= check_embedded_data_validity(); - failures |= embedded_vs_external_identifications(); - - if (failures) { - fprintf(stderr, "\nERRORS mgc_dict validation\n"); - return 1; - } else { - fprintf(stderr, "NO errors encountered in mgc_dict validation\n"); - return 0; - } + magic_close(magic_mimeenc_embedded); + magic_close(magic_mimeenc_external); + magic_close(magic_mimetyp_embedded); + magic_close(magic_mimetyp_external); } diff --git a/tiledb/sm/object/object.cc b/tiledb/sm/object/object.cc index 2bcbbe5e956..a353504d0a4 100644 --- a/tiledb/sm/object/object.cc +++ b/tiledb/sm/object/object.cc @@ -56,7 +56,6 @@ bool is_array(ContextResources& resources, const URI& uri) { auto&& [st, exists] = resources.rest_client()->check_array_exists_from_rest(uri); throw_if_not_ok(st); - assert(exists.has_value()); return exists.value(); } else { // Check if the schema directory exists or not @@ -77,32 +76,32 @@ bool is_array(ContextResources& resources, const URI& uri) { } } -Status is_group(ContextResources& resources, const URI& uri, bool* is_group) { +bool is_group(ContextResources& resources, const URI& uri) { // Handle remote array if (uri.is_tiledb()) { auto&& [st, exists] = resources.rest_client()->check_group_exists_from_rest(uri); throw_if_not_ok(st); - *is_group = *exists; + return exists.value(); } else { // Check for new group details directory auto& vfs = resources.vfs(); - throw_if_not_ok( - vfs.is_dir(uri.join_path(constants::group_detail_dir_name), is_group)); + bool dir_exists = false; + throw_if_not_ok(vfs.is_dir( + uri.join_path(constants::group_detail_dir_name), &dir_exists)); - if (*is_group) { - return Status::Ok(); + if (dir_exists) { + return true; } // Fall back to older group file to check for legacy (pre-format 12) groups throw_if_not_ok( - vfs.is_file(uri.join_path(constants::group_filename), is_group)); + vfs.is_file(uri.join_path(constants::group_filename), &dir_exists)); + return dir_exists; } - return Status::Ok(); } -Status object_type( - ContextResources& resources, const URI& uri, ObjectType* type) { +ObjectType object_type(ContextResources& resources, const URI& uri) { URI dir_uri = uri; if (uri.is_s3() || uri.is_azure() || uri.is_gcs()) { // Always add a trailing '/' in the S3/Azure/GCS case so that listing the @@ -116,62 +115,57 @@ Status object_type( bool is_dir = false; throw_if_not_ok(resources.vfs().is_dir(uri, &is_dir)); if (!is_dir) { - *type = ObjectType::INVALID; - return Status::Ok(); + return ObjectType::INVALID; } } - bool exists = is_array(resources, uri); - if (exists) { - *type = ObjectType::ARRAY; - return Status::Ok(); - } - throw_if_not_ok(is_group(resources, uri, &exists)); - if (exists) { - *type = ObjectType::GROUP; - return Status::Ok(); + if (is_array(resources, uri)) { + return ObjectType::ARRAY; + } + if (is_group(resources, uri)) { + return ObjectType::GROUP; } - *type = ObjectType::INVALID; - return Status::Ok(); + return ObjectType::INVALID; } -Status object_move( +void object_move( ContextResources& resources, const char* old_path, const char* new_path) { auto old_uri = URI(old_path); - if (old_uri.is_invalid()) + if (old_uri.is_invalid()) { throw ObjectException( "Cannot move object '" + std::string(old_path) + "'; Invalid URI"); + } auto new_uri = URI(new_path); - if (new_uri.is_invalid()) + if (new_uri.is_invalid()) { throw ObjectException( "Cannot move object to '" + std::string(new_path) + "'; Invalid URI"); + } - ObjectType obj_type; - throw_if_not_ok(object_type(resources, old_uri, &obj_type)); - if (obj_type == ObjectType::INVALID) + if (object_type(resources, old_uri) == ObjectType::INVALID) { throw ObjectException( "Cannot move object '" + std::string(old_path) + "'; Invalid TileDB object"); + } - return resources.vfs().move_dir(old_uri, new_uri); + throw_if_not_ok(resources.vfs().move_dir(old_uri, new_uri)); } -Status object_remove(ContextResources& resources, const char* path) { +void object_remove(ContextResources& resources, const char* path) { auto uri = URI(path); - if (uri.is_invalid()) + if (uri.is_invalid()) { throw ObjectException( "Cannot remove object '" + std::string(path) + "'; Invalid URI"); + } - ObjectType obj_type; - throw_if_not_ok(object_type(resources, uri, &obj_type)); - if (obj_type == ObjectType::INVALID) + if (object_type(resources, uri) == ObjectType::INVALID) { throw ObjectException( "Cannot remove object '" + std::string(path) + "'; Invalid TileDB object"); + } - return resources.vfs().remove_dir(uri); + throw_if_not_ok(resources.vfs().remove_dir(uri)); } } // namespace tiledb::sm diff --git a/tiledb/sm/object/object.h b/tiledb/sm/object/object.h index 108eb3ca0e0..c3edb73de6b 100644 --- a/tiledb/sm/object/object.h +++ b/tiledb/sm/object/object.h @@ -63,20 +63,18 @@ bool is_array(ContextResources& resources, const URI& uri); * @param uri the URI to be checked. * @param is_group Set to `true` if the URI is a group and `false` * otherwise. - * @return Status + * @return bool */ -Status is_group(ContextResources& resources, const URI& uri, bool* is_group); +bool is_group(ContextResources& resources, const URI& uri); /** * Returns the tiledb object type * * @param resources the context resources. - * @param uri Path to TileDB object resource - * @param type The ObjectType to be retrieved. - * @return Status + * @param uri Path to TileDB object resource. + * @return ObjectType */ -Status object_type( - ContextResources& resources, const URI& uri, ObjectType* type); +ObjectType object_type(ContextResources& resources, const URI& uri); /** * Moves a TileDB object. If `new_path` exists, it will be overwritten. @@ -84,9 +82,8 @@ Status object_type( * @param resources the context resources. * @param old_path the old path of the object. * @param new_path the new path of the object. - * @return Status */ -Status object_move( +void object_move( ContextResources& resources, const char* old_path, const char* new_path); /** @@ -94,9 +91,8 @@ Status object_move( * * @param resources the context resources. * @param path the path to the object to be removed. - * @return Status */ -Status object_remove(ContextResources& resources, const char* path); +void object_remove(ContextResources& resources, const char* path); } // namespace tiledb::sm diff --git a/tiledb/sm/object/object_iter.cc b/tiledb/sm/object/object_iter.cc new file mode 100644 index 00000000000..43599d6de22 --- /dev/null +++ b/tiledb/sm/object/object_iter.cc @@ -0,0 +1,225 @@ +/** + * @file object_iter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements object iterator functions. + */ + +#include "tiledb/sm/object/object_iter.h" +#include "tiledb/sm/enums/object_type.h" +#include "tiledb/sm/object/object.h" + +namespace tiledb::sm { + +class ObjectIterException : public StatusException { + public: + explicit ObjectIterException(const std::string& message) + : StatusException("ObjectIter", message) { + } +}; + +ObjectIter* object_iter_begin( + ContextResources& resources, const char* path, WalkOrder order) { + // Sanity check + URI path_uri(path); + if (path_uri.is_invalid()) { + throw ObjectIterException( + "Cannot create object iterator; Invalid input path"); + } + + // Get all contents of path + std::vector uris; + throw_if_not_ok(resources.vfs().ls(path_uri, &uris)); + + // Create a new object iterator + ObjectIter* obj_iter = tdb_new(ObjectIter); + obj_iter->order_ = order; + obj_iter->recursive_ = true; + + // Include the uris that are TileDB objects in the iterator state + ObjectType obj_type; + for (auto& uri : uris) { + try { + obj_type = object_type(resources, uri); + if (obj_type != ObjectType::INVALID) { + obj_iter->objs_.push_back(uri); + if (order == WalkOrder::POSTORDER) + obj_iter->expanded_.push_back(false); + } + } catch (...) { + tdb_delete(obj_iter); + throw; + } + } + + return obj_iter; +} + +ObjectIter* object_iter_begin(ContextResources& resources, const char* path) { + // Sanity check + URI path_uri(path); + if (path_uri.is_invalid()) { + throw ObjectIterException( + "Cannot create object iterator; Invalid input path"); + } + + // Get all contents of path + std::vector uris; + throw_if_not_ok(resources.vfs().ls(path_uri, &uris)); + + // Create a new object iterator + ObjectIter* obj_iter = tdb_new(ObjectIter); + obj_iter->order_ = WalkOrder::PREORDER; + obj_iter->recursive_ = false; + + // Include the uris that are TileDB objects in the iterator state + ObjectType obj_type; + for (auto& uri : uris) { + try { + obj_type = object_type(resources, uri); + if (obj_type != ObjectType::INVALID) { + obj_iter->objs_.push_back(uri); + } + } catch (...) { + tdb_delete(obj_iter); + throw; + } + } + + return obj_iter; +} + +void object_iter_free(ObjectIter* obj_iter) { + tdb_delete(obj_iter); +} + +Status object_iter_next( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next) { + // Handle case there is no next + if (obj_iter->objs_.empty()) { + *has_next = false; + return Status::Ok(); + } + + // Retrieve next object + switch (obj_iter->order_) { + case WalkOrder::PREORDER: + RETURN_NOT_OK( + object_iter_next_preorder(resources, obj_iter, path, type, has_next)); + break; + case WalkOrder::POSTORDER: + RETURN_NOT_OK(object_iter_next_postorder( + resources, obj_iter, path, type, has_next)); + break; + } + + return Status::Ok(); +} + +Status object_iter_next_postorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next) { + // Get all contents of the next URI recursively till the bottom, + // if the front of the list has not been expanded + if (obj_iter->expanded_.front() == false) { + uint64_t obj_num; + do { + obj_num = obj_iter->objs_.size(); + std::vector uris; + throw_if_not_ok(resources.vfs().ls(obj_iter->objs_.front(), &uris)); + obj_iter->expanded_.front() = true; + + // Push the new TileDB objects in the front of the iterator's list + ObjectType obj_type; + for (auto it = uris.rbegin(); it != uris.rend(); ++it) { + obj_type = object_type(resources, *it); + if (obj_type != ObjectType::INVALID) { + obj_iter->objs_.push_front(*it); + obj_iter->expanded_.push_front(false); + } + } + } while (obj_num != obj_iter->objs_.size()); + } + + // Prepare the values to be returned + URI front_uri = obj_iter->objs_.front(); + obj_iter->next_ = front_uri.to_string(); + *type = object_type(resources, front_uri); + *path = obj_iter->next_.c_str(); + *has_next = true; + + // Pop the front (next URI) of the iterator's object list + obj_iter->objs_.pop_front(); + obj_iter->expanded_.pop_front(); + + return Status::Ok(); +} + +Status object_iter_next_preorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next) { + // Prepare the values to be returned + URI front_uri = obj_iter->objs_.front(); + obj_iter->next_ = front_uri.to_string(); + *type = object_type(resources, front_uri); + *path = obj_iter->next_.c_str(); + *has_next = true; + + // Pop the front (next URI) of the iterator's object list + obj_iter->objs_.pop_front(); + + // Return if no recursion is needed + if (!obj_iter->recursive_) + return Status::Ok(); + + // Get all contents of the next URI + std::vector uris; + throw_if_not_ok(resources.vfs().ls(front_uri, &uris)); + + // Push the new TileDB objects in the front of the iterator's list + ObjectType obj_type; + for (auto it = uris.rbegin(); it != uris.rend(); ++it) { + obj_type = object_type(resources, *it); + if (obj_type != ObjectType::INVALID) + obj_iter->objs_.push_front(*it); + } + + return Status::Ok(); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/object/object_iter.h b/tiledb/sm/object/object_iter.h new file mode 100644 index 00000000000..b95a4e1ffc7 --- /dev/null +++ b/tiledb/sm/object/object_iter.h @@ -0,0 +1,149 @@ +/** + * @file object_iter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines object iterator functions. + */ + +#ifndef TILEDB_OBJECT_ITER_H +#define TILEDB_OBJECT_ITER_H + +#include "tiledb/common/common.h" +#include "tiledb/sm/enums/object_type.h" +#include "tiledb/sm/enums/walk_order.h" +#include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/storage_manager/context_resources.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** Enables iteration over TileDB objects in a path. */ +struct ObjectIter { + public: + /** + * There is a one-to-one correspondence between `expanded_` and `objs_`. + * An `expanded_` value is `true` if the corresponding `objs_` path + * has been expanded to the paths it contains in a post ored traversal. + * This is not used in a preorder traversal. + */ + std::list expanded_; + + /** The next URI in string format. */ + std::string next_; + + /** The next objects to be visited. */ + std::list objs_; + + /** The traversal order of the iterator. */ + WalkOrder order_; + + /** `True` if the iterator will recursively visit the directory tree. */ + bool recursive_; +}; + +/** + * Creates a new object iterator for the input path. The iteration + * in this case will be recursive in the entire directory tree rooted + * at `path`. + * + * @param resources The context resources. + * @param path The path the iterator will target at. + * @param order The traversal order of the iterator. + * @return The created object iterator. + */ +ObjectIter* object_iter_begin( + ContextResources& resources, const char* path, WalkOrder order); + +/** + * Creates a new object iterator for the input path. The iteration will + * not be recursive, and only the children of `path` will be visited. + * + * @param resources The context resources. + * @param path The path the iterator will target at. + * @return The created object iterator. + */ +ObjectIter* object_iter_begin(ContextResources& resources, const char* path); + +/** Frees the object iterator. */ +void object_iter_free(ObjectIter* obj_iter); + +/** + * Retrieves the next object path and type. + * + * @param resources The context resources. + * @param obj_iter The object iterator. + * @param path The object path that is retrieved. + * @param type The object type that is retrieved. + * @param has_next True if an object path was retrieved and false otherwise. + * @return Status + */ +Status object_iter_next( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next); + +/** + * Retrieves the next object in the post-order traversal. + * + * @param resources The context resources. + * @param obj_iter The object iterator. + * @param path The object path that is retrieved. + * @param type The object type that is retrieved. + * @param has_next True if an object path was retrieved and false otherwise. + * @return Status + */ +Status object_iter_next_postorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next); + +/** + * Retrieves the next object in the post-order traversal. + * + * @param resources The context resources. + * @param obj_iter The object iterator. + * @param path The object path that is retrieved. + * @param type The object type that is retrieved. + * @param has_next True if an object path was retrieved and false otherwise. + * @return Status + */ +Status object_iter_next_preorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next); + +} // namespace tiledb::sm + +#endif // TILEDB_OBJECT_ITER_H diff --git a/tiledb/sm/query/CMakeLists.txt b/tiledb/sm/query/CMakeLists.txt index cccb774ebad..4084966eaa4 100644 --- a/tiledb/sm/query/CMakeLists.txt +++ b/tiledb/sm/query/CMakeLists.txt @@ -27,6 +27,7 @@ include(common NO_POLICY_SCOPE) add_subdirectory(ast) add_subdirectory(deletes_and_updates) +add_subdirectory(external_sort) add_subdirectory(readers) # diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index 16dfad7c955..d49f68dd959 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -33,6 +33,7 @@ #include "query_ast.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/misc/integral_type_casts.h" +#include "tiledb/sm/query/query_condition.h" using namespace tiledb::common; @@ -317,8 +318,8 @@ Status ASTNodeVal::check_node_validity(const ArraySchema& array_schema) const { (op_ == QueryConditionOp::IN || op_ == QueryConditionOp::NOT_IN)) { for (auto& member : members_) { if (member.size() != cell_size) { - throw Status_QueryConditionError( - "Value node set memmber size mismatch: " + + throw QueryConditionException( + "Value node set member size mismatch: " + std::to_string(cell_size) + " != " + std::to_string(member.size())); } } diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index e57b67d1ca9..6444f9a4019 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -35,7 +35,6 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" @@ -45,9 +44,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class DeleteAndUpdateStatusException : public StatusException { +class DeleteAndUpdateException : public StatusException { public: - explicit DeleteAndUpdateStatusException(const std::string& message) + explicit DeleteAndUpdateException(const std::string& message) : StatusException("Deletes", message) { } }; @@ -65,28 +64,23 @@ DeletesAndUpdates::DeletesAndUpdates( , condition_(params.condition()) , update_values_(update_values) { // Sanity checks - if (storage_manager_ == nullptr) { - throw DeleteAndUpdateStatusException( - "Cannot initialize query; Storage manager not set"); - } - if (!buffers_.empty()) { - throw DeleteAndUpdateStatusException( + throw DeleteAndUpdateException( "Cannot initialize deletes; Buffers are set"); } if (array_schema_.dense()) { - throw DeleteAndUpdateStatusException( + throw DeleteAndUpdateException( "Cannot initialize deletes; Only supported for sparse arrays"); } if (subarray_.is_set()) { - throw DeleteAndUpdateStatusException( + throw DeleteAndUpdateException( "Cannot initialize deletes; Subarrays are not supported"); } if (!params.skip_checks_serialization() && !condition_.has_value()) { - throw DeleteAndUpdateStatusException( + throw DeleteAndUpdateException( "Cannot initialize deletes; One condition is needed"); } } @@ -112,7 +106,7 @@ Status DeletesAndUpdates::dowork() { if (condition_.has_value()) { RETURN_NOT_OK(condition_->check(array_schema_)); } else { - throw DeleteAndUpdateStatusException( + throw DeleteAndUpdateException( "Cannot process delete, no condition is set"); } @@ -137,7 +131,7 @@ Status DeletesAndUpdates::dowork() { auto fragment_timestamp_range{fragment_id.timestamp_range()}; if (timestamp >= fragment_timestamp_range.first && timestamp <= fragment_timestamp_range.second) { - throw DeleteAndUpdateStatusException( + throw DeleteAndUpdateException( "Cannot write a delete in the middle of a fragment consolidated " "without timestamps."); } diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc index 6330d25d098..4f961970d5a 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc @@ -50,13 +50,14 @@ using namespace tiledb::common; namespace tiledb::sm { ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( + ContextResources& resources, StorageManager* storage_manager, Array* array, const Subarray& subarray, const std::unordered_map& label_buffers, const std::unordered_map& array_buffers, const optional& fragment_name) - : resources_(storage_manager->resources()) + : resources_(resources) , storage_manager_(storage_manager) , label_range_queries_by_dim_idx_(subarray.dim_num(), nullptr) , label_data_queries_by_dim_idx_(subarray.dim_num()) @@ -79,7 +80,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( } else { // Cannot both read label ranges and write label data on the same write. if (subarray.has_label_ranges()) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to add dimension label queries. Cannot set both label " "buffer and label range on a write query."); } @@ -105,7 +106,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( case (QueryType::UPDATE): case (QueryType::MODIFY_EXCLUSIVE): if (!label_buffers.empty() || subarray.has_label_ranges()) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to add dimension label queries. Query type " + query_type_str(array->get_query_type()) + " is not supported for dimension labels."); @@ -113,7 +114,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( break; default: - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to add dimension label queries. Unknown query type " + query_type_str(array->get_query_type()) + "."); } @@ -132,7 +133,7 @@ bool ArrayDimensionLabelQueries::completed() const { DimensionLabelQuery* ArrayDimensionLabelQueries::get_range_query( ArrayDimensionLabelQueries::dimension_size_type dim_idx) const { if (!has_range_query(dim_idx)) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "No dimension label range query for dimension at index " + std::to_string(dim_idx)); } @@ -142,7 +143,7 @@ DimensionLabelQuery* ArrayDimensionLabelQueries::get_range_query( std::vector ArrayDimensionLabelQueries::get_data_query( ArrayDimensionLabelQueries::dimension_size_type dim_idx) const { if (!has_data_query(dim_idx)) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "No dimension label data query for dimension at index " + std::to_string(dim_idx)); } @@ -161,7 +162,7 @@ void ArrayDimensionLabelQueries::process_data_queries() { throw_if_not_ok(query->process()); return Status::Ok(); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to process data query for label '" + query->dim_label_name() + "'. " + err.what()); } @@ -182,7 +183,7 @@ void ArrayDimensionLabelQueries::process_range_queries(Query* parent_query) { range_query->init(); throw_if_not_ok(range_query->process()); if (!range_query->completed()) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Range query for label '" + range_query->dim_label_name() + "' failed to complete."); } @@ -202,7 +203,7 @@ void ArrayDimensionLabelQueries::process_range_queries(Query* parent_query) { } return Status::Ok(); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to process and update index ranges for label '" + range_query->dim_label_name() + "'. " + err.what()); } @@ -249,13 +250,14 @@ void ArrayDimensionLabelQueries::add_read_queries( // Create the range query. range_queries_.emplace_back(tdb_new( DimensionLabelQuery, + resources_, storage_manager_, dim_label, dim_label_ref, label_ranges)); label_range_queries_by_dim_idx_[dim_idx] = range_queries_.back().get(); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to initialize the query to read range data from label '" + label_name + "'. " + err.what()); } @@ -281,6 +283,7 @@ void ArrayDimensionLabelQueries::add_read_queries( // Create the data query. data_queries_.emplace_back(tdb_new( DimensionLabelQuery, + resources_, storage_manager_, dim_label, dim_label_ref, @@ -291,7 +294,7 @@ void ArrayDimensionLabelQueries::add_read_queries( label_data_queries_by_dim_idx_[dim_label_ref.dimension_index()].push_back( data_queries_.back().get()); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to initialize the data query for label '" + label_name + "'. " + err.what()); } @@ -312,7 +315,7 @@ void ArrayDimensionLabelQueries::add_write_queries( // Verify that this subarray is not set to use labels. if (subarray.has_label_ranges(dim_label_ref.dimension_index())) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Cannot write label data when subarray is set by label range."); } @@ -332,6 +335,7 @@ void ArrayDimensionLabelQueries::add_write_queries( // Create the dimension label query. data_queries_.emplace_back(tdb_new( DimensionLabelQuery, + resources_, storage_manager_, dim_label, dim_label_ref, @@ -344,7 +348,7 @@ void ArrayDimensionLabelQueries::add_write_queries( label_data_queries_by_dim_idx_[dim_label_ref.dimension_index()].push_back( data_queries_.back().get()); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to initialize the data query for label '" + label_name + "'. " + err.what()); } @@ -358,8 +362,7 @@ shared_ptr ArrayDimensionLabelQueries::open_dimension_label( const QueryType& query_type) { // Create the dimension label. auto label_iter = dimension_labels_.try_emplace( - dim_label_name, - make_shared(HERE(), storage_manager_->resources(), dim_label_uri)); + dim_label_name, make_shared(HERE(), resources_, dim_label_uri)); const auto dim_label = label_iter.first->second; // Open the dimension label with the same timestamps and encryption as the diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.h b/tiledb/sm/query/dimension_label/array_dimension_label_queries.h index 7746956ee85..0d73b42eed1 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.h +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.h @@ -69,6 +69,13 @@ class ArrayDimensionLabelQueries { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * ArrayDimensionLabelQueries(ContextResources&, Array*, ...). + * + * @param resources The context resources. * @param storage_manager Storage manager object. * @param array Parent array the dimension labels are defined on. * @param subarray Subarray for the query on the parent array. @@ -78,6 +85,7 @@ class ArrayDimensionLabelQueries { * @param fragment_name Optional fragment name for writing fragments. */ ArrayDimensionLabelQueries( + ContextResources& resources, StorageManager* storage_manager, Array* array, const Subarray& subarray, diff --git a/tiledb/sm/query/dimension_label/dimension_label_query.cc b/tiledb/sm/query/dimension_label/dimension_label_query.cc index e6563af7f14..bf810731fee 100644 --- a/tiledb/sm/query/dimension_label/dimension_label_query.cc +++ b/tiledb/sm/query/dimension_label/dimension_label_query.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,6 +50,7 @@ using namespace tiledb::common; namespace tiledb::sm { DimensionLabelQuery::DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, @@ -57,7 +58,7 @@ DimensionLabelQuery::DimensionLabelQuery( const QueryBuffer& label_buffer, const QueryBuffer& index_buffer, optional fragment_name) - : Query(storage_manager, dim_label, fragment_name) + : Query(resources, storage_manager, dim_label, fragment_name) , dim_label_name_{dim_label_ref.name()} { switch (dim_label->get_query_type()) { case (QueryType::READ): @@ -92,7 +93,7 @@ DimensionLabelQuery::DimensionLabelQuery( default: // Invalid label order type. - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Unrecognized label order " + data_order_str(dim_label_ref.label_order())); } @@ -100,18 +101,19 @@ DimensionLabelQuery::DimensionLabelQuery( } default: - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Query type " + query_type_str(dim_label->get_query_type()) + " not supported for dimension label queries."); } } DimensionLabelQuery::DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, const std::vector& label_ranges) - : Query(storage_manager, dim_label, nullopt) + : Query(resources, storage_manager, dim_label, nullopt) , dim_label_name_{dim_label_ref.name()} , index_data_{IndexDataCreate::make_index_data( array_schema().dimension_ptr(0)->type(), @@ -123,12 +125,12 @@ DimensionLabelQuery::DimensionLabelQuery( case (DataOrder::DECREASING_DATA): break; case (DataOrder::UNORDERED_DATA): - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Support for reading ranges from unordered labels is not yet " "implemented."); default: // Invalid label order type. - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Unrecognized label order " + data_order_str(dim_label_ref.label_order())); } @@ -194,7 +196,7 @@ void DimensionLabelQuery::initialize_ordered_write_query( Subarray subarray{*this->subarray()}; subarray.set_ranges_for_dim(0, parent_subarray.ranges_for_dim(dim_idx)); if (subarray.range_num() > 1) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Dimension label writes can only be set for a single range."); } set_subarray(subarray); @@ -210,7 +212,7 @@ void DimensionLabelQuery::initialize_ordered_write_query( subarray.set_coalesce_ranges(true); subarray.add_point_ranges(0, index_buffer.buffer_, count); if (subarray.range_num() > 1) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "The dimension data must contain consecutive points when writing to " "a dimension label."); } @@ -231,7 +233,7 @@ void DimensionLabelQuery::initialize_unordered_write_query( if (!parent_subarray.is_default(dim_idx)) { const auto& ranges = parent_subarray.ranges_for_dim(dim_idx); if (ranges.size() != 1) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Dimension label writes can only be set for a single range."); } } diff --git a/tiledb/sm/query/dimension_label/dimension_label_query.h b/tiledb/sm/query/dimension_label/dimension_label_query.h index aa03665be6c..5f8ae54b997 100644 --- a/tiledb/sm/query/dimension_label/dimension_label_query.h +++ b/tiledb/sm/query/dimension_label/dimension_label_query.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,9 +50,9 @@ class QueryBuffer; class Subarray; /** Class for dimension label query status exceptions. */ -class DimensionLabelQueryStatusException : public StatusException { +class DimensionLabelQueryException : public StatusException { public: - explicit DimensionLabelQueryStatusException(const std::string& msg) + explicit DimensionLabelQueryException(const std::string& msg) : StatusException("DimensionLabelQuery", msg) { } }; @@ -66,6 +66,13 @@ class DimensionLabelQuery : public Query { /** * Constructor for queries to read or write label data. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * DimensionLabelQuery(ContextResources&, shared_ptr, ...). + * + * @param resources The context resources. * @param storage_manager Storage manager object. * @param dim_label Opened dimension label for the query. * @param dim_label_ref Description of the dimension label. @@ -76,6 +83,7 @@ class DimensionLabelQuery : public Query { * @param fragment_name Name to use when writing the fragment. */ DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, @@ -87,12 +95,20 @@ class DimensionLabelQuery : public Query { /** * Constructor for range queries. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * DimensionLabelQuery(ContextResources&, shared_ptr, ...). + * + * @param resources The context resources. * @param storage_manager Storage manager object. * @param dim_label Opened dimension label for the query. * @param dim_label_ref Description of the dimension label. * @param label_ranges Label ranges to read index ranges from. */ DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, diff --git a/tiledb/sm/query/external_sort/CMakeLists.txt b/tiledb/sm/query/external_sort/CMakeLists.txt new file mode 100644 index 00000000000..ef98a1193d3 --- /dev/null +++ b/tiledb/sm/query/external_sort/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# tiledb/sm/query/external_sort/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) + +add_test_subdirectory() diff --git a/tiledb/sm/query/external_sort/README.md b/tiledb/sm/query/external_sort/README.md new file mode 100644 index 00000000000..65c9176d1fd --- /dev/null +++ b/tiledb/sm/query/external_sort/README.md @@ -0,0 +1,68 @@ + +### Definitions +* An *input block* is a fixed-size block of memory +* A *block* contains the sorted data of filled user buffers and is stored in a single temporary array. The tiles of the array are sized so that a single tile will fit completely into an input block. +* A *big block* is a set of blocks (arrays) that are sorted with respect to each other and is stored in a temporary group. +* A *pass* sorts a set of big blocks to create a new big block. +* An *iteration* comprises multiple passes, each of which sorts a subset of big blocks and produces a new set of big blocks. An iteration corresponds to the data in one query + + +### Algorithm: +#### Initialize: +* Iterate over incomplete queries to create a set of big blocks, each of which corresponds to one incomplete query and each of which consists of one block +#### Process +* Iterator until there is only one big block + * Partition the big blocks into subsets equal to the number of input blocks in the input block buffer + * Until all subsets are processed (new pass) + * Open a new big block + * Until input blocks are exhausted + * Merge sort input blocks into user buffers + * When an input block is depleted, read a new tile from the coresponding big block + * When user buffers are full, flush to a new block within the new big block + +### Directory Structure +The overall temp hierarchical structure from processing a complete query is thus envisioned to look like this: +``` +query-results +├── iter_000 (group, set of big blocks, corresponding to one query) +│ ├── pass_000 (group, one big block, set of blocks) +│ │ ├── block_000 (one array, sorted) +│ │ │ ├── attr_0 +│ │ │ │ ├── attr_0_data +│ │ │ │ ├── attr_0_offsets +│ │ │ │ └── attr_N_validity +│ │ │ ├── attr_1 +│ │ │ ... +│ │ │ └── attr_N +│ │ ├── block_001 +│ │ │ ... +│ │ └── block_M +│ ├── pass_001 +│ ... +│ └── pass_L +├── iter_001 +... +└── iter_K +``` + +#### Scenario 0: +There is enough memory to that entire query can be loaded into the user buffers. + +In this case, only the initialization step is required. The user buffers are sorted and comprise one big block and one block. +All of the sorted data fits in memory and can be used by the user, or it can be written to a single array that is completely sorted. + +(Since there will be no subsequent processing, the array does not have to be tiled.) In this case, there are is the initial iteration, one pass, one big block, one block, and one (final) array. + +#### Scenario 1: +There is not enough memory to load the entire query into the user buffers, there are more input blocks in the input block buffer than there are big blocks produced by the initialization step. + +In this case, there will be a sequence of incomplete queries. In the initialization step, each incomplete query creates a new big block, with one block inside of it. + +On the next iteration, all the big blocks can be processed in a single pass to create a single new big block. All the blocks in the big block will be sorted and will be sorted with respect to each other. This big block can be consolidated into a single (final) array, or it can fill user buffers directly. + + #### Scenario 3: +There is not enough memory to load the entire query into the user buffers. After initialization, there are more temporary big blocks than then are input blocks in the input block buffer. + + In this case, subsets of big blocks are processed at a time to create new big blocks, with each subset creating a new big block. If there are still more big blocks than there are input blocks in the input block buffer, the process is repeated until only a single output big buffer is created. Thi big block can be consolidated into a single (final) array, or it can be used to fill user buffers directly. + + diff --git a/tiledb/sm/query/external_sort/doc/figs/design.svg b/tiledb/sm/query/external_sort/doc/figs/design.svg new file mode 100644 index 00000000000..235a5945afb --- /dev/null +++ b/tiledb/sm/query/external_sort/doc/figs/design.svg @@ -0,0 +1,6786 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Case 0a: Initial query completes Sort buffers in memory + + + Case 0b: Initial query does complete but user makes no further queries + + + Case 0: Unordered, everything fits + + + + + Array + + + + + (In)complete query + + + + + + + + User- Specified Buffers + + + + + + + + User Processing + + + + + + + + User Output + + + + + + + + + + + + Data + + + + + Offsets + + + + + Data + + + + + TILEDB_ROW_MAJOR + + + + + + Case -1: Legacy Reader + + + + + Array + + + + + (In)complete query + + + + + + + + User-Specified Buffers + + + + + + + + In-Memory Sort + + + + + + + + User Processing + + + + + + + + Regularized Buffers + + + + + + + + proxy_sortpar_sortstd::sort + + + + + + zip_view + + + + + var_length_view + + + + + Data + + + + + Offsets + + + + + Data + + + + + + + + + + TILEDB_UNORDERED + + + + + User Output + + + + + + Case 1: Unordered, initial user query does not fit into memory budget + + + N elements per + + + M attributes + dimensions + + + proxy_sort will require single array of N size_t to hold permutation + + + Other sorts can be done in-place, with no additional memory required + + + Copying TileDB offset format data to Arrow for var_length_view will require additional N size_t per view + + + Regularizing with alt_var_length_view will require additional 2N size_t per view + + + Phase Ia: Incomplete query into user-specified buffersPhase 1b: Buffer regularizationPhase 1c: In-memory sortPhase Id: Abstract partitioningPhase 1e: Write blocks to dense array + + + A block view has a uniform number of elements per attribute /dimension, the total memory usage of which fits under a specified byte limit. + + + Sort entire user buffers + + + Once partitioned into blocks, each block will be internally sorted, and each block will be sorted with respect to other blocks + + + + + + Array + + + + + Incomplete query + + + + + + + + User-Specified Buffers + + + + + + + + In-Memory Sort + + + + + + + + Abstract Partitioning + + + + + + + + + + + + + + Regularized Buffers + + + + + + + + block_view + + + + + par_sortstd::sortproxy_sort + + + + + + zip_view + + + + + var_length_view + + + + + Data + + + + + Offsets + + + + + Data + + + + + + + + + + TILEDB_UNORDERED + + + + + + Dense Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Temporary storage + + + + The temporary dense array is a “meta-block” in some sense, it contains a sorted set of sorted blocks (each of which, in turn, is a “meta-element”) + + + Only a minimal amount of memory per block is required to represent a block_view No copies of data are made. + + + Each block gets its own tile (and vice-versa) + + + Phase II: Merge sort blocks from temporary arrays + + + Additional memory is required to buffer input blocks, equal to the size of a block times the number of temporary arrays + + + Because of memory constraints, we may not be able to sort all of the previously-created blocks in one pass, so we may need to do multiple additional passes — and then passes over those passes + + + The size of this array will be the combined size of all of the input temp arrays sorted in this pass + + + + + + + + + + In-Memory Merge Sort + + + + + Abstract Partitioning + + + + + + Temp Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Temp Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Temp Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Dense Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + block_buffer + + + + + + User Buffers + + + + + + + + + Schema?- Dimensions are 0 to M - M = num_blocks * block_size + + + Metadata?- Store size of each block + + + tiledb::Group?- each metablock is an array + + + group├── 0│ ├── 0│ ├── 1│ ├── ...│ └── n├── 1│ ├── 0│ ├── 1│ ├── ...│ └── n├── ...└── m ├── 0 ├── 1 ├── ... └── n + + + group└── iter └── metablockgroup└── iter └── iter └── metablock + + + The temporary dense array stores everything that was in the original user buffers (but sorted and tiled into blocks) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + User buffers (zipped) + + + chunk_view (of zipped), variable number of cells + + + Data blocks (wrapping chunks), constant bytes per block and constant cells per block (pad chunks) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Array (one block per tile) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Zoom in to temporary storage array format + + + Array = “metablock”, one per phase I pass + + + Case 1: One block from each temp array will fit into memory, only one Phase II pass needed + + + Case 2: One block from each temp array will not fit into memory, we need multiple Phase II passes, hierarchical + + + (Save chunk sizes as metadata) + + diff --git a/tiledb/sm/query/external_sort/partition.h b/tiledb/sm/query/external_sort/partition.h new file mode 100644 index 00000000000..08e7e79aae3 --- /dev/null +++ b/tiledb/sm/query/external_sort/partition.h @@ -0,0 +1,98 @@ +/** + * @file tiledb/sm/query/external_sort/partition.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#ifndef TILEDB_PARTITION_H +#define TILEDB_PARTITION_H + +#include +#include +#include +#include + +/** + * @brief Partition a list of sizes into bins that are less than or equal to a + * given number of bytes. The sizes are the number of elements in each cell, + * which are assumed to be of type `char`. + * @param bin_size The maximum number of bytes in a bin. + * @param num_cells The total number of cells to be partitioned. + * @param fixed_bytes_per_cell The number of fixed bytes per cell. This + * includes all of the non varlength elements in each cell, including the + * elements that specify the sizes. + * @param sizes The number of varlength elements in each cell. These are + * assumed to correspond to chars, so the number of bytes in each cell is + * the same as the number of elements in the cell. + * @return + */ +auto bin_partition( + size_t bin_size, + size_t num_cells, + size_t fixed_bytes_per_cell, + std::list::iterator>& sizes) { + assert(bin_size > 0); + assert(num_cells > 0); + assert(fixed_bytes_per_cell > 0); + + size_t current_index{0}; + // size_t next_index{0}; + size_t current_size{0}; + size_t next_size{0}; + + auto offsets_begin = sizes.begin(); + auto offsets_end = sizes.end(); + + std::vector bins{0}; + std::vector bin_sizes; + + while (true) { + next_size = current_size + fixed_bytes_per_cell; + for (auto o = offsets_begin; o != offsets_end; ++o) { + next_size += (*o)[current_index] * sizeof(char); + } + if (next_size > bin_size) { + bins.push_back(current_index); + bin_sizes.push_back(current_size); + + next_size = current_size = 0; + continue; + } else { + current_size = next_size; + } + if (++current_index == num_cells) { + bins.push_back(num_cells); + bin_sizes.push_back(current_size); + break; + } + } + + return std::make_tuple(std::move(bins), std::move(bin_sizes)); +} + +#endif // TILEDB_PARTITION_H diff --git a/tiledb/sm/query/external_sort/test/CMakeLists.txt b/tiledb/sm/query/external_sort/test/CMakeLists.txt new file mode 100644 index 00000000000..c47f50e8eb4 --- /dev/null +++ b/tiledb/sm/query/external_sort/test/CMakeLists.txt @@ -0,0 +1,33 @@ +# +# tiledb/sm/query/external_sort/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test partition) +this_target_sources( + unit_partition.cc +) +conclude(unit_test) \ No newline at end of file diff --git a/tiledb/sm/query/external_sort/test/unit_partition.cc b/tiledb/sm/query/external_sort/test/unit_partition.cc new file mode 100644 index 00000000000..f3ee50f2181 --- /dev/null +++ b/tiledb/sm/query/external_sort/test/unit_partition.cc @@ -0,0 +1,77 @@ +/** + * @file unit_partition.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the alt_var_length_view class. + */ + +#include +#include +#include "tiledb/common/util/var_length_util.h" +#include "tiledb/sm/query/external_sort/partition.h" + +TEST_CASE("partition: Null test", "[partition][null_test]") { + REQUIRE(true); +} + +TEST_CASE("partition: sized", "[partition]") { + std::vector o{8, 6, 7, 5, 3, 0, 9}; + std::vector p{3, 1, 4, 1, 5, 9, 2}; + + REQUIRE(o.size() == p.size()); + size_t num_cells = size(o); + size_t bin_size = 256; + auto fixed_bytes_per_cell = 24; + + std::vector o_bytes{64, 48, 56, 40, 24, 0, 72}; + std::vector p_bytes{24, 8, 32, 8, 40, 72, 16}; + for (size_t i = 0; i < num_cells; ++i) { + o_bytes[i] *= 8; + p_bytes[i] *= 8; + o[i] *= 8; + p[i] *= 8; + } + std::vector sum_bytes(num_cells); + for (size_t i = 0; i < num_cells; ++i) { + sum_bytes[i] = o_bytes[i] + p_bytes[i] + fixed_bytes_per_cell; + } + std::vector byte_offsets(num_cells + 1); + lengths_to_offsets(sum_bytes, byte_offsets); + // {112, 192, /**/ 304, 376, /**/ 464, 560, /**/ 672}; + // {112, 192, /**/ 112, 184, /**/ 88, 184, /**/ 112}; + + std::list::iterator> sizes{begin(o), begin(p)}; + + auto&& [x, y] = + bin_partition(bin_size, num_cells, fixed_bytes_per_cell, sizes); + std::vector expected_bins{0, 2, 4, 6, 7}; + std::vector expected_sizes{192, 184, 184, 112}; + + CHECK(x == expected_bins); + CHECK(y == expected_sizes); +} diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index c7582146f20..968e22a408b 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -48,7 +48,6 @@ #include "tiledb/sm/query/readers/result_tile.h" #include "tiledb/sm/query/update_value.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/cell_slab.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/type/apply_with_type.h" @@ -59,9 +58,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class ReaderStatusException : public StatusException { +class ReaderException : public StatusException { public: - explicit ReaderStatusException(const std::string& message) + explicit ReaderException(const std::string& message) : StatusException("Reader", message) { } }; @@ -110,23 +109,18 @@ Reader::Reader( bool remote_query) : ReaderBase(stats, logger->clone("Reader", ++logger_id_), params) { // Sanity checks - if (storage_manager_ == nullptr) { - throw ReaderStatusException( - "Cannot initialize reader; Storage manager not set"); - } - if (!params.default_channel_aggregates().empty()) { - throw ReaderStatusException( + throw ReaderException( "Cannot initialize reader; Reader cannot process aggregates"); } if (!params.skip_checks_serialization() && buffers_.empty()) { - throw ReaderStatusException("Cannot initialize reader; Buffers not set"); + throw ReaderException("Cannot initialize reader; Buffers not set"); } if (!params.skip_checks_serialization() && array_schema_.dense() && !subarray_.is_set()) { - throw ReaderStatusException( + throw ReaderException( "Cannot initialize reader; Dense reads must have a subarray set"); } @@ -596,9 +590,9 @@ Status Reader::compute_range_result_coords( range_result_coords[r].end(), range_result_coords[r].size(), sort_layout)); - this->throw_if_cancellation_requested(); + throw_if_cancelled(); throw_if_not_ok(dedup_result_coords(range_result_coords[r])); - this->throw_if_cancellation_requested(); + throw_if_cancelled(); } return Status::Ok(); @@ -1902,7 +1896,7 @@ void Reader::init_read_state() { // Check subarray if (subarray_.layout() == Layout::GLOBAL_ORDER && subarray_.range_num() != 1) { - throw ReaderStatusException( + throw ReaderException( "Cannot initialize read state; Multi-range subarrays do not support " "global order"); } @@ -1911,21 +1905,21 @@ void Reader::init_read_state() { bool found = false; uint64_t memory_budget = 0; if (!config_.get("sm.memory_budget", &memory_budget, &found).ok()) { - throw ReaderStatusException("Cannot get setting"); + throw ReaderException("Cannot get setting"); } assert(found); uint64_t memory_budget_var = 0; if (!config_.get("sm.memory_budget_var", &memory_budget_var, &found) .ok()) { - throw ReaderStatusException("Cannot get setting"); + throw ReaderException("Cannot get setting"); } assert(found); offsets_format_mode_ = config_.get("sm.var_offsets.mode", &found); assert(found); if (offsets_format_mode_ != "bytes" && offsets_format_mode_ != "elements") { - throw ReaderStatusException( + throw ReaderException( "Cannot initialize reader; Unsupported offsets" " format in configuration"); } @@ -1934,19 +1928,19 @@ void Reader::init_read_state() { .get( "sm.var_offsets.extra_element", &offsets_extra_element_, &found) .ok()) { - throw ReaderStatusException("Cannot get setting"); + throw ReaderException("Cannot get setting"); } assert(found); if (!config_ .get("sm.var_offsets.bitsize", &offsets_bitsize_, &found) .ok()) { - throw ReaderStatusException("Cannot get setting"); + throw ReaderException("Cannot get setting"); } assert(found); if (offsets_bitsize_ != 32 && offsets_bitsize_ != 64) { - throw ReaderStatusException( + throw ReaderException( "Cannot initialize reader; Unsupported offsets" " bitsize in configuration"); } @@ -1981,14 +1975,14 @@ void Reader::init_read_state() { if (!read_state_.partitioner_ .set_result_budget(attr_name.c_str(), *buffer_size) .ok()) { - throw ReaderStatusException("Cannot set result budget"); + throw ReaderException("Cannot set result budget"); } } else { if (!read_state_.partitioner_ .set_result_budget_nullable( attr_name.c_str(), *buffer_size, *buffer_validity_size) .ok()) { - throw ReaderStatusException("Cannot set result budget"); + throw ReaderException("Cannot set result budget"); } } } else { @@ -1997,7 +1991,7 @@ void Reader::init_read_state() { .set_result_budget( attr_name.c_str(), *buffer_size, *buffer_var_size) .ok()) { - throw ReaderStatusException("Cannot set result budget"); + throw ReaderException("Cannot set result budget"); } } else { if (!read_state_.partitioner_ @@ -2007,7 +2001,7 @@ void Reader::init_read_state() { *buffer_var_size, *buffer_validity_size) .ok()) { - throw ReaderStatusException("Cannot set result budget"); + throw ReaderException("Cannot set result budget"); } } } diff --git a/tiledb/sm/query/legacy/reader.h b/tiledb/sm/query/legacy/reader.h index 2c43922cf85..43e55f2123e 100644 --- a/tiledb/sm/query/legacy/reader.h +++ b/tiledb/sm/query/legacy/reader.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,13 +45,11 @@ #include "tiledb/sm/query/readers/reader_base.h" #include "tiledb/sm/query/readers/result_cell_slab.h" #include "tiledb/sm/query/readers/result_coords.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/subarray_partitioner.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class Tile; @@ -869,7 +867,6 @@ class Reader : public ReaderBase, public IQueryStrategy { std::vector& offsets) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_READER_H diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index f9bb85d6d00..111315a7aac 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -82,11 +82,12 @@ static uint64_t get_effective_memory_budget( /* ****************************** */ Query::Query( + ContextResources& resources, StorageManager* storage_manager, shared_ptr array, optional fragment_name, optional memory_budget) - : resources_(storage_manager->resources()) + : resources_(resources) , stats_(resources_.stats().create_child("Query")) , logger_(resources_.logger()->clone("Query", ++logger_id_)) , query_memory_tracker_(resources_.memory_tracker_manager().create_tracker( @@ -710,6 +711,7 @@ void Query::init() { // Initialize the dimension label queries. dim_label_queries_ = tdb_unique_ptr(tdb_new( ArrayDimensionLabelQueries, + resources_, storage_manager_, array_, subarray_, @@ -1777,6 +1779,7 @@ bool Query::is_aggregate(std::string output_field_name) const { Status Query::create_strategy(bool skip_checks_serialization) { auto params = StrategyParams( + resources_, array_->memory_tracker(), query_memory_tracker_, storage_manager_, diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index eb23873819c..b61dda30852 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -138,8 +138,16 @@ class Query { * case the query will be used as writes and the given URI should be used * for the name of the new fragment to be created. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * Query(ContextResources&, shared_ptr, ...). + * * @note Array must be a properly opened array. * + * @param resources The context resources. + * @param storage_manager Storage manager object. * @param array The array that is being queried. * @param fragment_uri The full URI for the new fragment. Only used for * writes. @@ -149,6 +157,7 @@ class Query { * will be obtained from the sm.mem.total_budget config option. */ Query( + ContextResources& resources, StorageManager* storage_manager, shared_ptr array, optional fragment_name = nullopt, diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 8ef3df48b01..6738063922b 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -47,6 +47,13 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +class QueryConditionException : public StatusException { + public: + explicit QueryConditionException(const std::string& message) + : StatusException("QueryCondition", message) { + } +}; + class FragmentMetadata; class MemoryTracker; struct ResultCellSlab; diff --git a/tiledb/sm/query/query_macros.h b/tiledb/sm/query/query_macros.h index 5d38fbc7a95..c222c6caf1a 100644 --- a/tiledb/sm/query/query_macros.h +++ b/tiledb/sm/query/query_macros.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -35,8 +35,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { #ifndef RETURN_CANCEL_OR_ERROR /** @@ -70,44 +69,6 @@ namespace sm { } while (false) #endif -#ifndef RETURN_CANCEL_OR_ERROR_ELSE -/** - * Returns an error status if the given Status is not Status::Ok, or - * if the StorageManager that owns this Query has requested cancellation. - * If an error status is returned, also execute the 'else' code. - */ -#define RETURN_CANCEL_OR_ERROR_ELSE(s, _else) \ - do { \ - Status _s = (s); \ - if (!_s.ok()) { \ - _else; \ - return _s; \ - } else if (storage_manager_->cancellation_in_progress()) { \ - _else; \ - return Status_QueryError("Query cancelled."); \ - } \ - } while (false) -#endif - -#ifndef BREAK_CANCEL_OR_ERROR -/** - * If the given status 's' is not Status::Ok, sets the Status variable - * 'outer_st' to 's' and breaks the containing loop. - */ -#define BREAK_CANCEL_OR_ERROR(outer_st, s) \ - do { \ - Status _s = (s); \ - if (!_s.ok()) { \ - outer_st = _s; \ - break; \ - } else if (storage_manager_->cancellation_in_progress()) { \ - outer_st = Status_QueryError("Query cancelled."); \ - break; \ - } \ - } while (false) -#endif - -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_QUERY_MACROS_H diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 511d7c43b5e..0378409565b 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -481,17 +481,17 @@ void basic_aggregation_test(std::vector expected_results) { 10, 10, &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), zero.data()); auto tile_metadata = TileMetadata( 10, 5, &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), full_tile_sum.data()); if (aggregator.has_value()) { // Regular attribute. diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index dae9a3570ef..9c8b68b51d1 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -44,7 +44,6 @@ #include "tiledb/sm/query/readers/filtered_data.h" #include "tiledb/sm/query/readers/result_tile.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/subarray.h" #include "tiledb/type/apply_with_type.h" @@ -56,9 +55,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class DenseReaderStatusException : public StatusException { +class DenseReaderException : public StatusException { public: - explicit DenseReaderStatusException(const std::string& message) + explicit DenseReaderException(const std::string& message) : StatusException("DenseReader", message) { } }; @@ -78,19 +77,14 @@ DenseReader::DenseReader( elements_mode_ = false; // Sanity checks. - if (storage_manager_ == nullptr) { - throw DenseReaderStatusException( - "Cannot initialize dense reader; Storage manager not set"); - } - if (!params.skip_checks_serialization() && buffers_.empty() && aggregate_buffers_.empty()) { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot initialize dense reader; Buffers not set"); } if (!params.skip_checks_serialization() && !subarray_.is_set()) { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot initialize reader; Dense reads must have a subarray set"); } @@ -134,8 +128,7 @@ void DenseReader::refresh_config() { // Set the memory budget for the array if (!array_memory_tracker_->set_budget(memory_budget_)) { - throw DenseReaderStatusException( - "Memory budget is too small to open array"); + throw DenseReaderException("Memory budget is too small to open array"); } } @@ -584,7 +577,7 @@ void DenseReader::init_read_state() { // Check subarray. if (subarray_.layout() == Layout::GLOBAL_ORDER && subarray_.range_num() != 1) { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot initialize read state; Multi-range subarrays do not support " "global order"); } @@ -594,7 +587,7 @@ void DenseReader::init_read_state() { offsets_format_mode_ = config_.get("sm.var_offsets.mode", &found); assert(found); if (offsets_format_mode_ != "bytes" && offsets_format_mode_ != "elements") { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot initialize reader; Unsupported offsets format in " "configuration"); } @@ -604,17 +597,17 @@ void DenseReader::init_read_state() { .get( "sm.var_offsets.extra_element", &offsets_extra_element_, &found) .ok()) { - throw DenseReaderStatusException("Cannot get setting"); + throw DenseReaderException("Cannot get setting"); } assert(found); if (!config_ .get("sm.var_offsets.bitsize", &offsets_bitsize_, &found) .ok()) { - throw DenseReaderStatusException("Cannot get setting"); + throw DenseReaderException("Cannot get setting"); } if (offsets_bitsize_ != 32 && offsets_bitsize_ != 64) { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot initialize reader; Unsupported offsets bitsize in " "configuration"); } @@ -623,12 +616,12 @@ void DenseReader::init_read_state() { if (!config_ .get("sm.query.dense.qc_coords_mode", &qc_coords_mode_, &found) .ok()) { - throw DenseReaderStatusException("Cannot get setting"); + throw DenseReaderException("Cannot get setting"); } assert(found); if (qc_coords_mode_ && !condition_.has_value()) { - throw DenseReaderStatusException( + throw DenseReaderException( "sm.query.dense.qc_coords_mode requires a query condition"); } @@ -656,14 +649,14 @@ void DenseReader::init_read_state() { if (!read_state_.partitioner_ .set_result_budget(attr_name.c_str(), *buffer_size) .ok()) { - throw DenseReaderStatusException("Cannot set result budget"); + throw DenseReaderException("Cannot set result budget"); } } else { if (!read_state_.partitioner_ .set_result_budget_nullable( attr_name.c_str(), *buffer_size, *buffer_validity_size) .ok()) { - throw DenseReaderStatusException("Cannot set result budget"); + throw DenseReaderException("Cannot set result budget"); } } } else { @@ -672,7 +665,7 @@ void DenseReader::init_read_state() { .set_result_budget( attr_name.c_str(), *buffer_size, *buffer_var_size) .ok()) { - throw DenseReaderStatusException("Cannot set result budget"); + throw DenseReaderException("Cannot set result budget"); } } else { if (!read_state_.partitioner_ @@ -682,7 +675,7 @@ void DenseReader::init_read_state() { *buffer_var_size, *buffer_validity_size) .ok()) { - throw DenseReaderStatusException("Cannot set result budget"); + throw DenseReaderException("Cannot set result budget"); } } } @@ -943,7 +936,7 @@ DenseReader::compute_result_space_tiles( // If a single tile doesn't fit in the available memory, we can't proceed. if (total_memory > available_memory) { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot process a single tile, increase memory budget"); } } @@ -960,7 +953,7 @@ DenseReader::compute_result_space_tiles( // If a single tile doesn't fit in the available memory, we can't proceed. if (total_memory_condition > available_memory) { - throw DenseReaderStatusException( + throw DenseReaderException( "Cannot process a single tile because of query condition, increase " "memory budget"); } @@ -2282,7 +2275,7 @@ void DenseReader::fill_dense_coords_row_col( // Iterate over all coordinates, retrieved in cell slabs. CellSlabIter iter(&subarray); if (!iter.begin().ok()) { - throw DenseReaderStatusException("Cannot begin iteration"); + throw DenseReaderException("Cannot begin iteration"); } while (!iter.end()) { auto cell_slab = iter.cell_slab(); diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index f3ccd3fbefa..d853c8b6f97 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,13 +42,11 @@ #include "tiledb/sm/query/iquery_strategy.h" #include "tiledb/sm/query/query_buffer.h" #include "tiledb/sm/query/readers/reader_base.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/tile_cell_slab_iter.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; @@ -603,7 +601,6 @@ class DenseReader : public ReaderBase, public IQueryStrategy { std::vector& offsets) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_DENSE_READER diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.cc b/tiledb/sm/query/readers/ordered_dim_label_reader.cc index 93df4650ed8..0f0ea291b6a 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.cc +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.cc @@ -43,7 +43,6 @@ #include "tiledb/sm/query/readers/filtered_data.h" #include "tiledb/sm/query/readers/result_tile.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/subarray.h" #include "tiledb/type/apply_with_type.h" @@ -55,9 +54,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class OrderedDimLabelReaderStatusException : public StatusException { +class OrderedDimLabelReaderException : public StatusException { public: - explicit OrderedDimLabelReaderStatusException(const std::string& message) + explicit OrderedDimLabelReaderException(const std::string& message) : StatusException("OrderedDimLabelReader", message) { } }; @@ -82,59 +81,54 @@ OrderedDimLabelReader::OrderedDimLabelReader( , index_dim_(array_schema_.domain().dimension_ptr(0)) , result_tiles_(fragment_metadata_.size()) { // Sanity checks. - if (storage_manager_ == nullptr) { - throw OrderedDimLabelReaderStatusException( - "Cannot initialize ordered dim label reader; Storage manager not set"); - } - if (!params.default_channel_aggregates().empty()) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize reader; Reader cannot process aggregates"); } if (!params.skip_checks_serialization() && buffers_.empty()) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize ordered dim label reader; Buffers not set"); } if (!params.skip_checks_serialization() && buffers_.size() != 1) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize ordered dim label reader with " + std::to_string(buffers_.size()) + " buffers; Only one buffer allowed"); } for (const auto& b : buffers_) { if (b.first != index_dim_->name()) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize ordered dim label reader; Wrong buffer set"); } if (*b.second.buffer_size_ != ranges_.size() * 2 * datatype_size(index_dim_->type())) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize ordered dim label reader; Wrong buffer size"); } if (b.second.buffer_var_size_ != 0) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize ordered dim label reader; Wrong buffer var size"); } } if (subarray_.is_set()) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot initialize ordered dim label reader; Subarray is set"); } if (condition_.has_value()) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Ordered dimension label reader cannot process query condition"); } bool found = false; if (!config_.get("sm.mem.total_budget", &memory_budget_, &found) .ok()) { - throw OrderedDimLabelReaderStatusException("Cannot get setting"); + throw OrderedDimLabelReaderException("Cannot get setting"); } assert(found); } @@ -196,7 +190,7 @@ void OrderedDimLabelReader::label_read() { if constexpr (tiledb::type::TileDBIntegral) { label_read(); } else { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot read ordered label array; Unsupported domain type"); } }; @@ -210,7 +204,7 @@ void OrderedDimLabelReader::label_read() { // Handle empty array. if (fragment_metadata_.empty()) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Cannot read dim label; Dimension label is empty"); } @@ -530,7 +524,7 @@ uint64_t OrderedDimLabelReader::create_result_tiles() { query_memory_tracker_)); } else { if (r == 0) { - throw OrderedDimLabelReaderStatusException( + throw OrderedDimLabelReaderException( "Can't process a single range, increase memory budget"); } return r; @@ -577,7 +571,7 @@ LabelType OrderedDimLabelReader::get_value_at( // We should always find the value before the last fragment. if (f == 0) { - throw OrderedDimLabelReaderStatusException("Couldn't find value"); + throw OrderedDimLabelReaderException("Couldn't find value"); } } } @@ -669,7 +663,7 @@ void OrderedDimLabelReader::compute_and_copy_range_indexes( LabelType value = get_range_as(r, 0); if (get_value_at(dest[0], dim_dom[0], tile_extent) < value) { - throw OrderedDimLabelReaderStatusException("Range contained no values"); + throw OrderedDimLabelReaderException("Range contained no values"); } } @@ -681,7 +675,7 @@ void OrderedDimLabelReader::compute_and_copy_range_indexes( LabelType value = get_range_as(r, 1); if (get_value_at(dest[1], dim_dom[0], tile_extent) > value) { - throw OrderedDimLabelReaderStatusException("Range contained no values"); + throw OrderedDimLabelReaderException("Range contained no values"); } } } else { @@ -694,7 +688,7 @@ void OrderedDimLabelReader::compute_and_copy_range_indexes( LabelType value = get_range_as(r, 1); if (get_value_at(dest[0], dim_dom[0], tile_extent) > value) { - throw OrderedDimLabelReaderStatusException("Range contained no values"); + throw OrderedDimLabelReaderException("Range contained no values"); } } @@ -706,14 +700,14 @@ void OrderedDimLabelReader::compute_and_copy_range_indexes( LabelType value = get_range_as(r, 0); if (get_value_at(dest[1], dim_dom[0], tile_extent) < value) { - throw OrderedDimLabelReaderStatusException("Range contained no values"); + throw OrderedDimLabelReaderException("Range contained no values"); } } } // If the range provided contained no values, throw an error. if (dest[0] > dest[1]) { - throw OrderedDimLabelReaderStatusException("Range contained no values"); + throw OrderedDimLabelReaderException("Range contained no values"); } } diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.h b/tiledb/sm/query/readers/ordered_dim_label_reader.h index afb8ad1fc3d..f96922e962f 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.h +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,12 +42,10 @@ #include "tiledb/sm/query/iquery_strategy.h" #include "tiledb/sm/query/query_buffer.h" #include "tiledb/sm/query/readers/reader_base.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; @@ -513,9 +511,8 @@ class OrderedDimLabelReader : public ReaderBase, public IQueryStrategy { */ template void compute_and_copy_range_indexes(uint64_t buffer_offset, uint64_t r); -}; // namespace sm +}; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_ORDERED_DIM_LABEL_READER diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index e80d5792c64..163f6911ff4 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -46,7 +46,6 @@ #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/query/readers/result_tile.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/subarray.h" using namespace tiledb; @@ -55,9 +54,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class SparseGlobalOrderReaderStatusException : public StatusException { +class SparseGlobalOrderReaderException : public StatusException { public: - explicit SparseGlobalOrderReaderStatusException(const std::string& message) + explicit SparseGlobalOrderReaderException(const std::string& message) : StatusException("SparseGlobalOrderReader", message) { } }; @@ -249,7 +248,7 @@ void SparseGlobalOrderReader::load_all_tile_offsets() { tile_offsets_size(subarray_.relevant_fragments()); uint64_t available_memory = array_memory_tracker_->get_memory_available(); if (total_tile_offset_usage > available_memory) { - throw SparseGlobalOrderReaderStatusException( + throw SparseGlobalOrderReaderException( "Cannot load tile offsets, computed size (" + std::to_string(total_tile_offset_usage) + ") is larger than available memory (" + @@ -385,7 +384,7 @@ SparseGlobalOrderReader::create_result_tiles( if (result_tiles[f].empty()) { auto tiles_size = get_coord_tiles_size(dim_num, f, t); - throw SparseGlobalOrderReaderStatusException( + throw SparseGlobalOrderReaderException( "Cannot load a single tile for fragment, increase " "memory " "budget, tile size : " + @@ -1711,7 +1710,7 @@ SparseGlobalOrderReader::respect_copy_memory_budget( })); if (max_cs_idx == 0) { - throw SparseGlobalOrderReaderStatusException( + throw SparseGlobalOrderReaderException( "Unable to copy one slab with current budget/buffers"); } @@ -1948,7 +1947,7 @@ void SparseGlobalOrderReader::process_slabs( if (aggregates_.count(name) != 0) { for (auto& aggregates : aggregates_[name]) { if (aggregates->need_recompute_on_overflow()) { - throw SparseGlobalOrderReaderStatusException( + throw SparseGlobalOrderReaderException( "Overflow happened after aggregate was computed, aggregate " "recompute pass is not yet implemented"); } diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.h b/tiledb/sm/query/readers/sparse_global_order_reader.h index c113dc6c612..d5da6f20ce2 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.h +++ b/tiledb/sm/query/readers/sparse_global_order_reader.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,12 +45,10 @@ #include "tiledb/sm/query/readers/result_cell_slab.h" #include "tiledb/sm/query/readers/result_coords.h" #include "tiledb/sm/query/readers/sparse_index_reader_base.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; @@ -628,7 +626,6 @@ class SparseGlobalOrderReader : public SparseIndexReaderBase, void end_iteration(std::vector& result_tiles); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SPARSE_GLOBAL_ORDER_READER diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index cf452614310..492ed3459bf 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -51,9 +51,9 @@ namespace tiledb::sm { -class SparseIndexReaderBaseStatusException : public StatusException { +class SparseIndexReaderBaseException : public StatusException { public: - explicit SparseIndexReaderBaseStatusException(const std::string& message) + explicit SparseIndexReaderBaseException(const std::string& message) : StatusException("SparseIndexReaderBase", message) { } }; @@ -78,14 +78,9 @@ SparseIndexReaderBase::SparseIndexReaderBase( buffers_.count(constants::delete_timestamps) != 0) , partial_tile_offsets_loading_(false) { // Sanity checks - if (storage_manager_ == nullptr) { - throw SparseIndexReaderBaseStatusException( - "Cannot initialize reader; Storage manager not set"); - } - if (!params.skip_checks_serialization() && buffers_.empty() && aggregate_buffers_.empty()) { - throw SparseIndexReaderBaseStatusException( + throw SparseIndexReaderBaseException( "Cannot initialize reader; Buffers not set"); } @@ -96,7 +91,7 @@ SparseIndexReaderBase::SparseIndexReaderBase( offsets_format_mode_ = config_.get("sm.var_offsets.mode", Config::must_find); if (offsets_format_mode_ != "bytes" && offsets_format_mode_ != "elements") { - throw SparseIndexReaderBaseStatusException( + throw SparseIndexReaderBaseException( "Cannot initialize reader; Unsupported offsets format in " "configuration"); } @@ -108,7 +103,7 @@ SparseIndexReaderBase::SparseIndexReaderBase( offsets_bitsize_ = config_.get("sm.var_offsets.bitsize", Config::must_find); if (offsets_bitsize_ != 32 && offsets_bitsize_ != 64) { - throw SparseIndexReaderBaseStatusException( + throw SparseIndexReaderBaseException( "Cannot initialize reader; Unsupported offsets bitsize in " "configuration"); } @@ -389,7 +384,7 @@ Status SparseIndexReaderBase::load_initial_data() { if (subarray_.is_set()) { // At this point, full memory budget is available. if (!array_memory_tracker_->set_budget(memory_budget_.total_budget())) { - throw SparseIndexReaderBaseStatusException( + throw SparseIndexReaderBaseException( "Cannot set array memory budget, already over limit."); } @@ -454,7 +449,7 @@ Status SparseIndexReaderBase::load_initial_data() { // Set a limit to the array memory. if (!array_memory_tracker_->set_budget( memory_budget_.total_budget() * memory_budget_.ratio_array_data())) { - throw SparseIndexReaderBaseStatusException( + throw SparseIndexReaderBaseException( "Cannot set array memory budget, already over limit."); } diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index 3ca26a1c18c..88e291b8aa8 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,10 +40,8 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/query/readers/result_cell_slab.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class ArraySchema; @@ -788,7 +786,6 @@ class SparseIndexReaderBase : public ReaderBase { void add_extra_offset(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SPARSE_INDEX_READER_BASE_H diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index b61595b5063..d33779de7d0 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -43,7 +43,6 @@ #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/query/readers/result_tile.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/subarray.h" using namespace tiledb; @@ -52,10 +51,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class SparseUnorderedWithDupsReaderStatusException : public StatusException { +class SparseUnorderedWithDupsReaderException : public StatusException { public: - explicit SparseUnorderedWithDupsReaderStatusException( - const std::string& message) + explicit SparseUnorderedWithDupsReaderException(const std::string& message) : StatusException("SparseUnorderedWithDupsReader", message) { } }; @@ -83,7 +81,7 @@ SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( &partial_tile_offsets_loading_, &found) .ok()) { - throw SparseUnorderedWithDupsReaderStatusException("Cannot get setting"); + throw SparseUnorderedWithDupsReaderException("Cannot get setting"); } assert(found); } @@ -262,7 +260,7 @@ void SparseUnorderedWithDupsReader::load_tile_offsets_data() { // tile offsets data. uint64_t total_tile_offset_usage = tile_offsets_size(relevant_fragments); if (total_tile_offset_usage > available_memory) { - throw SparseUnorderedWithDupsReaderStatusException( + throw SparseUnorderedWithDupsReaderException( "Cannot load tile offsets, computed size (" + std::to_string(total_tile_offset_usage) + ") is larger than available memory (" + @@ -312,7 +310,7 @@ void SparseUnorderedWithDupsReader::load_tile_offsets_data() { // Make sure plan to load tile offsets for at least one fragment. if (tile_offsets_min_frag_idx_ == tile_offsets_max_frag_idx_) { - throw SparseUnorderedWithDupsReaderStatusException( + throw SparseUnorderedWithDupsReaderException( "Cannot load tile offsets for only one fragment. Offsets size for " "the fragment (" + std::to_string( @@ -388,7 +386,7 @@ bool SparseUnorderedWithDupsReader::add_result_tile( // Make sure we can add at least one tile. if (result_tiles.empty()) { - throw SparseUnorderedWithDupsReaderStatusException( + throw SparseUnorderedWithDupsReaderException( "Cannot load a single tile, increase memory budget"); } @@ -1512,7 +1510,7 @@ SparseUnorderedWithDupsReader::respect_copy_memory_budget( })); if (max_rt_idx == 0) { - throw SparseUnorderedWithDupsReaderStatusException( + throw SparseUnorderedWithDupsReaderException( "Unable to copy one tile with current budget/buffers"); } @@ -1696,7 +1694,7 @@ bool SparseUnorderedWithDupsReader::process_tiles( if (aggregates_.count(name) != 0) { for (auto& aggregates : aggregates_[name]) { if (aggregates->need_recompute_on_overflow()) { - throw SparseUnorderedWithDupsReaderStatusException( + throw SparseUnorderedWithDupsReaderException( "Overflow happened after aggregate was computed, aggregate " "recompute pass is not yet implemented"); } diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h index 1ec82e2c70c..d4bf950ef02 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,12 +45,10 @@ #include "tiledb/sm/query/readers/result_cell_slab.h" #include "tiledb/sm/query/readers/result_coords.h" #include "tiledb/sm/query/readers/sparse_index_reader_base.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; @@ -552,7 +550,6 @@ class SparseUnorderedWithDupsReader : public SparseIndexReaderBase, void end_iteration(ResultTilesList& result_tiles); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SPARSE_UNORDERED_WITH_DUPS_READER diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index 1099c4eb8a7..742e8f782da 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -102,7 +102,7 @@ void StrategyBase::get_dim_attr_stats() const { } } -void StrategyBase::throw_if_cancellation_requested() const { +void StrategyBase::throw_if_cancelled() const { if (storage_manager_->cancellation_in_progress()) { throw QueryException("Query was cancelled"); } diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 2e029d0be11..1985cd7df6d 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -65,6 +65,7 @@ class StrategyParams { /* ********************************* */ StrategyParams( + ContextResources& resources, shared_ptr array_memory_tracker, shared_ptr query_memory_tracker, StorageManager* storage_manager, @@ -78,7 +79,7 @@ class StrategyParams { std::optional& condition, DefaultChannelAggregates& default_channel_aggregates, bool skip_checks_serialization) - : resources_(storage_manager->resources()) + : resources_(resources) , array_memory_tracker_(array_memory_tracker) , query_memory_tracker_(query_memory_tracker) , storage_manager_(storage_manager) @@ -340,9 +341,9 @@ class StrategyBase { void get_dim_attr_stats() const; /** - * Throws an exception if the query is canceled. + * Throws an exception if the query is cancelled. */ - void throw_if_cancellation_requested() const; + void throw_if_cancelled() const; }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index ca66dbe7eac..a10809236f6 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -4591,6 +4591,7 @@ TEST_CASE( offsets.push_back(curr_offset); curr_offset += elem.size(); } + offsets.push_back(curr_offset); // Initialize the result tile. ResultTile::TileSizes tile_sizes( diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 4a637e87794..cff8f31b69a 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -48,7 +48,6 @@ #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile_metadata_generator.h" #include "tiledb/sm/tile/writer_tile_tuple.h" @@ -60,9 +59,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class GlobalOrderWriterStatusException : public StatusException { +class GlobalOrderWriterException : public StatusException { public: - explicit GlobalOrderWriterStatusException(const std::string& message) + explicit GlobalOrderWriterException(const std::string& message) : StatusException("GlobalOrderWriter", message) { } }; @@ -96,14 +95,14 @@ GlobalOrderWriter::GlobalOrderWriter( , current_fragment_size_(0) { // Check the layout is global order. if (layout_ != Layout::GLOBAL_ORDER) { - throw GlobalOrderWriterStatusException( + throw GlobalOrderWriterException( "Failed to initialize global order writer. Layout " + layout_str(layout_) + " is not global order."); } // Check no ordered attributes. if (array_schema_.has_ordered_attributes()) { - throw GlobalOrderWriterStatusException( + throw GlobalOrderWriterException( "Failed to initialize global order writer. Global order writes to " "ordered attributes are not yet supported."); } @@ -186,8 +185,7 @@ Status GlobalOrderWriter::alloc_global_write_state() { // Alloc FragmentMetadata object global_write_state_->frag_meta_ = this->create_fragment_metadata(); // Used in serialization when FragmentMetadata is built from ground up - global_write_state_->frag_meta_->set_context_resources( - &storage_manager_->resources()); + global_write_state_->frag_meta_->set_context_resources(&resources_); return Status::Ok(); } @@ -797,7 +795,7 @@ Status GlobalOrderWriter::global_write() { frag_meta->set_num_tiles(new_num_tiles); if (new_num_tiles == 0) { - throw GlobalOrderWriterStatusException( + throw GlobalOrderWriterException( "Fragment size is too small to write a single tile"); } @@ -873,7 +871,7 @@ Status GlobalOrderWriter::prepare_full_tiles( std::advance(buff_it, i); const auto& name = buff_it->first; throw_if_not_ok(prepare_full_tiles(name, coord_dups, &tiles->at(name))); - this->throw_if_cancellation_requested(); + throw_if_cancelled(); return Status::Ok(); }); diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index f778b43e6ef..7c25c3fe598 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -48,7 +48,6 @@ #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile_metadata_generator.h" #include "tiledb/sm/tile/writer_tile_tuple.h" diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index b8cefabba39..8e1fa4e5c5f 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -48,7 +48,6 @@ #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile_metadata_generator.h" #include "tiledb/sm/tile/writer_tile_tuple.h" @@ -170,7 +169,7 @@ Status UnorderedWriter::alloc_frag_meta() { // Alloc FragmentMetadata object. frag_meta_ = this->create_fragment_metadata(); // Used in serialization when FragmentMetadata is built from ground up. - frag_meta_->set_context_resources(&storage_manager_->resources()); + frag_meta_->set_context_resources(&resources_); return Status::Ok(); } @@ -388,7 +387,7 @@ Status UnorderedWriter::prepare_tiles( auto tiles_it = tiles->begin(); std::advance(tiles_it, i); throw_if_not_ok(prepare_tiles(tiles_it->first, &(tiles_it->second))); - this->throw_if_cancellation_requested(); + throw_if_cancelled(); return Status::Ok(); }); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 5f5c65554b7..b6ff5ffddfd 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -50,7 +50,6 @@ #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile_metadata_generator.h" #include "tiledb/sm/tile/writer_tile_tuple.h" @@ -62,9 +61,9 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { -class WriterBaseStatusException : public StatusException { +class WriterBaseException : public StatusException { public: - explicit WriterBaseStatusException(const std::string& message) + explicit WriterBaseException(const std::string& message) : StatusException("WriterBase", message) { } }; @@ -92,21 +91,15 @@ WriterBase::WriterBase( , written_fragment_info_(written_fragment_info) , remote_query_(remote_query) { // Sanity checks - if (storage_manager_ == nullptr) { - throw WriterBaseStatusException( - "Cannot initialize query; Storage manager not set"); - } - if (!params.skip_checks_serialization() && buffers_.empty()) { - throw WriterBaseStatusException( - "Cannot initialize writer; Buffers not set"); + throw WriterBaseException("Cannot initialize writer; Buffers not set"); } if (array_schema_.dense() && (layout_ == Layout::ROW_MAJOR || layout_ == Layout::COL_MAJOR)) { for (const auto& b : buffers_) { if (array_schema_.is_dim(b.first)) { - throw WriterBaseStatusException( + throw WriterBaseException( "Cannot initialize writer; Sparse coordinates for dense arrays " "cannot be provided if the query layout is ROW_MAJOR or COL_MAJOR"); } @@ -117,19 +110,19 @@ WriterBase::WriterBase( const char *check_coord_dups, *check_coord_oob, *check_global_order; const char* dedup_coords; if (!config_.get("sm.check_coord_dups", &check_coord_dups).ok()) { - throw WriterBaseStatusException("Cannot get setting"); + throw WriterBaseException("Cannot get setting"); } if (!config_.get("sm.check_coord_oob", &check_coord_oob).ok()) { - throw WriterBaseStatusException("Cannot get setting"); + throw WriterBaseException("Cannot get setting"); } if (!config_.get("sm.check_global_order", &check_global_order).ok()) { - throw WriterBaseStatusException("Cannot get setting"); + throw WriterBaseException("Cannot get setting"); } if (!config_.get("sm.dedup_coords", &dedup_coords).ok()) { - throw WriterBaseStatusException("Cannot get setting"); + throw WriterBaseException("Cannot get setting"); } assert(check_coord_dups != nullptr && dedup_coords != nullptr); @@ -144,7 +137,7 @@ WriterBase::WriterBase( offsets_format_mode_ = config_.get("sm.var_offsets.mode", &found); assert(found); if (offsets_format_mode_ != "bytes" && offsets_format_mode_ != "elements") { - throw WriterBaseStatusException( + throw WriterBaseException( "Cannot initialize writer; Unsupported offsets format in " "configuration"); } @@ -152,19 +145,19 @@ WriterBase::WriterBase( .get( "sm.var_offsets.extra_element", &offsets_extra_element_, &found) .ok()) { - throw WriterBaseStatusException("Cannot get setting"); + throw WriterBaseException("Cannot get setting"); } assert(found); if (!config_ .get("sm.var_offsets.bitsize", &offsets_bitsize_, &found) .ok()) { - throw WriterBaseStatusException("Cannot get setting"); + throw WriterBaseException("Cannot get setting"); } assert(found); if (offsets_bitsize_ != 32 && offsets_bitsize_ != 64) { - throw WriterBaseStatusException( + throw WriterBaseException( "Cannot initialize writer; Unsupported offsets bitsize in " "configuration"); } @@ -172,12 +165,12 @@ WriterBase::WriterBase( // Check subarray is valid for strategy is set or set it to default if unset. if (subarray_.is_set()) { if (!array_schema_.dense()) { - throw WriterBaseStatusException( + throw WriterBaseException( "Cannot initialize write; Non-default subarray are not supported in " "sparse writes"); } if (subarray_.range_num() > 1) { - throw WriterBaseStatusException( + throw WriterBaseException( "Cannot initialize writer; Multi-range dense writes are not " "supported"); } @@ -270,7 +263,7 @@ void WriterBase::check_var_attr_offsets() const { // Allow the initial offset to be equal to the size, this indicates // the first and only value in the buffer is to be empty if (prev_offset > *buffer_val_size) { - throw WriterBaseStatusException( + throw WriterBaseException( "Invalid offsets for attribute " + attr + "; offset " + std::to_string(prev_offset) + " specified for buffer of size " + std::to_string(*buffer_val_size)); @@ -279,7 +272,7 @@ void WriterBase::check_var_attr_offsets() const { for (uint64_t i = 1; i < num_offsets; i++) { uint64_t cur_offset = get_offset_buffer_element(buffer_off, i); if (cur_offset < prev_offset) - throw WriterBaseStatusException( + throw WriterBaseException( "Invalid offsets for attribute " + attr + "; offsets must be given in " "strictly ascending order."); @@ -291,7 +284,7 @@ void WriterBase::check_var_attr_offsets() const { get_offset_buffer_element( buffer_off, (i < num_offsets - 1 ? i + 1 : i)) != *buffer_val_size)) - throw WriterBaseStatusException( + throw WriterBaseException( "Invalid offsets for attribute " + attr + "; offset " + std::to_string(cur_offset) + " specified at index " + std::to_string(i) + " for buffer of size " + @@ -383,7 +376,7 @@ void WriterBase::check_buffer_sizes() const { "given for "; ss << "attribute '" << attr << "'"; ss << " (" << expected_validity_num << " != " << cell_num << ")"; - throw WriterBaseStatusException(ss.str()); + throw WriterBaseException(ss.str()); } } else { if (expected_cell_num != cell_num) { @@ -391,7 +384,7 @@ void WriterBase::check_buffer_sizes() const { ss << "Buffer sizes check failed; Invalid number of cells given for "; ss << "attribute '" << attr << "'"; ss << " (" << expected_cell_num << " != " << cell_num << ")"; - throw WriterBaseStatusException(ss.str()); + throw WriterBaseException(ss.str()); } } } @@ -445,12 +438,11 @@ Status WriterBase::check_coord_oob() const { void WriterBase::check_subarray() const { if (array_schema_.dense()) { if (subarray_.range_num() != 1) { - throw WriterBaseStatusException( - "Multi-range dense writes are not supported"); + throw WriterBaseException("Multi-range dense writes are not supported"); } if (layout_ == Layout::GLOBAL_ORDER && !subarray_.coincides_with_tiles()) { - throw WriterBaseStatusException( + throw WriterBaseException( "Cannot initialize query; In global writes for dense arrays, the " "subarray must coincide with the tile bounds"); } @@ -529,7 +521,7 @@ bool is_sorted_buffer( buffer.is_sorted_str>() : buffer.is_sorted_str>(); default: - throw WriterBaseStatusException( + throw WriterBaseException( "Unexpected datatype '" + datatype_str(type) + "' for an ordered attribute."); } @@ -556,7 +548,7 @@ void WriterBase::check_attr_order() const { // Check the attribute sort. This assumes all ordered attributes are fixed // except STRING_ASCII which is assumed to always be variable. if (!is_sorted_buffer(buffer, attr->type(), increasing)) { - throw WriterBaseStatusException( + throw WriterBaseException( "The data for attribute '" + name + "' is not in the expected order."); } @@ -810,7 +802,7 @@ Status WriterBase::filter_tiles( auto tiles_it = tiles->begin(); std::advance(tiles_it, i); throw_if_not_ok(filter_tiles(tiles_it->first, &tiles_it->second)); - this->throw_if_cancellation_requested(); + throw_if_cancelled(); return Status::Ok(); }); @@ -899,7 +891,7 @@ Status WriterBase::filter_tile( } // Append an encryption filter when necessary. - RETURN_NOT_OK(FilterPipeline::append_encryption_filter( + throw_if_not_ok(FilterPipeline::append_encryption_filter( &filters, array_->get_encryption_key())); // Check if chunk or tile level filtering/unfiltering is appropriate @@ -907,8 +899,8 @@ Status WriterBase::filter_tile( array_schema_.var_size(name), array_schema_.version(), tile->type()); assert(!tile->filtered()); - RETURN_NOT_OK(filters.run_forward( - stats_, tile, offsets_tile, &resources_.compute_tp(), use_chunking)); + filters.run_forward( + stats_, tile, offsets_tile, &resources_.compute_tp(), use_chunking); assert(tile->filtered()); return Status::Ok(); @@ -982,7 +974,7 @@ void WriterBase::check_extra_element() { get_offset_buffer_element(buffer_off, num_offsets - 1); if (last_offset != max_offset) { - throw WriterBaseStatusException( + throw WriterBaseException( "Invalid offsets for attribute " + attr + "; the last offset: " + std::to_string(last_offset) + " is not equal to the size of the data buffer: " + diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 74fb4b8449c..7379e9c2fc2 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -95,7 +95,7 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - throw_if_not_ok(Array::create(resources_, uri, schema, key)); + Array::create(resources_, uri, schema, key); tdb_unique_ptr array(new Array{resources_, uri}); return array; @@ -126,7 +126,7 @@ TEST_CASE_METHOD(QueryPlanFx, "Query plan dump_json", "[query_plan][dump]") { REQUIRE(st.ok()); shared_ptr array_shared = std::move(array); - Query query(sm_.get(), array_shared); + Query query(resources_, sm_.get(), array_shared); REQUIRE(query.set_layout(Layout::ROW_MAJOR).ok()); stats::Stats stats("foo"); diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index c291323f39e..b7408fd6ffd 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -60,6 +60,7 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" +#include "tiledb/sm/config/config_iter.h" #include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/group/group.h" #include "tiledb/sm/misc/constants.h" @@ -148,6 +149,14 @@ Status RestClient::init( RETURN_NOT_OK(config_->get( "rest.resubmit_incomplete", &resubmit_incomplete_, &found)); + load_headers(*config_); + + if (auto payer = + config_->get("rest.payer_namespace", Config::must_find); + !payer.empty()) { + extra_headers_["X-Payer"] = std::move(payer); + } + return Status::Ok(); } @@ -318,7 +327,7 @@ shared_ptr RestClient::post_array_schema_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting array schema from REST; server returned no data."); } @@ -676,7 +685,7 @@ RestClient::post_enumerations_from_rest( const std::vector& enumeration_names, shared_ptr memory_tracker) { if (array == nullptr) { - throw Status_RestError( + throw RestClientException( "Error getting enumerations from REST; array is null."); } @@ -721,7 +730,7 @@ RestClient::post_enumerations_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting enumerations from REST; server returned no data."); } @@ -736,7 +745,8 @@ void RestClient::post_query_plan_from_rest( // Get array const Array* array = query.array(); if (array == nullptr) { - throw Status_RestError("Error submitting query plan to REST; null array."); + throw RestClientException( + "Error submitting query plan to REST; null array."); } Buffer buff; @@ -779,7 +789,7 @@ void RestClient::post_query_plan_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting query plan from REST; server returned no data."); } @@ -1650,7 +1660,7 @@ RestClient::post_consolidation_plan_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting query plan from REST; server returned no data."); } @@ -1660,6 +1670,17 @@ RestClient::post_consolidation_plan_from_rest( serialization_type_, returned_data); } +void RestClient::load_headers(const Config& cfg) { + for (auto iter = ConfigIter(cfg, constants::rest_header_prefix); !iter.end(); + iter.next()) { + auto key = iter.param(); + if (key.size() == 0) { + continue; + } + extra_headers_[key] = iter.value(); + } +} + #else RestClient::RestClient() { @@ -1831,6 +1852,10 @@ RestClient::post_consolidation_plan_from_rest( throw RestClientDisabledException(); } +void RestClient::load_headers(const Config&) { + throw RestClientDisabledException(); +} + #endif // TILEDB_SERIALIZATION } // namespace tiledb::sm diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 0f7cfae63f5..84b2535a820 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -418,6 +418,13 @@ class RestClient { std::vector> post_consolidation_plan_from_rest( const URI& uri, const Config& config, uint64_t fragment_size); + /** + * Constant accessor to `extra_headers_` member variable. + */ + const std::unordered_map& extra_headers() const { + return extra_headers_; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -566,6 +573,9 @@ class RestClient { * @return Status */ Status ensure_json_null_delimited_string(Buffer* buffer); + + /** Load all custom headers from the given config. */ + void load_headers(const Config& cfg); }; } // namespace tiledb::sm diff --git a/tiledb/sm/rtree/rtree.cc b/tiledb/sm/rtree/rtree.cc index 4a6737b1290..6392e64657f 100644 --- a/tiledb/sm/rtree/rtree.cc +++ b/tiledb/sm/rtree/rtree.cc @@ -349,17 +349,24 @@ void RTree::deserialize_v1_v4( auto level_num = deserializer.read(); levels_.clear(); - levels_.resize(level_num); + levels_.reserve(level_num); auto dim_num = domain->dim_num(); for (unsigned l = 0; l < level_num; ++l) { auto mbr_num = deserializer.read(); - levels_[l].resize(mbr_num); + // We use emplace_back to construct the level in place, which also + // implicitly passes the memory tracking allocator all the way down to the + // NDRange objects. + auto& level = levels_.emplace_back(); + level.reserve(mbr_num); for (uint64_t m = 0; m < mbr_num; ++m) { - levels_[l][m].resize(dim_num); + auto& mbr = level.emplace_back(); + mbr.reserve(dim_num); for (unsigned d = 0; d < dim_num; ++d) { auto r_size{2 * domain->dimension_ptr(d)->coord_size()}; auto data = deserializer.get_ptr(r_size); - levels_[l][m][d] = Range(data, r_size); + // NDRange is not yet a pmr vector, so we need to explicitly pass the + // grandparent container's allocator. + mbr.emplace_back(data, r_size, level.get_allocator()); } } } @@ -374,27 +381,36 @@ void RTree::deserialize_v5(Deserializer& deserializer, const Domain* domain) { auto level_num = deserializer.read(); levels_.clear(); - levels_.resize(level_num); + levels_.reserve(level_num); auto dim_num = domain->dim_num(); for (unsigned l = 0; l < level_num; ++l) { auto mbr_num = deserializer.read(); - levels_[l].resize(mbr_num); + // We use emplace_back to construct the level in place, which also + // implicitly passes the memory tracking allocator all the way down to the + // NDRange objects. + auto& level = levels_.emplace_back(); + level.reserve(mbr_num); for (uint64_t m = 0; m < mbr_num; ++m) { - levels_[l][m].resize(dim_num); + auto& mbr = level.emplace_back(); + mbr.reserve(dim_num); for (unsigned d = 0; d < dim_num; ++d) { auto dim{domain->dimension_ptr(d)}; if (!dim->var_size()) { // Fixed-sized auto r_size = 2 * dim->coord_size(); auto data = deserializer.get_ptr(r_size); - levels_[l][m][d] = Range(data, r_size); + // NDRange is not yet a pmr vector, so we need to explicitly pass the + // grandparent container's allocator. + mbr.emplace_back(data, r_size, level.get_allocator()); } else { // Var-sized // range_size | start_size | range auto r_size = deserializer.read(); auto start_size = deserializer.read(); auto data = deserializer.get_ptr(r_size); - levels_[l][m][d] = Range(data, r_size, start_size); + // NDRange is not yet a pmr vector, so we need to explicitly pass the + // grandparent container's allocator. + mbr.emplace_back(data, r_size, start_size, level.get_allocator()); } } } diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index dda5ccf47b9..1816fae0710 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -77,6 +77,21 @@ using namespace tiledb::common; namespace tiledb::sm::serialization { +class ArraySchemaSerializationException : public StatusException { + public: + explicit ArraySchemaSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][ArraySchema]", message) { + } +}; + +class ArraySchemaSerializationDisabledException + : public ArraySchemaSerializationException { + public: + explicit ArraySchemaSerializationDisabledException() + : ArraySchemaSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; #ifdef TILEDB_SERIALIZATION @@ -357,7 +372,7 @@ tuple>> filter_pipeline_from_capnp( void attribute_to_capnp( const Attribute* attribute, capnp::Attribute::Builder* attribute_builder) { if (attribute == nullptr) { - throw SerializationStatusException( + throw ArraySchemaSerializationException( "Error serializing attribute; attribute is null."); } @@ -770,7 +785,7 @@ void dimension_label_to_capnp( if (dimension_label.uri_is_relative()) { dim_label_builder->setUri(dimension_label.uri().to_string()); } else { - throw SerializationStatusException( + throw ArraySchemaSerializationException( "[Serialization::dimension_label_to_capnp] Serialization of absolute " "dimension label URIs not yet implemented."); } @@ -797,7 +812,7 @@ shared_ptr dimension_label_from_capnp( auto is_relative = dim_label_reader.getRelative(); if (!is_relative) { - throw SerializationStatusException( + throw ArraySchemaSerializationException( "[Deserialization::dimension_label_from_capnp] Deserialization of " "absolute dimension label URIs not yet implemented."); } @@ -1862,18 +1877,18 @@ void serialize_load_array_schema_request( break; } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema request; exception " + std::string(e.what())); } @@ -1907,17 +1922,17 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( return load_array_schema_request_from_capnp(reader); } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema request; exception " + std::string(e.what())); } @@ -1963,18 +1978,18 @@ void serialize_load_array_schema_response( break; } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema response; exception " + std::string(e.what())); } @@ -2012,17 +2027,17 @@ shared_ptr deserialize_load_array_schema_response( return load_array_schema_response_from_capnp(reader, memory_tracker); } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema response; exception " + std::string(e.what())); } @@ -2081,26 +2096,22 @@ Status max_buffer_sizes_deserialize( void serialize_load_array_schema_request( const Config&, const LoadArraySchemaRequest&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } LoadArraySchemaRequest deserialize_load_array_schema_request( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } void serialize_load_array_schema_response( const ArraySchema&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } shared_ptr deserialize_load_array_schema_response( SerializationType, const Buffer&, shared_ptr) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/consolidation.cc b/tiledb/sm/serialization/consolidation.cc index 7efababc271..e6c06c1ecaf 100644 --- a/tiledb/sm/serialization/consolidation.cc +++ b/tiledb/sm/serialization/consolidation.cc @@ -53,6 +53,22 @@ namespace tiledb { namespace sm { namespace serialization { +class ConsolidationSerializationException : public StatusException { + public: + explicit ConsolidationSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][Consolidation]", message) { + } +}; + +class ConsolidationSerializationDisabledException + : public ConsolidationSerializationException { + public: + explicit ConsolidationSerializationDisabledException() + : ConsolidationSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION Status array_consolidation_request_to_capnp( @@ -269,18 +285,18 @@ void serialize_consolidation_plan_request( break; } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan request; exception " + std::string(e.what())); } @@ -310,17 +326,17 @@ uint64_t deserialize_consolidation_plan_request( return consolidation_plan_request_from_capnp(reader); } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan request; exception " + std::string(e.what())); } @@ -360,18 +376,18 @@ void serialize_consolidation_plan_response( break; } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan response; exception " + std::string(e.what())); } @@ -401,17 +417,17 @@ std::vector> deserialize_consolidation_plan_response( return consolidation_plan_response_from_capnp(reader); } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan response; exception " + std::string(e.what())); } @@ -433,26 +449,22 @@ Status array_consolidation_request_deserialize( void serialize_consolidation_plan_request( uint64_t, const Config&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } uint64_t deserialize_consolidation_plan_request( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot deserialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } void serialize_consolidation_plan_response( const ConsolidationPlan&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } std::vector> deserialize_consolidation_plan_response( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot deserialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 197a2d8834d..99f771fc87b 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -48,6 +48,22 @@ using namespace tiledb::common; namespace tiledb::sm::serialization { +class EnumerationSerializationException : public StatusException { + public: + explicit EnumerationSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][Enumeration]", message) { + } +}; + +class EnumerationSerializationDisabledException + : public EnumerationSerializationException { + public: + explicit EnumerationSerializationDisabledException() + : EnumerationSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION void enumeration_to_capnp( @@ -198,18 +214,18 @@ void serialize_load_enumerations_request( break; } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations request; exception " + std::string(e.what())); } @@ -239,17 +255,17 @@ std::vector deserialize_load_enumerations_request( return load_enumerations_request_from_capnp(reader); } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations request; exception " + std::string(e.what())); } @@ -289,18 +305,18 @@ void serialize_load_enumerations_response( break; } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations response; exception " + std::string(e.what())); } @@ -333,17 +349,17 @@ deserialize_load_enumerations_response( return load_enumerations_response_from_capnp(reader, memory_tracker); } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations response; exception " + std::string(e.what())); } @@ -356,29 +372,25 @@ void serialize_load_enumerations_request( const std::vector&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } std::vector deserialize_load_enumerations_request( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } void serialize_load_enumerations_response( const std::vector>&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } std::vector> deserialize_load_enumerations_response( SerializationType, const Buffer&, shared_ptr) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index a6477626cfd..65127afa1d3 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -2280,7 +2280,7 @@ Status array_from_query_deserialize( const Buffer& serialized_buffer, SerializationType serialize_type, Array& array, - StorageManager* storage_manager, + ContextResources& resources, shared_ptr memory_tracker) { try { switch (serialize_type) { @@ -2296,11 +2296,7 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = query_builder.asReader(); // Deserialize array instance. array_from_capnp( - query_reader.getArray(), - storage_manager->resources(), - &array, - false, - memory_tracker); + query_reader.getArray(), resources, &array, false, memory_tracker); break; } case SerializationType::CAPNP: { @@ -2310,8 +2306,7 @@ Status array_from_query_deserialize( "Could not deserialize query; buffer is not 8-byte aligned.")); // Set traversal limit from config - uint64_t limit = storage_manager->resources() - .config() + uint64_t limit = resources.config() .get("rest.capnp_traversal_limit") .value(); ::capnp::ReaderOptions readerOptions; @@ -2329,11 +2324,7 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = reader.getRoot(); // Deserialize array instance. array_from_capnp( - query_reader.getArray(), - storage_manager->resources(), - &array, - false, - memory_tracker); + query_reader.getArray(), resources, &array, false, memory_tracker); break; } default: @@ -3198,7 +3189,7 @@ Status array_from_query_deserialize( const Buffer&, SerializationType, Array&, - StorageManager*, + ContextResources&, shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index 6f6ede1e059..2295a399c4b 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -52,6 +52,7 @@ namespace tiledb::sm { class Array; class Buffer; class BufferList; +class ContextResources; class Query; class GlobalOrderWriter; class UnorderedWriter; @@ -142,7 +143,7 @@ Status array_from_query_deserialize( const Buffer& serialized_buffer, SerializationType serialize_type, Array& array, - StorageManager* storage_manager, + ContextResources& resources, shared_ptr memory_tracker); /** diff --git a/tiledb/sm/serialization/query_plan.cc b/tiledb/sm/serialization/query_plan.cc index 6b604566cbe..9170522b31f 100644 --- a/tiledb/sm/serialization/query_plan.cc +++ b/tiledb/sm/serialization/query_plan.cc @@ -53,6 +53,22 @@ using namespace tiledb::common; namespace tiledb::sm::serialization { +class QueryPlanSerializationException : public StatusException { + public: + explicit QueryPlanSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][QueryPlan]", message) { + } +}; + +class QueryPlanSerializationDisabledException + : public QueryPlanSerializationException { + public: + explicit QueryPlanSerializationDisabledException() + : QueryPlanSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION void query_plan_request_to_capnp( @@ -178,18 +194,18 @@ void serialize_query_plan_request( break; } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan request; exception " + std::string(e.what())); } @@ -225,17 +241,17 @@ void deserialize_query_plan_request( break; } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan request; exception " + std::string(e.what())); } @@ -275,18 +291,18 @@ void serialize_query_plan_response( break; } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan response; exception " + std::string(e.what())); } @@ -318,17 +334,17 @@ QueryPlan deserialize_query_plan_response( return query_plan_response_from_capnp(reader, query); } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan response; exception " + std::string(e.what())); } @@ -338,26 +354,22 @@ QueryPlan deserialize_query_plan_response( void serialize_query_plan_request( const Config&, Query&, const SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } void deserialize_query_plan_request( const SerializationType, const Buffer&, ThreadPool&, Query&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } void serialize_query_plan_response( const QueryPlan&, const SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } QueryPlan deserialize_query_plan_response( Query&, const SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 0bf601483aa..76f3ce85ebc 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -199,163 +199,6 @@ void StorageManagerCanonical::increment_in_progress() { queries_in_progress_cv_.notify_all(); } -Status StorageManagerCanonical::object_iter_begin( - ObjectIter** obj_iter, const char* path, WalkOrder order) { - // Sanity check - URI path_uri(path); - if (path_uri.is_invalid()) { - throw StorageManagerException( - "Cannot create object iterator; Invalid input path"); - } - - // Get all contents of path - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(path_uri, &uris)); - - // Create a new object iterator - *obj_iter = tdb_new(ObjectIter); - (*obj_iter)->order_ = order; - (*obj_iter)->recursive_ = true; - - // Include the uris that are TileDB objects in the iterator state - ObjectType obj_type; - for (auto& uri : uris) { - RETURN_NOT_OK_ELSE( - object_type(resources_, uri, &obj_type), tdb_delete(*obj_iter)); - if (obj_type != ObjectType::INVALID) { - (*obj_iter)->objs_.push_back(uri); - if (order == WalkOrder::POSTORDER) - (*obj_iter)->expanded_.push_back(false); - } - } - - return Status::Ok(); -} - -Status StorageManagerCanonical::object_iter_begin( - ObjectIter** obj_iter, const char* path) { - // Sanity check - URI path_uri(path); - if (path_uri.is_invalid()) { - throw StorageManagerException( - "Cannot create object iterator; Invalid input path"); - } - - // Get all contents of path - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(path_uri, &uris)); - - // Create a new object iterator - *obj_iter = tdb_new(ObjectIter); - (*obj_iter)->order_ = WalkOrder::PREORDER; - (*obj_iter)->recursive_ = false; - - // Include the uris that are TileDB objects in the iterator state - ObjectType obj_type; - for (auto& uri : uris) { - throw_if_not_ok(object_type(resources_, uri, &obj_type)); - if (obj_type != ObjectType::INVALID) { - (*obj_iter)->objs_.push_back(uri); - } - } - - return Status::Ok(); -} - -void StorageManagerCanonical::object_iter_free(ObjectIter* obj_iter) { - tdb_delete(obj_iter); -} - -Status StorageManagerCanonical::object_iter_next( - ObjectIter* obj_iter, const char** path, ObjectType* type, bool* has_next) { - // Handle case there is no next - if (obj_iter->objs_.empty()) { - *has_next = false; - return Status::Ok(); - } - - // Retrieve next object - switch (obj_iter->order_) { - case WalkOrder::PREORDER: - RETURN_NOT_OK(object_iter_next_preorder(obj_iter, path, type, has_next)); - break; - case WalkOrder::POSTORDER: - RETURN_NOT_OK(object_iter_next_postorder(obj_iter, path, type, has_next)); - break; - } - - return Status::Ok(); -} - -Status StorageManagerCanonical::object_iter_next_postorder( - ObjectIter* obj_iter, const char** path, ObjectType* type, bool* has_next) { - // Get all contents of the next URI recursively till the bottom, - // if the front of the list has not been expanded - if (obj_iter->expanded_.front() == false) { - uint64_t obj_num; - do { - obj_num = obj_iter->objs_.size(); - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(obj_iter->objs_.front(), &uris)); - obj_iter->expanded_.front() = true; - - // Push the new TileDB objects in the front of the iterator's list - ObjectType obj_type; - for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - throw_if_not_ok(object_type(resources_, *it, &obj_type)); - if (obj_type != ObjectType::INVALID) { - obj_iter->objs_.push_front(*it); - obj_iter->expanded_.push_front(false); - } - } - } while (obj_num != obj_iter->objs_.size()); - } - - // Prepare the values to be returned - URI front_uri = obj_iter->objs_.front(); - obj_iter->next_ = front_uri.to_string(); - throw_if_not_ok(object_type(resources_, front_uri, type)); - *path = obj_iter->next_.c_str(); - *has_next = true; - - // Pop the front (next URI) of the iterator's object list - obj_iter->objs_.pop_front(); - obj_iter->expanded_.pop_front(); - - return Status::Ok(); -} - -Status StorageManagerCanonical::object_iter_next_preorder( - ObjectIter* obj_iter, const char** path, ObjectType* type, bool* has_next) { - // Prepare the values to be returned - URI front_uri = obj_iter->objs_.front(); - obj_iter->next_ = front_uri.to_string(); - throw_if_not_ok(object_type(resources_, front_uri, type)); - *path = obj_iter->next_.c_str(); - *has_next = true; - - // Pop the front (next URI) of the iterator's object list - obj_iter->objs_.pop_front(); - - // Return if no recursion is needed - if (!obj_iter->recursive_) - return Status::Ok(); - - // Get all contents of the next URI - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(front_uri, &uris)); - - // Push the new TileDB objects in the front of the iterator's list - ObjectType obj_type; - for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - throw_if_not_ok(object_type(resources_, *it, &obj_type)); - if (obj_type != ObjectType::INVALID) - obj_iter->objs_.push_front(*it); - } - - return Status::Ok(); -} - Status StorageManagerCanonical::query_submit(Query* query) { // Process the query QueryInProgress in_progress(this); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 67ab83edb02..16381e51bd6 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -74,30 +74,6 @@ enum class EncryptionType : uint8_t; /** The storage manager that manages pretty much everything in TileDB. */ class StorageManagerCanonical { public: - /* ********************************* */ - /* TYPE DEFINITIONS */ - /* ********************************* */ - - /** Enables iteration over TileDB objects in a path. */ - class ObjectIter { - public: - /** - * There is a one-to-one correspondence between `expanded_` and `objs_`. - * An `expanded_` value is `true` if the corresponding `objs_` path - * has been expanded to the paths it contains in a post ored traversal. - * This is not used in a preorder traversal. - */ - std::list expanded_; - /** The next URI in string format. */ - std::string next_; - /** The next objects to be visited. */ - std::list objs_; - /** The traversal order of the iterator. */ - WalkOrder order_; - /** `True` if the iterator will recursively visit the directory tree. */ - bool recursive_; - }; - /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -151,79 +127,6 @@ class StorageManagerCanonical { /** Returns the current map of any set tags. */ const std::unordered_map& tags() const; - /** - * Creates a new object iterator for the input path. The iteration - * in this case will be recursive in the entire directory tree rooted - * at `path`. - * - * @param obj_iter The object iterator to be created (memory is allocated for - * it by the function). - * @param path The path the iterator will target at. - * @param order The traversal order of the iterator. - * @return Status - */ - Status object_iter_begin( - ObjectIter** obj_iter, const char* path, WalkOrder order); - - /** - * Creates a new object iterator for the input path. The iteration will - * not be recursive, and only the children of `path` will be visited. - * - * @param obj_iter The object iterator to be created (memory is allocated for - * it by the function). - * @param path The path the iterator will target at. - * @return Status - */ - Status object_iter_begin(ObjectIter** obj_iter, const char* path); - - /** Frees the object iterator. */ - void object_iter_free(ObjectIter* obj_iter); - - /** - * Retrieves the next object path and type. - * - * @param obj_iter The object iterator. - * @param path The object path that is retrieved. - * @param type The object type that is retrieved. - * @param has_next True if an object path was retrieved and false otherwise. - * @return Status - */ - Status object_iter_next( - ObjectIter* obj_iter, - const char** path, - ObjectType* type, - bool* has_next); - - /** - * Retrieves the next object in the post-order traversal. - * - * @param obj_iter The object iterator. - * @param path The object path that is retrieved. - * @param type The object type that is retrieved. - * @param has_next True if an object path was retrieved and false otherwise. - * @return Status - */ - Status object_iter_next_postorder( - ObjectIter* obj_iter, - const char** path, - ObjectType* type, - bool* has_next); - - /** - * Retrieves the next object in the post-order traversal. - * - * @param obj_iter The object iterator. - * @param path The object path that is retrieved. - * @param type The object type that is retrieved. - * @param has_next True if an object path was retrieved and false otherwise. - * @return Status - */ - Status object_iter_next_preorder( - ObjectIter* obj_iter, - const char** path, - ObjectType* type, - bool* has_next); - /** Submits a query for (sync) execution. */ Status query_submit(Query* query); diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 063ffccaa1e..06fcf845338 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -91,8 +91,7 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { tiledb::sm::Array a(ctx.resources(), tiledb::sm::URI{"mem://junk"}); tiledb::sm::EncryptionKey ek; CHECK(ek.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0).ok()); - CHECK(tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek) - .ok()); + tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek); CHECK(a.open( tiledb::sm::QueryType::READ, tiledb::sm::EncryptionType::NO_ENCRYPTION, diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 976f333291c..7ebaef581d7 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -196,8 +196,8 @@ void GenericTileIO::write_generic( // Filter tile assert(!tile->filtered()); - throw_if_not_ok(header.filters.run_forward( - &resources_.stats(), tile.get(), nullptr, &resources_.compute_tp())); + header.filters.run_forward( + &resources_.stats(), tile.get(), nullptr, &resources_.compute_tp()); header.persisted_size = tile->filtered_buffer().size(); assert(tile->filtered()); diff --git a/tiledb/type/range/range.h b/tiledb/type/range/range.h index c3011983a76..143474f0d0e 100644 --- a/tiledb/type/range/range.h +++ b/tiledb/type/range/range.h @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/logger_public.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/tag.h" #include "tiledb/sm/enums/datatype.h" @@ -61,6 +62,9 @@ class Range { * The range is stored as a sequence of bytes with manual memory layout. The * memory layout is different for fixed-size and variable-size types. * + * All constructors accept an optional allocator argument, whose default value + * is an allocator using std::pmr::get_default_resource. + * * Fixed-size type T: * lower limit: sizeof(T) * upper limit: sizeof(T) @@ -68,7 +72,12 @@ class Range { * lower limit: range_start_size_ * upper limit: range_size() - range_start_size_ */ - std::vector range_; + tdb::pmr::vector range_; + + /** + * Alias to the allocator type used by the range vector. + */ + using allocator_type = decltype(range_)::allocator_type; /** * The byte size of the lower limit of the range. Applicable only to variable @@ -104,31 +113,40 @@ class Range { * * @param size Size of the storage array in bytes */ - explicit Range(size_t size) - : range_(size) { + explicit Range(size_t size, const allocator_type& alloc) + : range_(size, alloc) { } public: /** Default constructor. */ - Range() - : range_{} { + Range(const allocator_type& alloc = {}) + : range_(alloc) { } /** Constructs a range and sets fixed data. */ - Range(const void* range, uint64_t range_size) - : Range() { + Range( + const void* range, uint64_t range_size, const allocator_type& alloc = {}) + : Range(alloc) { set_range(range, range_size); } /** Constructs a range and sets variable data. */ - Range(const void* range, uint64_t range_size, uint64_t range_start_size) - : Range() { + Range( + const void* range, + uint64_t range_size, + uint64_t range_start_size, + const allocator_type& alloc = {}) + : Range(alloc) { set_range(range, range_size, range_start_size); } /** Constructs a range and sets fixed data. */ - Range(const void* start, const void* end, uint64_t type_size) - : Range() { + Range( + const void* start, + const void* end, + uint64_t type_size, + const allocator_type& alloc = {}) + : Range(alloc) { set_range_fixed(start, end, type_size); } @@ -137,14 +155,18 @@ class Range { const void* start, uint64_t start_size, const void* end, - uint64_t end_size) - : Range() { + uint64_t end_size, + const allocator_type& alloc = {}) + : Range(alloc) { set_range_var(start, start_size, end, end_size); } /** Constructs a range and sets variable data. */ - Range(const std::string& s1, const std::string& s2) - : Range() { + Range( + const std::string& s1, + const std::string& s2, + const allocator_type& alloc = {}) + : Range(alloc) { set_str_range(s1, s2); } @@ -156,8 +178,8 @@ class Range { * this constructor for language type that we use as fixed-length data. */ template - Range(Tag, T first, T second) - : Range(2 * sizeof(T)) { + Range(Tag, T first, T second, const allocator_type& alloc = {}) + : Range(2 * sizeof(T), alloc) { auto d{reinterpret_cast(range_.data())}; d[0] = first; d[1] = second;