diff --git a/.gitmodules b/.gitmodules index 7a9f6004..110773e1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "acquire-common"] path = acquire-common url = ../acquire-common.git +[submodule "acquire-zarr"] + path = acquire-zarr + url = ../acquire-zarr.git \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 0419a48b..ae6bd943 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,11 +19,22 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -option(NOTEST "Disable all tests" OFF) -option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) +option(NO_UNIT_TESTS "Disable unit tests" OFF) +option(WITH_EXAMPLES "Build examples" OFF) + +if (CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) + include(CTest) +endif () add_subdirectory(src) -add_subdirectory(tests) -add_subdirectory(examples) +if (BUILD_TESTING) + add_subdirectory(tests) +else () + message(STATUS "Skipping test targets") +endif () + +if (WITH_EXAMPLES) + add_subdirectory(examples) +endif () include(CPack) diff --git a/acquire-zarr b/acquire-zarr new file mode 160000 index 00000000..b2055947 --- /dev/null +++ b/acquire-zarr @@ -0,0 +1 @@ +Subproject commit b205594758d98a55e9da2b727a30df79677321b9 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5fe945b0..13cebcd3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,41 @@ -add_subdirectory(logger) -add_subdirectory(streaming) +if (NOT TARGET acquire-core-logger) + set(NOTEST ON) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) +endif () -if (BUILD_ACQUIRE_DRIVER_ZARR) - add_subdirectory(driver) +if (NOT TARGET acquire-zarr) + set(BUILD_TESTING_TMP ${BUILD_TESTING}) + set(BUILD_TESTING OFF) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-zarr ${CMAKE_CURRENT_BINARY_DIR}/acquire-zarr) + set(BUILD_TESTING ${BUILD_TESTING_TMP}) endif () + +set(tgt acquire-driver-zarr) +add_library(${tgt} MODULE + zarr.storage.hh + zarr.storage.cpp + zarr.driver.c +) + +target_include_directories(${tgt} PRIVATE + $ +) + +target_enable_simd(${tgt}) +target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + acquire-logger + acquire-zarr + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} LIBRARY DESTINATION lib) \ No newline at end of file diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt deleted file mode 100644 index 2ca6a2f2..00000000 --- a/src/driver/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -if (NOT TARGET acquire-core-logger) - add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) -endif () - -set(tgt acquire-driver-zarr) -add_library(${tgt} MODULE - zarr.storage.hh - zarr.storage.cpp - zarr.driver.c -) - -target_include_directories(${tgt} PRIVATE - $ -) - -target_enable_simd(${tgt}) -target_link_libraries(${tgt} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - acquire-zarr - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp -) - -set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS ${tgt} LIBRARY DESTINATION lib) \ No newline at end of file diff --git a/src/logger/CMakeLists.txt b/src/logger/CMakeLists.txt deleted file mode 100644 index 9d7c9998..00000000 --- a/src/logger/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -add_library(acquire-logger - logger.hh - logger.cpp -) - -target_include_directories(acquire-logger - PRIVATE - $ -) - -set_target_properties(acquire-logger PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS acquire-logger - LIBRARY DESTINATION lib -) \ No newline at end of file diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp deleted file mode 100644 index c7af5582..00000000 --- a/src/logger/logger.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "logger.hh" - -#include -#include -#include - -LogLevel Logger::current_level_ = LogLevel_Info; -std::mutex Logger::log_mutex_{}; - -void -Logger::set_log_level(LogLevel level) -{ - if (level < LogLevel_Debug || level > LogLevel_Error) { - throw std::invalid_argument("Invalid log level"); - } - - current_level_ = level; -} - -LogLevel -Logger::get_log_level() -{ - return current_level_; -} - -std::string -Logger::get_timestamp_() -{ - - auto now = std::chrono::system_clock::now(); - auto time = std::chrono::system_clock::to_time_t(now); - auto ms = std::chrono::duration_cast( - now.time_since_epoch()) % - 1000; - - std::tm tm{}; -#if defined(_WIN32) - localtime_s(&tm, &time); -#else - localtime_r(&time, &tm); -#endif - - std::ostringstream ss; - ss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S") << '.' << std::setfill('0') - << std::setw(3) << ms.count(); - - return ss.str(); -} \ No newline at end of file diff --git a/src/logger/logger.hh b/src/logger/logger.hh deleted file mode 100644 index c5046990..00000000 --- a/src/logger/logger.hh +++ /dev/null @@ -1,80 +0,0 @@ -#include "logger.types.h" - -#include -#include -#include -#include - -class Logger -{ - public: - static void set_log_level(LogLevel level); - static LogLevel get_log_level(); - - template - static std::string log(LogLevel level, - const char* file, - int line, - const char* func, - Args&&... args) - { - namespace fs = std::filesystem; - - std::scoped_lock lock(log_mutex_); - - std::string prefix; - auto stream = &std::cout; - - switch (level) { - case LogLevel_Debug: - prefix = "[DEBUG] "; - break; - case LogLevel_Info: - prefix = "[INFO] "; - break; - case LogLevel_Warning: - prefix = "[WARNING] "; - break; - default: - prefix = "[ERROR] "; - stream = &std::cerr; - break; - } - - fs::path filepath(file); - std::string filename = filepath.filename().string(); - - std::ostringstream ss; - ss << get_timestamp_() << " " << prefix << filename << ":" << line - << " " << func << ": "; - - format_arg_(ss, std::forward(args)...); - - std::string message = ss.str(); - *stream << message << std::endl; - - return message; - } - - private: - static LogLevel current_level_; - static std::mutex log_mutex_; - - static void format_arg_(std::ostream& ss) {}; // base case - template - static void format_arg_(std::ostream& ss, T&& arg, Args&&... args) { - ss << std::forward(arg); - format_arg_(ss, std::forward(args)...); - } - - static std::string get_timestamp_(); -}; - -#define LOG_DEBUG(...) \ - Logger::log(LogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) -#define LOG_INFO(...) \ - Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) -#define LOG_WARNING(...) \ - Logger::log(LogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) -#define LOG_ERROR(...) \ - Logger::log(LogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) diff --git a/src/logger/logger.types.h b/src/logger/logger.types.h deleted file mode 100644 index 86d9c0b1..00000000 --- a/src/logger/logger.types.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once - -typedef enum { - LogLevel_Debug, - LogLevel_Info, - LogLevel_Warning, - LogLevel_Error -} LogLevel; diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt deleted file mode 100644 index 3ab50ace..00000000 --- a/src/streaming/CMakeLists.txt +++ /dev/null @@ -1,65 +0,0 @@ -set(tgt acquire-zarr) - -add_library(${tgt} - macros.hh - acquire.zarr.cpp - zarr.dimension.hh - zarr.dimension.cpp - zarr.stream.hh - zarr.stream.cpp - zarr.common.hh - zarr.common.cpp - blosc.compression.params.hh - blosc.compression.params.cpp - thread.pool.hh - thread.pool.cpp - s3.connection.hh - s3.connection.cpp - sink.hh - sink.cpp - file.sink.hh - file.sink.cpp - s3.sink.hh - s3.sink.cpp - sink.creator.hh - sink.creator.cpp - array.writer.hh - array.writer.cpp - zarrv2.array.writer.hh - zarrv2.array.writer.cpp - zarrv3.array.writer.hh - zarrv3.array.writer.cpp -) - -target_include_directories(${tgt} - PUBLIC - $ - PRIVATE - $ - $ -) - -target_link_libraries(${tgt} PRIVATE - acquire-logger - blosc_static - miniocpp::miniocpp -) - -target_compile_definitions(${tgt} PRIVATE - "ACQUIRE_ZARR_API_VERSION=\"0.0.1\"" -) - -set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - POSITION_INDEPENDENT_CODE ON -) - -install(TARGETS ${tgt} - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib -) - -# Install public header files -install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/ - DESTINATION include -) \ No newline at end of file diff --git a/src/streaming/acquire.zarr.cpp b/src/streaming/acquire.zarr.cpp deleted file mode 100644 index 91f7f5fd..00000000 --- a/src/streaming/acquire.zarr.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include "acquire.zarr.h" -#include "zarr.stream.hh" -#include "macros.hh" - -#include // uint32_t - -extern "C" -{ - const char* Zarr_get_api_version() - { - return ACQUIRE_ZARR_API_VERSION; - } - - ZarrStatusCode Zarr_set_log_level(ZarrLogLevel level_) - { - LogLevel level; - switch (level_) { - case ZarrLogLevel_Debug: - level = LogLevel_Debug; - break; - case ZarrLogLevel_Info: - level = LogLevel_Info; - break; - case ZarrLogLevel_Warning: - level = LogLevel_Warning; - break; - case ZarrLogLevel_Error: - level = LogLevel_Error; - break; - default: - return ZarrStatusCode_InvalidArgument; - } - - try { - Logger::set_log_level(level); - } catch (const std::exception& e) { - LOG_ERROR("Error setting log level: ", e.what()); - return ZarrStatusCode_InternalError; - } - return ZarrStatusCode_Success; - } - - ZarrLogLevel Zarr_get_log_level() - { - ZarrLogLevel level; - switch (Logger::get_log_level()) { - case LogLevel_Debug: - level = ZarrLogLevel_Debug; - break; - case LogLevel_Info: - level = ZarrLogLevel_Info; - break; - case LogLevel_Warning: - level = ZarrLogLevel_Warning; - break; - default: - level = ZarrLogLevel_Error; - break; - } - return level; - } - - const char* Zarr_get_status_message(ZarrStatusCode code) - { - switch (code) { - case ZarrStatusCode_Success: - return "Success"; - case ZarrStatusCode_InvalidArgument: - return "Invalid argument"; - case ZarrStatusCode_Overflow: - return "Buffer overflow"; - case ZarrStatusCode_InvalidIndex: - return "Invalid index"; - case ZarrStatusCode_NotYetImplemented: - return "Not yet implemented"; - case ZarrStatusCode_InternalError: - return "Internal error"; - case ZarrStatusCode_OutOfMemory: - return "Out of memory"; - case ZarrStatusCode_IOError: - return "I/O error"; - case ZarrStatusCode_CompressionError: - return "Compression error"; - case ZarrStatusCode_InvalidSettings: - return "Invalid settings"; - default: - return "Unknown error"; - } - } - - ZarrStatusCode ZarrStreamSettings_create_dimension_array( - struct ZarrStreamSettings_s* settings, - size_t dimension_count) - { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(dimension_count >= 3, - "Invalid dimension count: ", - dimension_count); - - ZarrDimensionProperties* dimensions = nullptr; - - try { - dimensions = new ZarrDimensionProperties[dimension_count]; - } catch (const std::bad_alloc&) { - LOG_ERROR("Failed to allocate memory for dimensions"); - return ZarrStatusCode_OutOfMemory; - } - - ZarrStreamSettings_destroy_dimension_array(settings); - settings->dimensions = dimensions; - settings->dimension_count = dimension_count; - - return ZarrStatusCode_Success; - } - - void ZarrStreamSettings_destroy_dimension_array( - struct ZarrStreamSettings_s* settings) - { - if (settings == nullptr) { - return; - } - - if (settings->dimensions != nullptr) { - delete[] settings->dimensions; - settings->dimensions = nullptr; - } - settings->dimension_count = 0; - } - - ZarrStream_s* ZarrStream_create(struct ZarrStreamSettings_s* settings) - { - - ZarrStream_s* stream = nullptr; - - try { - stream = new ZarrStream_s(settings); - } catch (const std::bad_alloc&) { - LOG_ERROR("Failed to allocate memory for Zarr stream"); - } catch (const std::exception& e) { - LOG_ERROR("Error creating Zarr stream: ", e.what()); - } - - return stream; - } - - void ZarrStream_destroy(struct ZarrStream_s* stream) - { - if (!finalize_stream(stream)) { - return; - } - - delete stream; - } - - ZarrStatusCode ZarrStream_append(struct ZarrStream_s* stream, - const void* data, - size_t bytes_in, - size_t* bytes_out) - { - EXPECT_VALID_ARGUMENT(stream, "Null pointer: stream"); - EXPECT_VALID_ARGUMENT(data, "Null pointer: data"); - EXPECT_VALID_ARGUMENT(bytes_out, "Null pointer: bytes_out"); - - try { - *bytes_out = stream->append(data, bytes_in); - } catch (const std::exception& e) { - LOG_ERROR("Error appending data: ", e.what()); - return ZarrStatusCode_InternalError; - } - - return ZarrStatusCode_Success; - } -} \ No newline at end of file diff --git a/src/streaming/array.writer.cpp b/src/streaming/array.writer.cpp deleted file mode 100644 index f1a3b55b..00000000 --- a/src/streaming/array.writer.cpp +++ /dev/null @@ -1,466 +0,0 @@ -#include "macros.hh" -#include "array.writer.hh" -#include "zarr.common.hh" -#include "zarr.stream.hh" -#include "sink.creator.hh" - -#include -#include -#include -#include - -#ifdef min -#undef min -#endif - -bool -zarr::downsample(const ArrayWriterConfig& config, - ArrayWriterConfig& downsampled_config) -{ - // downsample dimensions - std::vector downsampled_dims(config.dimensions->ndims()); - for (auto i = 0; i < config.dimensions->ndims(); ++i) { - const auto& dim = config.dimensions->at(i); - // don't downsample channels - if (dim.type == ZarrDimensionType_Channel) { - downsampled_dims[i] = dim; - } else { - const uint32_t array_size_px = - (dim.array_size_px + (dim.array_size_px % 2)) / 2; - - const uint32_t chunk_size_px = - dim.array_size_px == 0 - ? dim.chunk_size_px - : std::min(dim.chunk_size_px, array_size_px); - - CHECK(chunk_size_px); - const uint32_t n_chunks = - (array_size_px + chunk_size_px - 1) / chunk_size_px; - - const uint32_t shard_size_chunks = - dim.array_size_px == 0 - ? 1 - : std::min(n_chunks, dim.shard_size_chunks); - - downsampled_dims[i] = { dim.name, - dim.type, - array_size_px, - chunk_size_px, - shard_size_chunks }; - } - } - downsampled_config.dimensions = std::make_shared( - std::move(downsampled_dims), config.dtype); - - downsampled_config.level_of_detail = config.level_of_detail + 1; - downsampled_config.bucket_name = config.bucket_name; - downsampled_config.store_path = config.store_path; - - downsampled_config.dtype = config.dtype; - - // copy the Blosc compression parameters - downsampled_config.compression_params = config.compression_params; - - // can we downsample downsampled_config? - for (auto i = 0; i < config.dimensions->ndims(); ++i) { - // downsampling made the chunk size strictly smaller - const auto& dim = config.dimensions->at(i); - const auto& downsampled_dim = downsampled_config.dimensions->at(i); - - if (dim.chunk_size_px > downsampled_dim.chunk_size_px) { - return false; - } - } - - return true; -} - -/// Writer -zarr::ArrayWriter::ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool) - : ArrayWriter(std::move(config), thread_pool, nullptr) -{ -} - -zarr::ArrayWriter::ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool) - : config_{ config } - , thread_pool_{ thread_pool } - , s3_connection_pool_{ s3_connection_pool } - , bytes_to_flush_{ 0 } - , frames_written_{ 0 } - , append_chunk_index_{ 0 } - , is_finalizing_{ false } -{ -} - -size_t -zarr::ArrayWriter::write_frame(std::span data) -{ - const auto nbytes_data = data.size(); - const auto nbytes_frame = - bytes_of_frame(*config_.dimensions, config_.dtype); - - if (nbytes_frame != nbytes_data) { - LOG_ERROR("Frame size mismatch: expected ", - nbytes_frame, - ", got ", - nbytes_data, - ". Skipping"); - return 0; - } - - if (chunk_buffers_.empty()) { - make_buffers_(); - } - - // split the incoming frame into tiles and write them to the chunk - // buffers - const auto bytes_written = write_frame_to_chunks_(data); - EXPECT(bytes_written == nbytes_data, "Failed to write frame to chunks"); - - LOG_DEBUG("Wrote ", bytes_written, " bytes of frame ", frames_written_); - bytes_to_flush_ += bytes_written; - ++frames_written_; - - if (should_flush_()) { - flush_(); - } - - return bytes_written; -} - -bool -zarr::ArrayWriter::is_s3_array_() const -{ - return config_.bucket_name.has_value(); -} - -bool -zarr::ArrayWriter::make_data_sinks_() -{ - std::string data_root; - std::function parts_along_dimension; - switch (version_()) { - case ZarrVersion_2: - parts_along_dimension = chunks_along_dimension; - data_root = config_.store_path + "/" + - std::to_string(config_.level_of_detail) + "/" + - std::to_string(append_chunk_index_); - break; - case ZarrVersion_3: - parts_along_dimension = shards_along_dimension; - data_root = config_.store_path + "/data/root/" + - std::to_string(config_.level_of_detail) + "/c" + - std::to_string(append_chunk_index_); - break; - default: - LOG_ERROR("Unsupported Zarr version"); - return false; - } - - SinkCreator creator(thread_pool_, s3_connection_pool_); - - if (is_s3_array_()) { - if (!creator.make_data_sinks(*config_.bucket_name, - data_root, - config_.dimensions.get(), - parts_along_dimension, - data_sinks_)) { - LOG_ERROR("Failed to create data sinks in ", - data_root, - " for bucket ", - *config_.bucket_name); - return false; - } - } else if (!creator.make_data_sinks(data_root, - config_.dimensions.get(), - parts_along_dimension, - data_sinks_)) { - LOG_ERROR("Failed to create data sinks in ", data_root); - return false; - } - - return true; -} - -bool -zarr::ArrayWriter::make_metadata_sink_() -{ - if (metadata_sink_) { - LOG_INFO("Metadata sink already exists"); - return true; - } - - std::string metadata_path; - switch (version_()) { - case ZarrVersion_2: - metadata_path = config_.store_path + "/" + - std::to_string(config_.level_of_detail) + - "/.zarray"; - break; - case ZarrVersion_3: - metadata_path = config_.store_path + "/meta/root/" + - std::to_string(config_.level_of_detail) + - ".array.json"; - break; - default: - LOG_ERROR("Unsupported Zarr version"); - return false; - } - - if (is_s3_array_()) { - SinkCreator creator(thread_pool_, s3_connection_pool_); - metadata_sink_ = - creator.make_sink(*config_.bucket_name, metadata_path); - } else { - metadata_sink_ = zarr::SinkCreator::make_sink(metadata_path); - } - - if (!metadata_sink_) { - LOG_ERROR("Failed to create metadata sink: ", metadata_path); - return false; - } - - return true; -} - -void -zarr::ArrayWriter::make_buffers_() noexcept -{ - LOG_DEBUG("Creating chunk buffers"); - - const size_t n_chunks = config_.dimensions->number_of_chunks_in_memory(); - chunk_buffers_.resize(n_chunks); // no-op if already the correct size - - const auto nbytes = config_.dimensions->bytes_per_chunk(); - - for (auto& buf : chunk_buffers_) { - buf.resize(nbytes); - std::fill(buf.begin(), buf.end(), std::byte(0)); - } -} - -size_t -zarr::ArrayWriter::write_frame_to_chunks_(std::span data) -{ - // break the frame into tiles and write them to the chunk buffers - const auto bytes_per_px = bytes_of_type(config_.dtype); - - const auto& dimensions = config_.dimensions; - - const auto& x_dim = dimensions->width_dim(); - const auto frame_cols = x_dim.array_size_px; - const auto tile_cols = x_dim.chunk_size_px; - - const auto& y_dim = dimensions->height_dim(); - const auto frame_rows = y_dim.array_size_px; - const auto tile_rows = y_dim.chunk_size_px; - - if (tile_cols == 0 || tile_rows == 0) { - return 0; - } - - const auto bytes_per_row = tile_cols * bytes_per_px; - - size_t bytes_written = 0; - - const auto n_tiles_x = (frame_cols + tile_cols - 1) / tile_cols; - const auto n_tiles_y = (frame_rows + tile_rows - 1) / tile_rows; - - // don't take the frame id from the incoming frame, as the camera may have - // dropped frames - const auto frame_id = frames_written_; - - // offset among the chunks in the lattice - const auto group_offset = dimensions->tile_group_offset(frame_id); - // offset within the chunk - const auto chunk_offset = - static_cast(dimensions->chunk_internal_offset(frame_id)); - - for (auto i = 0; i < n_tiles_y; ++i) { - // TODO (aliddell): we can optimize this when tiles_per_frame_x_ is 1 - for (auto j = 0; j < n_tiles_x; ++j) { - const auto c = group_offset + i * n_tiles_x + j; - auto& chunk = chunk_buffers_[c]; - auto chunk_it = chunk.begin() + chunk_offset; - - for (auto k = 0; k < tile_rows; ++k) { - const auto frame_row = i * tile_rows + k; - if (frame_row < frame_rows) { - const auto frame_col = j * tile_cols; - - const auto region_width = - std::min(frame_col + tile_cols, frame_cols) - frame_col; - - const auto region_start = static_cast( - bytes_per_px * (frame_row * frame_cols + frame_col)); - const auto nbytes = - static_cast(region_width * bytes_per_px); - const auto region_stop = region_start + nbytes; - if (region_stop > data.size()) { - LOG_ERROR("Buffer overflow"); - return bytes_written; - } - - // copy region - if (nbytes > std::distance(chunk_it, chunk.end())) { - LOG_ERROR("Buffer overflow"); - return bytes_written; - } - std::copy(data.begin() + region_start, - data.begin() + region_stop, - chunk_it); - - bytes_written += (region_stop - region_start); - } - chunk_it += static_cast(bytes_per_row); - } - } - } - - return bytes_written; -} - -bool -zarr::ArrayWriter::should_flush_() const -{ - const auto& dims = config_.dimensions; - size_t frames_before_flush = dims->final_dim().chunk_size_px; - for (auto i = 1; i < dims->ndims() - 2; ++i) { - frames_before_flush *= dims->at(i).array_size_px; - } - - CHECK(frames_before_flush > 0); - return frames_written_ % frames_before_flush == 0; -} - -void -zarr::ArrayWriter::compress_buffers_() -{ - if (!config_.compression_params.has_value()) { - return; - } - - LOG_DEBUG("Compressing"); - - BloscCompressionParams params = config_.compression_params.value(); - const auto bytes_per_px = bytes_of_type(config_.dtype); - - std::scoped_lock lock(buffers_mutex_); - std::latch latch(chunk_buffers_.size()); - for (auto& chunk : chunk_buffers_) { - EXPECT(thread_pool_->push_job( - [¶ms, buf = &chunk, bytes_per_px, &latch]( - std::string& err) -> bool { - bool success = false; - const size_t bytes_of_chunk = buf->size(); - - try { - const auto tmp_size = - bytes_of_chunk + BLOSC_MAX_OVERHEAD; - std::vector tmp(tmp_size); - const auto nb = - blosc_compress_ctx(params.clevel, - params.shuffle, - bytes_per_px, - bytes_of_chunk, - buf->data(), - tmp.data(), - tmp_size, - params.codec_id.c_str(), - 0 /* blocksize - 0:automatic */, - 1); - - tmp.resize(nb); - buf->swap(tmp); - - success = true; - } catch (const std::exception& exc) { - err = "Failed to compress chunk: " + - std::string(exc.what()); - } - latch.count_down(); - - return success; - }), - "Failed to push to job queue"); - } - - // wait for all threads to finish - latch.wait(); -} - -void -zarr::ArrayWriter::flush_() -{ - if (bytes_to_flush_ == 0) { - return; - } - - // compress buffers and write out - compress_buffers_(); - CHECK(flush_impl_()); - - const auto should_rollover = should_rollover_(); - if (should_rollover) { - rollover_(); - } - - if (should_rollover || is_finalizing_) { - CHECK(write_array_metadata_()); - } - - // reset buffers - make_buffers_(); - - // reset state - bytes_to_flush_ = 0; -} - -void -zarr::ArrayWriter::close_sinks_() -{ - for (auto i = 0; i < data_sinks_.size(); ++i) { - EXPECT(finalize_sink(std::move(data_sinks_[i])), - "Failed to finalize sink ", - i); - } - data_sinks_.clear(); -} - -void -zarr::ArrayWriter::rollover_() -{ - LOG_DEBUG("Rolling over"); - - close_sinks_(); - ++append_chunk_index_; -} - -bool -zarr::finalize_array(std::unique_ptr&& writer) -{ - if (writer == nullptr) { - LOG_INFO("Array writer is null. Nothing to finalize."); - return true; - } - - writer->is_finalizing_ = true; - try { - writer->flush_(); // data sinks finalized here - } catch (const std::exception& exc) { - LOG_ERROR("Failed to finalize array writer: ", exc.what()); - return false; - } - - if (!finalize_sink(std::move(writer->metadata_sink_))) { - LOG_ERROR("Failed to finalize metadata sink"); - return false; - } - - writer.reset(); - return true; -} diff --git a/src/streaming/array.writer.hh b/src/streaming/array.writer.hh deleted file mode 100644 index cf934f4d..00000000 --- a/src/streaming/array.writer.hh +++ /dev/null @@ -1,106 +0,0 @@ -#pragma once - -#include "zarr.dimension.hh" -#include "thread.pool.hh" -#include "s3.connection.hh" -#include "blosc.compression.params.hh" -#include "file.sink.hh" - -#include -#include - -namespace fs = std::filesystem; - -namespace zarr { -struct ArrayWriterConfig -{ - std::shared_ptr dimensions; - ZarrDataType dtype; - int level_of_detail; - std::optional bucket_name; - std::string store_path; - std::optional compression_params; -}; - -/** - * @brief Downsample the array writer configuration to a lower resolution. - * @param[in] config The original array writer configuration. - * @param[out] downsampled_config The downsampled array writer configuration. - * @return True if @p downsampled_config can be downsampled further. - * This is determined by the chunk size in @p config. This function will return - * false if and only if downsampling brings one or more dimensions lower than - * the chunk size along that dimension. - */ -[[nodiscard]] -bool -downsample(const ArrayWriterConfig& config, - ArrayWriterConfig& downsampled_config); - -class ArrayWriter -{ - public: - ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool); - - ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool); - - virtual ~ArrayWriter() = default; - - /** - * @brief Write a frame to the array. - * @param data The frame data. - * @return The number of bytes written. - */ - [[nodiscard]] size_t write_frame(std::span data); - - protected: - ArrayWriterConfig config_; - - /// Chunking - std::vector> chunk_buffers_; - - /// Filesystem - std::vector> data_sinks_; - std::unique_ptr metadata_sink_; - - /// Multithreading - std::shared_ptr thread_pool_; - std::mutex buffers_mutex_; - - /// Bookkeeping - uint64_t bytes_to_flush_; - uint32_t frames_written_; - uint32_t append_chunk_index_; - bool is_finalizing_; - - std::shared_ptr s3_connection_pool_; - - virtual ZarrVersion version_() const = 0; - - bool is_s3_array_() const; - - [[nodiscard]] bool make_data_sinks_(); - [[nodiscard]] bool make_metadata_sink_(); - void make_buffers_() noexcept; - - bool should_flush_() const; - virtual bool should_rollover_() const = 0; - - size_t write_frame_to_chunks_(std::span data); - void compress_buffers_(); - - void flush_(); - [[nodiscard]] virtual bool flush_impl_() = 0; - void rollover_(); - - [[nodiscard]] virtual bool write_array_metadata_() = 0; - - void close_sinks_(); - - friend bool finalize_array(std::unique_ptr&& writer); -}; - -bool finalize_array(std::unique_ptr&& writer); -} // namespace zarr diff --git a/src/streaming/blosc.compression.params.cpp b/src/streaming/blosc.compression.params.cpp deleted file mode 100644 index 19af8e8c..00000000 --- a/src/streaming/blosc.compression.params.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "blosc.compression.params.hh" - -const char* -zarr::blosc_codec_to_string(ZarrCompressionCodec codec) -{ - switch (codec) { - case ZarrCompressionCodec_BloscZstd: - return "zstd"; - case ZarrCompressionCodec_BloscLZ4: - return "lz4"; - default: - return "unrecognized codec"; - } -} - -zarr::BloscCompressionParams::BloscCompressionParams(std::string_view codec_id, - uint8_t clevel, - uint8_t shuffle) - : codec_id{ codec_id } - , clevel{ clevel } - , shuffle{ shuffle } -{ -} diff --git a/src/streaming/blosc.compression.params.hh b/src/streaming/blosc.compression.params.hh deleted file mode 100644 index dcd9cd42..00000000 --- a/src/streaming/blosc.compression.params.hh +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "acquire.zarr.h" - -#include - -#include -#include - -namespace zarr { -const char* -blosc_codec_to_string(ZarrCompressionCodec codec); - -struct BloscCompressionParams -{ - std::string codec_id; - uint8_t clevel{ 1 }; - uint8_t shuffle{ 1 }; - - BloscCompressionParams() = default; - BloscCompressionParams(std::string_view codec_id, - uint8_t clevel, - uint8_t shuffle); -}; -} // namespace zarr diff --git a/src/streaming/file.sink.cpp b/src/streaming/file.sink.cpp deleted file mode 100644 index 312d90ca..00000000 --- a/src/streaming/file.sink.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "file.sink.hh" - -#include - -namespace fs = std::filesystem; - -zarr::FileSink::FileSink(std::string_view filename) -: file_(filename.data(), std::ios::binary | std::ios::trunc) -{ -} - -bool -zarr::FileSink::write(size_t offset, std::span data) -{ - const auto bytes_of_buf = data.size(); - if (data.data() == nullptr || bytes_of_buf == 0) { - return true; - } - - file_.seekp(offset); - file_.write(reinterpret_cast(data.data()), bytes_of_buf); - return true; -} - -bool -zarr::FileSink::flush_() -{ - file_.flush(); - return true; -} diff --git a/src/streaming/file.sink.hh b/src/streaming/file.sink.hh deleted file mode 100644 index 1849f796..00000000 --- a/src/streaming/file.sink.hh +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "sink.hh" - -#include -#include - -namespace zarr { -class FileSink : public Sink -{ - public: - explicit FileSink(std::string_view filename); - - bool write(size_t offset, std::span data) override; - - protected: - bool flush_() override; - - private: - std::ofstream file_; -}; -} // namespace zarr diff --git a/src/streaming/macros.hh b/src/streaming/macros.hh deleted file mode 100644 index ac006165..00000000 --- a/src/streaming/macros.hh +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "logger.hh" - -#define EXPECT(e, ...) \ - do { \ - if (!(e)) { \ - const std::string __err = LOG_ERROR(__VA_ARGS__); \ - throw std::runtime_error(__err); \ - } \ - } while (0) -#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t", #e) - -#define EXPECT_VALID_ARGUMENT(e, ...) \ - do { \ - if (!(e)) { \ - LOG_ERROR(__VA_ARGS__); \ - return ZarrStatusCode_InvalidArgument; \ - } \ - } while (0) - -#define EXPECT_VALID_INDEX(e, ...) \ - do { \ - if (!(e)) { \ - LOG_ERROR(__VA_ARGS__); \ - return ZarrStatusCode_InvalidIndex; \ - } \ - } while (0) \ No newline at end of file diff --git a/src/streaming/s3.connection.cpp b/src/streaming/s3.connection.cpp deleted file mode 100644 index d4ea3f37..00000000 --- a/src/streaming/s3.connection.cpp +++ /dev/null @@ -1,264 +0,0 @@ -#include "macros.hh" -#include "s3.connection.hh" - -#include - -#include -#include -#include - -zarr::S3Connection::S3Connection(const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key) -{ - minio::s3::BaseUrl url(endpoint); - url.https = endpoint.starts_with("https"); - - provider_ = std::make_unique( - access_key_id, secret_access_key); - client_ = std::make_unique(url, provider_.get()); - - CHECK(client_); -} - -bool -zarr::S3Connection::is_connection_valid() -{ - return static_cast(client_->ListBuckets()); -} - -bool -zarr::S3Connection::bucket_exists(std::string_view bucket_name) -{ - minio::s3::BucketExistsArgs args; - args.bucket = bucket_name; - - auto response = client_->BucketExists(args); - return response.exist; -} - -bool -zarr::S3Connection::object_exists(std::string_view bucket_name, - std::string_view object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->StatObject(args); - // casts to true if response code in 200 range and error message is empty - return static_cast(response); -} - -std::string -zarr::S3Connection::put_object(std::string_view bucket_name, - std::string_view object_name, - std::span data) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - EXPECT(!data.empty(), "Data must not be empty."); - - minio::utils::CharBuffer buffer(reinterpret_cast(data.data()), - data.size()); - std::basic_istream stream(&buffer); - - LOG_DEBUG("Putting object ", object_name, " in bucket ", bucket_name); - minio::s3::PutObjectArgs args(stream, static_cast(data.size()), 0); - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->PutObject(args); - if (!response) { - LOG_ERROR("Failed to put object ", - object_name, - " in bucket ", - bucket_name, - ": ", - response.Error().String()); - return {}; - } - - return response.etag; -} - -bool -zarr::S3Connection::delete_object(std::string_view bucket_name, - std::string_view object_name) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - - LOG_DEBUG("Deleting object ", object_name, " from bucket ", bucket_name); - minio::s3::RemoveObjectArgs args; - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->RemoveObject(args); - if (!response) { - LOG_ERROR("Failed to delete object ", - object_name, - " from bucket ", - bucket_name, - ": ", - response.Error().String()); - return false; - } - - return true; -} - -std::string -zarr::S3Connection::create_multipart_object(std::string_view bucket_name, - std::string_view object_name) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - - LOG_DEBUG( - "Creating multipart object ", object_name, " in bucket ", bucket_name); - minio::s3::CreateMultipartUploadArgs args; - args.bucket = bucket_name; - args.object = object_name; - - auto response = client_->CreateMultipartUpload(args); - if (!response) { - LOG_ERROR("Failed to create multipart object ", - object_name, - " in bucket ", - bucket_name, - ": ", - response.Error().String()); - } - EXPECT(!response.upload_id.empty(), "Upload id returned empty."); - - return response.upload_id; -} - -std::string -zarr::S3Connection::upload_multipart_object_part(std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - std::span data, - unsigned int part_number) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - EXPECT(!data.empty(), "Number of bytes must be positive."); - EXPECT(part_number, "Part number must be positive."); - - LOG_DEBUG("Uploading multipart object part ", - part_number, - " for object ", - object_name, - " in bucket ", - bucket_name); - - std::string_view data_buffer(reinterpret_cast(data.data()), - data.size()); - - minio::s3::UploadPartArgs args; - args.bucket = bucket_name; - args.object = object_name; - args.part_number = part_number; - args.upload_id = upload_id; - args.data = data_buffer; - - auto response = client_->UploadPart(args); - if (!response) { - LOG_ERROR("Failed to upload part ", - part_number, - " for object ", - object_name, - " in bucket ", - bucket_name, - ": ", - response.Error().String()); - return {}; - } - - return response.etag; -} - -bool -zarr::S3Connection::complete_multipart_object( - std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - const std::list& parts) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_name.empty(), "Object name must not be empty."); - EXPECT(!upload_id.empty(), "Upload id must not be empty."); - EXPECT(!parts.empty(), "Parts list must not be empty."); - - LOG_DEBUG( - "Completing multipart object ", object_name, " in bucket ", bucket_name); - minio::s3::CompleteMultipartUploadArgs args; - args.bucket = bucket_name; - args.object = object_name; - args.upload_id = upload_id; - args.parts = parts; - - auto response = client_->CompleteMultipartUpload(args); - if (!response) { - LOG_ERROR("Failed to complete multipart object ", - object_name, - " in bucket ", - bucket_name, - ": ", - response.Error().String()); - return false; - } - - return true; -} - -zarr::S3ConnectionPool::S3ConnectionPool(size_t n_connections, - const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key) -{ - for (auto i = 0; i < n_connections; ++i) { - auto connection = std::make_unique( - endpoint, access_key_id, secret_access_key); - - if (connection->is_connection_valid()) { - connections_.push_back(std::make_unique( - endpoint, access_key_id, secret_access_key)); - } - } - - CHECK(!connections_.empty()); -} - -zarr::S3ConnectionPool::~S3ConnectionPool() -{ - is_accepting_connections_ = false; - cv_.notify_all(); -} - -std::unique_ptr -zarr::S3ConnectionPool::get_connection() -{ - std::unique_lock lock(connections_mutex_); - cv_.wait(lock, [this] { - return !is_accepting_connections_ || !connections_.empty(); - }); - - if (!is_accepting_connections_ || connections_.empty()) { - return nullptr; - } - - auto conn = std::move(connections_.back()); - connections_.pop_back(); - return conn; -} - -void -zarr::S3ConnectionPool::return_connection(std::unique_ptr&& conn) -{ - std::unique_lock lock(connections_mutex_); - connections_.push_back(std::move(conn)); - cv_.notify_one(); -} diff --git a/src/streaming/s3.connection.hh b/src/streaming/s3.connection.hh deleted file mode 100644 index 744fb933..00000000 --- a/src/streaming/s3.connection.hh +++ /dev/null @@ -1,138 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace zarr { -class S3Connection -{ - public: - S3Connection(const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key); - - /** - * @brief Test a connection by listing all buckets at this connection's - * endpoint. - * @returns True if the connection is valid, otherwise false. - */ - bool is_connection_valid(); - - /* Bucket operations */ - - /** - * @brief Check whether a bucket exists. - * @param bucket_name The name of the bucket. - * @returns True if the bucket exists, otherwise false. - */ - bool bucket_exists(std::string_view bucket_name); - - /* Object operations */ - - /** - * @brief Check whether an object exists. - * @param bucket_name The name of the bucket containing the object. - * @param object_name The name of the object. - * @returns True if the object exists, otherwise false. - */ - bool object_exists(std::string_view bucket_name, - std::string_view object_name); - - /** - * @brief Put an object. - * @param bucket_name The name of the bucket to put the object in. - * @param object_name The name of the object. - * @param data The data to put in the object. - * @returns The etag of the object. - * @throws std::runtime_error if the bucket name is empty, the object name - * is empty, or @p data is empty. - */ - [[nodiscard]] std::string put_object(std::string_view bucket_name, - std::string_view object_name, - std::span data); - - /** - * @brief Delete an object. - * @param bucket_name The name of the bucket containing the object. - * @param object_name The name of the object. - * @returns True if the object was successfully deleted, otherwise false. - * @throws std::runtime_error if the bucket name is empty or the object - * name is empty. - */ - [[nodiscard]] bool delete_object(std::string_view bucket_name, - std::string_view object_name); - - /* Multipart object operations */ - - /// @brief Create a multipart object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @returns The upload id of the multipart object. Nonempty if and only if - /// the operation succeeds. - /// @throws std::runtime_error if the bucket name is empty or the object - /// name is empty. - [[nodiscard]] std::string create_multipart_object( - std::string_view bucket_name, - std::string_view object_name); - - /// @brief Upload a part of a multipart object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @param upload_id The upload id of the multipart object. - /// @param data The data to upload. - /// @param part_number The part number of the object. - /// @returns The etag of the uploaded part. Nonempty if and only if the - /// operation is successful. - /// @throws std::runtime_error if the bucket name is empty, the object name - /// is empty, @p data is empty, or @p part_number is 0. - [[nodiscard]] std::string upload_multipart_object_part( - std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - std::span data, - unsigned int part_number); - - /// @brief Complete a multipart object. - /// @param bucket_name The name of the bucket containing the object. - /// @param object_name The name of the object. - /// @param upload_id The upload id of the multipart object. - /// @param parts List of the parts making up the object. - /// @returns True if the object was successfully completed, otherwise false. - [[nodiscard]] bool complete_multipart_object( - std::string_view bucket_name, - std::string_view object_name, - std::string_view upload_id, - const std::list& parts); - - private: - std::unique_ptr client_; - std::unique_ptr provider_; -}; - -class S3ConnectionPool -{ - public: - S3ConnectionPool(size_t n_connections, - const std::string& endpoint, - const std::string& access_key_id, - const std::string& secret_access_key); - ~S3ConnectionPool(); - - std::unique_ptr get_connection(); - void return_connection(std::unique_ptr&& conn); - - private: - std::vector> connections_; - std::mutex connections_mutex_; - std::condition_variable cv_; - - std::atomic is_accepting_connections_{true}; -}; -} // namespace zarr diff --git a/src/streaming/s3.sink.cpp b/src/streaming/s3.sink.cpp deleted file mode 100644 index 04ea3c7b..00000000 --- a/src/streaming/s3.sink.cpp +++ /dev/null @@ -1,205 +0,0 @@ -#include "macros.hh" -#include "s3.sink.hh" - -#include - -#ifdef min -#undef min -#endif - -zarr::S3Sink::S3Sink(std::string_view bucket_name, - std::string_view object_key, - std::shared_ptr connection_pool) - : bucket_name_{ bucket_name } - , object_key_{ object_key } - , connection_pool_{ connection_pool } -{ - EXPECT(!bucket_name_.empty(), "Bucket name must not be empty"); - EXPECT(!object_key_.empty(), "Object key must not be empty"); - EXPECT(connection_pool_, "Null pointer: connection_pool"); -} - -bool -zarr::S3Sink::flush_() -{ - if (is_multipart_upload_()) { - const auto& parts = multipart_upload_->parts; - if (nbytes_buffered_ > 0 && !flush_part_()) { - LOG_ERROR("Failed to upload part ", - parts.size() + 1, - " of object ", - object_key_); - return false; - } - if (!finalize_multipart_upload_()) { - LOG_ERROR("Failed to finalize multipart upload of object ", - object_key_); - return false; - } - } else if (nbytes_buffered_ > 0) { - if (!put_object_()) { - LOG_ERROR("Failed to upload object: ", object_key_); - return false; - } - } - - // cleanup - nbytes_buffered_ = 0; - - return true; -} - -bool -zarr::S3Sink::write(size_t offset, std::span data) -{ - if (data.data() == nullptr || data.empty()) { - return true; - } - - if (offset < nbytes_flushed_) { - LOG_ERROR("Cannot write data at offset ", - offset, - ", already flushed to ", - nbytes_flushed_); - return false; - } - nbytes_buffered_ = offset - nbytes_flushed_; - - size_t bytes_of_data = data.size(); - const std::byte* data_ptr = data.data(); - while (bytes_of_data > 0) { - const auto bytes_to_write = - std::min(bytes_of_data, part_buffer_.size() - nbytes_buffered_); - - if (bytes_to_write) { - std::copy_n(data_ptr, - bytes_to_write, - part_buffer_.begin() + nbytes_buffered_); - nbytes_buffered_ += bytes_to_write; - data_ptr += bytes_to_write; - bytes_of_data -= bytes_to_write; - } - - if (nbytes_buffered_ == part_buffer_.size() && !flush_part_()) { - return false; - } - } - - return true; -} - -bool -zarr::S3Sink::put_object_() -{ - if (nbytes_buffered_ == 0) { - return false; - } - - auto connection = connection_pool_->get_connection(); - std::span data(reinterpret_cast(part_buffer_.data()), - nbytes_buffered_); - - bool retval = false; - try { - std::string etag = - connection->put_object(bucket_name_, object_key_, data); - EXPECT(!etag.empty(), "Failed to upload object: ", object_key_); - - retval = true; - - nbytes_flushed_ = nbytes_buffered_; - nbytes_buffered_ = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Error: ", exc.what()); - } - - // cleanup - connection_pool_->return_connection(std::move(connection)); - - return retval; -} - -bool -zarr::S3Sink::is_multipart_upload_() const -{ - return multipart_upload_.has_value(); -} - -void -zarr::S3Sink::create_multipart_upload_() -{ - if (!is_multipart_upload_()) { - multipart_upload_ = {}; - } - - if (!multipart_upload_->upload_id.empty()) { - return; - } - - multipart_upload_->upload_id = - connection_pool_->get_connection()->create_multipart_object(bucket_name_, - object_key_); -} - -bool -zarr::S3Sink::flush_part_() -{ - if (nbytes_buffered_ == 0) { - return false; - } - - auto connection = connection_pool_->get_connection(); - - create_multipart_upload_(); - - bool retval = false; - try { - auto& parts = multipart_upload_->parts; - - minio::s3::Part part; - part.number = static_cast(parts.size()) + 1; - - std::span data(reinterpret_cast(part_buffer_.data()), - nbytes_buffered_); - part.etag = - connection->upload_multipart_object_part(bucket_name_, - object_key_, - multipart_upload_->upload_id, - data, - part.number); - EXPECT(!part.etag.empty(), - "Failed to upload part ", - part.number, - " of object ", - object_key_); - - parts.push_back(part); - - retval = true; - } catch (const std::exception& exc) { - LOG_ERROR("Error: ", exc.what()); - } - - // cleanup - connection_pool_->return_connection(std::move(connection)); - nbytes_flushed_ += nbytes_buffered_; - nbytes_buffered_ = 0; - - return retval; -} - -bool -zarr::S3Sink::finalize_multipart_upload_() -{ - auto connection = connection_pool_->get_connection(); - - const auto& upload_id = multipart_upload_->upload_id; - const auto& parts = multipart_upload_->parts; - - bool retval = connection->complete_multipart_object( - bucket_name_, object_key_, upload_id, parts); - - connection_pool_->return_connection(std::move(connection)); - - return retval; -} diff --git a/src/streaming/s3.sink.hh b/src/streaming/s3.sink.hh deleted file mode 100644 index 49d22194..00000000 --- a/src/streaming/s3.sink.hh +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once - -#include "sink.hh" -#include "s3.connection.hh" - -#include - -#include -#include -#include - -namespace zarr { -class S3Sink : public Sink -{ - public: - S3Sink(std::string_view bucket_name, - std::string_view object_key, - std::shared_ptr connection_pool); - - bool write(size_t offset, std::span data) override; - - protected: - bool flush_() override; - - private: - struct MultiPartUpload - { - std::string upload_id; - std::list parts; - }; - - static constexpr size_t max_part_size_ = 5 << 20; - std::string bucket_name_; - std::string object_key_; - - std::shared_ptr connection_pool_; - - std::array part_buffer_; - size_t nbytes_buffered_{ 0 }; - size_t nbytes_flushed_{ 0 }; - - std::optional multipart_upload_; - - /** - * @brief Upload the object to S3. - * @return True if the object was successfully uploaded, otherwise false. - */ - [[nodiscard]] bool put_object_(); - - /** - * @brief Check if a multipart upload is in progress. - * @return True if a multipart upload is in progress, otherwise false. - */ - bool is_multipart_upload_() const; - - /** - * @brief Create a new multipart upload. - */ - void create_multipart_upload_(); - - /** - * @brief Flush the current part to S3. - * @return True if the part was successfully flushed, otherwise false. - */ - [[nodiscard]] bool flush_part_(); - - /** - * @brief Finalize the multipart upload. - * @returns True if a multipart upload was successfully finalized, - * otherwise false. - */ - [[nodiscard]] bool finalize_multipart_upload_(); -}; -} // namespace zarr diff --git a/src/streaming/sink.cpp b/src/streaming/sink.cpp deleted file mode 100644 index e351a7b0..00000000 --- a/src/streaming/sink.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "sink.hh" -#include "macros.hh" - -bool -zarr::finalize_sink(std::unique_ptr&& sink) -{ - if (sink == nullptr) { - LOG_INFO("Sink is null. Nothing to finalize."); - return true; - } - - if (!sink->flush_()) { - return false; - } - - sink.reset(); - return true; -} \ No newline at end of file diff --git a/src/streaming/sink.creator.cpp b/src/streaming/sink.creator.cpp deleted file mode 100644 index d1343c2c..00000000 --- a/src/streaming/sink.creator.cpp +++ /dev/null @@ -1,482 +0,0 @@ -#include "macros.hh" -#include "sink.creator.hh" -#include "file.sink.hh" -#include "s3.sink.hh" -#include "acquire.zarr.h" - -#include -#include -#include -#include - -namespace fs = std::filesystem; - -zarr::SinkCreator::SinkCreator( - std::shared_ptr thread_pool_, - std::shared_ptr connection_pool) - : thread_pool_{ thread_pool_ } - , connection_pool_{ connection_pool } -{ -} - -std::unique_ptr -zarr::SinkCreator::make_sink(std::string_view file_path) -{ - if (file_path.starts_with("file://")) { - file_path = file_path.substr(7); - } - - EXPECT(!file_path.empty(), "File path must not be empty."); - - fs::path path(file_path); - EXPECT(!path.empty(), "Invalid file path: ", file_path); - - fs::path parent_path = path.parent_path(); - - if (!fs::is_directory(parent_path)) { - std::error_code ec; - if (!fs::create_directories(parent_path, ec)) { - LOG_ERROR("Failed to create directory '", parent_path, "': ", - ec.message()); - return nullptr; - } - } - - return std::make_unique(file_path); -} - -std::unique_ptr -zarr::SinkCreator::make_sink(std::string_view bucket_name, - std::string_view object_key) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!object_key.empty(), "Object key must not be empty."); - EXPECT(connection_pool_, "S3 connection pool not provided."); - if (!bucket_exists_(bucket_name)) { - LOG_ERROR("Bucket '", bucket_name, "' does not exist."); - return nullptr; - } - - return std::make_unique(bucket_name, object_key, connection_pool_); -} - -bool -zarr::SinkCreator::make_data_sinks( - std::string_view base_path, - const ArrayDimensions* dimensions, - const std::function& parts_along_dimension, - std::vector>& part_sinks) -{ - if (base_path.starts_with("file://")) { - base_path = base_path.substr(7); - } - - EXPECT(!base_path.empty(), "Base path must not be empty."); - - std::queue paths; - try { - paths = make_data_sink_paths_( - base_path, dimensions, parts_along_dimension, true); - } catch (const std::exception& exc) { - LOG_ERROR("Failed to create dataset paths: ", exc.what()); - return false; - } - - return make_files_(paths, part_sinks); -} - -bool -zarr::SinkCreator::make_data_sinks( - std::string_view bucket_name, - std::string_view base_path, - const ArrayDimensions* dimensions, - const std::function& parts_along_dimension, - std::vector>& part_sinks) -{ - EXPECT(!base_path.empty(), "Base path must not be empty."); - - auto paths = make_data_sink_paths_( - base_path, dimensions, parts_along_dimension, false); - - return make_s3_objects_(bucket_name, paths, part_sinks); -} - -bool -zarr::SinkCreator::make_metadata_sinks( - size_t version, - std::string_view base_path, - std::unordered_map>& metadata_sinks) -{ - if (base_path.starts_with("file://")) { - base_path = base_path.substr(7); - } - EXPECT(!base_path.empty(), "Base path must not be empty."); - - std::vector file_paths = - make_metadata_sink_paths_(version, base_path, true); - - return make_files_(base_path.data(), file_paths, metadata_sinks); -} - -bool -zarr::SinkCreator::make_metadata_sinks( - size_t version, - std::string_view bucket_name, - std::string_view base_path, - std::unordered_map>& metadata_sinks) -{ - EXPECT(!bucket_name.empty(), "Bucket name must not be empty."); - EXPECT(!base_path.empty(), "Base path must not be empty."); - if (!bucket_exists_(bucket_name)) { - LOG_ERROR("Bucket '", bucket_name, "' does not exist."); - return false; - } - - auto file_paths = make_metadata_sink_paths_(version, base_path, false); - return make_s3_objects_(bucket_name, base_path, file_paths, metadata_sinks); -} - -std::queue -zarr::SinkCreator::make_data_sink_paths_( - std::string_view base_path, - const ArrayDimensions* dimensions, - const std::function& parts_along_dimension, - bool create_directories) -{ - std::queue paths; - paths.emplace(base_path); - - if (create_directories) { - EXPECT( - make_dirs_(paths), "Failed to create directory '", base_path, "'."); - } - - // create intermediate paths - for (auto i = 1; // skip the last dimension - i < dimensions->ndims() - 1; // skip the x dimension - ++i) { - const auto& dim = dimensions->at(i); - const auto n_parts = parts_along_dimension(dim); - CHECK(n_parts); - - auto n_paths = paths.size(); - for (auto j = 0; j < n_paths; ++j) { - const auto path = paths.front(); - paths.pop(); - - for (auto k = 0; k < n_parts; ++k) { - const auto kstr = std::to_string(k); - paths.push(path + (path.empty() ? kstr : "/" + kstr)); - } - } - - if (create_directories) { - EXPECT(make_dirs_(paths), - "Failed to create directories for dimension '", - dim.name, - "'."); - } - } - - // create final paths - { - const auto& dim = dimensions->width_dim(); - const auto n_parts = parts_along_dimension(dim); - CHECK(n_parts); - - auto n_paths = paths.size(); - for (auto i = 0; i < n_paths; ++i) { - const auto path = paths.front(); - paths.pop(); - for (auto j = 0; j < n_parts; ++j) - paths.push(path + "/" + std::to_string(j)); - } - } - - return paths; -} - -std::vector -zarr::SinkCreator::make_metadata_sink_paths_(size_t version, - std::string_view base_path, - bool create_directories) -{ - std::vector paths; - - switch (version) { - case ZarrVersion_2: - paths.emplace_back(".zattrs"); - paths.emplace_back(".zgroup"); - paths.emplace_back("0/.zattrs"); - paths.emplace_back("acquire.json"); - break; - case ZarrVersion_3: - paths.emplace_back("zarr.json"); - paths.emplace_back("meta/root.group.json"); - paths.emplace_back("meta/acquire.json"); - break; - default: - throw std::runtime_error("Invalid Zarr version " + - std::to_string(static_cast(version))); - } - - if (create_directories) { - std::queue dir_paths; - dir_paths.emplace(base_path); - EXPECT(make_dirs_(dir_paths), - "Failed to create metadata directories."); - dir_paths.pop(); // remove the base path - - std::unordered_set parent_paths; - for (const auto& path : paths) { - fs::path parent = fs::path(path).parent_path(); - if (!parent.empty()) { - parent_paths.emplace((fs::path(base_path) / parent).string()); - } - } - - for (const auto& dir_path : parent_paths) { - dir_paths.push(dir_path); - } - - if (!dir_paths.empty()) { - EXPECT(make_dirs_(dir_paths), - "Failed to create metadata directories."); - } - } - - return paths; -} - -bool -zarr::SinkCreator::make_dirs_(std::queue& dir_paths) -{ - if (dir_paths.empty()) { - return true; - } - - std::atomic all_successful = 1; - - const auto n_dirs = dir_paths.size(); - std::latch latch(n_dirs); - - for (auto i = 0; i < n_dirs; ++i) { - const auto dirname = dir_paths.front(); - dir_paths.pop(); - - EXPECT(thread_pool_->push_job( - [dirname, &latch, &all_successful](std::string& err) -> bool { - if (dirname.empty()) { - err = "Directory name must not be empty."; - latch.count_down(); - all_successful.fetch_and(0); - return false; - } - - if (fs::is_directory(dirname)) { - latch.count_down(); - return true; - } else if (fs::exists(dirname)) { - err = - "'" + dirname + "' exists but is not a directory"; - latch.count_down(); - all_successful.fetch_and(0); - return false; - } - - if (all_successful) { - std::error_code ec; - if (!fs::create_directories(dirname, ec)) { - err = "Failed to create directory '" + dirname + - "': " + ec.message(); - latch.count_down(); - all_successful.fetch_and(0); - return false; - } - } - - latch.count_down(); - return true; - }), - "Failed to push job to thread pool."); - - dir_paths.push(dirname); - } - - latch.wait(); - - return (bool)all_successful; -} - -bool -zarr::SinkCreator::make_files_(std::queue& file_paths, - std::vector>& sinks) -{ - if (file_paths.empty()) { - return true; - } - - std::atomic all_successful = 1; - - const auto n_files = file_paths.size(); - sinks.resize(n_files); - std::fill(sinks.begin(), sinks.end(), nullptr); - std::latch latch(n_files); - - for (auto i = 0; i < n_files; ++i) { - const auto filename = file_paths.front(); - file_paths.pop(); - - std::unique_ptr* psink = sinks.data() + i; - - EXPECT(thread_pool_->push_job( - [filename, psink, &latch, &all_successful]( - std::string& err) -> bool { - bool success = false; - - try { - if (all_successful) { - *psink = std::make_unique(filename); - } - success = true; - } catch (const std::exception& exc) { - err = "Failed to create file '" + filename + - "': " + exc.what(); - } - - latch.count_down(); - all_successful.fetch_and((char)success); - - return success; - }), - "Failed to push job to thread pool."); - } - - latch.wait(); - - return (bool)all_successful; -} - -bool -zarr::SinkCreator::make_files_( - const std::string& base_dir, - const std::vector& file_paths, - std::unordered_map>& sinks) -{ - if (file_paths.empty()) { - return true; - } - - std::atomic all_successful = 1; - - const auto n_files = file_paths.size(); - std::latch latch(n_files); - - sinks.clear(); - for (const auto& filename : file_paths) { - sinks[filename] = nullptr; - std::unique_ptr* psink = &sinks[filename]; - - const std::string prefix = base_dir.empty() ? "" : base_dir + "/"; - const auto file_path = prefix + filename; - - EXPECT(thread_pool_->push_job( - [filename = file_path, psink, &latch, &all_successful]( - std::string& err) -> bool { - bool success = false; - - try { - if (all_successful) { - *psink = std::make_unique(filename); - } - success = true; - } catch (const std::exception& exc) { - err = "Failed to create file '" + filename + - "': " + exc.what(); - } - - latch.count_down(); - all_successful.fetch_and((char)success); - - return success; - }), - "Failed to push job to thread pool."); - } - - latch.wait(); - - return (bool)all_successful; -} - -bool -zarr::SinkCreator::bucket_exists_(std::string_view bucket_name) -{ - CHECK(!bucket_name.empty()); - EXPECT(connection_pool_, "S3 connection pool not provided."); - - auto conn = connection_pool_->get_connection(); - bool bucket_exists = conn->bucket_exists(bucket_name); - - connection_pool_->return_connection(std::move(conn)); - - return bucket_exists; -} - -bool -zarr::SinkCreator::make_s3_objects_(std::string_view bucket_name, - std::queue& object_keys, - std::vector>& sinks) -{ - if (object_keys.empty()) { - return true; - } - - if (bucket_name.empty()) { - LOG_ERROR("Bucket name not provided."); - return false; - } - if (!connection_pool_) { - LOG_ERROR("S3 connection pool not provided."); - return false; - } - - const auto n_objects = object_keys.size(); - sinks.resize(n_objects); - for (auto i = 0; i < n_objects; ++i) { - sinks[i] = std::make_unique( - bucket_name, object_keys.front(), connection_pool_); - object_keys.pop(); - } - - return true; -} - -bool -zarr::SinkCreator::make_s3_objects_( - std::string_view bucket_name, - std::string_view base_path, - std::vector& object_keys, - std::unordered_map>& sinks) -{ - if (object_keys.empty()) { - return true; - } - - if (bucket_name.empty()) { - LOG_ERROR("Bucket name not provided."); - return false; - } - - if (!connection_pool_) { - LOG_ERROR("S3 connection pool not provided."); - return false; - } - - sinks.clear(); - for (const auto& key : object_keys) { - sinks[key] = std::make_unique( - bucket_name, std::string(base_path) + "/" + key, connection_pool_); - } - - return true; -} diff --git a/src/streaming/sink.creator.hh b/src/streaming/sink.creator.hh deleted file mode 100644 index b7f6e689..00000000 --- a/src/streaming/sink.creator.hh +++ /dev/null @@ -1,181 +0,0 @@ -#pragma once - -#include "zarr.dimension.hh" -#include "sink.hh" -#include "thread.pool.hh" -#include "s3.connection.hh" - -#include -#include -#include - -namespace zarr { -class SinkCreator -{ - public: - SinkCreator(std::shared_ptr thread_pool_, - std::shared_ptr connection_pool); - ~SinkCreator() noexcept = default; - - /** - * @brief Create a sink from a file path. - * @param file_path The path to the file. - * @return Pointer to the sink created, or nullptr if the file cannot be - * opened. - * @throws std::runtime_error if the file path is not valid. - */ - static std::unique_ptr make_sink(std::string_view file_path); - - /** - * @brief Create a sink from an S3 bucket name and object key. - * @param bucket_name The name of the bucket in which the object is stored. - * @param object_key The key of the object to write to. - * @return Pointer to the sink created, or nullptr if the bucket does not - * exist. - * @throws std::runtime_error if the bucket name or object key is not valid, - * or if there is no connection pool. - */ - std::unique_ptr make_sink(std::string_view bucket_name, - std::string_view object_key); - - /** - * @brief Create a collection of file sinks for a Zarr dataset. - * @param[in] base_path The path to the base directory for the dataset. - * @param[in] dimensions The dimensions of the dataset. - * @param[in] parts_along_dimension Function to determine the number of - * parts (i.e., shards or chunks) along a dimension. - * @param[out] part_sinks The sinks created. - * @return True iff all file sinks were created successfully. - * @throws std::runtime_error if @p base_path is not valid, or if the number - * of parts along a dimension is zero. - */ - [[nodiscard]] bool make_data_sinks( - std::string_view base_path, - const ArrayDimensions* dimensions, - const std::function& parts_along_dimension, - std::vector>& part_sinks); - - /** - * @brief Create a collection of S3 sinks for a Zarr dataset. - * @param[in] bucket_name The name of the bucket in which the dataset is - * stored. - * @param[in] base_path The path to the base directory for the dataset. - * @param[in] dimensions The dimensions of the dataset. - * @param[in] parts_along_dimension Function to determine the number of - * parts (i.e., shards or chunks) along a dimension. - * @param[out] part_sinks The sinks created. - * @return True iff all file sinks were created successfully. - */ - [[nodiscard]] bool make_data_sinks( - std::string_view bucket_name, - std::string_view base_path, - const ArrayDimensions* dimensions, - const std::function& parts_along_dimension, - std::vector>& part_sinks); - - /** - * @brief Create a collection of metadata sinks for a Zarr dataset. - * @param[in] version The Zarr version. - * @param[in] base_path The base URI for the dataset. - * @param[out] metadata_sinks The sinks created, keyed by path. - * @return True iff all metadata sinks were created successfully. - * @throws std::runtime_error if @p base_uri is not valid, or if, for S3 - * sinks, the bucket does not exist. - */ - [[nodiscard]] bool make_metadata_sinks( - size_t version, - std::string_view base_path, - std::unordered_map>& metadata_sinks); - - /** - * @brief - * @param version - * @param bucket_name - * @param base_path - * @param metadata_sinks - * @return - * @throws std::runtime_error if @p version is invalid, if @p bucket_name is - * empty or does not exist, or if @p base_path is empty. - */ - [[nodiscard]] bool make_metadata_sinks( - size_t version, - std::string_view bucket_name, - std::string_view base_path, - std::unordered_map>& metadata_sinks); - - private: - std::shared_ptr thread_pool_; - std::shared_ptr connection_pool_; // could be null - - /** - * @brief Construct the paths for a Zarr dataset. - * @param base_path The base path for the dataset. - * @param dimensions The dimensions of the dataset. - * @param parts_along_dimension Function to determine the number of parts - * @param create_directories Whether to create intermediate directories. - * @return A queue of paths to the dataset components. - * @throws std::runtime_error if intermediate directories cannot be created, - * or if the number of parts along a dimension is zero. - */ - std::queue make_data_sink_paths_( - std::string_view base_path, - const ArrayDimensions* dimensions, - const std::function& parts_along_dimension, - bool create_directories); - - std::vector make_metadata_sink_paths_( - size_t version, - std::string_view base_path, - bool create_directories); - - /// @brief Parallel create a collection of directories. - /// @param[in] dir_paths The directories to create. - /// @return True iff all directories were created successfully. - [[nodiscard]] bool make_dirs_(std::queue& dir_paths); - - /// @brief Parallel create a collection of files. - /// @param[in,out] file_paths The files to create. Unlike `make_dirs_`, - /// this function drains the queue. - /// @param[out] files The files created. - /// @return True iff all files were created successfully. - [[nodiscard]] bool make_files_(std::queue& file_paths, - std::vector>& sinks); - - /// @brief Parallel create a collection of files, keyed by path. - /// @param[in] base_dir The base directory for the files. - /// @param[in] file_paths Paths to the files to create, relative to @p - /// base_dir. - /// @param[out] sinks The sinks created, keyed by path. - /// @return True iff all files were created successfully. - [[nodiscard]] bool make_files_( - const std::string& base_dir, - const std::vector& file_paths, - std::unordered_map>& sinks); - - /// @brief Check whether an S3 bucket exists. - /// @param[in] bucket_name The name of the bucket to check. - /// @return True iff the bucket exists. - bool bucket_exists_(std::string_view bucket_name); - - /// @brief Create a collection of S3 objects. - /// @param[in] bucket_name The name of the bucket. - /// @param[in,out] object_keys The keys of the objects to create. - /// @param[out] sinks The sinks created. - /// @return True iff all S3 objects were created successfully. - [[nodiscard]] bool make_s3_objects_( - std::string_view bucket_name, - std::queue& object_keys, - std::vector>& sinks); - - /// @brief Create a collection of S3 objects, keyed by object key. - /// @param[in] bucket_name The name of the bucket. - /// @param[in] object_keys The keys of the objects to create. - /// @param[out] sinks The sinks created, keyed by object key. - /// @return True iff all S3 objects were created successfully. - [[nodiscard]] bool make_s3_objects_( - std::string_view bucket_name, - std::string_view base_path, - std::vector& object_keys, - std::unordered_map>& sinks); -}; -} // namespace zarr diff --git a/src/streaming/sink.hh b/src/streaming/sink.hh deleted file mode 100644 index 502679fe..00000000 --- a/src/streaming/sink.hh +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include // size_t, std::byte -#include // std::unique_ptr -#include // std::span - -namespace zarr { -class Sink -{ - public: - virtual ~Sink() = default; - - /** - * @brief Write data to the sink. - * @param offset The offset in the sink to write to. - * @param buf The buffer to write to the sink. - * @param bytes_of_buf The number of bytes to write from @p buf. - * @return True if the write was successful, false otherwise. - */ - [[nodiscard]] virtual bool write(size_t offset, - std::span buf) = 0; - - protected: - [[nodiscard]] virtual bool flush_() = 0; - - friend bool finalize_sink(std::unique_ptr&& sink); -}; - -bool -finalize_sink(std::unique_ptr&& sink); -} // namespace zarr diff --git a/src/streaming/thread.pool.cpp b/src/streaming/thread.pool.cpp deleted file mode 100644 index 2550dd8a..00000000 --- a/src/streaming/thread.pool.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include "thread.pool.hh" - -zarr::ThreadPool::ThreadPool(unsigned int n_threads, ErrorCallback&& err) - : error_handler_{ std::move(err) } -{ - const auto max_threads = std::max(std::thread::hardware_concurrency(), 1u); - n_threads = std::clamp(n_threads, 1u, max_threads); - - for (auto i = 0; i < n_threads; ++i) { - threads_.emplace_back([this] { process_tasks_(); }); - } -} - -zarr::ThreadPool::~ThreadPool() noexcept -{ - { - std::unique_lock lock(jobs_mutex_); - while (!jobs_.empty()) { - jobs_.pop(); - } - } - - await_stop(); -} - -bool -zarr::ThreadPool::push_job(Task&& job) -{ - std::unique_lock lock(jobs_mutex_); - if (!is_accepting_jobs_) { - return false; - } - - jobs_.push(std::move(job)); - cv_.notify_one(); - - return true; -} - -void -zarr::ThreadPool::await_stop() noexcept -{ - { - std::scoped_lock lock(jobs_mutex_); - is_accepting_jobs_ = false; - - cv_.notify_all(); - } - - // spin down threads - for (auto& thread : threads_) { - if (thread.joinable()) { - thread.join(); - } - } -} - -std::optional -zarr::ThreadPool::pop_from_job_queue_() noexcept -{ - if (jobs_.empty()) { - return std::nullopt; - } - - auto job = std::move(jobs_.front()); - jobs_.pop(); - return job; -} - -bool -zarr::ThreadPool::should_stop_() const noexcept -{ - return !is_accepting_jobs_ && jobs_.empty(); -} - -void -zarr::ThreadPool::process_tasks_() -{ - while (true) { - std::unique_lock lock(jobs_mutex_); - cv_.wait(lock, [&] { return should_stop_() || !jobs_.empty(); }); - - if (should_stop_()) { - break; - } - - if (auto job = pop_from_job_queue_(); job.has_value()) { - lock.unlock(); - if (std::string err_msg; !job.value()(err_msg)) { - error_handler_(err_msg); - } - } - } -} \ No newline at end of file diff --git a/src/streaming/thread.pool.hh b/src/streaming/thread.pool.hh deleted file mode 100644 index c04c966b..00000000 --- a/src/streaming/thread.pool.hh +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace zarr { -class ThreadPool -{ - public: - using Task = std::function; - using ErrorCallback = std::function; - - // The error handler `err` is called when a job returns false. This - // can happen when the job encounters an error, or otherwise fails. The - // std::string& argument to the error handler is a diagnostic message from - // the failing job and is logged to the error stream by the Zarr driver when - // the next call to `append()` is made. - ThreadPool(unsigned int n_threads, ErrorCallback&& err); - ~ThreadPool() noexcept; - - /** - * @brief Push a job onto the job queue. - * - * @param job The job to push onto the queue. - * @return true if the job was successfully pushed onto the queue, false - * otherwise. - */ - [[nodiscard]] bool push_job(Task&& job); - - /** - * @brief Block until all jobs on the queue have processed, then spin down - * the threads. - * @note After calling this function, the job queue no longer accepts jobs. - */ - void await_stop() noexcept; - - private: - ErrorCallback error_handler_; - - std::vector threads_; - std::mutex jobs_mutex_; - std::condition_variable cv_; - std::queue jobs_; - - std::atomic is_accepting_jobs_{ true }; - - std::optional pop_from_job_queue_() noexcept; - [[nodiscard]] bool should_stop_() const noexcept; - void process_tasks_(); -}; -} // zarr diff --git a/src/streaming/zarr.common.cpp b/src/streaming/zarr.common.cpp deleted file mode 100644 index 74c7dfb5..00000000 --- a/src/streaming/zarr.common.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "macros.hh" -#include "zarr.common.hh" - -#include - -std::string -zarr::trim(std::string_view s) -{ - if (s.empty()) { - return {}; - } - - // trim left - std::string trimmed(s); - trimmed.erase(trimmed.begin(), - std::find_if(trimmed.begin(), trimmed.end(), [](char c) { - return !std::isspace(c); - })); - - // trim right - trimmed.erase(std::find_if(trimmed.rbegin(), - trimmed.rend(), - [](char c) { return !std::isspace(c); }) - .base(), - trimmed.end()); - - return trimmed; -} - -bool -zarr::is_empty_string(std::string_view s, std::string_view err_on_empty) -{ - auto trimmed = trim(s); - if (trimmed.empty()) { - LOG_ERROR(err_on_empty); - return true; - } - return false; -} - -size_t -zarr::bytes_of_type(ZarrDataType data_type) -{ - switch (data_type) { - case ZarrDataType_int8: - case ZarrDataType_uint8: - return 1; - case ZarrDataType_int16: - case ZarrDataType_uint16: - return 2; - case ZarrDataType_int32: - case ZarrDataType_uint32: - case ZarrDataType_float32: - return 4; - case ZarrDataType_int64: - case ZarrDataType_uint64: - case ZarrDataType_float64: - return 8; - default: - throw std::invalid_argument("Invalid data type: " + - std::to_string(data_type)); - } -} - -size_t -zarr::bytes_of_frame(const ArrayDimensions& dims, ZarrDataType type) -{ - const auto height = dims.height_dim().array_size_px; - const auto width = dims.width_dim().array_size_px; - return bytes_of_type(type) * height * width; -} - -uint32_t -zarr::chunks_along_dimension(const ZarrDimension& dimension) -{ - EXPECT(dimension.chunk_size_px > 0, "Invalid chunk size."); - - return (dimension.array_size_px + dimension.chunk_size_px - 1) / - dimension.chunk_size_px; -} - -uint32_t -zarr::shards_along_dimension(const ZarrDimension& dimension) -{ - if (dimension.shard_size_chunks == 0) { - return 0; - } - - const auto shard_size = dimension.shard_size_chunks; - const auto n_chunks = chunks_along_dimension(dimension); - return (n_chunks + shard_size - 1) / shard_size; -} diff --git a/src/streaming/zarr.common.hh b/src/streaming/zarr.common.hh deleted file mode 100644 index e45241df..00000000 --- a/src/streaming/zarr.common.hh +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include "zarr.dimension.hh" -#include "acquire.zarr.h" - -namespace zarr { -/** - * @brief Trim whitespace from a string. - * @param s The string to trim. - * @return The string with leading and trailing whitespace removed. - */ -[[nodiscard]] -std::string -trim(std::string_view s); - -/** - * @brief Check if a string is empty, including whitespace. - * @param s The string to check. - * @param err_on_empty The message to log if the string is empty. - * @return True if the string is empty, false otherwise. - */ -bool -is_empty_string(std::string_view s, std::string_view err_on_empty); - -/** - * @brief Get the number of bytes for a given data type. - * @param data_type The data type. - * @return The number of bytes for the data type. - * @throw std::invalid_argument if the data type is not recognized. - */ -size_t -bytes_of_type(ZarrDataType data_type); - -/** - * @brief Get the number of bytes for a frame with the given dimensions and - * data type. - * @param dims The dimensions of the full array. - * @param type The data type of the array. - * @return The number of bytes for a single frame. - * @throw std::invalid_argument if the data type is not recognized. - */ -size_t -bytes_of_frame(const ArrayDimensions& dims, ZarrDataType type); - -/** - * @brief Get the number of chunks along a dimension. - * @param dimension A dimension. - * @return The number of, possibly ragged, chunks along the dimension, given - * the dimension's array and chunk sizes. - * @throw std::runtime_error if the chunk size is zero. - */ -uint32_t -chunks_along_dimension(const ZarrDimension& dimension); - -/** - * @brief Get the number of shards along a dimension. - * @param dimension A dimension. - * @return The number of shards along the dimension, given the dimension's - * array, chunk, and shard sizes. - */ -uint32_t -shards_along_dimension(const ZarrDimension& dimension); -} // namespace zarr \ No newline at end of file diff --git a/src/streaming/zarr.dimension.cpp b/src/streaming/zarr.dimension.cpp deleted file mode 100644 index 4629f707..00000000 --- a/src/streaming/zarr.dimension.cpp +++ /dev/null @@ -1,258 +0,0 @@ -#include "zarr.dimension.hh" -#include "macros.hh" -#include "zarr.common.hh" - -ArrayDimensions::ArrayDimensions(std::vector&& dims, - ZarrDataType dtype) - : dims_(std::move(dims)) - , dtype_(dtype) -{ - EXPECT(dims_.size() > 2, "Array must have at least three dimensions."); -} - -size_t -ArrayDimensions::ndims() const -{ - return dims_.size(); -} - -const ZarrDimension& -ArrayDimensions::operator[](size_t idx) const -{ - return dims_[idx]; -} - -const ZarrDimension& -ArrayDimensions::final_dim() const -{ - return dims_[0]; -} - -const ZarrDimension& -ArrayDimensions::height_dim() const -{ - return dims_[ndims() - 2]; -} - -const ZarrDimension& -ArrayDimensions::width_dim() const -{ - return dims_.back(); -} - -uint32_t -ArrayDimensions::chunk_lattice_index(uint64_t frame_id, - uint32_t dim_index) const -{ - // the last two dimensions are special cases - EXPECT(dim_index < ndims() - 2, "Invalid dimension index: ", dim_index); - - // the first dimension is a special case - if (dim_index == 0) { - auto divisor = dims_.front().chunk_size_px; - for (auto i = 1; i < ndims() - 2; ++i) { - const auto& dim = dims_[i]; - divisor *= dim.array_size_px; - } - - CHECK(divisor); - return frame_id / divisor; - } - - size_t mod_divisor = 1, div_divisor = 1; - for (auto i = dim_index; i < ndims() - 2; ++i) { - const auto& dim = dims_[i]; - mod_divisor *= dim.array_size_px; - div_divisor *= (i == dim_index ? dim.chunk_size_px : dim.array_size_px); - } - - CHECK(mod_divisor); - CHECK(div_divisor); - - return (frame_id % mod_divisor) / div_divisor; -} - -uint32_t -ArrayDimensions::tile_group_offset(uint64_t frame_id) const -{ - std::vector strides(dims_.size(), 1); - for (auto i = dims_.size() - 1; i > 0; --i) { - const auto& dim = dims_[i]; - const auto a = dim.array_size_px, c = dim.chunk_size_px; - strides[i - 1] = strides[i] * ((a + c - 1) / c); - } - - size_t offset = 0; - for (auto i = ndims() - 3; i > 0; --i) { - const auto idx = chunk_lattice_index(frame_id, i); - const auto stride = strides[i]; - offset += idx * stride; - } - - return offset; -} - -uint64_t -ArrayDimensions::chunk_internal_offset(uint64_t frame_id) const -{ - const auto tile_size = zarr::bytes_of_type(dtype_) * - width_dim().chunk_size_px * - height_dim().chunk_size_px; - - uint64_t offset = 0; - std::vector array_strides(ndims() - 2, 1), - chunk_strides(ndims() - 2, 1); - - for (auto i = (int)ndims() - 3; i > 0; --i) { - const auto& dim = dims_[i]; - const auto internal_idx = - (frame_id / array_strides[i]) % dim.array_size_px % dim.chunk_size_px; - - array_strides[i - 1] = array_strides[i] * dim.array_size_px; - chunk_strides[i - 1] = chunk_strides[i] * dim.chunk_size_px; - offset += internal_idx * chunk_strides[i]; - } - - // final dimension - { - const auto& dim = dims_[0]; - const auto internal_idx = - (frame_id / array_strides.front()) % dim.chunk_size_px; - offset += internal_idx * chunk_strides.front(); - } - - return offset * tile_size; -} - -uint32_t -ArrayDimensions::number_of_chunks_in_memory() const -{ - uint32_t n_chunks = 1; - for (auto i = 1; i < ndims(); ++i) { - n_chunks *= zarr::chunks_along_dimension(dims_[i]); - } - - return n_chunks; -} - -size_t -ArrayDimensions::bytes_per_chunk() const -{ - auto n_bytes = zarr::bytes_of_type(dtype_); - for (const auto& d : dims_) { - n_bytes *= d.chunk_size_px; - } - - return n_bytes; -} - -uint32_t -ArrayDimensions::number_of_shards() const -{ - size_t n_shards = 1; - for (auto i = 1; i < ndims(); ++i) { - const auto& dim = dims_[i]; - n_shards *= zarr::shards_along_dimension(dim); - } - - return n_shards; -} - -uint32_t -ArrayDimensions::chunks_per_shard() const -{ - size_t n_chunks = 1; - for (const auto& dim : dims_) { - n_chunks *= dim.shard_size_chunks; - } - - return n_chunks; -} - -uint32_t -ArrayDimensions::shard_index_for_chunk(uint32_t chunk_index) const -{ - // make chunk strides - std::vector chunk_strides; - chunk_strides.resize(dims_.size()); - chunk_strides.back() = 1; - - for (auto i = dims_.size() - 1; i > 0; --i) { - const auto& dim = dims_[i]; - chunk_strides[i - 1] = - chunk_strides[i] * zarr::chunks_along_dimension(dim); - } - - // get chunk indices - std::vector chunk_lattice_indices(ndims()); - for (auto i = ndims() - 1; i > 0; --i) { - chunk_lattice_indices[i] = - chunk_index % chunk_strides[i - 1] / chunk_strides[i]; - } - - // make shard strides - std::vector shard_strides(ndims(), 1); - for (auto i = ndims() - 1; i > 0; --i) { - const auto& dim = dims_[i]; - shard_strides[i-1] = shard_strides[i] * zarr::shards_along_dimension(dim); - } - - std::vector shard_lattice_indices; - for (auto i = 0; i < ndims(); ++i) { - shard_lattice_indices.push_back(chunk_lattice_indices[i] / - dims_[i].shard_size_chunks); - } - - uint32_t index = 0; - for (auto i = 0; i < ndims(); ++i) { - index += shard_lattice_indices[i] * shard_strides[i]; - } - - return index; -} - -uint32_t -ArrayDimensions::shard_internal_index(uint32_t chunk_index) const -{ - // make chunk strides - std::vector chunk_strides; - chunk_strides.resize(dims_.size()); - chunk_strides.back() = 1; - - for (auto i = dims_.size() - 1; i > 0; --i) { - const auto& dim = dims_[i]; - chunk_strides[i - 1] = - chunk_strides[i] * zarr::chunks_along_dimension(dim); - } - - // get chunk indices - std::vector chunk_lattice_indices(ndims()); - for (auto i = ndims() - 1; i > 0; --i) { - chunk_lattice_indices[i] = - chunk_index % chunk_strides[i - 1] / chunk_strides[i]; - } - chunk_lattice_indices[0] = chunk_index / chunk_strides.front(); - - // make shard lattice indices - std::vector shard_lattice_indices; - for (auto i = 0; i < ndims(); ++i) { - shard_lattice_indices.push_back(chunk_lattice_indices[i] / - dims_[i].shard_size_chunks); - } - - std::vector chunk_internal_strides(ndims(), 1); - for (auto i = ndims() - 1; i > 0; --i) { - const auto& dim = dims_[i]; - chunk_internal_strides[i - 1] = - chunk_internal_strides[i] * dim.shard_size_chunks; - } - - size_t index = 0; - - for (auto i = 0; i < ndims(); ++i) { - index += (chunk_lattice_indices[i] % dims_[i].shard_size_chunks) * - chunk_internal_strides[i]; - } - - return index; -} \ No newline at end of file diff --git a/src/streaming/zarr.dimension.hh b/src/streaming/zarr.dimension.hh deleted file mode 100644 index 9c0174c0..00000000 --- a/src/streaming/zarr.dimension.hh +++ /dev/null @@ -1,114 +0,0 @@ -#pragma once - -#include "zarr.types.h" - -#include -#include -#include - -struct ZarrDimension -{ - ZarrDimension() = default; - ZarrDimension(std::string_view name, - ZarrDimensionType type, - uint32_t array_size_px, - uint32_t chunk_size_px, - uint32_t shard_size_chunks) - : name(name) - , type(type) - , array_size_px(array_size_px) - , chunk_size_px(chunk_size_px) - , shard_size_chunks(shard_size_chunks) - { - } - - std::string name; - ZarrDimensionType type{ ZarrDimensionType_Space }; - - uint32_t array_size_px{ 0 }; - uint32_t chunk_size_px{ 0 }; - uint32_t shard_size_chunks{ 0 }; -}; - -class ArrayDimensions -{ - public: - ArrayDimensions(std::vector&& dims, ZarrDataType dtype); - - size_t ndims() const; - - const ZarrDimension& operator[](size_t idx) const; - const ZarrDimension& at(size_t idx) const { return operator[](idx); } - - const ZarrDimension& final_dim() const; - const ZarrDimension& height_dim() const; - const ZarrDimension& width_dim() const; - - /** - * @brief Get the index of a chunk in the chunk lattice for a given frame - * and dimension. - * @param frame_id The frame ID. - * @param dimension_idx The index of the dimension in the dimension vector. - * @return The index of the chunk in the chunk lattice. - */ - uint32_t chunk_lattice_index(uint64_t frame_id, uint32_t dim_index) const; - - /** - * @brief Find the offset in the array of chunk buffers for the given frame. - * @param frame_id The frame ID. - * @return The offset in the array of chunk buffers. - */ - uint32_t tile_group_offset(uint64_t frame_id) const; - - /** - * @brief Find the byte offset inside a chunk for a given frame and data - * type. - * @param frame_id The frame ID. - * @param dims The dimensions of the array. - * @param type The data type of the array. - * @return The byte offset inside a chunk. - */ - uint64_t chunk_internal_offset(uint64_t frame_id) const; - - /** - * @brief Get the number of chunks to hold in memory. - * @return The number of chunks to buffer before writing out. - */ - uint32_t number_of_chunks_in_memory() const; - - /** - * @brief Get the size, in bytes, of a single raw chunk. - * @return The number of bytes to allocate for a chunk. - */ - size_t bytes_per_chunk() const; - - /** - * @brief Get the number of shards to write at one time. - * @return The number of shards to buffer and write out. - */ - uint32_t number_of_shards() const; - - /** - * @brief Get the number of chunks in a single shard. - * @return The number of chunks in a shard. - */ - uint32_t chunks_per_shard() const; - - /** - * @brief Get the shard index for a given chunk index, given array dimensions. - * @param chunk_index The index of the chunk. - * @return The index of the shard containing the chunk. - */ - uint32_t shard_index_for_chunk(uint32_t chunk_index) const; - - /** - * @brief Get the streaming index of a chunk within a shard. - * @param chunk_index The index of the chunk. - * @return The index of the chunk within the shard. - */ - uint32_t shard_internal_index(uint32_t chunk_index) const; - - private: - std::vector dims_; - ZarrDataType dtype_; -}; \ No newline at end of file diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp deleted file mode 100644 index ccb3d01f..00000000 --- a/src/streaming/zarr.stream.cpp +++ /dev/null @@ -1,1008 +0,0 @@ -#include "macros.hh" -#include "zarr.stream.hh" -#include "acquire.zarr.h" -#include "zarr.common.hh" -#include "zarrv2.array.writer.hh" -#include "zarrv3.array.writer.hh" -#include "sink.creator.hh" - -#include - -#include - -#ifdef min -#undef min -#endif - -namespace fs = std::filesystem; - -namespace { -bool -is_s3_acquisition(const struct ZarrStreamSettings_s* settings) -{ - return nullptr != settings->s3_settings; -} - -bool -is_compressed_acquisition(const struct ZarrStreamSettings_s* settings) -{ - return nullptr != settings->compression_settings; -} - -[[nodiscard]] -bool -validate_s3_settings(const ZarrS3Settings* settings) -{ - if (zarr::is_empty_string(settings->endpoint, "S3 endpoint is empty")) { - return false; - } - if (zarr::is_empty_string(settings->access_key_id, - "S3 access key ID is empty")) { - return false; - } - if (zarr::is_empty_string(settings->secret_access_key, - "S3 secret access key is empty")) { - return false; - } - - std::string trimmed = zarr::trim(settings->bucket_name); - if (trimmed.length() < 3 || trimmed.length() > 63) { - LOG_ERROR("Invalid length for S3 bucket name: ", - trimmed.length(), - ". Must be between 3 " - "and 63 characters"); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_filesystem_store_path(std::string_view data_root) -{ - fs::path path(data_root); - fs::path parent_path = path.parent_path(); - if (parent_path.empty()) { - parent_path = "."; - } - - // parent path must exist and be a directory - if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { - LOG_ERROR("Parent path '", - parent_path, - "' does not exist or is not a directory"); - return false; - } - - // parent path must be writable - const auto perms = fs::status(parent_path).permissions(); - const bool is_writable = - (perms & (fs::perms::owner_write | fs::perms::group_write | - fs::perms::others_write)) != fs::perms::none; - - if (!is_writable) { - LOG_ERROR("Parent path '", parent_path, "' is not writable"); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_compression_settings(const ZarrCompressionSettings* settings) -{ - if (settings->compressor >= ZarrCompressorCount) { - LOG_ERROR("Invalid compressor: ", settings->compressor); - return false; - } - - if (settings->codec >= ZarrCompressionCodecCount) { - LOG_ERROR("Invalid compression codec: ", settings->codec); - return false; - } - - // if compressing, we require a compression codec - if (settings->compressor != ZarrCompressor_None && - settings->codec == ZarrCompressionCodec_None) { - LOG_ERROR("Compression codec must be set when using a compressor"); - return false; - } - - if (settings->level > 9) { - LOG_ERROR("Invalid compression level: ", - settings->level, - ". Must be between 0 and 9"); - return false; - } - - if (settings->shuffle != BLOSC_NOSHUFFLE && - settings->shuffle != BLOSC_SHUFFLE && - settings->shuffle != BLOSC_BITSHUFFLE) { - LOG_ERROR("Invalid shuffle: ", - settings->shuffle, - ". Must be ", - BLOSC_NOSHUFFLE, - " (no shuffle), ", - BLOSC_SHUFFLE, - " (byte shuffle), or ", - BLOSC_BITSHUFFLE, - " (bit shuffle)"); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_custom_metadata(const char* metadata) -{ - if (metadata == nullptr || !*metadata) { - return true; // custom metadata is optional - } - - // parse the JSON - auto val = nlohmann::json::parse(metadata, - nullptr, // callback - false, // allow exceptions - true // ignore comments - ); - - if (val.is_discarded()) { - LOG_ERROR("Invalid JSON: ", metadata); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_dimension(const ZarrDimensionProperties* dimension, - ZarrVersion version, - bool is_append) -{ - if (zarr::is_empty_string(dimension->name, "Dimension name is empty")) { - return false; - } - - if (dimension->type >= ZarrDimensionTypeCount) { - LOG_ERROR("Invalid dimension type: ", dimension->type); - return false; - } - - if (!is_append && dimension->array_size_px == 0) { - LOG_ERROR("Array size must be nonzero"); - return false; - } - - if (dimension->chunk_size_px == 0) { - LOG_ERROR("Invalid chunk size: ", dimension->chunk_size_px); - return false; - } - - if (version == ZarrVersion_3 && dimension->shard_size_chunks == 0) { - LOG_ERROR("Shard size must be nonzero"); - return false; - } - - return true; -} - -[[nodiscard]] -bool -validate_settings(const struct ZarrStreamSettings_s* settings) -{ - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return false; - } - - auto version = settings->version; - if (version < ZarrVersion_2 || version >= ZarrVersionCount) { - LOG_ERROR("Invalid Zarr version: ", version); - return false; - } - - if (settings->store_path == nullptr) { - LOG_ERROR("Null pointer: store_path"); - return false; - } - std::string_view store_path(settings->store_path); - - // we require the store path (root of the dataset) to be nonempty - if (store_path.empty()) { - LOG_ERROR("Store path is empty"); - return false; - } - - if ((is_s3_acquisition(settings) && - !validate_s3_settings(settings->s3_settings)) || - (!is_s3_acquisition(settings) && - !validate_filesystem_store_path(store_path))) { - return false; - } - - if (settings->data_type >= ZarrDataTypeCount) { - LOG_ERROR("Invalid data type: ", settings->data_type); - return false; - } - - if (is_compressed_acquisition(settings) && - !validate_compression_settings(settings->compression_settings)) { - return false; - } - - if (!validate_custom_metadata(settings->custom_metadata)) { - return false; - } - - if (settings->dimensions == nullptr) { - LOG_ERROR("Null pointer: dimensions"); - return false; - } - - // we must have at least 3 dimensions - const size_t ndims = settings->dimension_count; - if (ndims < 3) { - LOG_ERROR( - "Invalid number of dimensions: ", ndims, ". Must be at least 3"); - return false; - } - - // check the final dimension (width), must be space - if (settings->dimensions[ndims - 1].type != ZarrDimensionType_Space) { - LOG_ERROR("Last dimension must be of type Space"); - return false; - } - - // check the penultimate dimension (height), must be space - if (settings->dimensions[ndims - 2].type != ZarrDimensionType_Space) { - LOG_ERROR("Second to last dimension must be of type Space"); - return false; - } - - // validate the dimensions individually - for (size_t i = 0; i < ndims; ++i) { - if (!validate_dimension(settings->dimensions + i, version, i == 0)) { - return false; - } - } - - return true; -} - -std::string -dimension_type_to_string(ZarrDimensionType type) -{ - switch (type) { - case ZarrDimensionType_Time: - return "time"; - case ZarrDimensionType_Channel: - return "channel"; - case ZarrDimensionType_Space: - return "space"; - case ZarrDimensionType_Other: - return "other"; - default: - return "(unknown)"; - } -} - -template -[[nodiscard]] -std::byte* -scale_image(const std::byte* const src, - size_t& bytes_of_src, - size_t& width, - size_t& height) -{ - CHECK(src); - - const size_t bytes_of_frame = width * height * sizeof(T); - EXPECT(bytes_of_src >= bytes_of_frame, - "Expecting at least %zu bytes, got %zu", - bytes_of_frame, - bytes_of_src); - - const int downscale = 2; - constexpr auto bytes_of_type = static_cast(sizeof(T)); - const double factor = 0.25; - - const auto w_pad = static_cast(width + (width % downscale)); - const auto h_pad = static_cast(height + (height % downscale)); - - const auto size_downscaled = - static_cast(w_pad * h_pad * factor * bytes_of_type); - - auto* dst = new T[size_downscaled]; - EXPECT(dst, - "Failed to allocate ", - size_downscaled, - " bytes for destination frame"); - - memset(dst, 0, size_downscaled); - - size_t dst_idx = 0; - for (auto row = 0; row < height; row += downscale) { - const bool pad_height = (row == height - 1 && height != h_pad); - - for (auto col = 0; col < width; col += downscale) { - size_t src_idx = row * width + col; - const bool pad_width = (col == width - 1 && width != w_pad); - - auto here = static_cast(src[src_idx]); - auto right = static_cast( - src[src_idx + (1 - static_cast(pad_width))]); - auto down = static_cast( - src[src_idx + width * (1 - static_cast(pad_height))]); - auto diag = static_cast( - src[src_idx + width * (1 - static_cast(pad_height)) + - (1 - static_cast(pad_width))]); - - dst[dst_idx++] = - static_cast(factor * (here + right + down + diag)); - } - } - - bytes_of_src = size_downscaled; - width = static_cast(w_pad) / 2; - height = static_cast(h_pad) / 2; - - return reinterpret_cast(dst); -} - -template -void -average_two_frames(void* dst_, - size_t bytes_of_dst, - const void* src_, - size_t bytes_of_src) -{ - CHECK(dst_); - CHECK(src_); - EXPECT(bytes_of_dst == bytes_of_src, - "Expecting %zu bytes in destination, got %zu", - bytes_of_src, - bytes_of_dst); - - T* dst = static_cast(dst_); - const T* src = static_cast(src_); - - const auto num_pixels = bytes_of_src / sizeof(T); - for (auto i = 0; i < num_pixels; ++i) { - dst[i] = static_cast(0.5 * (dst[i] + src[i])); - } -} -} // namespace - -/* ZarrStream_s implementation */ - -ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings) - : error_() - , frame_buffer_offset_(0) -{ - if (!validate_settings(settings)) { - throw std::runtime_error("Invalid Zarr stream settings"); - } - - commit_settings_(settings); - - // spin up thread pool - thread_pool_ = std::make_shared( - std::thread::hardware_concurrency(), - [this](const std::string& err) { this->set_error_(err); }); - - // allocate a frame buffer - frame_buffer_.resize( - zarr::bytes_of_frame(*dimensions_, static_cast(dtype_))); - - // create the data store - EXPECT(create_store_(), error_); - - // allocate writers - EXPECT(create_writers_(), error_); - - // allocate multiscale frame placeholders - if (multiscale_) { - create_scaled_frames_(); - } - - // allocate metadata sinks - EXPECT(create_metadata_sinks_(), error_); - - // write base metadata - EXPECT(write_base_metadata_(), error_); - - // write group metadata - EXPECT(write_group_metadata_(), error_); - - // write external metadata - EXPECT(write_external_metadata_(), error_); -} - -size_t -ZarrStream::append(const void* data_, size_t nbytes) -{ - EXPECT(error_.empty(), "Cannot append data: ", error_.c_str()); - - if (0 == nbytes) { - return 0; - } - - auto* data = static_cast(data_); - - const size_t bytes_of_frame = frame_buffer_.size(); - size_t bytes_written = 0; - - while (bytes_written < nbytes) { - const size_t bytes_remaining = nbytes - bytes_written; - if (frame_buffer_offset_ > 0) { // add to / finish a partial frame - const size_t bytes_to_copy = - std::min(bytes_of_frame - frame_buffer_offset_, bytes_remaining); - - memcpy(frame_buffer_.data() + frame_buffer_offset_, - data + bytes_written, - bytes_to_copy); - frame_buffer_offset_ += bytes_to_copy; - bytes_written += bytes_to_copy; - - // ready to flush the frame buffer - if (frame_buffer_offset_ == bytes_of_frame) { - const size_t bytes_written_this_frame = - writers_[0]->write_frame({ data, bytes_of_frame }); - if (bytes_written_this_frame == 0) { - break; - } - - bytes_written += bytes_to_copy; - data += bytes_to_copy; - frame_buffer_offset_ = 0; - } - } else if (bytes_remaining < bytes_of_frame) { // begin partial frame - memcpy(frame_buffer_.data(), data, bytes_remaining); - frame_buffer_offset_ = bytes_remaining; - bytes_written += bytes_remaining; - } else { // at least one full frame - const size_t bytes_written_this_frame = - writers_[0]->write_frame({ data, bytes_of_frame }); - if (bytes_written_this_frame == 0) { - break; - } - - write_multiscale_frames_(data, bytes_written_this_frame); - - bytes_written += bytes_written_this_frame; - data += bytes_written_this_frame; - } - } - - return bytes_written; -} - -bool -ZarrStream_s::is_s3_acquisition_() const -{ - return s3_settings_.has_value(); -} - -bool -ZarrStream_s::is_compressed_acquisition_() const -{ - return compression_settings_.has_value(); -} - -void -ZarrStream_s::commit_settings_(const struct ZarrStreamSettings_s* settings) -{ - version_ = settings->version; - store_path_ = zarr::trim(settings->store_path); - if (settings->custom_metadata) { - custom_metadata_ = zarr::trim(settings->custom_metadata); - } else { - custom_metadata_ = "{}"; - } - - if (is_s3_acquisition(settings)) { - s3_settings_ = { - .endpoint = zarr::trim(settings->s3_settings->endpoint), - .bucket_name = zarr::trim(settings->s3_settings->bucket_name), - .access_key_id = zarr::trim(settings->s3_settings->access_key_id), - .secret_access_key = - zarr::trim(settings->s3_settings->secret_access_key), - }; - } - - if (is_compressed_acquisition(settings)) { - compression_settings_ = { - .compressor = settings->compression_settings->compressor, - .codec = settings->compression_settings->codec, - .level = settings->compression_settings->level, - .shuffle = settings->compression_settings->shuffle, - }; - } - - dtype_ = settings->data_type; - - std::vector dims; - for (auto i = 0; i < settings->dimension_count; ++i) { - const auto& dim = settings->dimensions[i]; - dims.emplace_back(dim.name, - dim.type, - dim.array_size_px, - dim.chunk_size_px, - dim.shard_size_chunks); - } - dimensions_ = std::make_shared(std::move(dims), dtype_); - - multiscale_ = settings->multiscale; -} - -void -ZarrStream_s::set_error_(const std::string& msg) -{ - error_ = msg; -} - -bool -ZarrStream_s::create_store_() -{ - if (is_s3_acquisition_()) { - // spin up S3 connection pool - try { - s3_connection_pool_ = std::make_shared( - std::thread::hardware_concurrency(), - s3_settings_->endpoint, - s3_settings_->access_key_id, - s3_settings_->secret_access_key); - } catch (const std::exception& e) { - set_error_("Error creating S3 connection pool: " + - std::string(e.what())); - return false; - } - - // test the S3 connection - auto conn = s3_connection_pool_->get_connection(); - if (!conn->is_connection_valid()) { - set_error_("Failed to connect to S3"); - return false; - } - s3_connection_pool_->return_connection(std::move(conn)); - } else { - if (fs::exists(store_path_)) { - // remove everything inside the store path - std::error_code ec; - fs::remove_all(store_path_, ec); - - if (ec) { - set_error_("Failed to remove existing store path '" + - store_path_ + "': " + ec.message()); - return false; - } - } - - // create the store path - { - std::error_code ec; - if (!fs::create_directories(store_path_, ec)) { - set_error_("Failed to create store path '" + - store_path_ + "': " + ec.message()); - return false; - } - } - } - - return true; -} - -bool -ZarrStream_s::create_writers_() -{ - writers_.clear(); - - // construct Blosc compression parameters - std::optional blosc_compression_params; - if (is_compressed_acquisition_()) { - blosc_compression_params = zarr::BloscCompressionParams( - zarr::blosc_codec_to_string(compression_settings_->codec), - compression_settings_->level, - compression_settings_->shuffle); - } - - std::optional s3_bucket_name; - if (is_s3_acquisition_()) { - s3_bucket_name = s3_settings_->bucket_name; - } - - zarr::ArrayWriterConfig config = { - .dimensions = dimensions_, - .dtype = static_cast(dtype_), - .level_of_detail = 0, - .bucket_name = s3_bucket_name, - .store_path = store_path_, - .compression_params = blosc_compression_params, - }; - - if (version_ == 2) { - writers_.push_back(std::make_unique( - config, thread_pool_, s3_connection_pool_)); - } else { - writers_.push_back(std::make_unique( - config, thread_pool_, s3_connection_pool_)); - } - - if (multiscale_) { - zarr::ArrayWriterConfig downsampled_config; - - bool do_downsample = true; - while (do_downsample) { - do_downsample = downsample(config, downsampled_config); - - if (version_ == 2) { - writers_.push_back(std::make_unique( - downsampled_config, thread_pool_, s3_connection_pool_)); - } else { - writers_.push_back(std::make_unique( - downsampled_config, thread_pool_, s3_connection_pool_)); - } - // scaled_frames_.emplace(level++, std::nullopt); - - config = std::move(downsampled_config); - downsampled_config = {}; - } - } - - return true; -} - -void -ZarrStream_s::create_scaled_frames_() -{ - for (size_t level = 1; level < writers_.size(); ++level) { - scaled_frames_.emplace(level, std::nullopt); - } -} - -bool -ZarrStream_s::create_metadata_sinks_() -{ - zarr::SinkCreator creator(thread_pool_, s3_connection_pool_); - - try { - if (s3_connection_pool_) { - if (!creator.make_metadata_sinks(version_, - s3_settings_->bucket_name, - store_path_, - metadata_sinks_)) { - set_error_("Error creating metadata sinks"); - return false; - } - } else { - if (!creator.make_metadata_sinks( - version_, store_path_, metadata_sinks_)) { - set_error_("Error creating metadata sinks"); - return false; - } - } - } catch (const std::exception& e) { - set_error_("Error creating metadata sinks: " + std::string(e.what())); - return false; - } - - return true; -} - -bool -ZarrStream_s::write_base_metadata_() -{ - nlohmann::json metadata; - std::string metadata_key; - - if (version_ == 2) { - metadata["multiscales"] = make_multiscale_metadata_(); - - metadata_key = ".zattrs"; - } else { - metadata["extensions"] = nlohmann::json::array(); - metadata["metadata_encoding"] = - "https://purl.org/zarr/spec/protocol/core/3.0"; - metadata["metadata_key_suffix"] = ".json"; - metadata["zarr_format"] = - "https://purl.org/zarr/spec/protocol/core/3.0"; - - metadata_key = "zarr.json"; - } - - const std::unique_ptr& sink = metadata_sinks_.at(metadata_key); - if (!sink) { - set_error_("Metadata sink '" + metadata_key + "'not found"); - return false; - } - - const std::string metadata_str = metadata.dump(4); - std::span data{ reinterpret_cast(metadata_str.data()), - metadata_str.size() }; - - if (!sink->write(0, data)) { - set_error_("Error writing base metadata"); - return false; - } - - return true; -} - -bool -ZarrStream_s::write_group_metadata_() -{ - nlohmann::json metadata; - std::string metadata_key; - - if (version_ == 2) { - metadata = { { "zarr_format", 2 } }; - - metadata_key = ".zgroup"; - } else { - metadata["attributes"]["multiscales"] = make_multiscale_metadata_(); - - metadata_key = "meta/root.group.json"; - } - - const std::unique_ptr& sink = metadata_sinks_.at(metadata_key); - if (!sink) { - set_error_("Metadata sink '" + metadata_key + "'not found"); - return false; - } - - const std::string metadata_str = metadata.dump(4); - std::span data{ reinterpret_cast(metadata_str.data()), - metadata_str.size() }; - if (!sink->write(0, data)) { - set_error_("Error writing group metadata"); - return false; - } - - return true; -} - -bool -ZarrStream_s::write_external_metadata_() -{ - if (custom_metadata_.empty()) { - return true; - } - - auto metadata = nlohmann::json::parse(custom_metadata_, - nullptr, // callback - false, // allow exceptions - true // ignore comments - ); - std::string metadata_key = "acquire.json"; - - if (version_ == 3) { - metadata_key = "meta/" + metadata_key; - } - - const std::unique_ptr& sink = metadata_sinks_.at(metadata_key); - if (!sink) { - set_error_("Metadata sink '" + metadata_key + "'not found"); - return false; - } - - const std::string metadata_str = metadata.dump(4); - std::span data{ reinterpret_cast(metadata_str.data()), - metadata_str.size() }; - if (!sink->write(0, data)) { - set_error_("Error writing external metadata"); - return false; - } - - return true; -} - -nlohmann::json -ZarrStream_s::make_multiscale_metadata_() const -{ - nlohmann::json multiscales; - multiscales[0]["version"] = "0.4"; - - auto& axes = multiscales[0]["axes"]; - for (auto i = 0; i < dimensions_->ndims(); ++i) { - const auto& dim = dimensions_->at(i); - std::string type = dimension_type_to_string(dim.type); - - if (i < dimensions_->ndims() - 2) { - axes.push_back({ { "name", dim.name.c_str() }, { "type", type } }); - } else { - axes.push_back({ { "name", dim.name.c_str() }, - { "type", type }, - { "unit", "micrometer" } }); - } - } - - // spatial multiscale metadata - std::vector scales(dimensions_->ndims(), 1.0); - multiscales[0]["datasets"] = { - { - { "path", "0" }, - { "coordinateTransformations", - { - { - { "type", "scale" }, - { "scale", scales }, - }, - } }, - }, - }; - - for (auto i = 1; i < writers_.size(); ++i) { - scales.clear(); - scales.push_back(std::pow(2, i)); // append - for (auto k = 0; k < dimensions_->ndims() - 3; ++k) { - scales.push_back(1.); - } - scales.push_back(std::pow(2, i)); // y - scales.push_back(std::pow(2, i)); // x - - multiscales[0]["datasets"].push_back({ - { "path", std::to_string(i) }, - { "coordinateTransformations", - { - { - { "type", "scale" }, - { "scale", scales }, - }, - } }, - }); - - // downsampling metadata - multiscales[0]["type"] = "local_mean"; - multiscales[0]["metadata"] = { - { "description", - "The fields in the metadata describe how to reproduce this " - "multiscaling in scikit-image. The method and its parameters " - "are " - "given here." }, - { "method", "skimage.transform.downscale_local_mean" }, - { "version", "0.21.0" }, - { "args", "[2]" }, - { "kwargs", { "cval", 0 } }, - }; - } - - return multiscales; -} - -void -ZarrStream_s::write_multiscale_frames_(const std::byte* data, - size_t bytes_of_data) -{ - if (!multiscale_) { - return; - } - - std::function scale; - std::function average2; - - switch (dtype_) { - case ZarrDataType_uint8: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_uint16: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_uint32: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_uint64: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_int8: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_int16: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_int32: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_int64: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_float32: - scale = scale_image; - average2 = average_two_frames; - break; - case ZarrDataType_float64: - scale = scale_image; - average2 = average_two_frames; - break; - default: - throw std::runtime_error("Invalid data type: " + - std::to_string(dtype_)); - } - - size_t frame_width = dimensions_->width_dim().array_size_px; - size_t frame_height = dimensions_->height_dim().array_size_px; - - std::byte* dst; - for (auto i = 1; i < writers_.size(); ++i) { - dst = scale(data, bytes_of_data, frame_width, frame_height); - - // bytes_of data is now downscaled - // frame_width and frame_height are now the new dimensions - - if (scaled_frames_[i]) { - average2(dst, bytes_of_data, *scaled_frames_[i], bytes_of_data); - std::span frame_data{ reinterpret_cast(dst), - bytes_of_data }; - EXPECT(writers_[i]->write_frame(frame_data), - "Failed to write frame to writer %zu", - i); - - // clean up this LOD - delete[] *scaled_frames_[i]; - scaled_frames_[i].reset(); - - // set up for next iteration - if (i + 1 < writers_.size()) { - data = dst; - } else { - delete[] dst; - } - } else { - scaled_frames_[i] = dst; - break; - } - } -} - -bool -finalize_stream(struct ZarrStream_s* stream) -{ - if (stream == nullptr) { - LOG_INFO("Stream is null. Nothing to finalize."); - return true; - } - - if (!stream->write_group_metadata_()) { - LOG_ERROR("Error finalizing Zarr stream: ", stream->error_); - return false; - } - - for (auto& [sink_name, sink] : stream->metadata_sinks_) { - if (!finalize_sink(std::move(sink))) { - LOG_ERROR("Error finalizing Zarr stream. Failed to write ", - sink_name); - return false; - } - } - stream->metadata_sinks_.clear(); - - for (auto i = 0; i < stream->writers_.size(); ++i) { - if (!finalize_array(std::move(stream->writers_[i]))) { - LOG_ERROR("Error finalizing Zarr stream. Failed to write array ", - i); - return false; - } - } - stream->writers_.clear(); // flush before shutting down thread pool - stream->thread_pool_->await_stop(); - - for (auto& [_, frame] : stream->scaled_frames_) { - if (frame) { - delete[] *frame; - } - } - - return true; -} \ No newline at end of file diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh deleted file mode 100644 index d2fd9d83..00000000 --- a/src/streaming/zarr.stream.hh +++ /dev/null @@ -1,110 +0,0 @@ -#pragma once - -#include "zarr.dimension.hh" -#include "thread.pool.hh" -#include "s3.connection.hh" -#include "sink.hh" -#include "array.writer.hh" - -#include - -#include // size_t -#include // unique_ptr -#include - -struct ZarrStream_s -{ - public: - explicit ZarrStream_s(struct ZarrStreamSettings_s* settings); - - /** - * @brief Append data to the stream. - * @param data The data to append. - * @param nbytes The number of bytes to append. - * @return The number of bytes appended. - */ - size_t append(const void* data, size_t nbytes); - - private: - struct S3Settings { - std::string endpoint; - std::string bucket_name; - std::string access_key_id; - std::string secret_access_key; - }; - struct CompressionSettings { - ZarrCompressor compressor; - ZarrCompressionCodec codec; - uint8_t level; - uint8_t shuffle; - }; - - std::string error_; // error message. If nonempty, an error occurred. - - ZarrVersion version_; - std::string store_path_; - std::optional s3_settings_; - std::optional compression_settings_; - std::string custom_metadata_; - ZarrDataType dtype_; - std::shared_ptr dimensions_; - bool multiscale_; - - std::vector frame_buffer_; - size_t frame_buffer_offset_; - - std::shared_ptr thread_pool_; - std::shared_ptr s3_connection_pool_; - - std::vector> writers_; - std::unordered_map> - metadata_sinks_; - - std::unordered_map> scaled_frames_; - - bool is_s3_acquisition_() const; - bool is_compressed_acquisition_() const; - - /** - * @brief Copy settings to the stream. - * @param settings Struct containing settings to copy. - */ - void commit_settings_(const struct ZarrStreamSettings_s* settings); - - /** - * @brief Set an error message. - * @param msg The error message to set. - */ - void set_error_(const std::string& msg); - - /** @brief Create the data store. */ - [[nodiscard]] bool create_store_(); - - /** @brief Create the writers. */ - [[nodiscard]] bool create_writers_(); - - /** @brief Create placeholders for multiscale frames. */ - void create_scaled_frames_(); - - /** @brief Create the metadata sinks. */ - [[nodiscard]] bool create_metadata_sinks_(); - - /** @brief Write per-acquisition metadata. */ - [[nodiscard]] bool write_base_metadata_(); - - /** @brief Write Zarr group metadata. */ - [[nodiscard]] bool write_group_metadata_(); - - /** @brief Write external metadata. */ - [[nodiscard]] bool write_external_metadata_(); - - /** @brief Construct OME metadata pertaining to the multiscale pyramid. */ - [[nodiscard]] nlohmann::json make_multiscale_metadata_() const; - - void write_multiscale_frames_(const std::byte* data, size_t bytes_of_data); - - friend bool finalize_stream(struct ZarrStream_s* stream); -}; - -bool -finalize_stream(struct ZarrStream_s* stream); diff --git a/src/streaming/zarrv2.array.writer.cpp b/src/streaming/zarrv2.array.writer.cpp deleted file mode 100644 index 8b620ca1..00000000 --- a/src/streaming/zarrv2.array.writer.cpp +++ /dev/null @@ -1,183 +0,0 @@ -#include "macros.hh" -#include "zarrv2.array.writer.hh" -#include "sink.creator.hh" -#include "zarr.common.hh" - -#include - -#include -#include - -namespace { -[[nodiscard]] -bool -sample_type_to_dtype(ZarrDataType t, std::string& t_str) - -{ - const std::string dtype_prefix = - std::endian::native == std::endian::big ? ">" : "<"; - - switch (t) { - case ZarrDataType_uint8: - t_str = dtype_prefix + "u1"; - break; - case ZarrDataType_uint16: - t_str = dtype_prefix + "u2"; - break; - case ZarrDataType_uint32: - t_str = dtype_prefix + "u4"; - break; - case ZarrDataType_uint64: - t_str = dtype_prefix + "u8"; - break; - case ZarrDataType_int8: - t_str = dtype_prefix + "i1"; - break; - case ZarrDataType_int16: - t_str = dtype_prefix + "i2"; - break; - case ZarrDataType_int32: - t_str = dtype_prefix + "i4"; - break; - case ZarrDataType_int64: - t_str = dtype_prefix + "i8"; - break; - case ZarrDataType_float32: - t_str = dtype_prefix + "f4"; - break; - case ZarrDataType_float64: - t_str = dtype_prefix + "f8"; - break; - default: - LOG_ERROR("Unsupported sample type: ", t); - return false; - } - - return true; -} -} // namespace - -zarr::ZarrV2ArrayWriter::ZarrV2ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool) - : ArrayWriter(config, thread_pool) -{ -} - -zarr::ZarrV2ArrayWriter::ZarrV2ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool) - : ArrayWriter(config, thread_pool, s3_connection_pool) -{ -} - -bool -zarr::ZarrV2ArrayWriter::flush_impl_() -{ - // create chunk files - CHECK(data_sinks_.empty()); - if (!make_data_sinks_()) { - return false; - } - - CHECK(data_sinks_.size() == chunk_buffers_.size()); - - std::latch latch(chunk_buffers_.size()); - { - std::scoped_lock lock(buffers_mutex_); - for (auto i = 0; i < data_sinks_.size(); ++i) { - auto& chunk = chunk_buffers_.at(i); - EXPECT(thread_pool_->push_job( - std::move([&sink = data_sinks_.at(i), - data_ = chunk.data(), - size = chunk.size(), - &latch](std::string& err) -> bool { - bool success = false; - try { - std::span data{ - reinterpret_cast(data_), size - }; - CHECK(sink->write(0, data)); - success = true; - } catch (const std::exception& exc) { - err = "Failed to write chunk: " + - std::string(exc.what()); - } - - latch.count_down(); - return success; - })), - "Failed to push job to thread pool"); - } - } - - // wait for all threads to finish - latch.wait(); - - return true; -} - -bool -zarr::ZarrV2ArrayWriter::write_array_metadata_() -{ - if (!make_metadata_sink_()) { - return false; - } - - using json = nlohmann::json; - - std::string dtype; - if (!sample_type_to_dtype(config_.dtype, dtype)) { - return false; - } - - std::vector array_shape, chunk_shape; - - size_t append_size = frames_written_; - for (auto i = config_.dimensions->ndims() - 3; i > 0; --i) { - const auto& dim = config_.dimensions->at(i); - const auto& array_size_px = dim.array_size_px; - CHECK(array_size_px); - append_size = (append_size + array_size_px - 1) / array_size_px; - } - array_shape.push_back(append_size); - - chunk_shape.push_back(config_.dimensions->final_dim().chunk_size_px); - for (auto i = 1; i < config_.dimensions->ndims(); ++i) { - const auto& dim = config_.dimensions->at(i); - array_shape.push_back(dim.array_size_px); - chunk_shape.push_back(dim.chunk_size_px); - } - - json metadata; - metadata["zarr_format"] = 2; - metadata["shape"] = array_shape; - metadata["chunks"] = chunk_shape; - metadata["dtype"] = dtype; - metadata["fill_value"] = 0; - metadata["order"] = "C"; - metadata["filters"] = nullptr; - metadata["dimension_separator"] = "/"; - - if (config_.compression_params) { - const BloscCompressionParams bcp = *config_.compression_params; - metadata["compressor"] = json{ { "id", "blosc" }, - { "cname", bcp.codec_id }, - { "clevel", bcp.clevel }, - { "shuffle", bcp.shuffle } }; - } else { - metadata["compressor"] = nullptr; - } - - std::string metadata_str = metadata.dump(4); - std::span data{ reinterpret_cast(metadata_str.data()), - metadata_str.size() }; - return metadata_sink_->write(0, data); -} - -bool -zarr::ZarrV2ArrayWriter::should_rollover_() const -{ - return true; -} diff --git a/src/streaming/zarrv2.array.writer.hh b/src/streaming/zarrv2.array.writer.hh deleted file mode 100644 index 2a6039a4..00000000 --- a/src/streaming/zarrv2.array.writer.hh +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "array.writer.hh" - -namespace zarr { -class ZarrV2ArrayWriter final : public ArrayWriter -{ - public: - ZarrV2ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool); - - ZarrV2ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool); - - private: - ZarrVersion version_() const override { return ZarrVersion_2; }; - bool flush_impl_() override; - bool write_array_metadata_() override; - bool should_rollover_() const override; -}; -} // namespace zarr diff --git a/src/streaming/zarrv3.array.writer.cpp b/src/streaming/zarrv3.array.writer.cpp deleted file mode 100644 index d687076e..00000000 --- a/src/streaming/zarrv3.array.writer.cpp +++ /dev/null @@ -1,254 +0,0 @@ -#include "macros.hh" -#include "zarrv3.array.writer.hh" -#include "sink.creator.hh" -#include "zarr.common.hh" - -#include - -#include // std::fill -#include -#include - -#ifdef max -#undef max -#endif - -namespace { -std::string -sample_type_to_dtype(ZarrDataType t) -{ - switch (t) { - case ZarrDataType_uint8: - return "uint8"; - case ZarrDataType_uint16: - return "uint16"; - case ZarrDataType_uint32: - return "uint32"; - case ZarrDataType_uint64: - return "uint64"; - case ZarrDataType_int8: - return "int8"; - case ZarrDataType_int16: - return "int16"; - case ZarrDataType_int32: - return "int32"; - case ZarrDataType_int64: - return "int64"; - case ZarrDataType_float32: - return "float32"; - case ZarrDataType_float64: - return "float64"; - default: - throw std::runtime_error("Invalid ZarrDataType: " + - std::to_string(static_cast(t))); - } -} -} // namespace - -zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool) - : ZarrV3ArrayWriter(config, thread_pool, nullptr) -{ -} - -zarr::ZarrV3ArrayWriter::ZarrV3ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool) - : ArrayWriter(config, thread_pool, s3_connection_pool) -{ - const auto number_of_shards = config_.dimensions->number_of_shards(); - const auto chunks_per_shard = config_.dimensions->chunks_per_shard(); - - shard_file_offsets_.resize(number_of_shards, 0); - shard_tables_.resize(number_of_shards); - - for (auto& table : shard_tables_) { - table.resize(2 * chunks_per_shard); - std::fill( - table.begin(), table.end(), std::numeric_limits::max()); - } -} - -bool -zarr::ZarrV3ArrayWriter::flush_impl_() -{ - // create shard files if they don't exist - if (data_sinks_.empty() && !make_data_sinks_()) { - return false; - } - - const auto n_shards = config_.dimensions->number_of_shards(); - CHECK(data_sinks_.size() == n_shards); - - // get shard indices for each chunk - std::vector> chunk_in_shards(n_shards); - for (auto i = 0; i < chunk_buffers_.size(); ++i) { - const auto index = config_.dimensions->shard_index_for_chunk(i); - chunk_in_shards[index].push_back(i); - } - - // write out chunks to shards - auto write_table = is_finalizing_ || should_rollover_(); - std::latch latch(n_shards); - for (auto i = 0; i < n_shards; ++i) { - const auto& chunks = chunk_in_shards.at(i); - auto& chunk_table = shard_tables_.at(i); - auto* file_offset = &shard_file_offsets_.at(i); - - EXPECT(thread_pool_->push_job([&sink = data_sinks_.at(i), - &chunks, - &chunk_table, - file_offset, - write_table, - &latch, - this](std::string& err) { - bool success = false; - - try { - for (const auto& chunk_idx : chunks) { - auto& chunk = chunk_buffers_.at(chunk_idx); - std::span data{ reinterpret_cast(chunk.data()), - chunk.size() }; - success = sink->write(*file_offset, data); - if (!success) { - break; - } - - const auto internal_idx = - config_.dimensions->shard_internal_index(chunk_idx); - chunk_table.at(2 * internal_idx) = *file_offset; - chunk_table.at(2 * internal_idx + 1) = chunk.size(); - - *file_offset += chunk.size(); - } - - if (success && write_table) { - auto* table = - reinterpret_cast(chunk_table.data()); - std::span data{ table, - chunk_table.size() * sizeof(uint64_t) }; - success = sink->write(*file_offset, data); - } - } catch (const std::exception& exc) { - err = "Failed to write chunk: " + std::string(exc.what()); - } - - latch.count_down(); - return success; - }), - "Failed to push job to thread pool"); - } - - // wait for all threads to finish - latch.wait(); - - // reset shard tables and file offsets - if (write_table) { - for (auto& table : shard_tables_) { - std::fill( - table.begin(), table.end(), std::numeric_limits::max()); - } - - std::fill(shard_file_offsets_.begin(), shard_file_offsets_.end(), 0); - } - - return true; -} - -bool -zarr::ZarrV3ArrayWriter::write_array_metadata_() -{ - if (!make_metadata_sink_()) { - return false; - } - - using json = nlohmann::json; - - std::vector array_shape, chunk_shape, shard_shape; - - size_t append_size = frames_written_; - for (auto i = config_.dimensions->ndims() - 3; i > 0; --i) { - const auto& dim = config_.dimensions->at(i); - const auto& array_size_px = dim.array_size_px; - CHECK(array_size_px); - append_size = (append_size + array_size_px - 1) / array_size_px; - } - array_shape.push_back(append_size); - - const auto& final_dim = config_.dimensions->final_dim(); - chunk_shape.push_back(final_dim.chunk_size_px); - shard_shape.push_back(final_dim.shard_size_chunks); - for (auto i = 1; i < config_.dimensions->ndims(); ++i) { - const auto& dim = config_.dimensions->at(i); - array_shape.push_back(dim.array_size_px); - chunk_shape.push_back(dim.chunk_size_px); - shard_shape.push_back(dim.shard_size_chunks); - } - - json metadata; - metadata["attributes"] = json::object(); - metadata["chunk_grid"] = json::object({ - { "chunk_shape", chunk_shape }, - { "separator", "/" }, - { "type", "regular" }, - }); - - metadata["chunk_memory_layout"] = "C"; - metadata["data_type"] = sample_type_to_dtype(config_.dtype); - metadata["extensions"] = json::array(); - metadata["fill_value"] = 0; - metadata["shape"] = array_shape; - - if (config_.compression_params) { - const auto params = *config_.compression_params; - metadata["compressor"] = json::object({ - { "codec", "https://purl.org/zarr/spec/codec/blosc/1.0" }, - { "configuration", - json::object({ - { "blocksize", 0 }, - { "clevel", params.clevel }, - { "cname", params.codec_id }, - { "shuffle", params.shuffle }, - }) }, - }); - } else { - metadata["compressor"] = nullptr; - } - - // sharding storage transformer - // TODO (aliddell): - // https://github.com/zarr-developers/zarr-python/issues/877 - metadata["storage_transformers"] = json::array(); - metadata["storage_transformers"][0] = json::object({ - { "type", "indexed" }, - { "extension", - "https://purl.org/zarr/spec/storage_transformers/sharding/1.0" }, - { "configuration", - json::object({ - { "chunks_per_shard", shard_shape }, - }) }, - }); - - std::string metadata_str = metadata.dump(4); - std::span data = { reinterpret_cast(metadata_str.data()), - metadata_str.size() }; - - return metadata_sink_->write(0, data); -} - -bool -zarr::ZarrV3ArrayWriter::should_rollover_() const -{ - const auto& dims = config_.dimensions; - const auto& append_dim = dims->final_dim(); - size_t frames_before_flush = - append_dim.chunk_size_px * append_dim.shard_size_chunks; - for (auto i = 1; i < dims->ndims() - 2; ++i) { - frames_before_flush *= dims->at(i).array_size_px; - } - - CHECK(frames_before_flush > 0); - return frames_written_ % frames_before_flush == 0; -} diff --git a/src/streaming/zarrv3.array.writer.hh b/src/streaming/zarrv3.array.writer.hh deleted file mode 100644 index 3ad1b05c..00000000 --- a/src/streaming/zarrv3.array.writer.hh +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "array.writer.hh" - -namespace zarr { -struct ZarrV3ArrayWriter : public ArrayWriter -{ - public: - ZarrV3ArrayWriter(const ArrayWriterConfig& config, - std::shared_ptr thread_pool); - ZarrV3ArrayWriter( - const ArrayWriterConfig& config, - std::shared_ptr thread_pool, - std::shared_ptr s3_connection_pool); - - private: - std::vector shard_file_offsets_; - std::vector> shard_tables_; - - ZarrVersion version_() const override { return ZarrVersion_3; } - bool flush_impl_() override; - bool write_array_metadata_() override; - bool should_rollover_() const override; -}; -} // namespace zarr diff --git a/src/driver/zarr.driver.c b/src/zarr.driver.c similarity index 100% rename from src/driver/zarr.driver.c rename to src/zarr.driver.c diff --git a/src/driver/zarr.storage.cpp b/src/zarr.storage.cpp similarity index 100% rename from src/driver/zarr.storage.cpp rename to src/zarr.storage.cpp diff --git a/src/driver/zarr.storage.hh b/src/zarr.storage.hh similarity index 100% rename from src/driver/zarr.storage.hh rename to src/zarr.storage.hh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f369bb86..80df2335 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,10 +1,83 @@ -if (${NOTEST}) - message(STATUS "Skipping test targets") -else () - add_subdirectory(unit-tests) - add_subdirectory(integration) - - if (BUILD_ACQUIRE_DRIVER_ZARR) - add_subdirectory(driver) - endif () -endif () +set(NOTEST ON) +add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) + +# +# PARAMETERS +# +set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project + +# +# Tests +# +set(tests + list-devices + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 +) + +foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") +endforeach () + +# +# Copy driver to tests +# +list(POP_FRONT tests onename) + +foreach (driver + acquire-driver-common + acquire-driver-zarr +) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" + ) + + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + endforeach () +endforeach () \ No newline at end of file diff --git a/tests/driver/README.md b/tests/README.md similarity index 100% rename from tests/driver/README.md rename to tests/README.md diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt deleted file mode 100644 index 3fb6265d..00000000 --- a/tests/driver/CMakeLists.txt +++ /dev/null @@ -1,84 +0,0 @@ -set(NOTEST "TRUE") -add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) -set(NOTEST "FALSE") - -# -# PARAMETERS -# -set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - -# -# Tests -# -set(tests - list-devices - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 -) - -foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") -endforeach () - -# -# Copy driver to tests -# -list(POP_FRONT tests onename) - -foreach (driver - acquire-driver-common - acquire-driver-zarr -) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () -endforeach () \ No newline at end of file diff --git a/tests/driver/external-metadata-with-whitespace-ok.cpp b/tests/external-metadata-with-whitespace-ok.cpp similarity index 100% rename from tests/driver/external-metadata-with-whitespace-ok.cpp rename to tests/external-metadata-with-whitespace-ok.cpp diff --git a/tests/driver/get-meta.cpp b/tests/get-meta.cpp similarity index 100% rename from tests/driver/get-meta.cpp rename to tests/get-meta.cpp diff --git a/tests/driver/get-set-get.cpp b/tests/get-set-get.cpp similarity index 100% rename from tests/driver/get-set-get.cpp rename to tests/get-set-get.cpp diff --git a/tests/driver/get.cpp b/tests/get.cpp similarity index 100% rename from tests/driver/get.cpp rename to tests/get.cpp diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt deleted file mode 100644 index 0df6408b..00000000 --- a/tests/integration/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -set(project acquire-zarr) - -set(tests - stream-zarr-v2-raw-to-filesystem - stream-zarr-v2-compressed-to-filesystem - stream-zarr-v2-raw-to-s3 - stream-zarr-v2-compressed-to-s3 - stream-zarr-v3-raw-to-filesystem - stream-zarr-v3-compressed-to-filesystem - stream-zarr-v3-raw-to-s3 - stream-zarr-v3-compressed-to-s3 -) - -foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp test.macros.hh) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/logger - ) - target_link_libraries(${tgt} PRIVATE - acquire-logger - acquire-zarr - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-zarr") -endforeach () \ No newline at end of file diff --git a/tests/integration/stream-zarr-v2-compressed-to-filesystem.cpp b/tests/integration/stream-zarr-v2-compressed-to-filesystem.cpp deleted file mode 100644 index 4e43421a..00000000 --- a/tests/integration/stream-zarr-v2-compressed-to-filesystem.cpp +++ /dev/null @@ -1,313 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { -const std::string test_path = - (fs::temp_directory_path() / (TEST ".zarr")).string(); - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const size_t nbytes_px = sizeof(int32_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; -} // namespace/s - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = test_path.c_str(), - .s3_settings = nullptr, - .data_type = ZarrDataType_int32, - .version = ZarrVersion_2, - }; - - ZarrCompressionSettings compression_settings = { - .compressor = ZarrCompressor_Blosc1, - .codec = ZarrCompressionCodec_BloscZstd, - .level = 1, - .shuffle = 1, - }; - settings.compression_settings = &compression_settings; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, 0); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT( - type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT( - type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(double, scale[0].get(), 1.0); - EXPECT_EQ(double, scale[1].get(), 1.0); - EXPECT_EQ(double, scale[2].get(), 1.0); - EXPECT_EQ(double, scale[3].get(), 1.0); - EXPECT_EQ(double, scale[4].get(), 1.0); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto zarr_format = meta["zarr_format"].get(); - EXPECT_EQ(int, zarr_format, 2); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunks"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto dtype = meta["dtype"].get(); - EXPECT(dtype == "(); - EXPECT(compressor_id == "blosc", - "Expected compressor id to be blosc, but got ", - compressor_id); - - const auto cname = compressor["cname"].get(); - EXPECT( - cname == "zstd", "Expected compressor cname to be zstd, but got ", cname); - - const auto clevel = compressor["clevel"].get(); - EXPECT_EQ(int, clevel, 1); - - const auto shuffle = compressor["shuffle"].get(); - EXPECT_EQ(int, shuffle, 1); -} - -void -verify_file_data() -{ - const auto expected_file_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * - nbytes_px; - - fs::path data_root = fs::path(test_path) / "0"; - - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root / std::to_string(t); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < chunks_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_LT(size_t, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file(y_dir / - std::to_string(chunks_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(chunks_in_y))); - } - - CHECK(!fs::is_directory(c_dir / std::to_string(chunks_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(chunks_in_c))); - } - - CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_t))); -} - -void -verify() -{ - CHECK(std::filesystem::is_directory(test_path)); - - { - fs::path base_metadata_path = fs::path(test_path) / ".zattrs"; - std::ifstream f(base_metadata_path); - nlohmann::json base_metadata = nlohmann::json::parse(f); - - verify_base_metadata(base_metadata); - } - - { - fs::path group_metadata_path = fs::path(test_path) / ".zgroup"; - std::ifstream f = std::ifstream(group_metadata_path); - nlohmann::json group_metadata = nlohmann::json::parse(f); - - verify_group_metadata(group_metadata); - } - - { - fs::path array_metadata_path = fs::path(test_path) / "0" / ".zarray"; - std::ifstream f = std::ifstream(array_metadata_path); - nlohmann::json array_metadata = nlohmann::json::parse(f); - - verify_array_metadata(array_metadata); - } - - verify_file_data(); -} - -int -main() -{ - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", - i, - ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify(); - - // Clean up - fs::remove_all(test_path); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: ", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v2-compressed-to-s3.cpp b/tests/integration/stream-zarr-v2-compressed-to-s3.cpp deleted file mode 100644 index 8cabd4fa..00000000 --- a/tests/integration/stream-zarr-v2-compressed-to-s3.cpp +++ /dev/null @@ -1,447 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include -#include - -#include - -namespace { -std::string s3_endpoint, s3_bucket_name, s3_access_key_id, s3_secret_access_key; - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const size_t nbytes_px = sizeof(int32_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; - -bool -get_credentials() -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - s3_endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - s3_bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - s3_access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - s3_secret_access_key = env; - - return true; -} - -bool -object_exists(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - return (bool)response; -} - -size_t -get_object_size(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - if (!response) { - LOG_ERROR("Failed to get object size: %s", object_name.c_str()); - return 0; - } - - return response.size; -} - -std::string -get_object_contents(minio::s3::Client& client, const std::string& object_name) -{ - std::stringstream ss; - - minio::s3::GetObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - args.datafunc = [&ss](minio::http::DataFunctionArgs args) -> bool { - ss << args.datachunk; - return true; - }; - - // Call get object. - minio::s3::GetObjectResponse resp = client.GetObject(args); - - return ss.str(); -} - -bool -remove_items(minio::s3::Client& client, - const std::vector& item_keys) -{ - std::list objects; - for (const auto& key : item_keys) { - minio::s3::DeleteObject object; - object.name = key; - objects.push_back(object); - } - - minio::s3::RemoveObjectsArgs args; - args.bucket = s3_bucket_name; - - auto it = objects.begin(); - - args.func = [&objects = objects, - &i = it](minio::s3::DeleteObject& obj) -> bool { - if (i == objects.end()) - return false; - obj = *i; - i++; - return true; - }; - - minio::s3::RemoveObjectsResult result = client.RemoveObjects(args); - for (; result; result++) { - minio::s3::DeleteError err = *result; - if (!err) { - LOG_ERROR("Failed to delete object %s: %s", - err.object_name.c_str(), - err.message.c_str()); - return false; - } - } - - return true; -} -} // namespace/s - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = TEST, - .data_type = ZarrDataType_int32, - .version = ZarrVersion_2, - }; - - ZarrS3Settings s3_settings{ - .endpoint = s3_endpoint.c_str(), - .bucket_name = s3_bucket_name.c_str(), - .access_key_id = s3_access_key_id.c_str(), - .secret_access_key = s3_secret_access_key.c_str(), - }; - settings.s3_settings = &s3_settings; - - ZarrCompressionSettings compression_settings = { - .compressor = ZarrCompressor_Blosc1, - .codec = ZarrCompressionCodec_BloscZstd, - .level = 1, - .shuffle = 1, - }; - settings.compression_settings = &compression_settings; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, 0); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT( - type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT( - type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto zarr_format = meta["zarr_format"].get(); - EXPECT_EQ(int, zarr_format, 2); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunks"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto dtype = meta["dtype"].get(); - EXPECT(dtype == "(); - EXPECT(compressor_id == "blosc", - "Expected compressor id to be 'blosc', but got '%s'", - compressor_id.c_str()); - - const auto cname = compressor["cname"].get(); - EXPECT(cname == "zstd", - "Expected compressor cname to be 'zstd', but got '%s'", - cname.c_str()); - - const auto clevel = compressor["clevel"].get(); - EXPECT_EQ(int, clevel, 1); - - const auto shuffle = compressor["shuffle"].get(); - EXPECT_EQ(int, shuffle, 1); -} - -void -verify_and_cleanup() -{ - - minio::s3::BaseUrl url(s3_endpoint); - url.https = s3_endpoint.starts_with("https://"); - - minio::creds::StaticProvider provider(s3_access_key_id, - s3_secret_access_key); - minio::s3::Client client(url, &provider); - - std::string base_metadata_path = TEST "/.zattrs"; - std::string group_metadata_path = TEST "/.zgroup"; - std::string array_metadata_path = TEST "/0/.zarray"; - - { - EXPECT(object_exists(client, base_metadata_path), - "Object does not exist: %s", - base_metadata_path.c_str()); - std::string contents = get_object_contents(client, base_metadata_path); - nlohmann::json base_metadata = nlohmann::json::parse(contents); - - verify_base_metadata(base_metadata); - } - - { - EXPECT(object_exists(client, group_metadata_path), - "Object does not exist: %s", - group_metadata_path.c_str()); - std::string contents = get_object_contents(client, group_metadata_path); - nlohmann::json group_metadata = nlohmann::json::parse(contents); - - verify_group_metadata(group_metadata); - } - - { - EXPECT(object_exists(client, array_metadata_path), - "Object does not exist: %s", - array_metadata_path.c_str()); - std::string contents = get_object_contents(client, array_metadata_path); - nlohmann::json array_metadata = nlohmann::json::parse(contents); - - verify_array_metadata(array_metadata); - } - - CHECK(remove_items( - client, - { base_metadata_path, group_metadata_path, array_metadata_path })); - - const auto expected_file_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * - nbytes_px; - - // verify and clean up data files - std::vector data_files; - std::string data_root = TEST "/0"; - - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root + "/" + std::to_string(t); - - for (auto c = 0; c < chunks_in_c; ++c) { - const auto c_dir = t_dir + "/" + std::to_string(c); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = c_dir + "/" + std::to_string(z); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir + "/" + std::to_string(y); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir + "/" + std::to_string(x); - EXPECT(object_exists(client, x_file), - "Object does not exist: %s", - x_file.c_str()); - const auto file_size = get_object_size(client, x_file); - EXPECT_LT(size_t, file_size, expected_file_size); - data_files.push_back(x_file); - } - - CHECK(!object_exists( - client, y_dir + "/" + std::to_string(chunks_in_x))); - } - } - } - } - - CHECK(remove_items(client, data_files)); -} - -int -main() -{ - if (!get_credentials()) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", - i, - ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify_and_cleanup(); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: ", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v2-raw-to-filesystem.cpp b/tests/integration/stream-zarr-v2-raw-to-filesystem.cpp deleted file mode 100644 index 838ca735..00000000 --- a/tests/integration/stream-zarr-v2-raw-to-filesystem.cpp +++ /dev/null @@ -1,288 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { -const std::string test_path = - (fs::temp_directory_path() / (TEST ".zarr")).string(); - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const size_t nbytes_px = sizeof(int32_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; -} // namespace/s - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = test_path.c_str(), - .s3_settings = nullptr, - .compression_settings = nullptr, - .data_type = ZarrDataType_int32, - .version = ZarrVersion_2, - }; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, 0); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT(type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT(type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto zarr_format = meta["zarr_format"].get(); - EXPECT_EQ(int, zarr_format, 2); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunks"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto dtype = meta["dtype"].get(); - EXPECT(dtype == " frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", i, ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify(); - - // Clean up - fs::remove_all(test_path); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: %s", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v2-raw-to-s3.cpp b/tests/integration/stream-zarr-v2-raw-to-s3.cpp deleted file mode 100644 index f13f1a19..00000000 --- a/tests/integration/stream-zarr-v2-raw-to-s3.cpp +++ /dev/null @@ -1,429 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include -#include - -#include - -namespace { -std::string s3_endpoint, s3_bucket_name, s3_access_key_id, s3_secret_access_key; - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const size_t nbytes_px = sizeof(int32_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; - -bool -get_credentials() -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - s3_endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - s3_bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - s3_access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - s3_secret_access_key = env; - - return true; -} - -bool -object_exists(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - return (bool)response; -} - -size_t -get_object_size(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - if (!response) { - LOG_ERROR("Failed to get object size: %s", object_name.c_str()); - return 0; - } - - return response.size; -} - -std::string -get_object_contents(minio::s3::Client& client, const std::string& object_name) -{ - std::stringstream ss; - - minio::s3::GetObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - args.datafunc = [&ss](minio::http::DataFunctionArgs args) -> bool { - ss << args.datachunk; - return true; - }; - - // Call get object. - minio::s3::GetObjectResponse resp = client.GetObject(args); - - return ss.str(); -} - -bool -remove_items(minio::s3::Client& client, - const std::vector& item_keys) -{ - std::list objects; - for (const auto& key : item_keys) { - minio::s3::DeleteObject object; - object.name = key; - objects.push_back(object); - } - - minio::s3::RemoveObjectsArgs args; - args.bucket = s3_bucket_name; - - auto it = objects.begin(); - - args.func = [&objects = objects, - &i = it](minio::s3::DeleteObject& obj) -> bool { - if (i == objects.end()) - return false; - obj = *i; - i++; - return true; - }; - - minio::s3::RemoveObjectsResult result = client.RemoveObjects(args); - for (; result; result++) { - minio::s3::DeleteError err = *result; - if (!err) { - LOG_ERROR("Failed to delete object %s: %s", - err.object_name.c_str(), - err.message.c_str()); - return false; - } - } - - return true; -} -} // namespace - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = TEST, - .compression_settings = nullptr, - .data_type = ZarrDataType_int32, - .version = ZarrVersion_2, - }; - - ZarrS3Settings s3_settings{ - .endpoint = s3_endpoint.c_str(), - .bucket_name = s3_bucket_name.c_str(), - .access_key_id = s3_access_key_id.c_str(), - .secret_access_key = s3_secret_access_key.c_str(), - }; - - settings.s3_settings = &s3_settings; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, 0); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '%s'", name.c_str()); - EXPECT( - type == "time", "Expected type to be 'time', but got '%s'", type.c_str()); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '%s'", name.c_str()); - EXPECT(type == "channel", - "Expected type to be 'channel', but got '%s'", - type.c_str()); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '%s'", name.c_str()); - EXPECT(type == "space", - "Expected type to be 'space', but got '%s'", - type.c_str()); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '%s'", name.c_str()); - EXPECT(type == "space", - "Expected type to be 'space', but got '%s'", - type.c_str()); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '%s'", - unit.c_str()); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '%s'", name.c_str()); - EXPECT(type == "space", - "Expected type to be 'space', but got '%s'", - type.c_str()); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '%s'", - unit.c_str()); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '%s'", path.c_str()); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT(type == "scale", - "Expected type to be 'scale', but got '%s'", - type.c_str()); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto zarr_format = meta["zarr_format"].get(); - EXPECT_EQ(int, zarr_format, 2); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunks"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto dtype = meta["dtype"].get(); - EXPECT(dtype == " data_files; - std::string data_root = TEST "/0"; - - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root + "/" + std::to_string(t); - - for (auto c = 0; c < chunks_in_c; ++c) { - const auto c_dir = t_dir + "/" + std::to_string(c); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = c_dir + "/" + std::to_string(z); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir + "/" + std::to_string(y); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir + "/" + std::to_string(x); - EXPECT(object_exists(client, x_file), - "Object does not exist: %s", - x_file.c_str()); - const auto file_size = get_object_size(client, x_file); - EXPECT_EQ(size_t, file_size, expected_file_size); - data_files.push_back(x_file); - } - - CHECK(!object_exists( - client, y_dir + "/" + std::to_string(chunks_in_x))); - } - } - } - } - - CHECK(remove_items(client, data_files)); -} - -int -main() -{ - if (!get_credentials()) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame %d: %s", - i, - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify_and_cleanup(); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: ", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v3-compressed-to-filesystem.cpp b/tests/integration/stream-zarr-v3-compressed-to-filesystem.cpp deleted file mode 100644 index d778ec35..00000000 --- a/tests/integration/stream-zarr-v3-compressed-to-filesystem.cpp +++ /dev/null @@ -1,365 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { -const std::string test_path = - (fs::temp_directory_path() / (TEST ".zarr")).string(); - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_channels = 2, shard_timepoints = 2; -const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_channels * shard_timepoints; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards -const unsigned int shards_in_c = - (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard -const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - -const size_t nbytes_px = sizeof(uint16_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; -} // namespace/s - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = test_path.c_str(), - .s3_settings = nullptr, - .data_type = ZarrDataType_uint16, - .version = ZarrVersion_3, - }; - - ZarrCompressionSettings compression_settings = { - .compressor = ZarrCompressor_Blosc1, - .codec = ZarrCompressionCodec_BloscLZ4, - .level = 2, - .shuffle = 2, - }; - settings.compression_settings = &compression_settings; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, shard_timepoints); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, shard_channels); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, shard_planes); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, shard_height); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto extensions = meta["extensions"]; - EXPECT_EQ(size_t, extensions.size(), 0); - - const auto encoding = meta["metadata_encoding"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); - - const auto suffix = meta["metadata_key_suffix"].get(); - EXPECT(suffix == ".json", - "Expected suffix to be '.json', but got '%s'", - suffix.c_str()); - - const auto zarr_format = meta["zarr_format"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["attributes"]["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT( - type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT( - type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunk_grid"]["chunk_shape"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto& shards = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(size_t, shards.size(), 5); - EXPECT_EQ(int, shards[0].get(), shard_timepoints); - EXPECT_EQ(int, shards[1].get(), shard_channels); - EXPECT_EQ(int, shards[2].get(), shard_planes); - EXPECT_EQ(int, shards[3].get(), shard_height); - EXPECT_EQ(int, shards[4].get(), shard_width); - - const auto dtype = meta["data_type"].get(); - EXPECT(dtype == "uint16", - "Expected dtype to be 'uint16', but got '", - dtype, - "'"); - - const auto& compressor = meta["compressor"]; - EXPECT(!compressor.is_null(), "Expected compressor to be non-null"); - - const auto codec = compressor["codec"].get(); - EXPECT(codec == "https://purl.org/zarr/spec/codec/blosc/1.0", - "Expected codec to be 'https://purl.org/zarr/spec/codec/blosc/1.0', " - "but got '%s'", - codec.c_str()); - - const auto& configuration = compressor["configuration"]; - EXPECT_EQ(int, configuration["blocksize"].get(), 0); - EXPECT_EQ(int, configuration["clevel"].get(), 2); - EXPECT_EQ(int, configuration["shuffle"].get(), 2); - - const auto cname = configuration["cname"].get(); - EXPECT(cname == "lz4", "Expected cname to be 'lz4', but got '", cname, "'"); -} - -void -verify_file_data() -{ - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * shard_planes * - shard_channels * shard_timepoints * - chunk_size + - index_size; - - fs::path data_root = fs::path(test_path) / "data" / "root" / "0"; - - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root / ("c" + std::to_string(t)); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < shards_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_LT(size_t, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file(y_dir / - std::to_string(shards_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK(!fs::is_directory(c_dir / std::to_string(shards_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_c))); - } - - CHECK(!fs::is_directory(data_root / ("c" + std::to_string(shards_in_t)))); -} - -void -verify() -{ - CHECK(std::filesystem::is_directory(test_path)); - - { - fs::path base_metadata_path = fs::path(test_path) / "zarr.json"; - std::ifstream f(base_metadata_path); - nlohmann::json base_metadata = nlohmann::json::parse(f); - - verify_base_metadata(base_metadata); - } - - { - fs::path group_metadata_path = - fs::path(test_path) / "meta" / "root.group.json"; - std::ifstream f = std::ifstream(group_metadata_path); - nlohmann::json group_metadata = nlohmann::json::parse(f); - - verify_group_metadata(group_metadata); - } - - { - fs::path array_metadata_path = - fs::path(test_path) / "meta" / "root" / "0.array.json"; - std::ifstream f = std::ifstream(array_metadata_path); - nlohmann::json array_metadata = nlohmann::json::parse(f); - - verify_array_metadata(array_metadata); - } - - verify_file_data(); -} - -int -main() -{ - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", - i, - ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify(); - - // Clean up - fs::remove_all(test_path); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: %s", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v3-compressed-to-s3.cpp b/tests/integration/stream-zarr-v3-compressed-to-s3.cpp deleted file mode 100644 index 75a6eb1c..00000000 --- a/tests/integration/stream-zarr-v3-compressed-to-s3.cpp +++ /dev/null @@ -1,482 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include -#include - -#include - -namespace { -std::string s3_endpoint, s3_bucket_name, s3_access_key_id, s3_secret_access_key; - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_channels = 2, shard_timepoints = 2; -const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_channels * shard_timepoints; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards -const unsigned int shards_in_c = - (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard -const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - -const size_t nbytes_px = sizeof(uint16_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; - -bool -get_credentials() -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - s3_endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - s3_bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - s3_access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - s3_secret_access_key = env; - - return true; -} - -bool -object_exists(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - return (bool)response; -} - -size_t -get_object_size(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - if (!response) { - LOG_ERROR("Failed to get object size: %s", object_name.c_str()); - return 0; - } - - return response.size; -} - -std::string -get_object_contents(minio::s3::Client& client, const std::string& object_name) -{ - std::stringstream ss; - - minio::s3::GetObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - args.datafunc = [&ss](minio::http::DataFunctionArgs args) -> bool { - ss << args.datachunk; - return true; - }; - - // Call get object. - minio::s3::GetObjectResponse resp = client.GetObject(args); - - return ss.str(); -} - -bool -remove_items(minio::s3::Client& client, - const std::vector& item_keys) -{ - std::list objects; - for (const auto& key : item_keys) { - minio::s3::DeleteObject object; - object.name = key; - objects.push_back(object); - } - - minio::s3::RemoveObjectsArgs args; - args.bucket = s3_bucket_name; - - auto it = objects.begin(); - - args.func = [&objects = objects, - &i = it](minio::s3::DeleteObject& obj) -> bool { - if (i == objects.end()) - return false; - obj = *i; - i++; - return true; - }; - - minio::s3::RemoveObjectsResult result = client.RemoveObjects(args); - for (; result; result++) { - minio::s3::DeleteError err = *result; - if (!err) { - LOG_ERROR("Failed to delete object %s: %s", - err.object_name.c_str(), - err.message.c_str()); - return false; - } - } - - return true; -} -} // namespace - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = TEST, - .data_type = ZarrDataType_uint16, - .version = ZarrVersion_3, - }; - - ZarrS3Settings s3_settings{ - .endpoint = s3_endpoint.c_str(), - .bucket_name = s3_bucket_name.c_str(), - .access_key_id = s3_access_key_id.c_str(), - .secret_access_key = s3_secret_access_key.c_str(), - }; - settings.s3_settings = &s3_settings; - - ZarrCompressionSettings compression_settings = { - .compressor = ZarrCompressor_Blosc1, - .codec = ZarrCompressionCodec_BloscLZ4, - .level = 3, - .shuffle = 1, - }; - settings.compression_settings = &compression_settings; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, shard_timepoints); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, shard_channels); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, shard_planes); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, shard_height); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto extensions = meta["extensions"]; - EXPECT_EQ(size_t, extensions.size(), 0); - - const auto encoding = meta["metadata_encoding"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); - - const auto suffix = meta["metadata_key_suffix"].get(); - EXPECT(suffix == ".json", - "Expected suffix to be '.json', but got '%s'", - suffix.c_str()); - - const auto zarr_format = meta["zarr_format"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["attributes"]["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT(type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT(type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunk_grid"]["chunk_shape"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto& shards = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(size_t, shards.size(), 5); - EXPECT_EQ(int, shards[0].get(), shard_timepoints); - EXPECT_EQ(int, shards[1].get(), shard_channels); - EXPECT_EQ(int, shards[2].get(), shard_planes); - EXPECT_EQ(int, shards[3].get(), shard_height); - EXPECT_EQ(int, shards[4].get(), shard_width); - - const auto dtype = meta["data_type"].get(); - EXPECT(dtype == "uint16", - "Expected dtype to be 'uint16', but got '", - dtype, - "'"); - - const auto& compressor = meta["compressor"]; - EXPECT(!compressor.is_null(), "Expected compressor to be non-null"); - - const auto codec = compressor["codec"].get(); - EXPECT(codec == "https://purl.org/zarr/spec/codec/blosc/1.0", - "Expected codec to be 'https://purl.org/zarr/spec/codec/blosc/1.0', " - "but got '%s'", - codec.c_str()); - - const auto& configuration = compressor["configuration"]; - EXPECT_EQ(int, configuration["blocksize"].get(), 0); - EXPECT_EQ(int, configuration["clevel"].get(), 3); - EXPECT_EQ(int, configuration["shuffle"].get(), 1); - - const auto cname = configuration["cname"].get(); - EXPECT(cname == "lz4", "Expected cname to be 'lz4', but got '", cname, "'"); -} - -void -verify_and_cleanup() -{ - minio::s3::BaseUrl url(s3_endpoint); - url.https = s3_endpoint.starts_with("https://"); - - minio::creds::StaticProvider provider(s3_access_key_id, - s3_secret_access_key); - minio::s3::Client client(url, &provider); - - std::string base_metadata_path = TEST "/zarr.json"; - std::string group_metadata_path = TEST "/meta/root.group.json"; - std::string array_metadata_path = TEST "/meta/root/0.array.json"; - - { - EXPECT(object_exists(client, base_metadata_path), - "Object does not exist: %s", - base_metadata_path.c_str()); - std::string contents = get_object_contents(client, base_metadata_path); - nlohmann::json base_metadata = nlohmann::json::parse(contents); - - verify_base_metadata(base_metadata); - } - - { - EXPECT(object_exists(client, group_metadata_path), - "Object does not exist: %s", - group_metadata_path.c_str()); - std::string contents = get_object_contents(client, group_metadata_path); - nlohmann::json group_metadata = nlohmann::json::parse(contents); - - verify_group_metadata(group_metadata); - } - - { - EXPECT(object_exists(client, array_metadata_path), - "Object does not exist: %s", - array_metadata_path.c_str()); - std::string contents = get_object_contents(client, array_metadata_path); - nlohmann::json array_metadata = nlohmann::json::parse(contents); - - verify_array_metadata(array_metadata); - } - - CHECK(remove_items( - client, - { base_metadata_path, group_metadata_path, array_metadata_path })); - - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * shard_planes * - shard_channels * shard_timepoints * - chunk_size + - index_size; - - // verify and clean up data files - std::vector data_files; - std::string data_root = TEST "/data/root/0"; - - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root + "/" + ("c" + std::to_string(t)); - - for (auto c = 0; c < shards_in_c; ++c) { - const auto c_dir = t_dir + "/" + std::to_string(c); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = c_dir + "/" + std::to_string(z); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir + "/" + std::to_string(y); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir + "/" + std::to_string(x); - EXPECT(object_exists(client, x_file), - "Object does not exist: %s", - x_file.c_str()); - const auto file_size = get_object_size(client, x_file); - EXPECT_LT(size_t, file_size, expected_file_size); - } - } - } - } - } -} - -int -main() -{ - if (!get_credentials()) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", - i, - ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify_and_cleanup(); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: %s", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v3-raw-to-filesystem.cpp b/tests/integration/stream-zarr-v3-raw-to-filesystem.cpp deleted file mode 100644 index 31225a6a..00000000 --- a/tests/integration/stream-zarr-v3-raw-to-filesystem.cpp +++ /dev/null @@ -1,341 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { -const std::string test_path = - (fs::temp_directory_path() / (TEST ".zarr")).string(); - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_channels = 2, shard_timepoints = 2; -const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_channels * shard_timepoints; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards -const unsigned int shards_in_c = - (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard -const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - -const size_t nbytes_px = sizeof(uint16_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; -} // namespace/s - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = test_path.c_str(), - .s3_settings = nullptr, - .compression_settings = nullptr, - .data_type = ZarrDataType_uint16, - .version = ZarrVersion_3, - }; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, shard_timepoints); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, shard_channels); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, shard_planes); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, shard_height); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto extensions = meta["extensions"]; - EXPECT_EQ(size_t, extensions.size(), 0); - - const auto encoding = meta["metadata_encoding"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); - - const auto suffix = meta["metadata_key_suffix"].get(); - EXPECT(suffix == ".json", - "Expected suffix to be '.json', but got '%s'", - suffix.c_str()); - - const auto zarr_format = meta["zarr_format"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["attributes"]["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT(type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT(type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT(type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunk_grid"]["chunk_shape"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto& shards = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(size_t, shards.size(), 5); - EXPECT_EQ(int, shards[0].get(), shard_timepoints); - EXPECT_EQ(int, shards[1].get(), shard_channels); - EXPECT_EQ(int, shards[2].get(), shard_planes); - EXPECT_EQ(int, shards[3].get(), shard_height); - EXPECT_EQ(int, shards[4].get(), shard_width); - - const auto dtype = meta["data_type"].get(); - EXPECT(dtype == "uint16", - "Expected dtype to be 'uint16', but got '", - dtype, - "'"); - - const auto& compressor = meta["compressor"]; - EXPECT(compressor.is_null(), - "Expected compressor to be null, but got '%s'", - compressor.dump().c_str()); -} - -void -verify_file_data() -{ - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * shard_planes * - shard_channels * shard_timepoints * - chunk_size + - index_size; - - fs::path data_root = fs::path(test_path) / "data" / "root" / "0"; - - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root / ("c" + std::to_string(t)); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < shards_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(size_t, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file(y_dir / - std::to_string(shards_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK(!fs::is_directory(c_dir / std::to_string(shards_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_c))); - } - - CHECK(!fs::is_directory(data_root / ("c" + std::to_string(shards_in_t)))); -} - -void -verify() -{ - CHECK(std::filesystem::is_directory(test_path)); - - { - fs::path base_metadata_path = fs::path(test_path) / "zarr.json"; - std::ifstream f(base_metadata_path); - nlohmann::json base_metadata = nlohmann::json::parse(f); - - verify_base_metadata(base_metadata); - } - - { - fs::path group_metadata_path = - fs::path(test_path) / "meta" / "root.group.json"; - std::ifstream f = std::ifstream(group_metadata_path); - nlohmann::json group_metadata = nlohmann::json::parse(f); - - verify_group_metadata(group_metadata); - } - - { - fs::path array_metadata_path = - fs::path(test_path) / "meta" / "root" / "0.array.json"; - std::ifstream f = std::ifstream(array_metadata_path); - nlohmann::json array_metadata = nlohmann::json::parse(f); - - verify_array_metadata(array_metadata); - } - - verify_file_data(); -} - -int -main() -{ - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", - i, - ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify(); - - // Clean up - fs::remove_all(test_path); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: %s", e.what()); - } - - return retval; -} diff --git a/tests/integration/stream-zarr-v3-raw-to-s3.cpp b/tests/integration/stream-zarr-v3-raw-to-s3.cpp deleted file mode 100644 index f099a332..00000000 --- a/tests/integration/stream-zarr-v3-raw-to-s3.cpp +++ /dev/null @@ -1,469 +0,0 @@ -#include "acquire.zarr.h" -#include "test.macros.hh" - -#include -#include - -#include - -namespace { -std::string s3_endpoint, s3_bucket_name, s3_access_key_id, s3_secret_access_key; - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_channels = 2, shard_timepoints = 2; -const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_channels * shard_timepoints; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards -const unsigned int shards_in_c = - (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard -const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - -const size_t nbytes_px = sizeof(uint16_t); -const uint32_t frames_to_acquire = - array_planes * array_channels * array_timepoints; -const size_t bytes_of_frame = array_width * array_height * nbytes_px; - -bool -get_credentials() -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - s3_endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - s3_bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - s3_access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - s3_secret_access_key = env; - - return true; -} - -bool -object_exists(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - return (bool)response; -} - -size_t -get_object_size(minio::s3::Client& client, const std::string& object_name) -{ - minio::s3::StatObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - - minio::s3::StatObjectResponse response = client.StatObject(args); - - if (!response) { - LOG_ERROR("Failed to get object size: %s", object_name.c_str()); - return 0; - } - - return response.size; -} - -std::string -get_object_contents(minio::s3::Client& client, const std::string& object_name) -{ - std::stringstream ss; - - minio::s3::GetObjectArgs args; - args.bucket = s3_bucket_name; - args.object = object_name; - args.datafunc = [&ss](minio::http::DataFunctionArgs args) -> bool { - ss << args.datachunk; - return true; - }; - - // Call get object. - minio::s3::GetObjectResponse resp = client.GetObject(args); - - return ss.str(); -} - -bool -remove_items(minio::s3::Client& client, - const std::vector& item_keys) -{ - std::list objects; - for (const auto& key : item_keys) { - minio::s3::DeleteObject object; - object.name = key; - objects.push_back(object); - } - - minio::s3::RemoveObjectsArgs args; - args.bucket = s3_bucket_name; - - auto it = objects.begin(); - - args.func = [&objects = objects, - &i = it](minio::s3::DeleteObject& obj) -> bool { - if (i == objects.end()) - return false; - obj = *i; - i++; - return true; - }; - - minio::s3::RemoveObjectsResult result = client.RemoveObjects(args); - for (; result; result++) { - minio::s3::DeleteError err = *result; - if (!err) { - LOG_ERROR("Failed to delete object %s: %s", - err.object_name.c_str(), - err.message.c_str()); - return false; - } - } - - return true; -} -} // namespace - -ZarrStream* -setup() -{ - ZarrStreamSettings settings = { - .store_path = TEST, - .compression_settings = nullptr, - .data_type = ZarrDataType_uint16, - .version = ZarrVersion_3, - }; - - ZarrS3Settings s3_settings{ - .endpoint = s3_endpoint.c_str(), - .bucket_name = s3_bucket_name.c_str(), - .access_key_id = s3_access_key_id.c_str(), - .secret_access_key = s3_secret_access_key.c_str(), - }; - - settings.s3_settings = &s3_settings; - - CHECK_OK(ZarrStreamSettings_create_dimension_array(&settings, 5)); - - ZarrDimensionProperties* dim; - dim = settings.dimensions; - *dim = DIM("t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, shard_timepoints); - - dim = settings.dimensions + 1; - *dim = DIM("c", ZarrDimensionType_Channel, array_channels, chunk_channels, shard_channels); - - dim = settings.dimensions + 2; - *dim = DIM("z", ZarrDimensionType_Space, array_planes, chunk_planes, shard_planes); - - dim = settings.dimensions + 3; - *dim = DIM("y", ZarrDimensionType_Space, array_height, chunk_height, shard_height); - - dim = settings.dimensions + 4; - *dim = DIM("x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - auto* stream = ZarrStream_create(&settings); - - return stream; -} - -void -verify_base_metadata(const nlohmann::json& meta) -{ - const auto extensions = meta["extensions"]; - EXPECT_EQ(size_t, extensions.size(), 0); - - const auto encoding = meta["metadata_encoding"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); - - const auto suffix = meta["metadata_key_suffix"].get(); - EXPECT(suffix == ".json", - "Expected suffix to be '.json', but got '%s'", - suffix.c_str()); - - const auto zarr_format = meta["zarr_format"].get(); - EXPECT(encoding == "https://purl.org/zarr/spec/protocol/core/3.0", - "Expected encoding to be " - "'https://purl.org/zarr/spec/protocol/core/3.0', but got '%s'", - encoding.c_str()); -} - -void -verify_group_metadata(const nlohmann::json& meta) -{ - const auto multiscales = meta["attributes"]["multiscales"][0]; - - const auto axes = multiscales["axes"]; - EXPECT_EQ(size_t, axes.size(), 5); - std::string name, type, unit; - - name = axes[0]["name"]; - type = axes[0]["type"]; - EXPECT(name == "t", "Expected name to be 't', but got '", name, "'"); - EXPECT(type == "time", "Expected type to be 'time', but got '", type, "'"); - - name = axes[1]["name"]; - type = axes[1]["type"]; - EXPECT(name == "c", "Expected name to be 'c', but got '", name, "'"); - EXPECT( - type == "channel", "Expected type to be 'channel', but got '", type, "'"); - - name = axes[2]["name"]; - type = axes[2]["type"]; - EXPECT(name == "z", "Expected name to be 'z', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - - name = axes[3]["name"]; - type = axes[3]["type"]; - unit = axes[3]["unit"]; - EXPECT(name == "y", "Expected name to be 'y', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - name = axes[4]["name"]; - type = axes[4]["type"]; - unit = axes[4]["unit"]; - EXPECT(name == "x", "Expected name to be 'x', but got '", name, "'"); - EXPECT( - type == "space", "Expected type to be 'space', but got '", type, "'"); - EXPECT(unit == "micrometer", - "Expected unit to be 'micrometer', but got '", - unit, - "'"); - - const auto datasets = multiscales["datasets"][0]; - const std::string path = datasets["path"].get(); - EXPECT(path == "0", "Expected path to be '0', but got '", path, "'"); - - const auto coordinate_transformations = - datasets["coordinateTransformations"][0]; - - type = coordinate_transformations["type"].get(); - EXPECT( - type == "scale", "Expected type to be 'scale', but got '", type, "'"); - - const auto scale = coordinate_transformations["scale"]; - EXPECT_EQ(size_t, scale.size(), 5); - EXPECT_EQ(int, scale[0].get(), 1.0); - EXPECT_EQ(int, scale[1].get(), 1.0); - EXPECT_EQ(int, scale[2].get(), 1.0); - EXPECT_EQ(int, scale[3].get(), 1.0); - EXPECT_EQ(int, scale[4].get(), 1.0); -} - -void -verify_array_metadata(const nlohmann::json& meta) -{ - const auto& shape = meta["shape"]; - EXPECT_EQ(size_t, shape.size(), 5); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_channels); - EXPECT_EQ(int, shape[2].get(), array_planes); - EXPECT_EQ(int, shape[3].get(), array_height); - EXPECT_EQ(int, shape[4].get(), array_width); - - const auto& chunks = meta["chunk_grid"]["chunk_shape"]; - EXPECT_EQ(size_t, chunks.size(), 5); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_channels); - EXPECT_EQ(int, chunks[2].get(), chunk_planes); - EXPECT_EQ(int, chunks[3].get(), chunk_height); - EXPECT_EQ(int, chunks[4].get(), chunk_width); - - const auto& shards = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - EXPECT_EQ(size_t, shards.size(), 5); - EXPECT_EQ(int, shards[0].get(), shard_timepoints); - EXPECT_EQ(int, shards[1].get(), shard_channels); - EXPECT_EQ(int, shards[2].get(), shard_planes); - EXPECT_EQ(int, shards[3].get(), shard_height); - EXPECT_EQ(int, shards[4].get(), shard_width); - - const auto dtype = meta["data_type"].get(); - EXPECT(dtype == "uint16", - "Expected dtype to be 'uint16', but got '", - dtype, - "'"); - - const auto& compressor = meta["compressor"]; - EXPECT(compressor.is_null(), - "Expected compressor to be null, but got '%s'", - compressor.dump().c_str()); -} - -void -verify_and_cleanup() -{ - minio::s3::BaseUrl url(s3_endpoint); - url.https = s3_endpoint.starts_with("https://"); - - minio::creds::StaticProvider provider(s3_access_key_id, - s3_secret_access_key); - minio::s3::Client client(url, &provider); - - std::string base_metadata_path = TEST "/zarr.json"; - std::string group_metadata_path = TEST "/meta/root.group.json"; - std::string array_metadata_path = TEST "/meta/root/0.array.json"; - - { - EXPECT(object_exists(client, base_metadata_path), - "Object does not exist: %s", - base_metadata_path.c_str()); - std::string contents = get_object_contents(client, base_metadata_path); - nlohmann::json base_metadata = nlohmann::json::parse(contents); - - verify_base_metadata(base_metadata); - } - - { - EXPECT(object_exists(client, group_metadata_path), - "Object does not exist: %s", - group_metadata_path.c_str()); - std::string contents = get_object_contents(client, group_metadata_path); - nlohmann::json group_metadata = nlohmann::json::parse(contents); - - verify_group_metadata(group_metadata); - } - - { - EXPECT(object_exists(client, array_metadata_path), - "Object does not exist: %s", - array_metadata_path.c_str()); - std::string contents = get_object_contents(client, array_metadata_path); - nlohmann::json array_metadata = nlohmann::json::parse(contents); - - verify_array_metadata(array_metadata); - } - - CHECK(remove_items( - client, - { base_metadata_path, group_metadata_path, array_metadata_path })); - - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * shard_planes * - shard_channels * shard_timepoints * - chunk_size + - index_size; - - // verify and clean up data files - std::vector data_files; - std::string data_root = TEST "/data/root/0"; - - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root + "/" + ("c" + std::to_string(t)); - - for (auto c = 0; c < shards_in_c; ++c) { - const auto c_dir = t_dir + "/" + std::to_string(c); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = c_dir + "/" + std::to_string(z); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir + "/" + std::to_string(y); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir + "/" + std::to_string(x); - EXPECT(object_exists(client, x_file), - "Object does not exist: %s", - x_file.c_str()); - const auto file_size = get_object_size(client, x_file); - EXPECT_EQ(size_t, file_size, expected_file_size); - } - } - } - } - } -} - -int -main() -{ - if (!get_credentials()) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - Zarr_set_log_level(ZarrLogLevel_Debug); - - auto* stream = setup(); - std::vector frame(array_width * array_height, 0); - - int retval = 1; - - try { - size_t bytes_out; - for (auto i = 0; i < frames_to_acquire; ++i) { - ZarrStatusCode status = ZarrStream_append( - stream, frame.data(), bytes_of_frame, &bytes_out); - EXPECT(status == ZarrStatusCode_Success, - "Failed to append frame ", - i, - ": ", - Zarr_get_status_message(status)); - EXPECT_EQ(size_t, bytes_out, bytes_of_frame); - } - - ZarrStream_destroy(stream); - - verify_and_cleanup(); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: %s", e.what()); - } - - return retval; -} diff --git a/tests/integration/test.macros.hh b/tests/integration/test.macros.hh deleted file mode 100644 index 3c79024a..00000000 --- a/tests/integration/test.macros.hh +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once - -#include "logger.hh" - -#define EXPECT(e, ...) \ - do { \ - if (!(e)) { \ - const std::string __err = LOG_ERROR(__VA_ARGS__); \ - throw std::runtime_error(__err); \ - } \ - } while (0) -#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t", #e) - -/// Check that a==b -/// example: `ASSERT_EQ(int,42,meaning_of_life())` -#define EXPECT_EQ(T, a, b) \ - do { \ - T a_ = (T)(a); \ - T b_ = (T)(b); \ - EXPECT( \ - a_ == b_, "Expected ", #a, " == ", #b, " but ", a_, " != ", b_); \ - } while (0) - -#define CHECK_OK(e) CHECK((e) == ZarrStatusCode_Success) - -#define EXPECT_STR_EQ(a, b) \ - do { \ - std::string a_ = (a) ? (a) : ""; \ - std::string b_ = (b) ? (b) : ""; \ - EXPECT(a_ == b_, \ - "Expected ", \ - #a, \ - " == ", \ - #b, \ - " but ", \ - a_, \ - " != ", \ - b_, \ - #a, \ - #b, \ - a_, \ - b_); \ - } while (0) - -#define EXPECT_LT(T, a, b) \ - do { \ - T a_ = (T)(a); \ - T b_ = (T)(b); \ - EXPECT(a_ < b_, "Expected ", #a, " < ", #b, " but ", a_, " >= ", b_); \ - } while (0) - -#define SIZED(str) str, sizeof(str) -#define DIM(name_, type_, array_size, chunk_size, shard_size) \ - { \ - .name = (name_), .type = (type_), \ - .array_size_px = (array_size), .chunk_size_px = (chunk_size), \ - .shard_size_chunks = (shard_size) \ - } \ No newline at end of file diff --git a/tests/driver/list-devices.cpp b/tests/list-devices.cpp similarity index 100% rename from tests/driver/list-devices.cpp rename to tests/list-devices.cpp diff --git a/tests/driver/metadata-dimension-sizes.cpp b/tests/metadata-dimension-sizes.cpp similarity index 100% rename from tests/driver/metadata-dimension-sizes.cpp rename to tests/metadata-dimension-sizes.cpp diff --git a/tests/driver/multiscales-metadata.cpp b/tests/multiscales-metadata.cpp similarity index 100% rename from tests/driver/multiscales-metadata.cpp rename to tests/multiscales-metadata.cpp diff --git a/tests/driver/repeat-start.cpp b/tests/repeat-start.cpp similarity index 100% rename from tests/driver/repeat-start.cpp rename to tests/repeat-start.cpp diff --git a/tests/driver/restart-stopped-zarr-resets-threadpool.cpp b/tests/restart-stopped-zarr-resets-threadpool.cpp similarity index 100% rename from tests/driver/restart-stopped-zarr-resets-threadpool.cpp rename to tests/restart-stopped-zarr-resets-threadpool.cpp diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt deleted file mode 100644 index 8269c633..00000000 --- a/tests/unit-tests/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -set(project acquire-zarr) - -set(tests - create-stream - array-dimensions-chunk-lattice-index - array-dimensions-tile-group-offset - array-dimensions-chunk-internal-offset - array-dimensions-shard-index-for-chunk - array-dimensions-shard-internal-index - thread-pool-push-to-job-queue - s3-connection-bucket-exists - s3-connection-object-exists-check-false-positives - s3-connection-put-object - s3-connection-upload-multipart-object - file-sink-write - s3-sink-write - sink-creator-make-metadata-sinks - sink-creator-make-data-sinks - array-writer-downsample-writer-config - array-writer-write-frame-to-chunks - zarrv2-writer-write-even - zarrv2-writer-write-ragged-append-dim - zarrv2-writer-write-ragged-internal-dim - zarrv3-writer-write-even - zarrv3-writer-write-ragged-append-dim - zarrv3-writer-write-ragged-internal-dim -) - -foreach (name ${tests}) - set(tgt "${project}-unit-test-${name}") - add_executable(${tgt} ${name}.cpp unit.test.macros.hh) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/src/logger - ${CMAKE_SOURCE_DIR}/src/streaming - ) - target_link_libraries(${tgt} PRIVATE - acquire-logger - acquire-zarr - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;unit-tests;acquire-zarr") -endforeach () \ No newline at end of file diff --git a/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp b/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp deleted file mode 100644 index 02b58cd8..00000000 --- a/tests/unit-tests/array-dimensions-chunk-internal-offset.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "zarr.dimension.hh" -#include "unit.test.macros.hh" - -#include - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint16); - - try { - EXPECT_EQ(int, dimensions.chunk_internal_offset(0), 0); - EXPECT_EQ(int, dimensions.chunk_internal_offset(1), 512); - EXPECT_EQ(int, dimensions.chunk_internal_offset(2), 0); - EXPECT_EQ(int, dimensions.chunk_internal_offset(3), 512); - EXPECT_EQ(int, dimensions.chunk_internal_offset(4), 0); - EXPECT_EQ(int, dimensions.chunk_internal_offset(5), 1024); - EXPECT_EQ(int, dimensions.chunk_internal_offset(6), 1536); - EXPECT_EQ(int, dimensions.chunk_internal_offset(7), 1024); - EXPECT_EQ(int, dimensions.chunk_internal_offset(8), 1536); - EXPECT_EQ(int, dimensions.chunk_internal_offset(9), 1024); - EXPECT_EQ(int, dimensions.chunk_internal_offset(10), 0); - EXPECT_EQ(int, dimensions.chunk_internal_offset(11), 512); - EXPECT_EQ(int, dimensions.chunk_internal_offset(12), 0); - EXPECT_EQ(int, dimensions.chunk_internal_offset(13), 512); - EXPECT_EQ(int, dimensions.chunk_internal_offset(14), 0); - EXPECT_EQ(int, dimensions.chunk_internal_offset(15), 2048); - EXPECT_EQ(int, dimensions.chunk_internal_offset(16), 2560); - EXPECT_EQ(int, dimensions.chunk_internal_offset(17), 2048); - EXPECT_EQ(int, dimensions.chunk_internal_offset(18), 2560); - EXPECT_EQ(int, dimensions.chunk_internal_offset(19), 2048); - EXPECT_EQ(int, dimensions.chunk_internal_offset(20), 3072); - EXPECT_EQ(int, dimensions.chunk_internal_offset(21), 3584); - EXPECT_EQ(int, dimensions.chunk_internal_offset(22), 3072); - EXPECT_EQ(int, dimensions.chunk_internal_offset(23), 3584); - EXPECT_EQ(int, dimensions.chunk_internal_offset(24), 3072); - EXPECT_EQ(int, dimensions.chunk_internal_offset(25), 2048); - EXPECT_EQ(int, dimensions.chunk_internal_offset(26), 2560); - EXPECT_EQ(int, dimensions.chunk_internal_offset(27), 2048); - EXPECT_EQ(int, dimensions.chunk_internal_offset(28), 2560); - EXPECT_EQ(int, dimensions.chunk_internal_offset(29), 2048); - EXPECT_EQ(int, dimensions.chunk_internal_offset(30), 4096); - EXPECT_EQ(int, dimensions.chunk_internal_offset(31), 4608); - EXPECT_EQ(int, dimensions.chunk_internal_offset(32), 4096); - EXPECT_EQ(int, dimensions.chunk_internal_offset(33), 4608); - EXPECT_EQ(int, dimensions.chunk_internal_offset(34), 4096); - EXPECT_EQ(int, dimensions.chunk_internal_offset(35), 5120); - EXPECT_EQ(int, dimensions.chunk_internal_offset(36), 5632); - EXPECT_EQ(int, dimensions.chunk_internal_offset(37), 5120); - EXPECT_EQ(int, dimensions.chunk_internal_offset(38), 5632); - EXPECT_EQ(int, dimensions.chunk_internal_offset(39), 5120); - EXPECT_EQ(int, dimensions.chunk_internal_offset(40), 4096); - EXPECT_EQ(int, dimensions.chunk_internal_offset(41), 4608); - EXPECT_EQ(int, dimensions.chunk_internal_offset(42), 4096); - EXPECT_EQ(int, dimensions.chunk_internal_offset(43), 4608); - EXPECT_EQ(int, dimensions.chunk_internal_offset(44), 4096); - EXPECT_EQ(int, dimensions.chunk_internal_offset(45), 6144); - EXPECT_EQ(int, dimensions.chunk_internal_offset(46), 6656); - EXPECT_EQ(int, dimensions.chunk_internal_offset(47), 6144); - EXPECT_EQ(int, dimensions.chunk_internal_offset(48), 6656); - EXPECT_EQ(int, dimensions.chunk_internal_offset(49), 6144); - EXPECT_EQ(int, dimensions.chunk_internal_offset(50), 7168); - EXPECT_EQ(int, dimensions.chunk_internal_offset(51), 7680); - EXPECT_EQ(int, dimensions.chunk_internal_offset(52), 7168); - EXPECT_EQ(int, dimensions.chunk_internal_offset(53), 7680); - EXPECT_EQ(int, dimensions.chunk_internal_offset(54), 7168); - EXPECT_EQ(int, dimensions.chunk_internal_offset(55), 6144); - EXPECT_EQ(int, dimensions.chunk_internal_offset(56), 6656); - EXPECT_EQ(int, dimensions.chunk_internal_offset(57), 6144); - EXPECT_EQ(int, dimensions.chunk_internal_offset(58), 6656); - EXPECT_EQ(int, dimensions.chunk_internal_offset(59), 6144); - EXPECT_EQ(int, dimensions.chunk_internal_offset(60), 8192); - EXPECT_EQ(int, dimensions.chunk_internal_offset(61), 8704); - EXPECT_EQ(int, dimensions.chunk_internal_offset(62), 8192); - EXPECT_EQ(int, dimensions.chunk_internal_offset(63), 8704); - EXPECT_EQ(int, dimensions.chunk_internal_offset(64), 8192); - EXPECT_EQ(int, dimensions.chunk_internal_offset(65), 9216); - EXPECT_EQ(int, dimensions.chunk_internal_offset(66), 9728); - EXPECT_EQ(int, dimensions.chunk_internal_offset(67), 9216); - EXPECT_EQ(int, dimensions.chunk_internal_offset(68), 9728); - EXPECT_EQ(int, dimensions.chunk_internal_offset(69), 9216); - EXPECT_EQ(int, dimensions.chunk_internal_offset(70), 8192); - EXPECT_EQ(int, dimensions.chunk_internal_offset(71), 8704); - EXPECT_EQ(int, dimensions.chunk_internal_offset(72), 8192); - EXPECT_EQ(int, dimensions.chunk_internal_offset(73), 8704); - EXPECT_EQ(int, dimensions.chunk_internal_offset(74), 8192); - EXPECT_EQ(int, dimensions.chunk_internal_offset(75), 0); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp b/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp deleted file mode 100644 index 2e9f06ce..00000000 --- a/tests/unit-tests/array-dimensions-chunk-lattice-index.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "zarr.dimension.hh" -#include "unit.test.macros.hh" - -#include - -int -main() -{ - int retval = 1; - - try { - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint8); - - EXPECT_EQ(int, dimensions.chunk_lattice_index(0, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(0, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(0, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(1, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(1, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(1, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(2, 2), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(2, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(2, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(3, 2), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(3, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(3, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(4, 2), 2); - EXPECT_EQ(int, dimensions.chunk_lattice_index(4, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(4, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(5, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(5, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(5, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(12, 2), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(12, 1), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(12, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(19, 2), 2); - EXPECT_EQ(int, dimensions.chunk_lattice_index(19, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(19, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(26, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(26, 1), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(26, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(33, 2), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(33, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(33, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(40, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(40, 1), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(40, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(47, 2), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(47, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(47, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(54, 2), 2); - EXPECT_EQ(int, dimensions.chunk_lattice_index(54, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(54, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(61, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(61, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(61, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(68, 2), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(68, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(68, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(74, 2), 2); - EXPECT_EQ(int, dimensions.chunk_lattice_index(74, 1), 1); - EXPECT_EQ(int, dimensions.chunk_lattice_index(74, 0), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(75, 2), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(75, 1), 0); - EXPECT_EQ(int, dimensions.chunk_lattice_index(75, 0), 1); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp b/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp deleted file mode 100644 index 307e9dda..00000000 --- a/tests/unit-tests/array-dimensions-shard-index-for-chunk.cpp +++ /dev/null @@ -1,191 +0,0 @@ -#include "zarr.dimension.hh" -#include "unit.test.macros.hh" - -#include - -int -main() -{ - int retval = 1; - - try { - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 5, // 5 timepoints / chunk - 2); // 2 chunks / shard - dims.emplace_back("c", - ZarrDimensionType_Channel, - 8, - 4, // 8 / 4 = 2 chunks - 2); // 4 / 2 = 2 shards - dims.emplace_back("z", - ZarrDimensionType_Space, - 6, - 2, // 6 / 2 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("y", - ZarrDimensionType_Space, - 48, - 16, // 48 / 16 = 3 chunks - 1); // 3 / 1 = 3 shards - dims.emplace_back("x", - ZarrDimensionType_Space, - 64, - 16, // 64 / 16 = 4 chunks - 2); // 4 / 2 = 2 shards - ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint32); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(0), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(1), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(2), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(3), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(4), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(5), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(6), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(7), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(8), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(9), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(10), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(11), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(12), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(13), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(14), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(15), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(16), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(17), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(18), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(19), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(20), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(21), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(22), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(23), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(24), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(25), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(26), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(27), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(28), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(29), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(30), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(31), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(32), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(33), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(34), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(35), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(36), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(37), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(38), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(39), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(40), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(41), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(42), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(43), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(44), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(45), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(46), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(47), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(48), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(49), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(50), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(51), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(52), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(53), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(54), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(55), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(56), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(57), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(58), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(59), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(60), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(61), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(62), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(63), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(64), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(65), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(66), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(67), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(68), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(69), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(70), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(71), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(72), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(73), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(74), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(75), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(76), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(77), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(78), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(79), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(80), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(81), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(82), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(83), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(84), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(85), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(86), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(87), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(88), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(89), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(90), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(91), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(92), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(93), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(94), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(95), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(96), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(97), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(98), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(99), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(100), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(101), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(102), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(103), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(104), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(105), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(106), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(107), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(108), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(109), 0); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(110), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(111), 1); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(112), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(113), 2); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(114), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(115), 3); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(116), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(117), 4); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(118), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(119), 5); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(120), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(121), 6); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(122), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(123), 7); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(124), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(125), 8); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(126), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(127), 9); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(128), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(129), 10); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(130), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(131), 11); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(132), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(133), 12); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(134), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(135), 13); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(136), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(137), 14); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(138), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(139), 15); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(140), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(141), 16); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(142), 17); - EXPECT_EQ(int, dimensions.shard_index_for_chunk(143), 17); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/array-dimensions-shard-internal-index.cpp b/tests/unit-tests/array-dimensions-shard-internal-index.cpp deleted file mode 100644 index ed4591e4..00000000 --- a/tests/unit-tests/array-dimensions-shard-internal-index.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include "zarr.dimension.hh" -#include "unit.test.macros.hh" - -#include - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 32, // 32 timepoints / chunk - 1); // 1 shard - dims.emplace_back("y", - ZarrDimensionType_Space, - 960, - 320, // 3 chunks - 2); // 2 ragged shards - dims.emplace_back("x", - ZarrDimensionType_Space, - 1080, - 270, // 4 chunks - 3); // 2 ragged shards - ArrayDimensions dimensions(std::move(dims), ZarrDataType_uint64); - - try { - EXPECT_EQ(int, dimensions.shard_index_for_chunk(0), 0); - EXPECT_EQ(int, dimensions.shard_internal_index(0), 0); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(1), 0); - EXPECT_EQ(int, dimensions.shard_internal_index(1), 1); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(2), 0); - EXPECT_EQ(int, dimensions.shard_internal_index(2), 2); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(3), 1); - EXPECT_EQ(int, dimensions.shard_internal_index(3), 0); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(4), 0); - EXPECT_EQ(int, dimensions.shard_internal_index(4), 3); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(5), 0); - EXPECT_EQ(int, dimensions.shard_internal_index(5), 4); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(6), 0); - EXPECT_EQ(int, dimensions.shard_internal_index(6), 5); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(7), 1); - EXPECT_EQ(int, dimensions.shard_internal_index(7), 3); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(8), 2); - EXPECT_EQ(int, dimensions.shard_internal_index(8), 0); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(9), 2); - EXPECT_EQ(int, dimensions.shard_internal_index(9), 1); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(10), 2); - EXPECT_EQ(int, dimensions.shard_internal_index(10), 2); - - EXPECT_EQ(int, dimensions.shard_index_for_chunk(11), 3); - EXPECT_EQ(int, dimensions.shard_internal_index(11), 0); - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/array-dimensions-tile-group-offset.cpp b/tests/unit-tests/array-dimensions-tile-group-offset.cpp deleted file mode 100644 index c62368db..00000000 --- a/tests/unit-tests/array-dimensions-tile-group-offset.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "zarr.common.hh" -#include "unit.test.macros.hh" - -#include - -int -main() -{ - int retval = 1; - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk - dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks - dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks - dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks - dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks - ArrayDimensions dimensions(std::move(dims), ZarrDataType_float32); - - try { - EXPECT_EQ(int, dimensions.tile_group_offset(0), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(1), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(2), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(3), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(4), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(5), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(6), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(7), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(8), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(9), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(10), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(11), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(12), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(13), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(14), 60); - EXPECT_EQ(int, dimensions.tile_group_offset(15), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(16), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(17), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(18), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(19), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(20), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(21), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(22), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(23), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(24), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(25), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(26), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(27), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(28), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(29), 60); - EXPECT_EQ(int, dimensions.tile_group_offset(30), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(31), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(32), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(33), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(34), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(35), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(36), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(37), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(38), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(39), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(40), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(41), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(42), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(43), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(44), 60); - EXPECT_EQ(int, dimensions.tile_group_offset(45), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(46), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(47), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(48), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(49), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(50), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(51), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(52), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(53), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(54), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(55), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(56), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(57), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(58), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(59), 60); - EXPECT_EQ(int, dimensions.tile_group_offset(60), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(61), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(62), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(63), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(64), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(65), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(66), 0); - EXPECT_EQ(int, dimensions.tile_group_offset(67), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(68), 12); - EXPECT_EQ(int, dimensions.tile_group_offset(69), 24); - EXPECT_EQ(int, dimensions.tile_group_offset(70), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(71), 36); - EXPECT_EQ(int, dimensions.tile_group_offset(72), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(73), 48); - EXPECT_EQ(int, dimensions.tile_group_offset(74), 60); - EXPECT_EQ(int, dimensions.tile_group_offset(75), 0); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/array-writer-downsample-writer-config.cpp b/tests/unit-tests/array-writer-downsample-writer-config.cpp deleted file mode 100644 index 249c2728..00000000 --- a/tests/unit-tests/array-writer-downsample-writer-config.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include "array.writer.hh" -#include "unit.test.macros.hh" - -#include - -namespace fs = std::filesystem; - -int -main() -{ - int retval = 1; - - try { - const fs::path base_dir = "acquire"; - - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - 0, - 5, - 1); // 5 timepoints / chunk, 1 shard - dims.emplace_back( - "c", ZarrDimensionType_Channel, 2, 1, 1); // 2 chunks, 2 shards - dims.emplace_back( - "z", ZarrDimensionType_Space, 7, 3, 3); // 3 chunks, 3 shards - dims.emplace_back( - "y", ZarrDimensionType_Space, 48, 16, 3); // 3 chunks, 1 shard - dims.emplace_back( - "x", ZarrDimensionType_Space, 64, 16, 2); // 4 chunks, 2 shards - - zarr::ArrayWriterConfig config{ - .dimensions = std::make_unique(std::move(dims), - ZarrDataType_uint8), - .dtype = ZarrDataType_uint8, - .level_of_detail = 0, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt - }; - - zarr::ArrayWriterConfig downsampled_config; - CHECK(zarr::downsample(config, downsampled_config)); - - // check dimensions - CHECK(downsampled_config.dimensions->ndims() == 5); - - CHECK(downsampled_config.dimensions->at(0).name == "t"); - CHECK(downsampled_config.dimensions->at(0).array_size_px == 0); - CHECK(downsampled_config.dimensions->at(0).chunk_size_px == 5); - CHECK(downsampled_config.dimensions->at(0).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions->at(1).name == "c"); - // we don't downsample channels - CHECK(downsampled_config.dimensions->at(1).array_size_px == 2); - CHECK(downsampled_config.dimensions->at(1).chunk_size_px == 1); - CHECK(downsampled_config.dimensions->at(1).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions->at(2).name == "z"); - CHECK(downsampled_config.dimensions->at(2).array_size_px == 4); - CHECK(downsampled_config.dimensions->at(2).chunk_size_px == 3); - CHECK(downsampled_config.dimensions->at(2).shard_size_chunks == 2); - - CHECK(downsampled_config.dimensions->at(3).name == "y"); - CHECK(downsampled_config.dimensions->at(3).array_size_px == 24); - CHECK(downsampled_config.dimensions->at(3).chunk_size_px == 16); - CHECK(downsampled_config.dimensions->at(3).shard_size_chunks == 2); - - CHECK(downsampled_config.dimensions->at(4).name == "x"); - CHECK(downsampled_config.dimensions->at(4).array_size_px == 32); - CHECK(downsampled_config.dimensions->at(4).chunk_size_px == 16); - CHECK(downsampled_config.dimensions->at(4).shard_size_chunks == 2); - - // check level of detail - CHECK(downsampled_config.level_of_detail == 1); - - // check store path - CHECK(downsampled_config.store_path == config.store_path); - - // check compression params - CHECK(!downsampled_config.compression_params.has_value()); - - // downsample again - config = std::move(downsampled_config); - - // can't downsample anymore - CHECK(!zarr::downsample(config, downsampled_config)); - - // check dimensions - CHECK(downsampled_config.dimensions->ndims() == 5); - - CHECK(downsampled_config.dimensions->at(0).name == "t"); - CHECK(downsampled_config.dimensions->at(0).array_size_px == 0); - CHECK(downsampled_config.dimensions->at(0).chunk_size_px == 5); - CHECK(downsampled_config.dimensions->at(0).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions->at(1).name == "c"); - // we don't downsample channels - CHECK(downsampled_config.dimensions->at(1).array_size_px == 2); - CHECK(downsampled_config.dimensions->at(1).chunk_size_px == 1); - CHECK(downsampled_config.dimensions->at(1).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions->at(2).name == "z"); - CHECK(downsampled_config.dimensions->at(2).array_size_px == 2); - CHECK(downsampled_config.dimensions->at(2).chunk_size_px == 2); - CHECK(downsampled_config.dimensions->at(2).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions->at(3).name == "y"); - CHECK(downsampled_config.dimensions->at(3).array_size_px == 12); - CHECK(downsampled_config.dimensions->at(3).chunk_size_px == 12); - CHECK(downsampled_config.dimensions->at(3).shard_size_chunks == 1); - - CHECK(downsampled_config.dimensions->at(4).name == "x"); - CHECK(downsampled_config.dimensions->at(4).array_size_px == 16); - CHECK(downsampled_config.dimensions->at(4).chunk_size_px == 16); - CHECK(downsampled_config.dimensions->at(4).shard_size_chunks == 1); - - // check level of detail - CHECK(downsampled_config.level_of_detail == 2); - - // check data root - CHECK(downsampled_config.store_path == config.store_path); - - // check compression params - CHECK(!downsampled_config.compression_params.has_value()); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/array-writer-write-frame-to-chunks.cpp b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp deleted file mode 100644 index 9c426340..00000000 --- a/tests/unit-tests/array-writer-write-frame-to-chunks.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -namespace { -class TestWriter : public zarr::ArrayWriter -{ - public: - TestWriter(zarr::ArrayWriterConfig&& array_spec, - std::shared_ptr thread_pool) - : zarr::ArrayWriter(std::move(array_spec), thread_pool) - { - } - - private: - ZarrVersion version_() const override { return ZarrVersionCount; } - bool should_rollover_() const override { return false; } - bool flush_impl_() override { return true; } - bool write_array_metadata_() override { return true; } -}; -} // namespace - -int -main() -{ - const auto base_dir = fs::temp_directory_path() / TEST; - int retval = 1; - - const unsigned int array_width = 64, array_height = 48, array_planes = 2, - array_channels = 1, array_timepoints = 2; - const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 1, - chunk_channels = 1, chunk_timepoints = 1; - - const unsigned int n_frames = - array_planes * array_channels * array_timepoints; - - const ZarrDataType dtype = ZarrDataType_uint16; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOG_ERROR("Error: ", err); }); - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - dims.emplace_back( - "c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); - dims.emplace_back( - "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "y", ZarrDimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - zarr::ArrayWriterConfig config = { - .dimensions = - std::make_unique(std::move(dims), dtype), - .dtype = dtype, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - TestWriter writer(std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data_(frame_size, std::byte(0)); - std::span data(data_.data(), data_.size()); - - for (auto i = 0; i < n_frames; ++i) { - CHECK(writer.write_frame(data) == frame_size); - } - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/create-stream.cpp b/tests/unit-tests/create-stream.cpp deleted file mode 100644 index 2dc8e39f..00000000 --- a/tests/unit-tests/create-stream.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "acquire.zarr.h" -#include "zarr.stream.hh" -#include "unit.test.macros.hh" - -#include - -namespace fs = std::filesystem; - -void -configure_stream_dimensions(ZarrStreamSettings* settings) -{ - CHECK(ZarrStatusCode_Success == ZarrStreamSettings_create_dimension_array(settings, 3)); - ZarrDimensionProperties *dim = settings->dimensions; - - *dim = ZarrDimensionProperties{ - .name = "t", - .type = ZarrDimensionType_Time, - .array_size_px = 100, - .chunk_size_px = 10, - }; - - dim = settings->dimensions + 1; - *dim = ZarrDimensionProperties{ - .name = "y", - .type = ZarrDimensionType_Space, - .array_size_px = 200, - .chunk_size_px = 20, - }; - - dim = settings->dimensions + 2; - *dim = ZarrDimensionProperties{ - .name = "x", - .type = ZarrDimensionType_Space, - .array_size_px = 300, - .chunk_size_px = 30, - }; -} - -int -main() -{ - int retval = 1; - - ZarrStream* stream; - ZarrStreamSettings settings; - memset(&settings, 0, sizeof(settings)); - settings.version = ZarrVersion_2; - - try { - // try to create a stream with no store path - stream = ZarrStream_create(&settings); - CHECK(nullptr == stream); - - // try to create a stream with no dimensions - settings.store_path = static_cast(TEST ".zarr"); - stream = ZarrStream_create(&settings); - CHECK(nullptr == stream); - CHECK(!fs::exists(settings.store_path)); - - // allocate dimensions - configure_stream_dimensions(&settings); - stream = ZarrStream_create(&settings); - CHECK(nullptr != stream); - CHECK(fs::is_directory(settings.store_path)); - - retval = 0; - } catch (const std::exception& exception) { - LOG_ERROR(exception.what()); - } - - // cleanup - ZarrStream_destroy(stream); - - std::error_code ec; - if (fs::is_directory(settings.store_path) && !fs::remove_all(settings.store_path, ec)) { - LOG_ERROR("Failed to remove store path: ", ec.message().c_str()); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/file-sink-write.cpp b/tests/unit-tests/file-sink-write.cpp deleted file mode 100644 index 9df1e8f4..00000000 --- a/tests/unit-tests/file-sink-write.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "file.sink.hh" -#include "unit.test.macros.hh" - -#include -#include -#include - -namespace fs = std::filesystem; - -int -main() -{ - int retval = 0; - fs::path tmp_path = fs::temp_directory_path() / TEST; - - try { - CHECK(!fs::exists(tmp_path)); - { - char str[] = "Hello, Acquire!"; - auto sink = std::make_unique(tmp_path.string()); - - std::span data = { reinterpret_cast(str), - sizeof(str) - 1 }; - CHECK(sink->write(0, data)); - CHECK(zarr::finalize_sink(std::move(sink))); - } - - // The file tmp_path should now contain the string "Hello, world!\n". - CHECK(fs::exists(tmp_path)); - - std::ifstream ifs(tmp_path); - CHECK(ifs.is_open()); - - std::string contents; - while (!ifs.eof()) { - std::getline(ifs, contents); - } - ifs.close(); - - EXPECT_STR_EQ(contents.c_str(), "Hello, Acquire!"); - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: ", e.what()); - retval = 1; - } - - std::error_code ec; - if (!fs::remove(tmp_path, ec)) { - LOG_ERROR("Failed to remove file: ", ec.message()); - retval = 1; - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/s3-connection-bucket-exists.cpp b/tests/unit-tests/s3-connection-bucket-exists.cpp deleted file mode 100644 index 86047471..00000000 --- a/tests/unit-tests/s3-connection-bucket-exists.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "s3.connection.hh" -#include "unit.test.macros.hh" - -#include -#include - -namespace { -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -int -main() -{ - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - int retval = 1; - - try { - zarr::S3Connection conn{ s3_endpoint, - s3_access_key_id, - s3_secret_access_key }; - - if (!conn.is_connection_valid()) { - LOG_ERROR("Failed to connect to S3."); - return 1; - } - - if (conn.bucket_exists("")) { - LOG_ERROR("False positive response for empty bucket name."); - return 1; - } - - CHECK(conn.bucket_exists(bucket_name)); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp b/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp deleted file mode 100644 index b2afd643..00000000 --- a/tests/unit-tests/s3-connection-object-exists-check-false-positives.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "s3.connection.hh" -#include "unit.test.macros.hh" - -#include -#include - -namespace { -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -int -main() -{ - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - int retval = 1; - const std::string object_name = "test-object"; - - try { - zarr::S3Connection conn{ s3_endpoint, - s3_access_key_id, - s3_secret_access_key }; - - if (!conn.is_connection_valid()) { - LOG_ERROR("Failed to connect to S3."); - return 1; - } - - CHECK(conn.bucket_exists(bucket_name)); - - if (conn.object_exists("", object_name)) { - LOG_ERROR("False positive for empty bucket name."); - return 1; - } - - if (conn.object_exists(bucket_name, "")) { - LOG_ERROR("False positive for empty object name."); - return 1; - } - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/s3-connection-put-object.cpp b/tests/unit-tests/s3-connection-put-object.cpp deleted file mode 100644 index 678fdf8a..00000000 --- a/tests/unit-tests/s3-connection-put-object.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "s3.connection.hh" -#include "unit.test.macros.hh" - -#include - -namespace { -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -int -main() -{ - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - int retval = 1; - const std::string object_name = "test-object"; - - try { - zarr::S3Connection conn{ s3_endpoint, - s3_access_key_id, - s3_secret_access_key }; - - if (!conn.is_connection_valid()) { - LOG_ERROR("Failed to connect to S3."); - return 1; - } - CHECK(conn.bucket_exists(bucket_name)); - CHECK(conn.delete_object(bucket_name, object_name)); - CHECK(!conn.object_exists(bucket_name, object_name)); - - std::vector data(1024, 0); - - std::string etag = - conn.put_object(bucket_name, - object_name, - std::span(data.data(), data.size())); - CHECK(!etag.empty()); - - CHECK(conn.object_exists(bucket_name, object_name)); - - // cleanup - CHECK(conn.delete_object(bucket_name, object_name)); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/s3-connection-upload-multipart-object.cpp b/tests/unit-tests/s3-connection-upload-multipart-object.cpp deleted file mode 100644 index 684569b8..00000000 --- a/tests/unit-tests/s3-connection-upload-multipart-object.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include "s3.connection.hh" -#include "unit.test.macros.hh" - -#include - -namespace { -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -int -main() -{ - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - int retval = 1; - const std::string object_name = "test-object"; - - try { - zarr::S3Connection conn( - s3_endpoint, s3_access_key_id, s3_secret_access_key); - - if (!conn.is_connection_valid()) { - LOG_ERROR("Failed to connect to S3."); - return 1; - } - CHECK(conn.bucket_exists(bucket_name)); - CHECK(conn.delete_object(bucket_name, object_name)); - CHECK(!conn.object_exists(bucket_name, object_name)); - - std::string upload_id = - conn.create_multipart_object(bucket_name, object_name); - CHECK(!upload_id.empty()); - - std::list parts; - - // parts need to be at least 5MiB, except the last part - std::vector data(5 << 20, 0); - for (auto i = 0; i < 4; ++i) { - std::string etag = conn.upload_multipart_object_part( - bucket_name, - object_name, - upload_id, - std::span(data.data(), data.size()), - i + 1); - CHECK(!etag.empty()); - - minio::s3::Part part; - part.number = i + 1; - part.etag = etag; - part.size = data.size(); - - parts.push_back(part); - } - - // last part is 1MiB - { - const unsigned int part_number = parts.size() + 1; - const size_t part_size = 1 << 20; // 1MiB - std::string etag = conn.upload_multipart_object_part( - bucket_name, - object_name, - upload_id, - std::span(data.data(), data.size()), - part_number); - CHECK(!etag.empty()); - - minio::s3::Part part; - part.number = part_number; - part.etag = etag; - part.size = part_size; - - parts.push_back(part); - } - - CHECK(conn.complete_multipart_object( - bucket_name, object_name, upload_id, parts)); - - CHECK(conn.object_exists(bucket_name, object_name)); - - // cleanup - CHECK(conn.delete_object(bucket_name, object_name)); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/s3-sink-write.cpp b/tests/unit-tests/s3-sink-write.cpp deleted file mode 100644 index 68326163..00000000 --- a/tests/unit-tests/s3-sink-write.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include "s3.sink.hh" -#include "unit.test.macros.hh" - -#include -#include - -namespace { -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -int -main() -{ - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping test."); - return 0; - } - - int retval = 1; - const std::string object_name = "test-object"; - - try { - auto pool = std::make_shared( - 1, s3_endpoint, s3_access_key_id, s3_secret_access_key); - - auto conn = pool->get_connection(); - if (!conn->is_connection_valid()) { - LOG_ERROR("Failed to connect to S3."); - return 1; - } - CHECK(conn->bucket_exists(bucket_name)); - CHECK(conn->delete_object(bucket_name, object_name)); - CHECK(!conn->object_exists(bucket_name, object_name)); - - pool->return_connection(std::move(conn)); - - { - char str[] = "Hello, Acquire!"; - auto sink = - std::make_unique(bucket_name, object_name, pool); - std::span data{ reinterpret_cast(str), - sizeof(str) - 1 }; - CHECK(sink->write(0, data)); - CHECK(zarr::finalize_sink(std::move(sink))); - } - - conn = pool->get_connection(); - CHECK(conn->object_exists(bucket_name, object_name)); - pool->return_connection(std::move(conn)); - - // Verify the object contents. - { - minio::s3::BaseUrl url(s3_endpoint); - url.https = s3_endpoint.starts_with("https://"); - - minio::creds::StaticProvider provider(s3_access_key_id, - s3_secret_access_key); - - minio::s3::Client client(url, &provider); - minio::s3::GetObjectArgs args; - args.bucket = bucket_name; - args.object = object_name; - - std::string contents; - args.datafunc = - [&contents](minio::http::DataFunctionArgs args) -> bool { - contents = args.datachunk; - return true; - }; - - // Call get object. - minio::s3::GetObjectResponse resp = client.GetObject(args); - - if (contents != "Hello, Acquire!") { - LOG_ERROR( - "Expected 'Hello, Acquire!' but got '", contents, "'"); - return 1; - } - } - - // cleanup - conn = pool->get_connection(); - CHECK(conn->delete_object(bucket_name, object_name)); - - retval = 0; - } catch (const std::exception& e) { - LOG_ERROR("Exception: ", e.what()); - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-data-sinks.cpp b/tests/unit-tests/sink-creator-make-data-sinks.cpp deleted file mode 100644 index cac596a3..00000000 --- a/tests/unit-tests/sink-creator-make-data-sinks.cpp +++ /dev/null @@ -1,279 +0,0 @@ -#include "sink.creator.hh" -#include "s3.connection.hh" -#include "zarr.common.hh" -#include "acquire.zarr.h" -#include "unit.test.macros.hh" - -#include -#include - -namespace fs = std::filesystem; - -namespace { -const std::string test_dir = TEST "-data"; - -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -void -sink_creator_make_chunk_sinks(std::shared_ptr thread_pool, - const ArrayDimensions* dimensions) -{ - zarr::SinkCreator sink_creator(thread_pool, nullptr); - - // create the sinks, then let them go out of scope to close the handles - { - std::vector> sinks; - CHECK(sink_creator.make_data_sinks( - test_dir, dimensions, zarr::chunks_along_dimension, sinks)); - } - - const auto chunks_in_y = zarr::chunks_along_dimension(dimensions->height_dim()); - const auto chunks_in_x = zarr::chunks_along_dimension(dimensions->width_dim()); - - const fs::path base_path(test_dir); - for (auto i = 0; i < chunks_in_y; ++i) { - const fs::path y_dir = base_path / std::to_string(i); - - for (auto j = 0; j < chunks_in_x; ++j) { - const fs::path x_file = y_dir / std::to_string(j); - CHECK(fs::is_regular_file(x_file)); - - // cleanup - fs::remove(x_file); - } - CHECK(!fs::is_regular_file(y_dir / std::to_string(chunks_in_x))); - fs::remove(y_dir); - } - CHECK(!fs::is_directory(base_path / std::to_string(chunks_in_y))); -} - -void -sink_creator_make_chunk_sinks( - std::shared_ptr thread_pool, - std::shared_ptr connection_pool, - const std::string& bucket_name, - const ArrayDimensions* dimensions) -{ - zarr::SinkCreator sink_creator(thread_pool, connection_pool); - - // create the sinks, then let them go out of scope to close the handles - { - char data_[] = { 0, 0 }; - std::span data(reinterpret_cast(data_), sizeof(data_)); - std::vector> sinks; - CHECK(sink_creator.make_data_sinks(bucket_name, - test_dir, - dimensions, - zarr::chunks_along_dimension, - sinks)); - - for (auto& sink : sinks) { - CHECK(sink); - // we need to write some data to the sink to ensure it is created - CHECK(sink->write(0, data)); - CHECK(zarr::finalize_sink(std::move(sink))); - } - } - - const auto chunks_in_y = zarr::chunks_along_dimension(dimensions->height_dim()); - const auto chunks_in_x = zarr::chunks_along_dimension(dimensions->width_dim()); - - auto conn = connection_pool->get_connection(); - - const std::string base_path(test_dir); - for (auto i = 0; i < chunks_in_y; ++i) { - const std::string y_dir = base_path + "/" + std::to_string(i); - - for (auto j = 0; j < chunks_in_x; ++j) { - const std::string x_file = y_dir + "/" + std::to_string(j); - CHECK(conn->object_exists(bucket_name, x_file)); - - // cleanup - CHECK(conn->delete_object(bucket_name, x_file)); - } - CHECK(!conn->object_exists(bucket_name, - y_dir + "/" + std::to_string(chunks_in_x))); - CHECK(conn->delete_object(bucket_name, y_dir)); - } - CHECK(!conn->object_exists(bucket_name, - base_path + "/" + std::to_string(chunks_in_y))); - CHECK(conn->delete_object(bucket_name, base_path)); -} - -void -sink_creator_make_shard_sinks(std::shared_ptr thread_pool, - const ArrayDimensions* dimensions) -{ - zarr::SinkCreator sink_creator(thread_pool, nullptr); - - // create the sinks, then let them go out of scope to close the handles - { - std::vector> sinks; - CHECK(sink_creator.make_data_sinks( - test_dir, dimensions, zarr::shards_along_dimension, sinks)); - } - - const auto shards_in_y = zarr::shards_along_dimension(dimensions->height_dim()); - const auto shards_in_x = zarr::shards_along_dimension(dimensions->width_dim()); - - const fs::path base_path(test_dir); - for (auto i = 0; i < shards_in_y; ++i) { - const fs::path y_dir = base_path / std::to_string(i); - - for (auto j = 0; j < shards_in_x; ++j) { - const fs::path x_file = y_dir / std::to_string(j); - CHECK(fs::is_regular_file(x_file)); - - // cleanup - fs::remove(x_file); - } - CHECK(!fs::is_regular_file(y_dir / std::to_string(shards_in_x))); - fs::remove(y_dir); - } - CHECK(!fs::is_directory(base_path / std::to_string(shards_in_y))); -} - -void -sink_creator_make_shard_sinks( - std::shared_ptr thread_pool, - std::shared_ptr connection_pool, - const std::string& bucket_name, - const ArrayDimensions* dimensions) -{ - zarr::SinkCreator sink_creator(thread_pool, connection_pool); - - // create the sinks, then let them go out of scope to close the handles - { - char data_[] = { 0, 0 }; - std::span data(reinterpret_cast(data_), sizeof(data_)); - std::vector> sinks; - CHECK(sink_creator.make_data_sinks(bucket_name, - test_dir, - dimensions, - zarr::shards_along_dimension, - sinks)); - - for (auto& sink : sinks) { - CHECK(sink); - // we need to write some data to the sink to ensure it is created - CHECK(sink->write(0, data)); - CHECK(zarr::finalize_sink(std::move(sink))); - } - } - - const auto shards_in_y = zarr::shards_along_dimension(dimensions->height_dim()); - const auto shards_in_x = zarr::shards_along_dimension(dimensions->width_dim()); - - auto conn = connection_pool->get_connection(); - - const std::string base_path(test_dir); - for (auto i = 0; i < shards_in_y; ++i) { - const std::string y_dir = base_path + "/" + std::to_string(i); - - for (auto j = 0; j < shards_in_x; ++j) { - const std::string x_file = y_dir + "/" + std::to_string(j); - CHECK(conn->object_exists(bucket_name, x_file)); - - // cleanup - CHECK(conn->delete_object(bucket_name, x_file)); - } - CHECK(!conn->object_exists(bucket_name, - y_dir + "/" + std::to_string(shards_in_x))); - CHECK(conn->delete_object(bucket_name, y_dir)); - } - CHECK(!conn->object_exists(bucket_name, - base_path + "/" + std::to_string(shards_in_y))); - CHECK(conn->delete_object(bucket_name, base_path)); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - std::vector dims; - dims.emplace_back("z", - ZarrDimensionType_Space, - 0, - 3, // 3 planes per chunk - 1); // 1 chunk per shard (3 planes per shard) - dims.emplace_back("y", - ZarrDimensionType_Space, - 4, - 2, // 2 rows per chunk, 2 chunks - 2); // 2 chunks per shard (4 rows per shard, 1 shard) - dims.emplace_back("x", - ZarrDimensionType_Space, - 12, - 3, // 3 columns per chunk, 4 chunks - 2); // 2 chunks per shard (6 columns per shard, 2 shards) - ArrayDimensions dimensions(std::move(dims), ZarrDataType_int8); - - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOG_ERROR("Failed: ", err.c_str()); }); - - try { - sink_creator_make_chunk_sinks(thread_pool, &dimensions); - sink_creator_make_shard_sinks(thread_pool, &dimensions); - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - return 1; - } - - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping S3 portion of test."); - return 0; - } - - auto connection_pool = std::make_shared( - 4, s3_endpoint, s3_access_key_id, s3_secret_access_key); - - try { - sink_creator_make_chunk_sinks( - thread_pool, connection_pool, bucket_name, &dimensions); - sink_creator_make_shard_sinks( - thread_pool, connection_pool, bucket_name, &dimensions); - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - return 1; - } - - return 0; -} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-metadata-sinks.cpp b/tests/unit-tests/sink-creator-make-metadata-sinks.cpp deleted file mode 100644 index c4000797..00000000 --- a/tests/unit-tests/sink-creator-make-metadata-sinks.cpp +++ /dev/null @@ -1,216 +0,0 @@ -#include "sink.creator.hh" -#include "s3.connection.hh" -#include "unit.test.macros.hh" - -#include -#include - -namespace fs = std::filesystem; - -namespace { -const std::string test_dir = TEST "-data"; - -bool -get_credentials(std::string& endpoint, - std::string& bucket_name, - std::string& access_key_id, - std::string& secret_access_key) -{ - char* env = nullptr; - if (!(env = std::getenv("ZARR_S3_ENDPOINT"))) { - LOG_ERROR("ZARR_S3_ENDPOINT not set."); - return false; - } - endpoint = env; - - if (!(env = std::getenv("ZARR_S3_BUCKET_NAME"))) { - LOG_ERROR("ZARR_S3_BUCKET_NAME not set."); - return false; - } - bucket_name = env; - - if (!(env = std::getenv("ZARR_S3_ACCESS_KEY_ID"))) { - LOG_ERROR("ZARR_S3_ACCESS_KEY_ID not set."); - return false; - } - access_key_id = env; - - if (!(env = std::getenv("ZARR_S3_SECRET_ACCESS_KEY"))) { - LOG_ERROR("ZARR_S3_SECRET_ACCESS_KEY not set."); - return false; - } - secret_access_key = env; - - return true; -} -} // namespace - -void -sink_creator_make_v2_metadata_sinks( - std::shared_ptr thread_pool) -{ - zarr::SinkCreator sink_creator(thread_pool, nullptr); - - std::unordered_map> metadata_sinks; - CHECK(sink_creator.make_metadata_sinks(2, test_dir, metadata_sinks)); - - CHECK(metadata_sinks.size() == 4); - CHECK(metadata_sinks.contains(".zattrs")); - CHECK(metadata_sinks.contains(".zgroup")); - CHECK(metadata_sinks.contains("0/.zattrs")); - CHECK(metadata_sinks.contains("acquire.json")); - - for (auto& [key, sink] : metadata_sinks) { - CHECK(sink); - sink.reset(nullptr); // close the file - - fs::path file_path(test_dir + "/" + key); - CHECK(fs::is_regular_file(file_path)); - // cleanup - fs::remove(file_path); - } - - fs::remove(test_dir + "/0"); -} - -void -sink_creator_make_v2_metadata_sinks( - std::shared_ptr thread_pool, - std::shared_ptr connection_pool, - const std::string& bucket_name) -{ - zarr::SinkCreator sink_creator(thread_pool, connection_pool); - - std::unordered_map> metadata_sinks; - CHECK( - sink_creator.make_metadata_sinks(2, bucket_name, test_dir, metadata_sinks)); - - CHECK(metadata_sinks.size() == 4); - CHECK(metadata_sinks.contains(".zattrs")); - CHECK(metadata_sinks.contains(".zgroup")); - CHECK(metadata_sinks.contains("0/.zattrs")); - CHECK(metadata_sinks.contains("acquire.json")); - - auto conn = connection_pool->get_connection(); - - char data_[] = { 0, 0 }; - std::span data(reinterpret_cast(data_), sizeof(data_)); - for (auto& [key, sink] : metadata_sinks) { - CHECK(sink); - // we need to write some data to the sink to ensure it is created - CHECK(sink->write(0, data)); - - CHECK(zarr::finalize_sink(std::move(sink))); // close the connection - - std::string path = test_dir + "/" + key; - CHECK(conn->object_exists(bucket_name, path)); - // cleanup - CHECK(conn->delete_object(bucket_name, path)); - } - - CHECK(conn->delete_object(bucket_name, "0")); -} - -void -sink_creator_make_v3_metadata_sinks( - std::shared_ptr thread_pool) -{ - zarr::SinkCreator sink_creator(thread_pool, nullptr); - - std::unordered_map> metadata_sinks; - CHECK(sink_creator.make_metadata_sinks(3, test_dir, metadata_sinks)); - - CHECK(metadata_sinks.size() == 3); - CHECK(metadata_sinks.contains("zarr.json")); - CHECK(metadata_sinks.contains("meta/root.group.json")); - CHECK(metadata_sinks.contains("meta/acquire.json")); - - for (auto& [key, sink] : metadata_sinks) { - CHECK(sink); - sink.reset(nullptr); // close the file - - fs::path file_path(test_dir + "/" + key); - CHECK(fs::is_regular_file(file_path)); - // cleanup - fs::remove(file_path); - } - - fs::remove(test_dir + "/meta"); -} - -void -sink_creator_make_v3_metadata_sinks( - std::shared_ptr thread_pool, - std::shared_ptr connection_pool, - const std::string& bucket_name) -{ - zarr::SinkCreator sink_creator(thread_pool, connection_pool); - - std::unordered_map> metadata_sinks; - CHECK( - sink_creator.make_metadata_sinks(3, bucket_name, test_dir, metadata_sinks)); - - CHECK(metadata_sinks.size() == 3); - CHECK(metadata_sinks.contains("zarr.json")); - CHECK(metadata_sinks.contains("meta/root.group.json")); - CHECK(metadata_sinks.contains("meta/acquire.json")); - - auto conn = connection_pool->get_connection(); - - char data_[] = { 0, 0 }; - std::span data(reinterpret_cast(data_), sizeof(data_)); - for (auto& [key, sink] : metadata_sinks) { - CHECK(sink); - // we need to write some data to the sink to ensure it is created - CHECK(sink->write(0, data)); - CHECK(zarr::finalize_sink(std::move(sink))); // close the connection - - std::string path = test_dir + "/" + key; - CHECK(conn->object_exists(bucket_name, path)); - // cleanup - CHECK(conn->delete_object(bucket_name, path)); - } - - CHECK(conn->delete_object(bucket_name, "meta")); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOG_ERROR("Failed: ", err.c_str()); }); - - try { - sink_creator_make_v2_metadata_sinks(thread_pool); - sink_creator_make_v3_metadata_sinks(thread_pool); - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - return 1; - } - - std::string s3_endpoint, bucket_name, s3_access_key_id, - s3_secret_access_key; - if (!get_credentials( - s3_endpoint, bucket_name, s3_access_key_id, s3_secret_access_key)) { - LOG_WARNING("Failed to get credentials. Skipping S3 portion of test."); - return 0; - } - - auto connection_pool = std::make_shared( - 4, s3_endpoint, s3_access_key_id, s3_secret_access_key); - - try { - sink_creator_make_v2_metadata_sinks( - thread_pool, connection_pool, bucket_name); - sink_creator_make_v3_metadata_sinks( - thread_pool, connection_pool, bucket_name); - } catch (const std::exception& e) { - LOG_ERROR("Failed: ", e.what()); - return 1; - } - - return 0; -} \ No newline at end of file diff --git a/tests/unit-tests/thread-pool-push-to-job-queue.cpp b/tests/unit-tests/thread-pool-push-to-job-queue.cpp deleted file mode 100644 index 72ef6ca8..00000000 --- a/tests/unit-tests/thread-pool-push-to-job-queue.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "thread.pool.hh" -#include "unit.test.macros.hh" - -#include -#include -#include - -namespace fs = std::filesystem; - -int -main() -{ - int retval = 0; - - fs::path tmp_path = fs::temp_directory_path() / TEST; - - try { - CHECK(!fs::exists(tmp_path)); - - zarr::ThreadPool pool{ 1, [](const std::string&) {} }; - - CHECK(pool.push_job([&tmp_path](std::string&) { - std::ofstream ofs(tmp_path); - ofs << "Hello, Acquire!"; - ofs.close(); - return true; - })); - pool.await_stop(); - - CHECK(fs::exists(tmp_path)); - - std::ifstream ifs(tmp_path); - CHECK(ifs.is_open()); - - std::string contents; - while (!ifs.eof()) { - std::getline(ifs, contents); - } - ifs.close(); - - EXPECT_STR_EQ(contents.c_str(), "Hello, Acquire!"); - } catch (const std::exception& e) { - LOG_ERROR("Caught exception: ", e.what()); - retval = 1; - } - - std::error_code ec; - if (!fs::remove(tmp_path, ec)) { - LOG_ERROR("Failed to remove file: ", ec.message()); - retval = 1; - } - - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/unit.test.macros.hh b/tests/unit-tests/unit.test.macros.hh deleted file mode 100644 index 302a033a..00000000 --- a/tests/unit-tests/unit.test.macros.hh +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include "logger.hh" - -#define EXPECT(e, ...) \ - do { \ - if (!(e)) { \ - const std::string __err = LOG_ERROR(__VA_ARGS__); \ - throw std::runtime_error(__err); \ - } \ - } while (0) -#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t", #e) - -/// Check that a==b -/// example: `ASSERT_EQ(int,42,meaning_of_life())` -#define EXPECT_EQ(T, a, b) \ - do { \ - T a_ = (T)(a); \ - T b_ = (T)(b); \ - EXPECT( \ - a_ == b_, "Expected ", #a, " == ", #b, " but ", a_, " != ", b_); \ - } while (0) - -#define EXPECT_STR_EQ(a, b) \ - do { \ - std::string a_ = (a) ? (a) : ""; \ - std::string b_ = (b) ? (b) : ""; \ - EXPECT(a_ == b_, \ - "Expected ", \ - #a, \ - " == ", \ - #b, \ - " but ", \ - a_, \ - " != ", \ - b_, \ - #a, \ - #b, \ - a_, \ - b_); \ - } while (0) diff --git a/tests/unit-tests/zarrv2-writer-write-even.cpp b/tests/unit-tests/zarrv2-writer-write-even.cpp deleted file mode 100644 index 6f94f191..00000000 --- a/tests/unit-tests/zarrv2-writer-write-even.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#include "zarrv2.array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -#include - -#include - -namespace fs = std::filesystem; - -namespace { -const fs::path base_dir = fs::temp_directory_path() / TEST; - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; -const unsigned int n_frames = array_planes * array_channels * array_timepoints; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; // 2 chunks - -const int level_of_detail = 0; -} // namespace - -void -check_json() -{ - fs::path meta_path = - base_dir / std::to_string(level_of_detail) / ".zarray"; - CHECK(fs::is_regular_file(meta_path)); - - std::ifstream f(meta_path); - nlohmann::json meta = nlohmann::json::parse(f); - - EXPECT(meta["dtype"].get() == "()); - - EXPECT_EQ(int, meta["zarr_format"].get(), 2); - - const auto& array_shape = meta["shape"]; - const auto& chunk_shape = meta["chunks"]; - - EXPECT_EQ(int, array_shape.size(), 5); - EXPECT_EQ(int, array_shape[0].get(), array_timepoints); - EXPECT_EQ(int, array_shape[1].get(), array_channels); - EXPECT_EQ(int, array_shape[2].get(), array_planes); - EXPECT_EQ(int, array_shape[3].get(), array_height); - EXPECT_EQ(int, array_shape[4].get(), array_width); - - EXPECT_EQ(int, chunk_shape.size(), 5); - EXPECT_EQ(int, chunk_shape[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunk_shape[1].get(), chunk_channels); - EXPECT_EQ(int, chunk_shape[2].get(), chunk_planes); - EXPECT_EQ(int, chunk_shape[3].get(), chunk_height); - EXPECT_EQ(int, chunk_shape[4].get(), chunk_width); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - int retval = 1; - - const ZarrDataType dtype = ZarrDataType_uint16; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), [](const std::string& err) { - LOG_ERROR("Error: ", err); - }); - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - dims.emplace_back( - "c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); - dims.emplace_back( - "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "y", ZarrDimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - zarr::ArrayWriterConfig config = { - .dimensions = std::make_shared(std::move(dims), dtype), - .dtype = dtype, - .level_of_detail = level_of_detail, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - { - auto writer = std::make_unique( - std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, std::byte(0)); - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer->write_frame(data)); - } - - CHECK(finalize_array(std::move(writer))); - } - - check_json(); - - const auto expected_file_size = chunk_width * chunk_height * - chunk_planes * chunk_channels * - chunk_timepoints * nbytes_px; - - const fs::path data_root = - base_dir / std::to_string(config.level_of_detail); - - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root / std::to_string(t); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < chunks_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(int, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(chunks_in_x))); - } - - CHECK( - !fs::is_directory(z_dir / std::to_string(chunks_in_y))); - } - - CHECK(!fs::is_directory(c_dir / std::to_string(chunks_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(chunks_in_c))); - } - - CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_t))); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp deleted file mode 100644 index 245c35fc..00000000 --- a/tests/unit-tests/zarrv2-writer-write-ragged-append-dim.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include "zarrv2.array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -#include -#include - -namespace fs = std::filesystem; - -namespace { -const fs::path base_dir = fs::temp_directory_path() / TEST; - -const unsigned int array_width = 64, array_height = 48, array_planes = 5; -const unsigned int n_frames = array_planes; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks - -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged - -const int level_of_detail = 1; -} // namespace - -void -check_json() -{ - fs::path zarray_path = - base_dir / std::to_string(level_of_detail) / ".zarray"; - CHECK(fs::is_regular_file(zarray_path)); - - std::ifstream f(zarray_path); - nlohmann::json zarray = nlohmann::json::parse(f); - - EXPECT(zarray["dtype"].get() == "().c_str()); - - EXPECT_EQ(int, zarray["zarr_format"].get(), 2); - - const auto& chunks = zarray["chunks"]; - EXPECT_EQ(int, chunks.size(), 3); - EXPECT_EQ(int, chunks[0].get(), chunk_planes); - EXPECT_EQ(int, chunks[1].get(), chunk_height); - EXPECT_EQ(int, chunks[2].get(), chunk_width); - - const auto& shape = zarray["shape"]; - EXPECT_EQ(int, shape.size(), 3); - EXPECT_EQ(int, shape[0].get(), array_planes); - EXPECT_EQ(int, shape[1].get(), array_height); - EXPECT_EQ(int, shape[2].get(), array_width); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - int retval = 1; - - const ZarrDataType dtype = ZarrDataType_uint8; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), [](const std::string& err) { - LOG_ERROR("Error: ", err); - }); - - std::vector dims; - dims.emplace_back( - "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "y", ZarrDimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - zarr::ArrayWriterConfig config = { - .dimensions = std::make_shared(std::move(dims), dtype), - .dtype = dtype, - .level_of_detail = level_of_detail, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - { - auto writer = std::make_unique( - std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, std::byte(0)); - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer->write_frame(data)); - } - - CHECK(finalize_array(std::move(writer))); - } - - check_json(); - - const auto expected_file_size = - chunk_width * chunk_height * chunk_planes * nbytes_px; - - const fs::path data_root = - base_dir / std::to_string(config.level_of_detail); - - CHECK(fs::is_directory(data_root)); - - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = data_root / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(int, file_size, expected_file_size); - } - - CHECK( - !fs::is_regular_file(y_dir / std::to_string(chunks_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(chunks_in_y))); - } - - CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_z))); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp deleted file mode 100644 index 0f39b616..00000000 --- a/tests/unit-tests/zarrv2-writer-write-ragged-internal-dim.cpp +++ /dev/null @@ -1,167 +0,0 @@ -#include "zarrv2.array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -#include -#include - -namespace fs = std::filesystem; - -namespace { -const fs::path base_dir = fs::temp_directory_path() / TEST; - -const unsigned int array_width = 64, array_height = 48, array_planes = 5, - array_timepoints = 5; -const unsigned int n_frames = array_planes * array_timepoints; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_timepoints = 5; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; // 1 chunk - -const int level_of_detail = 2; -} // namespace - -void -check_json() -{ - fs::path zarray_path = - base_dir / std::to_string(level_of_detail) / ".zarray"; - CHECK(fs::is_regular_file(zarray_path)); - - std::ifstream f(zarray_path); - nlohmann::json zarray = nlohmann::json::parse(f); - - EXPECT(zarray["dtype"].get() == "()); - - EXPECT_EQ(int, zarray["zarr_format"].get(), 2); - - const auto& chunks = zarray["chunks"]; - EXPECT_EQ(int, chunks.size(), 4); - EXPECT_EQ(int, chunks[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunks[1].get(), chunk_planes); - EXPECT_EQ(int, chunks[2].get(), chunk_height); - EXPECT_EQ(int, chunks[3].get(), chunk_width); - - const auto& shape = zarray["shape"]; - EXPECT_EQ(int, shape.size(), 4); - EXPECT_EQ(int, shape[0].get(), array_timepoints); - EXPECT_EQ(int, shape[1].get(), array_planes); - EXPECT_EQ(int, shape[2].get(), array_height); - EXPECT_EQ(int, shape[3].get(), array_width); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - int retval = 1; - - const ZarrDataType dtype = ZarrDataType_float64; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), [](const std::string& err) { - LOG_ERROR("Error: ", err); - }); - - std::vector dims; - dims.emplace_back( - "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); - dims.emplace_back( - "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); - dims.emplace_back( - "y", ZarrDimensionType_Space, array_height, chunk_height, 0); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, 0); - - zarr::ArrayWriterConfig config = { - .dimensions = std::make_shared(std::move(dims), dtype), - .dtype = dtype, - .level_of_detail = level_of_detail, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - { - auto writer = std::make_unique( - std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, std::byte(0)); - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer->write_frame(data)); - } - - CHECK(finalize_array(std::move(writer))); - } - - check_json(); - - const auto expected_file_size = chunk_width * chunk_height * - chunk_planes * chunk_timepoints * - nbytes_px; - - const fs::path data_root = - base_dir / std::to_string(config.level_of_detail); - - CHECK(fs::is_directory(data_root)); - - for (auto t = 0; t < chunks_in_t; ++t) { - const auto t_dir = data_root / std::to_string(t); - CHECK(fs::is_directory(t_dir)); - { - for (auto z = 0; z < chunks_in_z; ++z) { - const auto z_dir = t_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < chunks_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < chunks_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(int, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(chunks_in_x))); - } - - CHECK( - !fs::is_directory(z_dir / std::to_string(chunks_in_y))); - } - - CHECK( - !fs::is_directory(t_dir / std::to_string(chunks_in_z))); - } - - CHECK(!fs::is_directory(data_root / std::to_string(chunks_in_t))); - } - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/zarrv3-writer-write-even.cpp b/tests/unit-tests/zarrv3-writer-write-even.cpp deleted file mode 100644 index ae062735..00000000 --- a/tests/unit-tests/zarrv3-writer-write-even.cpp +++ /dev/null @@ -1,219 +0,0 @@ -#include "zarrv3.array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -#include - -#include - -namespace fs = std::filesystem; - -namespace { -const fs::path base_dir = fs::temp_directory_path() / TEST; - -const unsigned int array_width = 64, array_height = 48, array_planes = 6, - array_channels = 8, array_timepoints = 10; -const unsigned int n_frames = array_planes * array_channels * array_timepoints; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_channels = 4, chunk_timepoints = 5; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_channels = 2, shard_timepoints = 2; -const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_channels * shard_timepoints; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks -const unsigned int chunks_in_c = - (array_channels + chunk_channels - 1) / chunk_channels; // 2 chunks -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards -const unsigned int shards_in_c = - (chunks_in_c + shard_channels - 1) / shard_channels; // 1 shard -const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - -const int level_of_detail = 3; -} // namespace - -void -check_json() -{ - fs::path meta_path = base_dir / "meta" / "root" / - (std::to_string(level_of_detail) + ".array.json"); - CHECK(fs::is_regular_file(meta_path)); - - std::ifstream f(meta_path); - nlohmann::json meta = nlohmann::json::parse(f); - - EXPECT(meta["data_type"].get() == "uint16", - "Expected dtype to be uint16, but got ", - meta["data_type"].get()); - - const auto& array_shape = meta["shape"]; - const auto& chunk_shape = meta["chunk_grid"]["chunk_shape"]; - const auto& shard_shape = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - - EXPECT_EQ(int, array_shape.size(), 5); - EXPECT_EQ(int, array_shape[0].get(), array_timepoints); - EXPECT_EQ(int, array_shape[1].get(), array_channels); - EXPECT_EQ(int, array_shape[2].get(), array_planes); - EXPECT_EQ(int, array_shape[3].get(), array_height); - EXPECT_EQ(int, array_shape[4].get(), array_width); - - EXPECT_EQ(int, chunk_shape.size(), 5); - EXPECT_EQ(int, chunk_shape[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunk_shape[1].get(), chunk_channels); - EXPECT_EQ(int, chunk_shape[2].get(), chunk_planes); - EXPECT_EQ(int, chunk_shape[3].get(), chunk_height); - EXPECT_EQ(int, chunk_shape[4].get(), chunk_width); - - EXPECT_EQ(int, shard_shape.size(), 5); - EXPECT_EQ(int, shard_shape[0].get(), shard_timepoints); - EXPECT_EQ(int, shard_shape[1].get(), shard_channels); - EXPECT_EQ(int, shard_shape[2].get(), shard_planes); - EXPECT_EQ(int, shard_shape[3].get(), shard_height); - EXPECT_EQ(int, shard_shape[4].get(), shard_width); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - int retval = 1; - - const ZarrDataType dtype = ZarrDataType_uint16; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOG_ERROR("Error: ", err.c_str()); }); - - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - array_timepoints, - chunk_timepoints, - shard_timepoints); - dims.emplace_back("c", - ZarrDimensionType_Channel, - array_channels, - chunk_channels, - shard_channels); - dims.emplace_back("z", - ZarrDimensionType_Space, - array_planes, - chunk_planes, - shard_planes); - dims.emplace_back("y", - ZarrDimensionType_Space, - array_height, - chunk_height, - shard_height); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - zarr::ArrayWriterConfig config = { - .dimensions = std::make_shared(std::move(dims), dtype), - .dtype = dtype, - .level_of_detail = level_of_detail, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - { - auto writer = std::make_unique( - std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, std::byte(0)); - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer->write_frame(data)); - } - - CHECK(finalize_array(std::move(writer))); - } - - check_json(); - - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_channels * chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * - shard_planes * shard_channels * - shard_timepoints * chunk_size + - index_size; - - const fs::path data_root = - base_dir / "data/root" / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root / ("c" + std::to_string(t)); - CHECK(fs::is_directory(t_dir)); - - for (auto c = 0; c < shards_in_c; ++c) { - const auto c_dir = t_dir / std::to_string(c); - CHECK(fs::is_directory(c_dir)); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = c_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(int, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file( - y_dir / std::to_string(shards_in_x))); - } - - CHECK( - !fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK(!fs::is_directory(c_dir / std::to_string(shards_in_z))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_c))); - } - - CHECK( - !fs::is_directory(data_root / ("c" + std::to_string(shards_in_t)))); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp deleted file mode 100644 index f31d7e7c..00000000 --- a/tests/unit-tests/zarrv3-writer-write-ragged-append-dim.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include "zarrv3.array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -#include - -#include - -namespace fs = std::filesystem; - -namespace { -const fs::path base_dir = fs::temp_directory_path() / TEST; - -const unsigned int array_width = 64, array_height = 48, array_planes = 5; -const unsigned int n_frames = array_planes; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1; -const unsigned int chunks_per_shard = shard_width * shard_height * shard_planes; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards - -const int level_of_detail = 4; -} // namespace - -void -check_json() -{ - fs::path meta_path = base_dir / "meta" / "root" / - (std::to_string(level_of_detail) + ".array.json"); - CHECK(fs::is_regular_file(meta_path)); - - std::ifstream f(meta_path); - nlohmann::json meta = nlohmann::json::parse(f); - - EXPECT(meta["data_type"].get() == "int32", - "Expected dtype to be int32, but got ", - meta["data_type"].get()); - - const auto& array_shape = meta["shape"]; - const auto& chunk_shape = meta["chunk_grid"]["chunk_shape"]; - const auto& shard_shape = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - - EXPECT_EQ(int, array_shape.size(), 3); - EXPECT_EQ(int, array_shape[0].get(), array_planes); - EXPECT_EQ(int, array_shape[1].get(), array_height); - EXPECT_EQ(int, array_shape[2].get(), array_width); - - EXPECT_EQ(int, chunk_shape.size(), 3); - EXPECT_EQ(int, chunk_shape[0].get(), chunk_planes); - EXPECT_EQ(int, chunk_shape[1].get(), chunk_height); - EXPECT_EQ(int, chunk_shape[2].get(), chunk_width); - - EXPECT_EQ(int, shard_shape.size(), 3); - EXPECT_EQ(int, shard_shape[0].get(), shard_planes); - EXPECT_EQ(int, shard_shape[1].get(), shard_height); - EXPECT_EQ(int, shard_shape[2].get(), shard_width); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - int retval = 1; - - const ZarrDataType dtype = ZarrDataType_int32; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOG_ERROR("Error: ", err.c_str()); }); - - std::vector dims; - dims.emplace_back("z", - ZarrDimensionType_Space, - array_planes, - chunk_planes, - shard_planes); - dims.emplace_back("y", - ZarrDimensionType_Space, - array_height, - chunk_height, - shard_height); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - zarr::ArrayWriterConfig config = { - .dimensions = std::make_shared(std::move(dims), dtype), - .dtype = dtype, - .level_of_detail = 4, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - { - auto writer = std::make_unique( - std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, std::byte(0)); - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer->write_frame(data)); - } - - CHECK(finalize_array(std::move(writer))); - } - - check_json(); - - const auto chunk_size = - chunk_width * chunk_height * chunk_planes * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = - shard_width * shard_height * shard_planes * chunk_size + index_size; - - const fs::path data_root = - base_dir / "data/root" / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = data_root / ("c" + std::to_string(z)); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(int, file_size, expected_file_size); - } - - CHECK( - !fs::is_regular_file(y_dir / std::to_string(shards_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK( - !fs::is_directory(data_root / ("c" + std::to_string(shards_in_z)))); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp b/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp deleted file mode 100644 index 0affa4ed..00000000 --- a/tests/unit-tests/zarrv3-writer-write-ragged-internal-dim.cpp +++ /dev/null @@ -1,197 +0,0 @@ -#include "zarrv3.array.writer.hh" -#include "unit.test.macros.hh" -#include "zarr.common.hh" - -#include - -#include - -namespace fs = std::filesystem; - -namespace { -const fs::path base_dir = fs::temp_directory_path() / TEST; - -const unsigned int array_width = 64, array_height = 48, array_planes = 5, - array_timepoints = 10; -const unsigned int n_frames = array_planes * array_timepoints; - -const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 2, - chunk_timepoints = 5; - -const unsigned int shard_width = 2, shard_height = 1, shard_planes = 1, - shard_timepoints = 2; -const unsigned int chunks_per_shard = - shard_width * shard_height * shard_planes * shard_timepoints; - -const unsigned int chunks_in_x = - (array_width + chunk_width - 1) / chunk_width; // 4 chunks -const unsigned int chunks_in_y = - (array_height + chunk_height - 1) / chunk_height; // 3 chunks -const unsigned int chunks_in_z = - (array_planes + chunk_planes - 1) / chunk_planes; // 3 chunks, ragged -const unsigned int chunks_in_t = - (array_timepoints + chunk_timepoints - 1) / chunk_timepoints; - -const unsigned int shards_in_x = - (chunks_in_x + shard_width - 1) / shard_width; // 2 shards -const unsigned int shards_in_y = - (chunks_in_y + shard_height - 1) / shard_height; // 3 shards -const unsigned int shards_in_z = - (chunks_in_z + shard_planes - 1) / shard_planes; // 3 shards -const unsigned int shards_in_t = - (chunks_in_t + shard_timepoints - 1) / shard_timepoints; // 1 shard - -const int level_of_detail = 5; -} // namespace - -void -check_json() -{ - fs::path meta_path = base_dir / "meta" / "root" / - (std::to_string(level_of_detail) + ".array.json"); - CHECK(fs::is_regular_file(meta_path)); - - std::ifstream f(meta_path); - nlohmann::json meta = nlohmann::json::parse(f); - - EXPECT(meta["data_type"].get() == "float64", - "Expected dtype to be uint16, but got ", - meta["data_type"].get()); - - const auto& array_shape = meta["shape"]; - const auto& chunk_shape = meta["chunk_grid"]["chunk_shape"]; - const auto& shard_shape = - meta["storage_transformers"][0]["configuration"]["chunks_per_shard"]; - - EXPECT_EQ(int, array_shape.size(), 4); - EXPECT_EQ(int, array_shape[0].get(), array_timepoints); - EXPECT_EQ(int, array_shape[1].get(), array_planes); - EXPECT_EQ(int, array_shape[2].get(), array_height); - EXPECT_EQ(int, array_shape[3].get(), array_width); - - EXPECT_EQ(int, chunk_shape.size(), 4); - EXPECT_EQ(int, chunk_shape[0].get(), chunk_timepoints); - EXPECT_EQ(int, chunk_shape[1].get(), chunk_planes); - EXPECT_EQ(int, chunk_shape[2].get(), chunk_height); - EXPECT_EQ(int, chunk_shape[3].get(), chunk_width); - - EXPECT_EQ(int, shard_shape.size(), 4); - EXPECT_EQ(int, shard_shape[0].get(), shard_timepoints); - EXPECT_EQ(int, shard_shape[1].get(), shard_planes); - EXPECT_EQ(int, shard_shape[2].get(), shard_height); - EXPECT_EQ(int, shard_shape[3].get(), shard_width); -} - -int -main() -{ - Logger::set_log_level(LogLevel_Debug); - - int retval = 1; - - const ZarrDataType dtype = ZarrDataType_float64; - const unsigned int nbytes_px = zarr::bytes_of_type(dtype); - - try { - auto thread_pool = std::make_shared( - std::thread::hardware_concurrency(), - [](const std::string& err) { LOG_ERROR("Error: ", err.c_str()); }); - - std::vector dims; - dims.emplace_back("t", - ZarrDimensionType_Time, - array_timepoints, - chunk_timepoints, - shard_timepoints); - dims.emplace_back("z", - ZarrDimensionType_Space, - array_planes, - chunk_planes, - shard_planes); - dims.emplace_back("y", - ZarrDimensionType_Space, - array_height, - chunk_height, - shard_height); - dims.emplace_back( - "x", ZarrDimensionType_Space, array_width, chunk_width, shard_width); - - zarr::ArrayWriterConfig config = { - .dimensions = std::make_shared(std::move(dims), dtype), - .dtype = dtype, - .level_of_detail = 5, - .bucket_name = std::nullopt, - .store_path = base_dir.string(), - .compression_params = std::nullopt, - }; - - { - auto writer = std::make_unique( - std::move(config), thread_pool); - - const size_t frame_size = array_width * array_height * nbytes_px; - std::vector data(frame_size, std::byte(0));; - - for (auto i = 0; i < n_frames; ++i) { // 2 time points - CHECK(writer->write_frame(data)); - } - - CHECK(finalize_array(std::move(writer))); - } - - const auto chunk_size = chunk_width * chunk_height * chunk_planes * - chunk_timepoints * nbytes_px; - const auto index_size = chunks_per_shard * - sizeof(uint64_t) * // indices are 64 bits - 2; // 2 indices per chunk - const auto expected_file_size = shard_width * shard_height * - shard_planes * shard_timepoints * - chunk_size + - index_size; - - const fs::path data_root = - base_dir / "data/root" / std::to_string(config.level_of_detail); - CHECK(fs::is_directory(data_root)); - for (auto t = 0; t < shards_in_t; ++t) { - const auto t_dir = data_root / ("c" + std::to_string(t)); - CHECK(fs::is_directory(t_dir)); - - for (auto z = 0; z < shards_in_z; ++z) { - const auto z_dir = t_dir / std::to_string(z); - CHECK(fs::is_directory(z_dir)); - - for (auto y = 0; y < shards_in_y; ++y) { - const auto y_dir = z_dir / std::to_string(y); - CHECK(fs::is_directory(y_dir)); - - for (auto x = 0; x < shards_in_x; ++x) { - const auto x_file = y_dir / std::to_string(x); - CHECK(fs::is_regular_file(x_file)); - const auto file_size = fs::file_size(x_file); - EXPECT_EQ(int, file_size, expected_file_size); - } - - CHECK(!fs::is_regular_file(y_dir / - std::to_string(shards_in_x))); - } - - CHECK(!fs::is_directory(z_dir / std::to_string(shards_in_y))); - } - - CHECK(!fs::is_directory(t_dir / std::to_string(shards_in_z))); - } - - CHECK( - !fs::is_directory(data_root / ("c" + std::to_string(shards_in_t)))); - - retval = 0; - } catch (const std::exception& exc) { - LOG_ERROR("Exception: ", exc.what()); - } - - // cleanup - if (fs::exists(base_dir)) { - fs::remove_all(base_dir); - } - return retval; -} \ No newline at end of file diff --git a/tests/driver/write-zarr-v2-compressed-multiscale.cpp b/tests/write-zarr-v2-compressed-multiscale.cpp similarity index 100% rename from tests/driver/write-zarr-v2-compressed-multiscale.cpp rename to tests/write-zarr-v2-compressed-multiscale.cpp diff --git a/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp b/tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp similarity index 100% rename from tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp rename to tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp diff --git a/tests/driver/write-zarr-v2-compressed-with-chunking.cpp b/tests/write-zarr-v2-compressed-with-chunking.cpp similarity index 100% rename from tests/driver/write-zarr-v2-compressed-with-chunking.cpp rename to tests/write-zarr-v2-compressed-with-chunking.cpp diff --git a/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp b/tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp rename to tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp diff --git a/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp b/tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp rename to tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp diff --git a/tests/driver/write-zarr-v2-raw-multiscale.cpp b/tests/write-zarr-v2-raw-multiscale.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw-multiscale.cpp rename to tests/write-zarr-v2-raw-multiscale.cpp diff --git a/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp b/tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp rename to tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp diff --git a/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp b/tests/write-zarr-v2-raw-with-even-chunking.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw-with-even-chunking.cpp rename to tests/write-zarr-v2-raw-with-even-chunking.cpp diff --git a/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp b/tests/write-zarr-v2-raw-with-ragged-chunking.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp rename to tests/write-zarr-v2-raw-with-ragged-chunking.cpp diff --git a/tests/driver/write-zarr-v2-raw.cpp b/tests/write-zarr-v2-raw.cpp similarity index 100% rename from tests/driver/write-zarr-v2-raw.cpp rename to tests/write-zarr-v2-raw.cpp diff --git a/tests/driver/write-zarr-v2-to-s3.cpp b/tests/write-zarr-v2-to-s3.cpp similarity index 100% rename from tests/driver/write-zarr-v2-to-s3.cpp rename to tests/write-zarr-v2-to-s3.cpp diff --git a/tests/driver/write-zarr-v2-with-lz4-compression.cpp b/tests/write-zarr-v2-with-lz4-compression.cpp similarity index 100% rename from tests/driver/write-zarr-v2-with-lz4-compression.cpp rename to tests/write-zarr-v2-with-lz4-compression.cpp diff --git a/tests/driver/write-zarr-v2-with-zstd-compression.cpp b/tests/write-zarr-v2-with-zstd-compression.cpp similarity index 100% rename from tests/driver/write-zarr-v2-with-zstd-compression.cpp rename to tests/write-zarr-v2-with-zstd-compression.cpp diff --git a/tests/driver/write-zarr-v3-compressed.cpp b/tests/write-zarr-v3-compressed.cpp similarity index 100% rename from tests/driver/write-zarr-v3-compressed.cpp rename to tests/write-zarr-v3-compressed.cpp diff --git a/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp b/tests/write-zarr-v3-raw-chunk-exceeds-array.cpp similarity index 100% rename from tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp rename to tests/write-zarr-v3-raw-chunk-exceeds-array.cpp diff --git a/tests/driver/write-zarr-v3-raw-multiscale.cpp b/tests/write-zarr-v3-raw-multiscale.cpp similarity index 100% rename from tests/driver/write-zarr-v3-raw-multiscale.cpp rename to tests/write-zarr-v3-raw-multiscale.cpp diff --git a/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp b/tests/write-zarr-v3-raw-with-ragged-sharding.cpp similarity index 100% rename from tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp rename to tests/write-zarr-v3-raw-with-ragged-sharding.cpp diff --git a/tests/driver/write-zarr-v3-raw.cpp b/tests/write-zarr-v3-raw.cpp similarity index 100% rename from tests/driver/write-zarr-v3-raw.cpp rename to tests/write-zarr-v3-raw.cpp diff --git a/tests/driver/write-zarr-v3-to-s3.cpp b/tests/write-zarr-v3-to-s3.cpp similarity index 100% rename from tests/driver/write-zarr-v3-to-s3.cpp rename to tests/write-zarr-v3-to-s3.cpp