diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 85f4a97e..4473a46a 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -23,6 +23,8 @@ add_library(${tgt} s3.sink.cpp sink.creator.hh sink.creator.cpp + array.writer.hh + array.writer.cpp ) target_include_directories(${tgt} diff --git a/src/streaming/array.writer.cpp b/src/streaming/array.writer.cpp new file mode 100644 index 00000000..1c0275c4 --- /dev/null +++ b/src/streaming/array.writer.cpp @@ -0,0 +1,438 @@ +#include "macros.hh" +#include "array.writer.hh" +#include "zarr.common.hh" +#include "zarr.stream.hh" +#include "sink.creator.hh" + +#include +#include +#include +#include + +#ifdef min +#undef min +#endif + +bool +zarr::downsample(const ArrayWriterConfig& config, + ArrayWriterConfig& downsampled_config) +{ + // downsample dimensions + std::vector downsampled_dims(config.dimensions->ndims()); + for (auto i = 0; i < config.dimensions->ndims(); ++i) { + const auto& dim = config.dimensions->at(i); + // don't downsample channels + if (dim.type == ZarrDimensionType_Channel) { + downsampled_dims[i] = dim; + } else { + const uint32_t array_size_px = + (dim.array_size_px + (dim.array_size_px % 2)) / 2; + + const uint32_t chunk_size_px = + dim.array_size_px == 0 + ? dim.chunk_size_px + : std::min(dim.chunk_size_px, array_size_px); + + CHECK(chunk_size_px); + const uint32_t n_chunks = + (array_size_px + chunk_size_px - 1) / chunk_size_px; + + const uint32_t shard_size_chunks = + dim.array_size_px == 0 + ? 1 + : std::min(n_chunks, dim.shard_size_chunks); + + downsampled_dims[i] = { dim.name, + dim.type, + array_size_px, + chunk_size_px, + shard_size_chunks }; + } + } + downsampled_config.dimensions = std::make_unique( + std::move(downsampled_dims), config.dtype); + + downsampled_config.level_of_detail = config.level_of_detail + 1; + downsampled_config.bucket_name = config.bucket_name; + downsampled_config.store_path = config.store_path; + + downsampled_config.dtype = config.dtype; + + // copy the Blosc compression parameters + downsampled_config.compression_params = config.compression_params; + + // can we downsample downsampled_config? + for (auto i = 0; i < config.dimensions->ndims(); ++i) { + // downsampling made the chunk size strictly smaller + const auto& dim = config.dimensions->at(i); + const auto& downsampled_dim = downsampled_config.dimensions->at(i); + + if (dim.chunk_size_px > downsampled_dim.chunk_size_px) { + return false; + } + } + + return true; +} + +/// Writer +zarr::ArrayWriter::ArrayWriter(ArrayWriterConfig&& config, + std::shared_ptr thread_pool) + : ArrayWriter(std::move(config), thread_pool, nullptr) +{ +} + +zarr::ArrayWriter::ArrayWriter( + ArrayWriterConfig&& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool) + : config_{ std::move(config) } + , thread_pool_{ thread_pool } + , s3_connection_pool_{ s3_connection_pool } + , bytes_to_flush_{ 0 } + , frames_written_{ 0 } + , append_chunk_index_{ 0 } + , is_finalizing_{ false } +{ +} + +size_t +zarr::ArrayWriter::write_frame(std::span data) +{ + const auto nbytes_data = data.size(); + const auto nbytes_frame = + bytes_of_frame(*config_.dimensions, config_.dtype); + + if (nbytes_frame != nbytes_data) { + LOG_ERROR("Frame size mismatch: expected ", + nbytes_frame, + ", got ", + nbytes_data, + ". Skipping"); + return 0; + } + + if (chunk_buffers_.empty()) { + make_buffers_(); + } + + // split the incoming frame into tiles and write them to the chunk + // buffers + const auto bytes_written = write_frame_to_chunks_(data); + EXPECT(bytes_written == nbytes_data, "Failed to write frame to chunks"); + + LOG_DEBUG("Wrote ", bytes_written, " bytes of frame ", frames_written_); + bytes_to_flush_ += bytes_written; + ++frames_written_; + + if (should_flush_()) { + flush_(); + } + + return bytes_written; +} + +bool +zarr::ArrayWriter::is_s3_array_() const +{ + return config_.bucket_name.has_value(); +} + +bool +zarr::ArrayWriter::make_data_sinks_() +{ + std::string data_root; + std::function parts_along_dimension; + switch (version_()) { + case ZarrVersion_2: + parts_along_dimension = chunks_along_dimension; + data_root = config_.store_path + "/" + + std::to_string(config_.level_of_detail) + "/" + + std::to_string(append_chunk_index_); + break; + case ZarrVersion_3: + parts_along_dimension = shards_along_dimension; + data_root = config_.store_path + "/data/root/" + + std::to_string(config_.level_of_detail) + "/c" + + std::to_string(append_chunk_index_); + break; + default: + LOG_ERROR("Unsupported Zarr version"); + return false; + } + + SinkCreator creator(thread_pool_, s3_connection_pool_); + + if (is_s3_array_()) { + if (!creator.make_data_sinks(*config_.bucket_name, + data_root, + config_.dimensions.get(), + parts_along_dimension, + data_sinks_)) { + LOG_ERROR("Failed to create data sinks in ", + data_root, + " for bucket ", + *config_.bucket_name); + return false; + } + } else if (!creator.make_data_sinks(data_root, + config_.dimensions.get(), + parts_along_dimension, + data_sinks_)) { + LOG_ERROR("Failed to create data sinks in ", data_root); + return false; + } + + return true; +} + +bool +zarr::ArrayWriter::make_metadata_sink_() +{ + if (metadata_sink_) { + LOG_INFO("Metadata sink already exists"); + return true; + } + + std::string metadata_path; + switch (version_()) { + case ZarrVersion_2: + metadata_path = config_.store_path + "/" + + std::to_string(config_.level_of_detail) + + "/.zarray"; + break; + case ZarrVersion_3: + metadata_path = config_.store_path + "/meta/root/" + + std::to_string(config_.level_of_detail) + + ".array.json"; + break; + default: + LOG_ERROR("Unsupported Zarr version"); + return false; + } + + if (is_s3_array_()) { + SinkCreator creator(thread_pool_, s3_connection_pool_); + metadata_sink_ = + creator.make_sink(*config_.bucket_name, metadata_path); + } else { + metadata_sink_ = zarr::SinkCreator::make_sink(metadata_path); + } + + if (!metadata_sink_) { + LOG_ERROR("Failed to create metadata sink: ", metadata_path); + return false; + } + + return true; +} + +void +zarr::ArrayWriter::make_buffers_() noexcept +{ + LOG_DEBUG("Creating chunk buffers"); + + const size_t n_chunks = config_.dimensions->number_of_chunks_in_memory(); + chunk_buffers_.resize(n_chunks); // no-op if already the correct size + + const auto nbytes = config_.dimensions->bytes_per_chunk(); + + for (auto& buf : chunk_buffers_) { + buf.resize(nbytes); + std::fill_n(buf.begin(), nbytes, std::byte(0)); + } +} + +size_t +zarr::ArrayWriter::write_frame_to_chunks_(std::span data) +{ + // break the frame into tiles and write them to the chunk buffers + const auto bytes_per_px = bytes_of_type(config_.dtype); + + const auto& dimensions = config_.dimensions; + + const auto& x_dim = dimensions->width_dim(); + const auto frame_cols = x_dim.array_size_px; + const auto tile_cols = x_dim.chunk_size_px; + + const auto& y_dim = dimensions->height_dim(); + const auto frame_rows = y_dim.array_size_px; + const auto tile_rows = y_dim.chunk_size_px; + + if (tile_cols == 0 || tile_rows == 0) { + return 0; + } + + const auto bytes_per_row = tile_cols * bytes_per_px; + + size_t bytes_written = 0; + + const auto n_tiles_x = (frame_cols + tile_cols - 1) / tile_cols; + const auto n_tiles_y = (frame_rows + tile_rows - 1) / tile_rows; + + // don't take the frame id from the incoming frame, as the camera may have + // dropped frames + const auto frame_id = frames_written_; + + // offset among the chunks in the lattice + const auto group_offset = dimensions->tile_group_offset(frame_id); + // offset within the chunk + const auto chunk_offset = + static_cast(dimensions->chunk_internal_offset(frame_id)); + + for (auto i = 0; i < n_tiles_y; ++i) { + // TODO (aliddell): we can optimize this when tiles_per_frame_x_ is 1 + for (auto j = 0; j < n_tiles_x; ++j) { + const auto c = group_offset + i * n_tiles_x + j; + auto& chunk = chunk_buffers_[c]; + auto chunk_it = chunk.begin() + chunk_offset; + + for (auto k = 0; k < tile_rows; ++k) { + const auto frame_row = i * tile_rows + k; + if (frame_row < frame_rows) { + const auto frame_col = j * tile_cols; + + const auto region_width = + std::min(frame_col + tile_cols, frame_cols) - frame_col; + + const auto region_start = static_cast( + bytes_per_px * (frame_row * frame_cols + frame_col)); + const auto nbytes = + static_cast(region_width * bytes_per_px); + const auto region_stop = region_start + nbytes; + if (region_stop > data.size()) { + LOG_ERROR("Buffer overflow"); + return bytes_written; + } + + // copy region + if (nbytes > std::distance(chunk_it, chunk.end())) { + LOG_ERROR("Buffer overflow"); + return bytes_written; + } + std::copy(data.begin() + region_start, + data.begin() + region_stop, + chunk_it); + + bytes_written += (region_stop - region_start); + } + chunk_it += static_cast(bytes_per_row); + } + } + } + + return bytes_written; +} + +bool +zarr::ArrayWriter::should_flush_() const +{ + const auto& dims = config_.dimensions; + size_t frames_before_flush = dims->final_dim().chunk_size_px; + for (auto i = 1; i < dims->ndims() - 2; ++i) { + frames_before_flush *= dims->at(i).array_size_px; + } + + CHECK(frames_before_flush > 0); + return frames_written_ % frames_before_flush == 0; +} + +void +zarr::ArrayWriter::compress_buffers_() +{ + if (!config_.compression_params.has_value()) { + return; + } + + LOG_DEBUG("Compressing"); + + BloscCompressionParams params = config_.compression_params.value(); + const auto bytes_per_px = bytes_of_type(config_.dtype); + + std::scoped_lock lock(buffers_mutex_); + std::latch latch(chunk_buffers_.size()); + for (auto& chunk : chunk_buffers_) { + EXPECT(thread_pool_->push_job( + [¶ms, buf = &chunk, bytes_per_px, &latch]( + std::string& err) -> bool { + bool success = false; + const size_t bytes_of_chunk = buf->size(); + + try { + const auto tmp_size = + bytes_of_chunk + BLOSC_MAX_OVERHEAD; + std::vector tmp(tmp_size); + const auto nb = + blosc_compress_ctx(params.clevel, + params.shuffle, + bytes_per_px, + bytes_of_chunk, + buf->data(), + tmp.data(), + tmp_size, + params.codec_id.c_str(), + 0 /* blocksize - 0:automatic */, + 1); + + tmp.resize(nb); + buf->swap(tmp); + + success = true; + } catch (const std::exception& exc) { + err = "Failed to compress chunk: " + + std::string(exc.what()); + } catch (...) { + err = "Failed to compress chunk (unknown)"; + } + latch.count_down(); + + return success; + }), + "Failed to push to job queue"); + } + + // wait for all threads to finish + latch.wait(); +} + +void +zarr::ArrayWriter::flush_() +{ + if (bytes_to_flush_ == 0) { + return; + } + + // compress buffers and write out + compress_buffers_(); + CHECK(flush_impl_()); + + const auto should_rollover = should_rollover_(); + if (should_rollover) { + rollover_(); + } + + if (should_rollover || is_finalizing_) { + CHECK(write_array_metadata_()); + } + + // reset buffers + make_buffers_(); + + // reset state + bytes_to_flush_ = 0; +} + +void +zarr::ArrayWriter::close_sinks_() +{ + data_sinks_.clear(); +} + +void +zarr::ArrayWriter::rollover_() +{ + LOG_DEBUG("Rolling over"); + + close_sinks_(); + ++append_chunk_index_; +} diff --git a/src/streaming/array.writer.hh b/src/streaming/array.writer.hh new file mode 100644 index 00000000..0eb19681 --- /dev/null +++ b/src/streaming/array.writer.hh @@ -0,0 +1,103 @@ +#pragma once + +#include "zarr.stream.hh" +#include "thread.pool.hh" +#include "s3.connection.hh" +#include "blosc.compression.params.hh" +#include "file.sink.hh" + +#include +#include + +namespace fs = std::filesystem; + +namespace zarr { + +struct ArrayWriterConfig +{ + std::unique_ptr dimensions; + ZarrDataType dtype; + int level_of_detail; + std::optional bucket_name; + std::string store_path; + std::optional compression_params; +}; + +/** + * @brief Downsample the array writer configuration to a lower resolution. + * @param[in] config The original array writer configuration. + * @param[out] downsampled_config The downsampled array writer configuration. + * @return True if @p downsampled_config can be downsampled further. + * This is determined by the chunk size in @p config. This function will return + * false if and only if downsampling brings one or more dimensions lower than + * the chunk size along that dimension. + */ +[[nodiscard]] +bool +downsample(const ArrayWriterConfig& config, + ArrayWriterConfig& downsampled_config); + +class ArrayWriter +{ + public: + ArrayWriter(ArrayWriterConfig&& config, + std::shared_ptr thread_pool); + + ArrayWriter(ArrayWriterConfig&& config, + std::shared_ptr thread_pool, + std::shared_ptr s3_connection_pool); + + virtual ~ArrayWriter() = default; + + /** + * @brief Write a frame to the array. + * @param data The frame data. + * @return The number of bytes written. + */ + [[nodiscard]] size_t write_frame(std::span data); + + protected: + ArrayWriterConfig config_; + + /// Chunking + std::vector> chunk_buffers_; + + /// Filesystem + std::vector> data_sinks_; + std::unique_ptr metadata_sink_; + + /// Multithreading + std::shared_ptr thread_pool_; + std::mutex buffers_mutex_; + + /// Bookkeeping + uint64_t bytes_to_flush_; + uint32_t frames_written_; + uint32_t append_chunk_index_; + bool is_finalizing_; + + std::shared_ptr s3_connection_pool_; + + virtual ZarrVersion version_() const = 0; + + bool is_s3_array_() const; + + [[nodiscard]] bool make_data_sinks_(); + [[nodiscard]] bool make_metadata_sink_(); + void make_buffers_() noexcept; + + bool should_flush_() const; + virtual bool should_rollover_() const = 0; + + size_t write_frame_to_chunks_(std::span data); + void compress_buffers_(); + + void flush_(); + [[nodiscard]] virtual bool flush_impl_() = 0; + void rollover_(); + + [[nodiscard]] virtual bool write_array_metadata_() = 0; + + void close_sinks_(); +}; +} // namespace zarr diff --git a/src/streaming/sink.creator.cpp b/src/streaming/sink.creator.cpp index 2eadc35c..7d2b4e6c 100644 --- a/src/streaming/sink.creator.cpp +++ b/src/streaming/sink.creator.cpp @@ -63,7 +63,7 @@ zarr::SinkCreator::make_sink(std::string_view bucket_name, bool zarr::SinkCreator::make_data_sinks( std::string_view base_path, - std::shared_ptr dimensions, + const ArrayDimensions* dimensions, const std::function& parts_along_dimension, std::vector>& part_sinks) { @@ -89,7 +89,7 @@ bool zarr::SinkCreator::make_data_sinks( std::string_view bucket_name, std::string_view base_path, - std::shared_ptr dimensions, + const ArrayDimensions* dimensions, const std::function& parts_along_dimension, std::vector>& part_sinks) { @@ -139,7 +139,7 @@ zarr::SinkCreator::make_metadata_sinks( std::queue zarr::SinkCreator::make_data_sink_paths_( std::string_view base_path, - std::shared_ptr dimensions, + const ArrayDimensions* dimensions, const std::function& parts_along_dimension, bool create_directories) { diff --git a/src/streaming/sink.creator.hh b/src/streaming/sink.creator.hh index 85c29e4d..b7f6e689 100644 --- a/src/streaming/sink.creator.hh +++ b/src/streaming/sink.creator.hh @@ -51,7 +51,7 @@ class SinkCreator */ [[nodiscard]] bool make_data_sinks( std::string_view base_path, - std::shared_ptr dimensions, + const ArrayDimensions* dimensions, const std::function& parts_along_dimension, std::vector>& part_sinks); @@ -69,7 +69,7 @@ class SinkCreator [[nodiscard]] bool make_data_sinks( std::string_view bucket_name, std::string_view base_path, - std::shared_ptr dimensions, + const ArrayDimensions* dimensions, const std::function& parts_along_dimension, std::vector>& part_sinks); @@ -119,7 +119,7 @@ class SinkCreator */ std::queue make_data_sink_paths_( std::string_view base_path, - std::shared_ptr dimensions, + const ArrayDimensions* dimensions, const std::function& parts_along_dimension, bool create_directories); diff --git a/src/streaming/zarr.dimension.hh b/src/streaming/zarr.dimension.hh index a28d7530..76e86cca 100644 --- a/src/streaming/zarr.dimension.hh +++ b/src/streaming/zarr.dimension.hh @@ -3,11 +3,13 @@ #include "zarr.types.h" #include +#include #include struct ZarrDimension { - ZarrDimension(const char* name, + ZarrDimension() = default; + ZarrDimension(std::string_view name, ZarrDimensionType type, uint32_t array_size_px, uint32_t chunk_size_px, diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt index 7740470b..5b51d84d 100644 --- a/tests/unit-tests/CMakeLists.txt +++ b/tests/unit-tests/CMakeLists.txt @@ -16,6 +16,8 @@ set(tests s3-sink-write sink-creator-make-metadata-sinks sink-creator-make-data-sinks + array-writer-downsample-writer-config + array-writer-write-frame-to-chunks ) foreach (name ${tests}) diff --git a/tests/unit-tests/array-writer-downsample-writer-config.cpp b/tests/unit-tests/array-writer-downsample-writer-config.cpp new file mode 100644 index 00000000..0755471a --- /dev/null +++ b/tests/unit-tests/array-writer-downsample-writer-config.cpp @@ -0,0 +1,134 @@ +#include "array.writer.hh" +#include "unit.test.macros.hh" + +#include + +namespace fs = std::filesystem; + +int +main() +{ + int retval = 1; + + try { + const fs::path base_dir = "acquire"; + + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + 0, + 5, + 1); // 5 timepoints / chunk, 1 shard + dims.emplace_back( + "c", ZarrDimensionType_Channel, 2, 1, 1); // 2 chunks, 2 shards + dims.emplace_back( + "z", ZarrDimensionType_Space, 7, 3, 3); // 3 chunks, 3 shards + dims.emplace_back( + "y", ZarrDimensionType_Space, 48, 16, 3); // 3 chunks, 1 shard + dims.emplace_back( + "x", ZarrDimensionType_Space, 64, 16, 2); // 4 chunks, 2 shards + + zarr::ArrayWriterConfig config{ + .dimensions = std::make_unique(std::move(dims), + ZarrDataType_uint8), + .dtype = ZarrDataType_uint8, + .level_of_detail = 0, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt + }; + + zarr::ArrayWriterConfig downsampled_config; + CHECK(zarr::downsample(config, downsampled_config)); + + // check dimensions + CHECK(downsampled_config.dimensions->ndims() == 5); + + CHECK(downsampled_config.dimensions->at(0).name == "t"); + CHECK(downsampled_config.dimensions->at(0).array_size_px == 0); + CHECK(downsampled_config.dimensions->at(0).chunk_size_px == 5); + CHECK(downsampled_config.dimensions->at(0).shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions->at(1).name == "c"); + // we don't downsample channels + CHECK(downsampled_config.dimensions->at(1).array_size_px == 2); + CHECK(downsampled_config.dimensions->at(1).chunk_size_px == 1); + CHECK(downsampled_config.dimensions->at(1).shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions->at(2).name == "z"); + CHECK(downsampled_config.dimensions->at(2).array_size_px == 4); + CHECK(downsampled_config.dimensions->at(2).chunk_size_px == 3); + CHECK(downsampled_config.dimensions->at(2).shard_size_chunks == 2); + + CHECK(downsampled_config.dimensions->at(3).name == "y"); + CHECK(downsampled_config.dimensions->at(3).array_size_px == 24); + CHECK(downsampled_config.dimensions->at(3).chunk_size_px == 16); + CHECK(downsampled_config.dimensions->at(3).shard_size_chunks == 2); + + CHECK(downsampled_config.dimensions->at(4).name == "x"); + CHECK(downsampled_config.dimensions->at(4).array_size_px == 32); + CHECK(downsampled_config.dimensions->at(4).chunk_size_px == 16); + CHECK(downsampled_config.dimensions->at(4).shard_size_chunks == 2); + + // check level of detail + CHECK(downsampled_config.level_of_detail == 1); + + // check store path + CHECK(downsampled_config.store_path == config.store_path); + + // check compression params + CHECK(!downsampled_config.compression_params.has_value()); + + // downsample again + config = std::move(downsampled_config); + + // can't downsample anymore + CHECK(!zarr::downsample(config, downsampled_config)); + + // check dimensions + CHECK(downsampled_config.dimensions->ndims() == 5); + + CHECK(downsampled_config.dimensions->at(0).name == "t"); + CHECK(downsampled_config.dimensions->at(0).array_size_px == 0); + CHECK(downsampled_config.dimensions->at(0).chunk_size_px == 5); + CHECK(downsampled_config.dimensions->at(0).shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions->at(1).name == "c"); + // we don't downsample channels + CHECK(downsampled_config.dimensions->at(1).array_size_px == 2); + CHECK(downsampled_config.dimensions->at(1).chunk_size_px == 1); + CHECK(downsampled_config.dimensions->at(1).shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions->at(2).name == "z"); + CHECK(downsampled_config.dimensions->at(2).array_size_px == 2); + CHECK(downsampled_config.dimensions->at(2).chunk_size_px == 2); + CHECK(downsampled_config.dimensions->at(2).shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions->at(3).name == "y"); + CHECK(downsampled_config.dimensions->at(3).array_size_px == 12); + CHECK(downsampled_config.dimensions->at(3).chunk_size_px == 12); + CHECK(downsampled_config.dimensions->at(3).shard_size_chunks == 1); + + CHECK(downsampled_config.dimensions->at(4).name == "x"); + CHECK(downsampled_config.dimensions->at(4).array_size_px == 16); + CHECK(downsampled_config.dimensions->at(4).chunk_size_px == 16); + CHECK(downsampled_config.dimensions->at(4).shard_size_chunks == 1); + + // check level of detail + CHECK(downsampled_config.level_of_detail == 2); + + // check data root + CHECK(downsampled_config.store_path == config.store_path); + + // check compression params + CHECK(!downsampled_config.compression_params.has_value()); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/array-writer-write-frame-to-chunks.cpp b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp new file mode 100644 index 00000000..46d406cc --- /dev/null +++ b/tests/unit-tests/array-writer-write-frame-to-chunks.cpp @@ -0,0 +1,88 @@ +#include "array.writer.hh" +#include "unit.test.macros.hh" +#include "zarr.common.hh" + +namespace { +class TestWriter : public zarr::ArrayWriter +{ + public: + TestWriter(zarr::ArrayWriterConfig&& array_spec, + std::shared_ptr thread_pool) + : zarr::ArrayWriter(std::move(array_spec), thread_pool) + { + } + + private: + ZarrVersion version_() const override { return ZarrVersionCount; } + bool should_rollover_() const override { return false; } + bool flush_impl_() override { return true; } + bool write_array_metadata_() override { return true; } +}; +} // namespace + +int +main() +{ + const auto base_dir = fs::temp_directory_path() / TEST; + int retval = 1; + + const unsigned int array_width = 64, array_height = 48, array_planes = 2, + array_channels = 1, array_timepoints = 2; + const unsigned int chunk_width = 16, chunk_height = 16, chunk_planes = 1, + chunk_channels = 1, chunk_timepoints = 1; + + const unsigned int n_frames = + array_planes * array_channels * array_timepoints; + + const ZarrDataType dtype = ZarrDataType_uint16; + const unsigned int nbytes_px = zarr::bytes_of_type(dtype); + + try { + auto thread_pool = std::make_shared( + std::thread::hardware_concurrency(), + [](const std::string& err) { LOG_ERROR("Error: %s", err.c_str()); }); + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, array_timepoints, chunk_timepoints, 0); + dims.emplace_back( + "c", ZarrDimensionType_Channel, array_channels, chunk_channels, 0); + dims.emplace_back( + "z", ZarrDimensionType_Space, array_planes, chunk_planes, 0); + dims.emplace_back( + "y", ZarrDimensionType_Space, array_height, chunk_height, 0); + dims.emplace_back( + "x", ZarrDimensionType_Space, array_width, chunk_width, 0); + + zarr::ArrayWriterConfig config = { + .dimensions = + std::make_unique(std::move(dims), dtype), + .dtype = dtype, + .bucket_name = std::nullopt, + .store_path = base_dir.string(), + .compression_params = std::nullopt, + }; + + TestWriter writer(std::move(config), thread_pool); + + const size_t frame_size = array_width * array_height * nbytes_px; + std::vector data_(frame_size, std::byte(0)); + std::span data(data_.data(), data_.size()); + + for (auto i = 0; i < n_frames; ++i) { + CHECK(writer.write_frame(data) == frame_size); + } + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + // cleanup + if (fs::exists(base_dir)) { + fs::remove_all(base_dir); + } + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-chunk-internal-offset.cpp b/tests/unit-tests/common-chunk-internal-offset.cpp new file mode 100644 index 00000000..ff45eb01 --- /dev/null +++ b/tests/unit-tests/common-chunk-internal-offset.cpp @@ -0,0 +1,184 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk + dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks + dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks + dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks + dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks + + try { + EXPECT_INT_EQ(zarr::chunk_internal_offset(0, dims, ZarrDataType_uint16), + 0); + EXPECT_INT_EQ(zarr::chunk_internal_offset(1, dims, ZarrDataType_uint16), + 512); + EXPECT_INT_EQ(zarr::chunk_internal_offset(2, dims, ZarrDataType_uint16), + 0); + EXPECT_INT_EQ(zarr::chunk_internal_offset(3, dims, ZarrDataType_uint16), + 512); + EXPECT_INT_EQ(zarr::chunk_internal_offset(4, dims, ZarrDataType_uint16), + 0); + EXPECT_INT_EQ(zarr::chunk_internal_offset(5, dims, ZarrDataType_uint16), + 1024); + EXPECT_INT_EQ(zarr::chunk_internal_offset(6, dims, ZarrDataType_uint16), + 1536); + EXPECT_INT_EQ(zarr::chunk_internal_offset(7, dims, ZarrDataType_uint16), + 1024); + EXPECT_INT_EQ(zarr::chunk_internal_offset(8, dims, ZarrDataType_uint16), + 1536); + EXPECT_INT_EQ(zarr::chunk_internal_offset(9, dims, ZarrDataType_uint16), + 1024); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(10, dims, ZarrDataType_uint16), 0); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(11, dims, ZarrDataType_uint16), 512); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(12, dims, ZarrDataType_uint16), 0); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(13, dims, ZarrDataType_uint16), 512); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(14, dims, ZarrDataType_uint16), 0); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(15, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(16, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(17, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(18, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(19, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(20, dims, ZarrDataType_uint16), 3072); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(21, dims, ZarrDataType_uint16), 3584); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(22, dims, ZarrDataType_uint16), 3072); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(23, dims, ZarrDataType_uint16), 3584); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(24, dims, ZarrDataType_uint16), 3072); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(25, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(26, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(27, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(28, dims, ZarrDataType_uint16), 2560); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(29, dims, ZarrDataType_uint16), 2048); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(30, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(31, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(32, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(33, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(34, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(35, dims, ZarrDataType_uint16), 5120); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(36, dims, ZarrDataType_uint16), 5632); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(37, dims, ZarrDataType_uint16), 5120); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(38, dims, ZarrDataType_uint16), 5632); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(39, dims, ZarrDataType_uint16), 5120); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(40, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(41, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(42, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(43, dims, ZarrDataType_uint16), 4608); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(44, dims, ZarrDataType_uint16), 4096); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(45, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(46, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(47, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(48, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(49, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(50, dims, ZarrDataType_uint16), 7168); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(51, dims, ZarrDataType_uint16), 7680); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(52, dims, ZarrDataType_uint16), 7168); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(53, dims, ZarrDataType_uint16), 7680); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(54, dims, ZarrDataType_uint16), 7168); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(55, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(56, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(57, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(58, dims, ZarrDataType_uint16), 6656); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(59, dims, ZarrDataType_uint16), 6144); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(60, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(61, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(62, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(63, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(64, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(65, dims, ZarrDataType_uint16), 9216); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(66, dims, ZarrDataType_uint16), 9728); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(67, dims, ZarrDataType_uint16), 9216); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(68, dims, ZarrDataType_uint16), 9728); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(69, dims, ZarrDataType_uint16), 9216); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(70, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(71, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(72, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(73, dims, ZarrDataType_uint16), 8704); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(74, dims, ZarrDataType_uint16), 8192); + EXPECT_INT_EQ( + zarr::chunk_internal_offset(75, dims, ZarrDataType_uint16), 0); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-chunk-lattice-index.cpp b/tests/unit-tests/common-chunk-lattice-index.cpp new file mode 100644 index 00000000..3dc94d44 --- /dev/null +++ b/tests/unit-tests/common-chunk-lattice-index.cpp @@ -0,0 +1,83 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %d != %d", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + try { + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk + dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks + dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks + dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks + dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks + + EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(0, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(1, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(2, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(3, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(4, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(5, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(12, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(19, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(26, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(33, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(40, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(47, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(54, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(61, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 2, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(68, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 2, dims), 2); + EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 1, dims), 1); + EXPECT_INT_EQ(zarr::chunk_lattice_index(74, 0, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 2, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 1, dims), 0); + EXPECT_INT_EQ(zarr::chunk_lattice_index(75, 0, dims), 1); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-index-for-chunk.cpp b/tests/unit-tests/common-shard-index-for-chunk.cpp new file mode 100644 index 00000000..6c012b43 --- /dev/null +++ b/tests/unit-tests/common-shard-index-for-chunk.cpp @@ -0,0 +1,195 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + try { + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + 0, + 5, // 5 timepoints / chunk + 2); // 2 chunks / shard + dims.emplace_back("c", + ZarrDimensionType_Channel, + 8, + 4, // 8 / 4 = 2 chunks + 2); // 4 / 2 = 2 shards + dims.emplace_back("z", + ZarrDimensionType_Space, + 6, + 2, // 6 / 2 = 3 chunks + 1); // 3 / 1 = 3 shards + dims.emplace_back("y", + ZarrDimensionType_Space, + 48, + 16, // 48 / 16 = 3 chunks + 1); // 3 / 1 = 3 shards + dims.emplace_back("x", + ZarrDimensionType_Space, + 64, + 16, // 64 / 16 = 4 chunks + 2); // 4 / 2 = 2 shards + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(12, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(13, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(14, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(15, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(16, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(17, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(18, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(19, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(20, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(21, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(22, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(23, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(24, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(25, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(26, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(27, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(28, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(29, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(30, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(31, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(32, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(33, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(34, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(35, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(36, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(37, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(38, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(39, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(40, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(41, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(42, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(43, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(44, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(45, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(46, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(47, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(48, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(49, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(50, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(51, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(52, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(53, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(54, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(55, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(56, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(57, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(58, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(59, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(60, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(61, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(62, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(63, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(64, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(65, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(66, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(67, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(68, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(69, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(70, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(71, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(72, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(73, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(74, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(75, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(76, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(77, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(78, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(79, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(80, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(81, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(82, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(83, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(84, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(85, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(86, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(87, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(88, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(89, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(90, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(91, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(92, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(93, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(94, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(95, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(96, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(97, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(98, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(99, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(100, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(101, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(102, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(103, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(104, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(105, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(106, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(107, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(108, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(109, dims), 0); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(110, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(111, dims), 1); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(112, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(113, dims), 2); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(114, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(115, dims), 3); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(116, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(117, dims), 4); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(118, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(119, dims), 5); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(120, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(121, dims), 6); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(122, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(123, dims), 7); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(124, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(125, dims), 8); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(126, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(127, dims), 9); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(128, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(129, dims), 10); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(130, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(131, dims), 11); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(132, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(133, dims), 12); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(134, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(135, dims), 13); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(136, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(137, dims), 14); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(138, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(139, dims), 15); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(140, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(141, dims), 16); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(142, dims), 17); + EXPECT_INT_EQ(zarr::shard_index_for_chunk(143, dims), 17); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-shard-internal-index.cpp b/tests/unit-tests/common-shard-internal-index.cpp new file mode 100644 index 00000000..346c13bf --- /dev/null +++ b/tests/unit-tests/common-shard-internal-index.cpp @@ -0,0 +1,75 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + std::vector dims; + dims.emplace_back("t", + ZarrDimensionType_Time, + 0, + 32, // 32 timepoints / chunk + 1); // 1 shard + dims.emplace_back("y", + ZarrDimensionType_Space, + 960, + 320, // 3 chunks + 2); // 2 ragged shards + dims.emplace_back("x", + ZarrDimensionType_Space, + 1080, + 270, // 4 chunks + 3); // 2 ragged shards + + try { + EXPECT_INT_EQ(zarr::shard_index_for_chunk(0, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(0, dims), 0); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(1, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(1, dims), 1); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(2, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(2, dims), 2); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(3, dims), 1); + EXPECT_INT_EQ(zarr::shard_internal_index(3, dims), 0); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(4, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(4, dims), 3); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(5, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(5, dims), 4); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(6, dims), 0); + EXPECT_INT_EQ(zarr::shard_internal_index(6, dims), 5); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(7, dims), 1); + EXPECT_INT_EQ(zarr::shard_internal_index(7, dims), 3); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(8, dims), 2); + EXPECT_INT_EQ(zarr::shard_internal_index(8, dims), 0); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(9, dims), 2); + EXPECT_INT_EQ(zarr::shard_internal_index(9, dims), 1); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(10, dims), 2); + EXPECT_INT_EQ(zarr::shard_internal_index(10, dims), 2); + + EXPECT_INT_EQ(zarr::shard_index_for_chunk(11, dims), 3); + EXPECT_INT_EQ(zarr::shard_internal_index(11, dims), 0); + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/common-tile-group-offset.cpp b/tests/unit-tests/common-tile-group-offset.cpp new file mode 100644 index 00000000..be9efecc --- /dev/null +++ b/tests/unit-tests/common-tile-group-offset.cpp @@ -0,0 +1,108 @@ +#include "zarr.common.hh" +#include "unit.test.macros.hh" + +#include + +#define EXPECT_INT_EQ(a, b) \ + EXPECT((a) == (b), "Expected %s == %s, but %zu != %zu", #a, #b, a, b) + +int +main() +{ + int retval = 1; + + std::vector dims; + dims.emplace_back( + "t", ZarrDimensionType_Time, 0, 5, 0); // 5 timepoints / chunk + dims.emplace_back("c", ZarrDimensionType_Channel, 3, 2, 0); // 2 chunks + dims.emplace_back("z", ZarrDimensionType_Space, 5, 2, 0); // 3 chunks + dims.emplace_back("y", ZarrDimensionType_Space, 48, 16, 0); // 3 chunks + dims.emplace_back("x", ZarrDimensionType_Space, 64, 16, 0); // 4 chunks + + try { + EXPECT_INT_EQ(zarr::tile_group_offset(0, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(1, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(2, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(3, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(4, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(5, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(6, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(7, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(8, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(9, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(10, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(11, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(12, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(13, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(14, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(15, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(16, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(17, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(18, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(19, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(20, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(21, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(22, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(23, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(24, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(25, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(26, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(27, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(28, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(29, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(30, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(31, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(32, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(33, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(34, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(35, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(36, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(37, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(38, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(39, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(40, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(41, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(42, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(43, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(44, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(45, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(46, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(47, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(48, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(49, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(50, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(51, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(52, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(53, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(54, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(55, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(56, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(57, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(58, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(59, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(60, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(61, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(62, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(63, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(64, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(65, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(66, dims), 0); + EXPECT_INT_EQ(zarr::tile_group_offset(67, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(68, dims), 12); + EXPECT_INT_EQ(zarr::tile_group_offset(69, dims), 24); + EXPECT_INT_EQ(zarr::tile_group_offset(70, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(71, dims), 36); + EXPECT_INT_EQ(zarr::tile_group_offset(72, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(73, dims), 48); + EXPECT_INT_EQ(zarr::tile_group_offset(74, dims), 60); + EXPECT_INT_EQ(zarr::tile_group_offset(75, dims), 0); + + retval = 0; + } catch (const std::exception& exc) { + LOG_ERROR("Exception: %s\n", exc.what()); + } catch (...) { + LOG_ERROR("Exception: (unknown)"); + } + + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/sink-creator-make-data-sinks.cpp b/tests/unit-tests/sink-creator-make-data-sinks.cpp index 09cee1e9..cac596a3 100644 --- a/tests/unit-tests/sink-creator-make-data-sinks.cpp +++ b/tests/unit-tests/sink-creator-make-data-sinks.cpp @@ -49,7 +49,7 @@ get_credentials(std::string& endpoint, void sink_creator_make_chunk_sinks(std::shared_ptr thread_pool, - std::shared_ptr dimensions) + const ArrayDimensions* dimensions) { zarr::SinkCreator sink_creator(thread_pool, nullptr); @@ -85,7 +85,7 @@ sink_creator_make_chunk_sinks( std::shared_ptr thread_pool, std::shared_ptr connection_pool, const std::string& bucket_name, - std::shared_ptr dimensions) + const ArrayDimensions* dimensions) { zarr::SinkCreator sink_creator(thread_pool, connection_pool); @@ -135,7 +135,7 @@ sink_creator_make_chunk_sinks( void sink_creator_make_shard_sinks(std::shared_ptr thread_pool, - std::shared_ptr dimensions) + const ArrayDimensions* dimensions) { zarr::SinkCreator sink_creator(thread_pool, nullptr); @@ -171,7 +171,7 @@ sink_creator_make_shard_sinks( std::shared_ptr thread_pool, std::shared_ptr connection_pool, const std::string& bucket_name, - std::shared_ptr dimensions) + const ArrayDimensions* dimensions) { zarr::SinkCreator sink_creator(thread_pool, connection_pool); @@ -240,16 +240,15 @@ main() 12, 3, // 3 columns per chunk, 4 chunks 2); // 2 chunks per shard (6 columns per shard, 2 shards) - auto dimensions = - std::make_shared(std::move(dims), ZarrDataType_int8); + ArrayDimensions dimensions(std::move(dims), ZarrDataType_int8); auto thread_pool = std::make_shared( std::thread::hardware_concurrency(), [](const std::string& err) { LOG_ERROR("Failed: ", err.c_str()); }); try { - sink_creator_make_chunk_sinks(thread_pool, dimensions); - sink_creator_make_shard_sinks(thread_pool, dimensions); + sink_creator_make_chunk_sinks(thread_pool, &dimensions); + sink_creator_make_shard_sinks(thread_pool, &dimensions); } catch (const std::exception& e) { LOG_ERROR("Failed: ", e.what()); return 1; @@ -268,9 +267,9 @@ main() try { sink_creator_make_chunk_sinks( - thread_pool, connection_pool, bucket_name, dimensions); + thread_pool, connection_pool, bucket_name, &dimensions); sink_creator_make_shard_sinks( - thread_pool, connection_pool, bucket_name, dimensions); + thread_pool, connection_pool, bucket_name, &dimensions); } catch (const std::exception& e) { LOG_ERROR("Failed: ", e.what()); return 1;