diff --git a/src/README.md b/src/README.md new file mode 100644 index 00000000..7a9fba97 --- /dev/null +++ b/src/README.md @@ -0,0 +1,42 @@ +# The Zarr Storage device + +## Components + +### The `StorageInterface` class. + +Defines the interface that all Acquire `Storage` devices must implement, namely + +- `set`: Set the storage properties. +- `get`: Get the storage properties. +- `get_meta`: Get metadata for the storage properties. +- `start`: Signal to the `Storage` device that it should start accepting frames. +- `stop`: Signal to the `Storage` device that it should stop accepting frames. +- `append`: Write incoming frames to the filesystem or other storage layer. +- `reserve_image_shape`: Set the image shape for allocating chunk writers. + +### The `Zarr` class + +An abstract class that implements the `StorageInterface`. +Zarr is "[a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification.](https://zarr.readthedocs.io/en/stable/index.html)" + +### The `ZarrV2` class + +Subclass of the `Zarr` class. +Implements abstract methods for writer allocation and metadata. +Specifically, `ZarrV2` allocates one writer of type `ChunkWriter` for each multiscale level-of-detail +and writes metadata in the format specified by the [Zarr V2 spec](https://zarr.readthedocs.io/en/stable/spec/v2.html). + +### The `Writer` class + +An abstract class that writes frames to the filesystem or other storage layer. +In general, frames are chunked and potentially compressed. +The `Writer` handles chunking, chunk compression, and writing. + +### The `ChunkWriter` class + +Subclass of the `Writer` class. +Implements abstract methods relating to writing and flushing chunk buffers. + +### The `BloscCompressionParams` struct + +Stores parameters for compression using [C-Blosc](https://github.com/Blosc/c-blosc). diff --git a/src/frame.scaler.cpp b/src/frame.scaler.cpp deleted file mode 100644 index 5e7eef6f..00000000 --- a/src/frame.scaler.cpp +++ /dev/null @@ -1,442 +0,0 @@ -#include "frame.scaler.hh" -#include "zarr.hh" - -#include -#include - -namespace { -namespace zarr = acquire::sink::zarr; - -size_t -bytes_of_type(const SampleType& type) -{ - CHECK(type < SampleTypeCount); - static size_t table[SampleTypeCount]; // = { 1, 2, 1, 2, 4, 2, 2, 2 }; -#define XXX(s, b) table[(s)] = (b) - XXX(SampleType_u8, 1); - XXX(SampleType_u16, 2); - XXX(SampleType_i8, 1); - XXX(SampleType_i16, 2); - XXX(SampleType_f32, 4); - XXX(SampleType_u10, 2); - XXX(SampleType_u12, 2); - XXX(SampleType_u14, 2); -#undef XXX - return table[type]; -} - -template -void -average_one_frame(std::shared_ptr dst, - std::shared_ptr src) -{ - CHECK(dst); - CHECK(src); - - const auto& src_shape = src->image_shape(); - const int downscale = 2; - const auto factor = 0.125f; - - const auto width = src_shape.dims.width; - const auto w_pad = width + (width % downscale); - - const auto height = src_shape.dims.height; - const auto h_pad = height + (height % downscale); - - const auto planes = src_shape.dims.planes; - const auto p_pad = planes > 1 ? planes + (planes % downscale) : 1; - - CHECK(dst->bytes_of_image() >= w_pad * h_pad * p_pad * factor * sizeof(T)); - - const auto* src_img = (T*)src->image(); - auto* dst_img = (T*)dst->data(); - - size_t dst_idx = 0; - for (auto plane = 0; plane < planes; plane += downscale) { - const bool pad_plane = (plane == planes - 1); - - for (auto row = 0; row < height; row += downscale) { - const bool pad_height = (row == height - 1 && height != h_pad); - - for (auto col = 0; col < width; col += downscale) { - const bool pad_width = (col == width - 1 && width != w_pad); - - size_t idx = plane * width * height + row * width + col; - dst_img[dst_idx++] = - (T)(factor * - ((float)src_img[idx] + - (float)src_img[idx + (1 - (int)pad_width)] + - (float)src_img[idx + width * (1 - (int)pad_height)] + - (float)src_img[idx + width * (1 - (int)pad_height) + - (1 - (int)pad_width)] + - (float) - src_img[idx + width * height * (1 - (int)pad_plane)] + - (float) - src_img[idx + width * height * (1 - (int)pad_plane) + - (1 - (int)pad_width)] + - (float) - src_img[idx + width * height * (1 - (int)pad_plane) + - width * (1 - (int)pad_height)] + - (float) - src_img[idx + width * height * (1 - (int)pad_plane) + - width * (1 - (int)pad_height) + - (1 - (int)pad_width)])); - } - } - } -} - -template -void -average_two_frames(std::shared_ptr dst, - std::shared_ptr src1, - std::shared_ptr src2) -{ - CHECK(dst); - CHECK(src1); - CHECK(src2); - - CHECK(dst->bytes_of_image() == src1->bytes_of_image() && - dst->bytes_of_image() == src2->bytes_of_image()); - - const float factor = 0.5f; - const size_t npx = dst->bytes_of_image() / sizeof(T); - - const auto* src1_img = (T*)src1->image(); - const auto* src2_img = (T*)src2->image(); - auto* dst_img = (T*)dst->data(); - - for (auto i = 0; i < npx; ++i) { - dst_img[i] = (T)(factor * ((float)src1_img[i] + (float)src2_img[i])); - } -} -} // :: namespace - -namespace acquire::sink::zarr { -ScalingParameters::ScalingParameters(const ImageShape& image_shape, - const TileShape& tile_shape) - : image_shape{ image_shape } - , tile_shape{ tile_shape } -{ -} - -FrameScaler::FrameScaler(Zarr* zarr, - const ImageShape& image_shape, - const TileShape& tile_shape) - : zarr_{ zarr } -{ - CHECK(zarr_); - scaling_params_ = make_scaling_parameters(image_shape, tile_shape); - for (int16_t i = 1; i < scaling_params_.size(); ++i) { - accumulators_.insert({ i, {} }); - } -} - -bool -FrameScaler::push_frame(std::shared_ptr frame) -{ - std::unique_lock lock(mutex_); - try { - zarr_->push_frame_to_writers(frame); - if (accumulators_.contains(1)) { - downsample_and_accumulate(frame, 1); - } - return true; - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - } catch (...) { - LOGE("Exception: (unknown)"); - } - - return false; -} - -void -FrameScaler::downsample_and_accumulate(std::shared_ptr frame, - int16_t layer) -{ - std::vector>& accumulator = - accumulators_.at(layer); - - const ImageShape& image_shape = scaling_params_.at(layer - 1).image_shape; - auto dst = - std::make_shared(frame->frame_id(), - layer, - scaling_params_.at(layer).image_shape, - scaling_params_.at(layer).tile_shape); - - switch (image_shape.type) { - case SampleType_u10: - case SampleType_u12: - case SampleType_u14: - case SampleType_u16: - average_one_frame(dst, frame); - if (accumulator.size() == 1) { - auto averaged = std::make_shared(dst->frame_id(), - dst->layer(), - dst->image_shape(), - dst->tile_shape()); - average_two_frames( - averaged, accumulator.front(), dst); - accumulator.clear(); - - zarr_->push_frame_to_writers(averaged); - if (layer < scaling_params_.size() - 1) { - downsample_and_accumulate(averaged, layer + 1); - } - } else { - accumulator.push_back(dst); - } - break; - case SampleType_i8: - average_one_frame(dst, frame); - if (accumulator.size() == 1) { - auto averaged = std::make_shared(dst->frame_id(), - dst->layer(), - dst->image_shape(), - dst->tile_shape()); - average_two_frames(averaged, accumulator.front(), dst); - accumulator.clear(); - - zarr_->push_frame_to_writers(averaged); - if (layer < scaling_params_.size() - 1) { - downsample_and_accumulate(averaged, layer + 1); - } - } else { - accumulator.push_back(dst); - } - break; - case SampleType_i16: - average_one_frame(dst, frame); - if (accumulator.size() == 1) { - auto averaged = std::make_shared(dst->frame_id(), - dst->layer(), - dst->image_shape(), - dst->tile_shape()); - average_two_frames(averaged, accumulator.front(), dst); - accumulator.clear(); - - zarr_->push_frame_to_writers(averaged); - if (layer < scaling_params_.size() - 1) { - downsample_and_accumulate(averaged, layer + 1); - } - } else { - accumulator.push_back(dst); - } - break; - case SampleType_f32: - average_one_frame(dst, frame); - if (accumulator.size() == 1) { - auto averaged = std::make_shared(dst->frame_id(), - dst->layer(), - dst->image_shape(), - dst->tile_shape()); - average_two_frames(averaged, accumulator.front(), dst); - accumulator.clear(); - - zarr_->push_frame_to_writers(averaged); - if (layer < scaling_params_.size() - 1) { - downsample_and_accumulate(averaged, layer + 1); - } - } else { - accumulator.push_back(dst); - } - break; - case SampleType_u8: - default: - average_one_frame(dst, frame); - if (accumulator.size() == 1) { - auto averaged = std::make_shared(dst->frame_id(), - dst->layer(), - dst->image_shape(), - dst->tile_shape()); - average_two_frames(averaged, accumulator.front(), dst); - accumulator.clear(); - - zarr_->push_frame_to_writers(averaged); - if (layer < scaling_params_.size() - 1) { - downsample_and_accumulate(averaged, layer + 1); - } - } else { - accumulator.push_back(dst); - } - break; - } -} - -std::vector -make_scaling_parameters(const ImageShape& base_image_shape, - const TileShape& base_tile_shape) -{ - std::vector shapes; - shapes.emplace_back(base_image_shape, base_tile_shape); - - const int downscale = 2; - - uint32_t w = base_image_shape.dims.width; - uint32_t h = base_image_shape.dims.height; - - while (w > base_tile_shape.width || h > base_tile_shape.height) { - w = (w + (w % downscale)) / downscale; - h = (h + (h % downscale)) / downscale; - - ImageShape im_shape = base_image_shape; - im_shape.dims.width = w; - im_shape.dims.height = h; - im_shape.strides.width = im_shape.strides.channels; - im_shape.strides.height = im_shape.strides.width * w; - im_shape.strides.planes = im_shape.strides.height * h; - - TileShape tile_shape = base_tile_shape; - if (tile_shape.width > w) - tile_shape.width = w; - - if (tile_shape.height > h) - tile_shape.height = h; - - shapes.emplace_back(im_shape, tile_shape); - } - - return shapes; -} -} // namespace acquire::sink::zarr - -#ifndef NO_UNIT_TESTS - -#ifdef _WIN32 -#define acquire_export __declspec(dllexport) -#else -#define acquire_export -#endif - -///< Test that a single frame with 1 plane is padded and averaged correctly. -template -void -test_average_frame_inner(const SampleType& stype) -{ - ImageShape image_shape { - .dims = { - .channels = 1, - .width = 3, - .height = 3, - .planes = 1, - }, - .strides = { - .channels = 1, - .width = 1, - .height = 3, - .planes = 9 - }, - .type = stype - }; - zarr::TileShape tile_shape{ .width = 3, .height = 3, .planes = 1 }; - - auto src = - std::make_shared(0, 0, image_shape, tile_shape); - for (auto i = 0; i < 9; ++i) { - ((T*)src->data())[i] = (T)(i + 1); - } - - image_shape.dims = { .channels = 1, .width = 2, .height = 2, .planes = 1 }; - image_shape.strides = { - .channels = 1, .width = 1, .height = 2, .planes = 4 - }; - tile_shape = { - .width = 2, - .height = 2, - .planes = 1, - }; - - auto dst = - std::make_shared(0, 0, image_shape, tile_shape); - - average_one_frame(dst, src); - CHECK(((T*)dst->image())[0] == (T)3); - CHECK(((T*)dst->image())[1] == (T)4.5); - CHECK(((T*)dst->image())[2] == (T)7.5); - CHECK(((T*)dst->image())[3] == (T)9); -} - -///< Test that a single frame with 3 planes is padded and averaged correctly. -template -void -test_average_planes_inner(const SampleType& stype) -{ - ImageShape image_shape { - .dims = { - .channels = 1, - .width = 4, - .height = 4, - .planes = 3, - }, - .strides = { - .channels = 1, - .width = 1, - .height = 4, - .planes = 16 - }, - .type = stype - }; - zarr::TileShape tile_shape{ .width = 4, .height = 4, .planes = 1 }; - - auto src = - std::make_shared(0, 0, image_shape, tile_shape); - for (auto i = 0; i < 48; ++i) { - ((T*)src->data())[i] = (T)(i + 1); - } - - image_shape.dims = { .channels = 1, .width = 2, .height = 2, .planes = 2 }; - image_shape.strides = { - .channels = 1, .width = 1, .height = 2, .planes = 4 - }; - tile_shape = { - .width = 2, - .height = 2, - .planes = 2, - }; - - auto dst = - std::make_shared(0, 0, image_shape, tile_shape); - - average_one_frame(dst, src); - CHECK(((T*)dst->image())[0] == (T)11.5); - CHECK(((T*)dst->image())[1] == (T)13.5); - CHECK(((T*)dst->image())[2] == (T)19.5); - CHECK(((T*)dst->image())[3] == (T)21.5); - CHECK(((T*)dst->image())[4] == (T)35.5); - CHECK(((T*)dst->image())[5] == (T)37.5); - CHECK(((T*)dst->image())[6] == (T)43.5); - CHECK(((T*)dst->image())[7] == (T)45.5); -} - -extern "C" -{ - acquire_export int unit_test__average_frame() - { - try { - test_average_frame_inner(SampleType_u8); - test_average_planes_inner(SampleType_u8); - - test_average_frame_inner(SampleType_i8); - test_average_planes_inner(SampleType_i8); - - test_average_frame_inner(SampleType_u16); - test_average_planes_inner(SampleType_u16); - - test_average_frame_inner(SampleType_i16); - test_average_planes_inner(SampleType_i16); - - test_average_frame_inner(SampleType_f32); - test_average_planes_inner(SampleType_f32); - } catch (const std::exception& exc) { - LOGE("Exception: %s\n", exc.what()); - return 0; - } catch (...) { - LOGE("Exception: (unknown)"); - return 0; - } - - return 1; - } -} -#endif diff --git a/src/frame.scaler.hh b/src/frame.scaler.hh deleted file mode 100644 index 7e533a26..00000000 --- a/src/frame.scaler.hh +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef H_ACQUIRE_ZARR_FRAME_SCALER_V0 -#define H_ACQUIRE_ZARR_FRAME_SCALER_V0 - -#ifdef __cplusplus - -#include -#include -#include -#include -#include - -#include "prelude.h" -#include "tiled.frame.hh" -#include "chunk.writer.hh" - -namespace acquire::sink::zarr { -class Zarr; - -struct ScalingParameters -{ - ImageShape image_shape; - TileShape tile_shape; - ScalingParameters(const ImageShape& image_shape, - const TileShape& tile_shape); -}; - -struct FrameScaler final -{ - public: - FrameScaler() = delete; - FrameScaler(Zarr* zarr, - const ImageShape& image_shape, - const TileShape& tile_shape); - FrameScaler(const FrameScaler&) = delete; - ~FrameScaler() = default; - - [[nodiscard]] bool push_frame(std::shared_ptr frame); - - private: - Zarr* zarr_; // non-owning - - std::vector scaling_params_; - - // Accumulate downsampled layers until we have enough to average and write. - std::unordered_map>> - accumulators_; - - mutable std::mutex mutex_; - - void downsample_and_accumulate(std::shared_ptr frame, - int16_t layer); -}; - -std::vector -make_scaling_parameters(const ImageShape& base_image_shape, - const TileShape& base_tile_shape); -} // namespace acquire::sink::zarr - -#endif // __cplusplus -#endif // H_ACQUIRE_ZARR_FRAME_SCALER_V0 diff --git a/src/writers/chunk.writer.cpp b/src/writers/chunk.writer.cpp index f360bab2..c94cf921 100644 --- a/src/writers/chunk.writer.cpp +++ b/src/writers/chunk.writer.cpp @@ -157,7 +157,7 @@ zarr::ChunkWriter::flush_() noexcept auto buf_sizes = compress_buffers_(); std::fill(buffers_ready_, buffers_ready_ + chunk_buffers_.size(), false); { - std::scoped_lock lock(mutex_); + std::scoped_lock lock(buffers_mutex_); for (auto i = 0; i < files_.size(); ++i) { auto& buf = chunk_buffers_.at(i); zarr_->push_to_job_queue(std::move( diff --git a/src/writers/writer.cpp b/src/writers/writer.cpp index 8df926c6..2c93d11c 100644 --- a/src/writers/writer.cpp +++ b/src/writers/writer.cpp @@ -315,7 +315,7 @@ zarr::Writer::compress_buffers_() noexcept const auto bytes_of_type = common::bytes_of_type(pixel_type_); - std::scoped_lock lock(mutex_); + std::scoped_lock lock(buffers_mutex_); for (auto i = 0; i < chunk_buffers_.size(); ++i) { auto& buf = chunk_buffers_.at(i); diff --git a/src/writers/writer.hh b/src/writers/writer.hh index d8894c32..2653f952 100644 --- a/src/writers/writer.hh +++ b/src/writers/writer.hh @@ -77,7 +77,6 @@ struct Writer /// Compression std::optional blosc_compression_params_; - // std::optional zstd_compression_params_; // TODO /// Filesystem FileCreator file_creator_; @@ -87,7 +86,7 @@ struct Writer /// Multithreading std::vector> chunk_buffers_; bool* buffers_ready_; - std::mutex mutex_; + std::mutex buffers_mutex_; /// Bookkeeping uint64_t bytes_to_flush_; diff --git a/src/zarr.v2.cpp b/src/zarr.v2.cpp index 256faeaf..6d9c2aef 100644 --- a/src/zarr.v2.cpp +++ b/src/zarr.v2.cpp @@ -100,16 +100,14 @@ zarr::ZarrV2::write_array_metadata_(size_t level) const { "shape", { frame_count, // t - // TODO (aliddell): c? - 1, // z + 1, // c image_dims.rows, // y image_dims.cols, // x } }, { "chunks", { frames_per_chunk, // t - // TODO (aliddell): c? - 1, // z + 1, // c tile_dims.rows, // y tile_dims.cols, // x } }, @@ -208,8 +206,7 @@ zarr::ZarrV2::write_group_metadata_() const "scale", { std::pow(2, i), // t - // TODO (aliddell): c? - 1, // z + 1, // c std::pow(2, i) * pixel_scale_um_.y, // y std::pow(2, i) * pixel_scale_um_.x // x },