Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Zarr many dims #135

Closed
wants to merge 6 commits into from
Closed

Zarr many dims #135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
075f958
Hacky oen chunk test passing
andy-sweet Oct 10, 2023
06a552c
Basic 5D zarr writing
andy-sweet Oct 10, 2023
e3b9f04
Fix path and use random in test
andy-sweet Oct 12, 2023
1ae6dd8
Remove unused member
andy-sweet Oct 12, 2023
15341f5
Fix zarr indices
andy-sweet Oct 12, 2023
45ad0ef
Write fixed number of time points out immediately
andy-sweet Oct 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions src/chunk.writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#include "device/props/components.h"
#include "platform.h"
#include "prelude.h"

/// Check that a==b
/// example: `ASSERT_EQ(int,"%d",42,meaning_of_life())`
Expand Down Expand Up @@ -72,13 +73,17 @@ ChunkWriter::ChunkWriter(BaseEncoder* encoder,
uint32_t tile_col,
uint32_t tile_row,
uint32_t tile_plane,
uint8_t num_channels,
uint8_t num_slices,
uint64_t max_bytes_per_chunk,
char dimension_separator,
const std::string& base_directory)
: encoder_{ encoder }
, bytes_per_chunk_{ 0 }
, tiles_per_chunk_{ 0 }
, bytes_written_{ 0 }
, num_channels_{ num_channels }
, num_slices_{ num_slices }
, current_chunk_{ 0 }
, dimension_separator_{ dimension_separator }
, base_dir_{ base_directory }
Expand All @@ -95,13 +100,15 @@ ChunkWriter::ChunkWriter(BaseEncoder* encoder,
EXPECT(bpt > 0, "Computed zero bytes per tile.", bpt);

tiles_per_chunk_ = std::floor((float)max_bytes_per_chunk / bpt);
LOGE("tiles_per_chunk = %d", tiles_per_chunk_);
EXPECT(tiles_per_chunk_ > 0,
"Given %lu bytes per chunk, %lu bytes per tile.",
max_bytes_per_chunk,
::bytes_of_type(image_shape.type));

// this is guaranteed to be positive
bytes_per_chunk_ = tiles_per_chunk_ * (size_t)bpt;
LOGE("bytes_per_chunk = %d", bytes_per_chunk_);

EXPECT('.' == dimension_separator || '/' == dimension_separator,
"Expecting either '.' or '/' for dimension separator, got '%c'.",
Expand Down Expand Up @@ -193,15 +200,23 @@ ChunkWriter::write(const uint8_t* beg, const uint8_t* end)
void
ChunkWriter::open_chunk_file()
{
const uint32_t frames_written = this->frames_written();
const int current_z = current_chunk_ % int(std::ceil(num_slices_ / tiles_per_chunk_));
const int channels_written = frames_written / num_slices_;
const int current_channel = channels_written % num_channels_;
const int current_time = channels_written / num_channels_;
char file_path[512];
// Modified the path to include hard-coded append split of tcz.
snprintf(file_path,
sizeof(file_path) - 1,
"%d%c%d%c%d%c%d%c%d",
"%d%c%d%c%d%c%d%c%d%c%d",
layer_,
dimension_separator_,
current_chunk_,
current_time, // index over t
dimension_separator_,
tile_plane_,
current_channel, // index over c
dimension_separator_,
current_z, // index over z
dimension_separator_,
tile_row_,
dimension_separator_,
Expand Down
6 changes: 6 additions & 0 deletions src/chunk.writer.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ struct ChunkWriter final
uint32_t tile_col,
uint32_t tile_row,
uint32_t tile_plane,
uint8_t num_channels,
uint8_t num_slices,
uint64_t max_bytes_per_chunk,
char dimension_separator,
const std::string& base_directory);
Expand All @@ -57,8 +59,12 @@ struct ChunkWriter final
uint32_t tiles_per_chunk_;
uint64_t bytes_written_;

uint8_t num_slices_;
uint8_t num_channels_;

std::string base_dir_;
uint32_t layer_;
// index over z
int current_chunk_;
char dimension_separator_;
std::optional<struct file> current_file_;
Expand Down
39 changes: 33 additions & 6 deletions src/zarr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,9 @@ zarr::Zarr::Zarr()
, max_bytes_per_chunk_{ 0 }
, image_shape_{ 0 }
, tile_shape_{ 0 }
, max_frame_count_{ 0 }
, num_channels_{ 1 }
, num_slices_{ 1 }
, thread_pool_(std::thread::hardware_concurrency())
{
start_threads_();
Expand All @@ -209,6 +212,9 @@ zarr::Zarr::Zarr(CompressionParams&& compression_params)
, max_bytes_per_chunk_{ 0 }
, image_shape_{ 0 }
, tile_shape_{ 0 }
, max_frame_count_{ 0 }
, num_channels_{ 1 }
, num_slices_{ 1 }
, thread_pool_(std::thread::hardware_concurrency())
{
compression_params_ = std::move(compression_params);
Expand Down Expand Up @@ -246,6 +252,10 @@ zarr::Zarr::set(const StorageProperties* props)

// hang on to this until we have the image shape
enable_multiscale_ = (bool)props->enable_multiscale;

num_channels_ = props->num_channels;
num_slices_ = props->num_slices;
max_frame_count_ = props->max_frame_count;
}

void
Expand Down Expand Up @@ -273,7 +283,7 @@ zarr::Zarr::get_meta(StoragePropertyMetadata* meta) const
.supported = 1,
.max_bytes_per_chunk = {
.writable = 1,
.low = (float)(16 << 20),
.low = (float)(1 << 8),
.high = (float)(1 << 30),
.type = PropertyType_FixedPrecision },
},
Expand Down Expand Up @@ -515,25 +525,28 @@ zarr::Zarr::write_zarray_json_inner_(size_t layer,
return;
}

const uint64_t frame_count = writers_.at(layer).front()->frames_written();
const uint64_t frame_count = max_frame_count_ > 0 ? max_frame_count_ : writers_.at(layer).front()->frames_written();
const auto frames_per_chunk =
std::min(frame_count,
(uint64_t)get_tiles_per_chunk(
image_shape, tile_shape, max_bytes_per_chunk_));
const uint64_t num_times = frame_count / (num_channels_ * num_slices_);

json zarray_attrs = {
{ "zarr_format", 2 },
{ "shape",
{
frame_count,
image_shape.dims.channels,
num_times,
num_channels_,
num_slices_,
image_shape.dims.height,
image_shape.dims.width,
} },
{ "chunks",
{
frames_per_chunk,
1,
1,
frames_per_chunk,
tile_shape.height,
tile_shape.width,
} },
Expand Down Expand Up @@ -577,11 +590,18 @@ zarr::Zarr::write_group_zattrs_json_() const
{
{ "name", "t" },
{ "type", "time" },
// TODO: add support for frame time in some de-facto or specified units
},
{
{ "name", "c" },
{ "type", "channel" },
},
{
{ "name", "z" },
{ "type", "space" },
{ "unit", "micrometer" },
// TODO: allow spatial units to be specified
},
{
{ "name", "y" },
{ "type", "space" },
Expand All @@ -603,7 +623,7 @@ zarr::Zarr::write_group_zattrs_json_() const
{
{
{ "type", "scale" },
{ "scale", { 1, 1, pixel_scale_um_.y, pixel_scale_um_.x } },
{ "scale", { 1, 1, 1, pixel_scale_um_.y, pixel_scale_um_.x } },
},
} },
},
Expand All @@ -619,6 +639,7 @@ zarr::Zarr::write_group_zattrs_json_() const
{ "scale",
{ std::pow(2, layer),
1,
1, // TODO: want to downsample over z?
std::pow(2, layer) * pixel_scale_um_.y,
std::pow(2, layer) * pixel_scale_um_.x } },
},
Expand Down Expand Up @@ -686,6 +707,10 @@ zarr::Zarr::allocate_writers_()
size_t tile_planes =
std::ceil((float)img_px_p / (float)tile_shape_.planes);

// TODO: need to allocate more writers to account for append dimensions?
// No, we're going to use the same number of writers (based on the multi-scale
// levels and plane-y-x tiles), rollover z (instead of t) and always append
// unitary chunks to t and c.
TRACE("Allocating %llu writers for layer %d",
tile_cols * tile_rows * tile_planes,
layer);
Expand Down Expand Up @@ -718,6 +743,8 @@ zarr::Zarr::allocate_writers_()
col,
row,
plane,
num_channels_,
num_slices_,
max_bytes_per_chunk_,
dimension_separator_,
data_dir_));
Expand Down
3 changes: 3 additions & 0 deletions src/zarr.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ struct Zarr final : StorageInterface
ImageShape image_shape_;
TileShape tile_shape_;
bool enable_multiscale_;
uint64_t max_frame_count_;
uint8_t num_slices_;
uint8_t num_channels_;

/// Downsampling of incoming frames.
std::optional<FrameScaler> frame_scaler_;
Expand Down
1 change: 1 addition & 0 deletions tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ else()
write-zarr-raw-with-chunking-and-rollover
write-zarr-raw-with-ragged-tiling
write-zarr-with-defaults
write-zarr-tczyx
write-zarr-with-lz4-compression
write-zarr-with-zstd-compression
)
Expand Down
Loading