Skip to content

Commit

Permalink
Support multiscale arrays in Zarr V3 with provisional metadata (#269)
Browse files Browse the repository at this point in the history 
This PR uses the same OME metadata as our Zarr V2 writer to express
multiscale semantics. The following changes were made:

- A protected method, `Zarr::make_multiscale_metadata_()`, was
implemented by moving the code from `ZarrV2::write_group_metadata_()` so
that it could be made accessible to both `ZarrV2` and `ZarrV3`
instances.
- Call `make_multiscale_metadata_()` from both overrides of
`write_group_metadata_()`.
- Update the metadata returned by `Zarr::get_meta()` to reflect that
both types of Zarr store support multiscale.

Basic functionality is verified in the
`write-zarr-v3-raw-multiscale.cpp` test file. The test scheme is the
same as in `write-zarr-v2-raw-multiscale.cpp`, but the paths are changed
to reflect the different layout.
  • Loading branch information
@aliddell
aliddell authored Jul 23, 2024
1 parent 597eed5 commit d2510d5
Show file tree
Hide file tree
Showing 9 changed files with 472 additions and 114 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Calling `acquire_get_configuration` with a Zarr storage device now returns a URI of the storage device, with file://
scheme indicator and absolute path, assuming localhost.

### Added

- Support for multiscale in Zarr V3 stores.

## [0.1.11](https://github.com/acquire-project/acquire-driver-zarr/compare/v0.1.10..v0.1.11) - 2024-04-22

### Fixed
Expand Down
113 changes: 102 additions & 11 deletions src/zarr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,9 +353,6 @@ zarr::Zarr::set(const StorageProperties* props)
"Cannot set properties while running.");
CHECK(props);

StoragePropertyMetadata meta{};
get_meta(&meta);

// checks the directory exists and is writable
validate_props(props);

Expand Down Expand Up @@ -386,14 +383,7 @@ zarr::Zarr::set(const StorageProperties* props)
pixel_scale_um_ = props->pixel_scale_um;

set_dimensions_(props);

if (props->enable_multiscale && !meta.multiscale_is_supported) {
// TODO (aliddell): https://github.com/ome/ngff/pull/206
LOGE("OME-Zarr multiscale not yet supported in Zarr v3. "
"Multiscale arrays will not be written.");
}
enable_multiscale_ = meta.multiscale_is_supported &&
props->enable_multiscale &&
enable_multiscale_ = props->enable_multiscale &&
is_multiscale_supported(acquisition_dimensions_);
}

Expand Down Expand Up @@ -471,6 +461,7 @@ zarr::Zarr::get_meta(StoragePropertyMetadata* meta) const
memset(meta, 0, sizeof(*meta));

meta->chunking_is_supported = 1;
meta->multiscale_is_supported = 1;
meta->s3_is_supported = 1;
}

Expand Down Expand Up @@ -709,6 +700,106 @@ zarr::Zarr::write_mutable_metadata_() const
}
}

json
zarr::Zarr::make_multiscale_metadata_() const
{
json multiscales = json::array({ json::object() });
// write multiscale metadata
multiscales[0]["version"] = "0.4";

auto& axes = multiscales[0]["axes"];
for (auto dim = acquisition_dimensions_.rbegin();
dim != acquisition_dimensions_.rend();
++dim) {
std::string type;
switch (dim->kind) {
case DimensionType_Space:
type = "space";
break;
case DimensionType_Channel:
type = "channel";
break;
case DimensionType_Time:
type = "time";
break;
case DimensionType_Other:
type = "other";
break;
default:
throw std::runtime_error("Unknown dimension type");
}

if (dim < acquisition_dimensions_.rend() - 2) {
axes.push_back({ { "name", dim->name }, { "type", type } });
} else {
axes.push_back({ { "name", dim->name },
{ "type", type },
{ "unit", "micrometer" } });
}
}

// spatial multiscale metadata
if (writers_.empty()) {
std::vector<double> scales;
for (auto i = 0; i < acquisition_dimensions_.size() - 2; ++i) {
scales.push_back(1.);
}
scales.push_back(pixel_scale_um_.y);
scales.push_back(pixel_scale_um_.x);

multiscales[0]["datasets"] = {
{
{ "path", "0" },
{ "coordinateTransformations",
{
{
{ "type", "scale" },
{ "scale", scales },
},
}
},
},
};
} else {
for (auto i = 0; i < writers_.size(); ++i) {
std::vector<double> scales;
scales.push_back(std::pow(2, i)); // append
for (auto k = 0; k < acquisition_dimensions_.size() - 3; ++k) {
scales.push_back(1.);
}
scales.push_back(std::pow(2, i) * pixel_scale_um_.y); // y
scales.push_back(std::pow(2, i) * pixel_scale_um_.x); // x

multiscales[0]["datasets"].push_back({
{ "path", std::to_string(i) },
{ "coordinateTransformations",
{
{
{ "type", "scale" },
{ "scale", scales },
},
}
},
});
}

// downsampling metadata
multiscales[0]["type"] = "local_mean";
multiscales[0]["metadata"] = {
{ "description",
"The fields in the metadata describe how to reproduce this "
"multiscaling in scikit-image. The method and its parameters are "
"given here." },
{ "method", "skimage.transform.downscale_local_mean" },
{ "version", "0.21.0" },
{ "args", "[2]" },
{ "kwargs", { "cval", 0 } },
};
}

return multiscales;
}

void
zarr::Zarr::write_multiscale_frames_(const uint8_t* const data_,
size_t bytes_of_data,
Expand Down
5 changes: 4 additions & 1 deletion src/zarr.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
#include <utility> // std::pair
#include <vector>

#include <nlohmann/json.hpp>

namespace fs = std::filesystem;
using json = nlohmann::json;

namespace acquire::sink::zarr {

struct Zarr : public Storage
{
public:
Expand Down Expand Up @@ -95,6 +97,7 @@ struct Zarr : public Storage
virtual void write_array_metadata_(size_t level) const = 0;

/// Multiscale
json make_multiscale_metadata_() const;
void write_multiscale_frames_(const uint8_t* data_,
size_t bytes_of_data,
const ImageShape& shape_);
Expand Down
96 changes: 1 addition & 95 deletions src/zarr.v2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ zarr::ZarrV2::get_meta(StoragePropertyMetadata* meta) const
{
Zarr::get_meta(meta);
meta->sharding_is_supported = 0;
meta->multiscale_is_supported = 1;
}

void
Expand Down Expand Up @@ -111,7 +110,6 @@ void
zarr::ZarrV2::write_base_metadata_() const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

const json metadata = { { "zarr_format", 2 } };
const std::string metadata_str = metadata.dump(4);
Expand All @@ -126,7 +124,6 @@ void
zarr::ZarrV2::write_external_metadata_() const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

std::string metadata_str = external_metadata_json_.empty()
? "{}"
Expand All @@ -147,99 +144,9 @@ void
zarr::ZarrV2::write_group_metadata_() const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

json metadata;
metadata["multiscales"] = json::array({ json::object() });
metadata["multiscales"][0]["version"] = "0.4";

auto& axes = metadata["multiscales"][0]["axes"];
for (auto dim = acquisition_dimensions_.rbegin();
dim != acquisition_dimensions_.rend();
++dim) {
std::string type;
switch (dim->kind) {
case DimensionType_Space:
type = "space";
break;
case DimensionType_Channel:
type = "channel";
break;
case DimensionType_Time:
type = "time";
break;
case DimensionType_Other:
type = "other";
break;
default:
throw std::runtime_error("Unknown dimension type");
}

if (dim < acquisition_dimensions_.rend() - 2) {
axes.push_back({ { "name", dim->name }, { "type", type } });
} else {
axes.push_back({ { "name", dim->name },
{ "type", type },
{ "unit", "micrometer" } });
}
}

// spatial multiscale metadata
if (writers_.empty()) {
std::vector<double> scales;
for (auto i = 0; i < acquisition_dimensions_.size() - 2; ++i) {
scales.push_back(1.);
}
scales.push_back(pixel_scale_um_.y);
scales.push_back(pixel_scale_um_.x);

metadata["multiscales"][0]["datasets"] = {
{
{ "path", "0" },
{ "coordinateTransformations",
{
{
{ "type", "scale" },
{ "scale", scales },
},
} },
},
};
} else {
for (auto i = 0; i < writers_.size(); ++i) {
std::vector<double> scales;
scales.push_back(std::pow(2, i)); // append
for (auto k = 0; k < acquisition_dimensions_.size() - 3; ++k) {
scales.push_back(1.);
}
scales.push_back(std::pow(2, i) * pixel_scale_um_.y); // y
scales.push_back(std::pow(2, i) * pixel_scale_um_.x); // x

metadata["multiscales"][0]["datasets"].push_back({
{ "path", std::to_string(i) },
{ "coordinateTransformations",
{
{
{ "type", "scale" },
{ "scale", scales },
},
} },
});
}

// downsampling metadata
metadata["multiscales"][0]["type"] = "local_mean";
metadata["multiscales"][0]["metadata"] = {
{ "description",
"The fields in the metadata describe how to reproduce this "
"multiscaling in scikit-image. The method and its parameters are "
"given here." },
{ "method", "skimage.transform.downscale_local_mean" },
{ "version", "0.21.0" },
{ "args", "[2]" },
{ "kwargs", { "cval", 0 } },
};
}
metadata["multiscales"] = make_multiscale_metadata_();

const std::string metadata_str = metadata.dump(4);
const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str();
Expand All @@ -253,7 +160,6 @@ void
zarr::ZarrV2::write_array_metadata_(size_t level) const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

CHECK(level < writers_.size());
const auto& writer = writers_.at(level);
Expand Down
5 changes: 1 addition & 4 deletions src/zarr.v3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ zarr::ZarrV3::get_meta(StoragePropertyMetadata* meta) const
{
Zarr::get_meta(meta);
meta->sharding_is_supported = 1;
meta->multiscale_is_supported = 0;
}

void
Expand All @@ -108,7 +107,6 @@ void
zarr::ZarrV3::write_base_metadata_() const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

json metadata;
metadata["extensions"] = json::array();
Expand Down Expand Up @@ -141,7 +139,6 @@ void
zarr::ZarrV3::write_group_metadata_() const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

json metadata;
metadata["attributes"]["acquire"] =
Expand All @@ -151,6 +148,7 @@ zarr::ZarrV3::write_group_metadata_() const
true, // allow exceptions
true // ignore comments
);
metadata["attributes"]["multiscales"] = make_multiscale_metadata_();

const std::string metadata_str = metadata.dump(4);
const auto* metadata_bytes = (const uint8_t*)metadata_str.c_str();
Expand All @@ -163,7 +161,6 @@ void
zarr::ZarrV3::write_array_metadata_(size_t level) const
{
namespace fs = std::filesystem;
using json = nlohmann::json;

CHECK(level < writers_.size());
const auto& writer = writers_.at(level);
Expand Down
1 change: 1 addition & 0 deletions tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ else ()
write-zarr-v3-raw-with-ragged-sharding
write-zarr-v3-raw-chunk-exceeds-array
write-zarr-v3-compressed
write-zarr-v3-raw-multiscale
write-zarr-v3-to-s3
)

Expand Down
3 changes: 1 addition & 2 deletions tests/get-meta.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,10 @@ main()
CHECK(Device_Ok == storage_get_meta(storage, &metadata));

CHECK(metadata.chunking_is_supported);
CHECK(metadata.multiscale_is_supported);
CHECK(metadata.s3_is_supported);
CHECK((bool)metadata.sharding_is_supported ==
name.starts_with("ZarrV3"));
CHECK((bool)metadata.multiscale_is_supported !=
name.starts_with("ZarrV3"));

CHECK(Device_Ok == driver_close_device(device));
}
Expand Down
2 changes: 1 addition & 1 deletion tests/get.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ main()

CHECK(props.first_frame_id == 0); // this is ignored

CHECK(props.enable_multiscale == !name.starts_with("ZarrV3"));
CHECK(props.enable_multiscale);

storage_properties_destroy(&props);

Expand Down
Loading

0 comments on commit d2510d5

Please sign in to comment.