From d23fe1a4aed5b219573222dfbfdda3854e10a7c7 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 13:35:06 -0400 Subject: [PATCH 1/3] Implement and test stream settings, with API setters and getters. --- include/zarr.types.h | 2 +- src/CMakeLists.txt | 1 + src/streaming/CMakeLists.txt | 36 ++ src/streaming/macros.hh | 28 ++ src/streaming/stream.settings.cpp | 513 +++++++++++++++++++++ src/streaming/stream.settings.hh | 46 ++ tests/CMakeLists.txt | 1 + tests/unit-tests/CMakeLists.txt | 28 ++ tests/unit-tests/get-stream-parameters.cpp | 236 ++++++++++ tests/unit-tests/set-stream-parameters.cpp | 274 +++++++++++ tests/unit-tests/unit.test.macros.hh | 34 ++ 11 files changed, 1198 insertions(+), 1 deletion(-) create mode 100644 src/streaming/CMakeLists.txt create mode 100644 src/streaming/macros.hh create mode 100644 src/streaming/stream.settings.cpp create mode 100644 src/streaming/stream.settings.hh create mode 100644 tests/unit-tests/CMakeLists.txt create mode 100644 tests/unit-tests/get-stream-parameters.cpp create mode 100644 tests/unit-tests/set-stream-parameters.cpp create mode 100644 tests/unit-tests/unit.test.macros.hh diff --git a/include/zarr.types.h b/include/zarr.types.h index fdb6c715..56a54a56 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -117,7 +117,7 @@ extern "C" { const char* name; /**< Name of the dimension */ size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ - ZarrDimensionType kind; /**< Type of the dimension */ + ZarrDimensionType type; /**< Type of the dimension */ uint32_t array_size_px; /**< Size of the array along this dimension in pixels */ uint32_t chunk_size_px; /**< Size of the chunks along this dimension in diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1323f310..0c51ab98 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_subdirectory(logger) +add_subdirectory(streaming) add_subdirectory(driver) diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt new file mode 100644 index 00000000..2f9fb4fd --- /dev/null +++ b/src/streaming/CMakeLists.txt @@ -0,0 +1,36 @@ +set(tgt acquire-zarr) + +add_library(${tgt} + macros.hh + stream.settings.hh + stream.settings.cpp +) + +target_include_directories(${tgt} + PUBLIC + $ + PRIVATE + $ + $ +) + +target_link_libraries(${tgt} PRIVATE + acquire-zarr-logger + blosc_static + miniocpp::miniocpp +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) + +# Install public header files +install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/ + DESTINATION include + FILES_MATCHING PATTERN "*.h" +) \ No newline at end of file diff --git a/src/streaming/macros.hh b/src/streaming/macros.hh new file mode 100644 index 00000000..05b094b3 --- /dev/null +++ b/src/streaming/macros.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "logger.hh" + +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + const std::string __err = LOG_ERROR(__VA_ARGS__); \ + throw std::runtime_error(__err); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) + +#define EXPECT_VALID_ARGUMENT(e, ...) \ + do { \ + if (!(e)) { \ + LOG_ERROR(__VA_ARGS__); \ + return ZarrStatus_InvalidArgument; \ + } \ + } while (0) + +#define EXPECT_VALID_INDEX(e, ...) \ + do { \ + if (!(e)) { \ + LOG_ERROR(__VA_ARGS__); \ + return ZarrStatus_InvalidIndex; \ + } \ + } while (0) \ No newline at end of file diff --git a/src/streaming/stream.settings.cpp b/src/streaming/stream.settings.cpp new file mode 100644 index 00000000..09c8bdb8 --- /dev/null +++ b/src/streaming/stream.settings.cpp @@ -0,0 +1,513 @@ +#include "macros.hh" +#include "stream.settings.hh" +#include "acquire.zarr.h" + +#include +#include + +#include // memcpy, strnlen +#include + +#define SETTINGS_GET_STRING(settings, member) \ + do { \ + if (!settings) { \ + LOG_ERROR("Null pointer: %s", #settings); \ + return nullptr; \ + } \ + return settings->member.c_str(); \ + } while (0) + +namespace fs = std::filesystem; + +namespace { +const size_t zarr_dimension_min = 3; +const size_t zarr_dimension_max = 32; + +[[nodiscard]] +std::string +trim(const char* s, size_t bytes_of_s) +{ + // trim left + std::string trimmed(s, bytes_of_s); + trimmed.erase(trimmed.begin(), + std::find_if(trimmed.begin(), trimmed.end(), [](char c) { + return !std::isspace(c); + })); + + // trim right + trimmed.erase(std::find_if(trimmed.rbegin(), + trimmed.rend(), + [](char c) { return !std::isspace(c); }) + .base(), + trimmed.end()); + + return trimmed; +} + +[[nodiscard]] +bool +validate_s3_settings(const ZarrS3Settings* settings) +{ + size_t len; + + if (len = strnlen(settings->endpoint, settings->bytes_of_endpoint); + len == 0) { + LOG_ERROR("S3 endpoint is empty"); + return false; + } + + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html + if (len = strnlen(settings->bucket_name, settings->bytes_of_bucket_name); + len < 4 || len > 64) { + LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " + "and 63 characters", + len); + return false; + } + + if (len = + strnlen(settings->access_key_id, settings->bytes_of_access_key_id); + len == 0) { + LOG_ERROR("S3 access key ID is empty"); + return false; + } + + if (len = strnlen(settings->secret_access_key, + settings->bytes_of_secret_access_key); + len == 0) { + LOG_ERROR("S3 secret access key is empty"); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_filesystem_store_path(std::string_view data_root) +{ + fs::path path(data_root); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) { + parent_path = "."; + } + + // parent path must exist and be a directory + if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { + LOG_ERROR("Parent path '%s' does not exist or is not a directory", + parent_path.c_str()); + return false; + } + + // parent path must be writable + const auto perms = fs::status(parent_path).permissions(); + const bool is_writable = + (perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none; + + if (!is_writable) { + LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_compression_settings(const ZarrCompressionSettings* settings) +{ + if (settings->compressor >= ZarrCompressorCount) { + LOG_ERROR("Invalid compressor: %d", settings->compressor); + return false; + } + + if (settings->codec >= ZarrCompressionCodecCount) { + LOG_ERROR("Invalid compression codec: %d", settings->codec); + return false; + } + + // if compressing, we require a compression codec + if (settings->compressor != ZarrCompressor_None && + settings->codec == ZarrCompressionCodec_None) { + LOG_ERROR("Compression codec must be set when using a compressor"); + return false; + } + + if (settings->level > 9) { + LOG_ERROR("Invalid compression level: %d. Must be between 0 and 9", + settings->level); + return false; + } + + if (settings->shuffle != BLOSC_NOSHUFFLE && + settings->shuffle != BLOSC_SHUFFLE && + settings->shuffle != BLOSC_BITSHUFFLE) { + LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " + "shuffle), or %d (bit shuffle)", + settings->shuffle, + BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, + BLOSC_BITSHUFFLE); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_dimension(const ZarrDimensionProperties* dimension) +{ + std::string trimmed = trim(dimension->name, dimension->bytes_of_name - 1); + if (trimmed.empty()) { + LOG_ERROR("Invalid name. Must not be empty"); + return false; + } + + if (dimension->type >= ZarrDimensionTypeCount) { + LOG_ERROR("Invalid dimension type: %d", dimension->type); + return false; + } + + if (dimension->chunk_size_px == 0) { + LOG_ERROR("Invalid chunk size: %zu", dimension->chunk_size_px); + return false; + } + + return true; +} +} // namespace + +ZarrStreamSettings_s::ZarrStreamSettings_s() + : store_path() + , s3_endpoint() + , s3_bucket_name() + , s3_access_key_id() + , s3_secret_access_key() + , custom_metadata("{}") + , dtype(ZarrDataType_uint8) + , compressor(ZarrCompressor_None) + , compression_codec(ZarrCompressionCodec_None) + , compression_level(0) + , compression_shuffle(BLOSC_NOSHUFFLE) + , dimensions() + , multiscale(false) +{ +} + +/* Lifecycle */ +ZarrStreamSettings* +ZarrStreamSettings_create() +{ + try { + return new ZarrStreamSettings(); + } catch (const std::bad_alloc&) { + return nullptr; + } +} + +void +ZarrStreamSettings_destroy(ZarrStreamSettings* settings) +{ + delete settings; +} + +ZarrStreamSettings* +ZarrStreamSettings_copy(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return nullptr; + } + + ZarrStreamSettings* copy = ZarrStreamSettings_create(); + if (!copy) { + LOG_ERROR("Failed to allocate memory for copy"); + return nullptr; + } + + *copy = *settings; + + return copy; +} + +/* Setters */ +ZarrStatus +ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); + + bytes_of_store_path = strnlen(store_path, bytes_of_store_path); + EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, + "Invalid store path. Must not be empty"); + + std::string_view store_path_sv(store_path, bytes_of_store_path); + if (store_path_sv.empty()) { + LOG_ERROR("Invalid store path. Must not be empty"); + return ZarrStatus_InvalidArgument; + } + + if (nullptr != s3_settings) { + if (!validate_s3_settings(s3_settings)) { + return ZarrStatus_InvalidArgument; + } + } else if (!validate_filesystem_store_path(store_path)) { + return ZarrStatus_InvalidArgument; + } + + if (nullptr != s3_settings) { + settings->s3_endpoint = s3_settings->endpoint; + settings->s3_bucket_name = s3_settings->bucket_name; + settings->s3_access_key_id = s3_settings->access_key_id; + settings->s3_secret_access_key = s3_settings->secret_access_key; + } + + settings->store_path = store_path; + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(compression_settings, + "Null pointer: compression_settings"); + + if (!validate_compression_settings(compression_settings)) { + return ZarrStatus_InvalidArgument; + } + + settings->compressor = compression_settings->compressor; + settings->compression_codec = compression_settings->codec; + settings->compression_level = compression_settings->level; + settings->compression_shuffle = compression_settings->shuffle; + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_custom_metadata(ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata) +{ + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return ZarrStatus_InvalidArgument; + } + + if (!external_metadata) { + LOG_ERROR("Null pointer: custom_metadata"); + return ZarrStatus_InvalidArgument; + } + + if (bytes_of_external_metadata == 0) { + LOG_ERROR("Invalid length: %zu. Must be greater than 0", + bytes_of_external_metadata); + return ZarrStatus_InvalidArgument; + } + + size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); + if (nbytes < 2) { + settings->custom_metadata = "{}"; + return ZarrStatus_Success; + } + + auto val = nlohmann::json::parse(external_metadata, + external_metadata + nbytes, + nullptr, // callback + false, // allow exceptions + true // ignore comments + ); + + if (val.is_discarded()) { + LOG_ERROR("Invalid JSON: %s", external_metadata); + return ZarrStatus_InvalidArgument; + } + settings->custom_metadata = val.dump(); + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); + + settings->dtype = data_type; + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_reserve_dimensions(ZarrStreamSettings* settings, + size_t count) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(count >= zarr_dimension_min && + count <= zarr_dimension_max, + "Invalid count: %zu. Count must be between %d and %d", + count, + zarr_dimension_min, + zarr_dimension_max); + + settings->dimensions.resize(count); + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); + EXPECT_VALID_INDEX(index < settings->dimensions.size(), + "Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + + if (!validate_dimension(dimension)) { + return ZarrStatus_InvalidArgument; + } + + struct ZarrDimension_s& dim = settings->dimensions[index]; + + dim.name = trim(dimension->name, dimension->bytes_of_name - 1); + dim.type = dimension->type; + dim.array_size_px = dimension->array_size_px; + dim.chunk_size_px = dimension->chunk_size_px; + dim.shard_size_chunks = dimension->shard_size_chunks; + + return ZarrStatus_Success; +} + +ZarrStatus +ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + + settings->multiscale = multiscale > 0; + return ZarrStatus_Success; +} + +/* Getters */ +const char* +ZarrStreamSettings_get_store_path(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, store_path); +} + +ZarrS3Settings +ZarrStreamSettings_get_s3_settings(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); + return {}; + } + + ZarrS3Settings s3_settings = { + settings->s3_endpoint.c_str(), + settings->s3_endpoint.length() + 1, + settings->s3_bucket_name.c_str(), + settings->s3_bucket_name.length() + 1, + settings->s3_access_key_id.c_str(), + settings->s3_access_key_id.length() + 1, + settings->s3_secret_access_key.c_str(), + settings->s3_secret_access_key.length() + 1, + }; + return s3_settings; +} + +const char* +ZarrStreamSettings_get_custom_metadata(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, custom_metadata); +} + +ZarrDataType +ZarrStreamSettings_get_data_type(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); + return ZarrDataType_uint8; + } + return static_cast(settings->dtype); +} + +ZarrCompressionSettings +ZarrStreamSettings_get_compression(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty compression."); + return {}; + } + + ZarrCompressionSettings compression = { + .compressor = settings->compressor, + .codec = static_cast(settings->compression_codec), + .level = settings->compression_level, + .shuffle = settings->compression_shuffle, + }; + return compression; +} + +size_t +ZarrStreamSettings_get_dimension_count(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->dimensions.size(); +} + +ZarrDimensionProperties +ZarrStreamSettings_get_dimension(const ZarrStreamSettings* settings, + size_t index) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty dimension."); + return {}; + } + + if (index >= settings->dimensions.size()) { + LOG_ERROR("Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + return {}; + } + + const auto& dim = settings->dimensions[index]; + + ZarrDimensionProperties dimension = { + .name = dim.name.c_str(), + .bytes_of_name = dim.name.size() + 1, + .type = dim.type, + .array_size_px = dim.array_size_px, + .chunk_size_px = dim.chunk_size_px, + .shard_size_chunks = dim.shard_size_chunks, + }; + + return dimension; +} + +bool +ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning false."); + return false; + } + return settings->multiscale; +} diff --git a/src/streaming/stream.settings.hh b/src/streaming/stream.settings.hh new file mode 100644 index 00000000..a23e48cf --- /dev/null +++ b/src/streaming/stream.settings.hh @@ -0,0 +1,46 @@ +#pragma once + +#include "zarr.types.h" + +#include // size_t +#include // uint8_t +#include +#include + +struct ZarrDimension_s +{ + std::string name; /* Name of the dimension */ + ZarrDimensionType type; /* Type of dimension */ + + uint32_t array_size_px; /* Size of the array along this dimension */ + uint32_t chunk_size_px; /* Size of a chunk along this dimension */ + uint32_t shard_size_chunks; /* Number of chunks in a shard along this + dimension */ +}; + +struct ZarrStreamSettings_s +{ + public: + ZarrStreamSettings_s(); + std::string store_path; /* Path to the Zarr store on the local filesystem */ + + std::string s3_endpoint; /* Endpoint for the S3 service */ + std::string s3_bucket_name; /* Name of the S3 bucket */ + std::string s3_access_key_id; /* Access key ID for the S3 service */ + std::string s3_secret_access_key; /* Secret access key for the S3 service */ + + std::string custom_metadata; /* JSON formatted external metadata for the + base array */ + + ZarrDataType dtype; /* Data type of the base array */ + + ZarrCompressor compressor; /* Compression library to use */ + ZarrCompressionCodec compression_codec; /* Compression codec to use */ + uint8_t compression_level; /* Compression level to use */ + uint8_t compression_shuffle; /* Whether and how to shuffle the data before + compressing */ + + std::vector dimensions; /* Dimensions of the base array */ + + bool multiscale; /* Whether to stream to multiple resolutions */ +}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87878b37..f7713371 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,6 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () + add_subdirectory(unit-tests) add_subdirectory(driver) endif () diff --git a/tests/unit-tests/CMakeLists.txt b/tests/unit-tests/CMakeLists.txt new file mode 100644 index 00000000..f5d9264b --- /dev/null +++ b/tests/unit-tests/CMakeLists.txt @@ -0,0 +1,28 @@ +set(project acquire-zarr) + +set(tests + set-stream-parameters + get-stream-parameters +) + +foreach (name ${tests}) + set(tgt "${project}-unit-test-${name}") + add_executable(${tgt} ${name}.cpp unit.test.macros.hh) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src/logger + ${CMAKE_SOURCE_DIR}/src/streaming + ) + target_link_libraries(${tgt} PRIVATE + acquire-zarr-logger + acquire-zarr + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;unit-tests;acquire-zarr") +endforeach () \ No newline at end of file diff --git a/tests/unit-tests/get-stream-parameters.cpp b/tests/unit-tests/get-stream-parameters.cpp new file mode 100644 index 00000000..c3d287a4 --- /dev/null +++ b/tests/unit-tests/get-stream-parameters.cpp @@ -0,0 +1,236 @@ +#include "acquire.zarr.h" +#include "stream.settings.hh" +#include "unit.test.macros.hh" + +void +check_preliminaries(ZarrStreamSettings* settings) +{ + CHECK(settings); + + CHECK(settings->store_path.empty()); + + CHECK(settings->s3_endpoint.empty()); + CHECK(settings->s3_bucket_name.empty()); + CHECK(settings->s3_access_key_id.empty()); + CHECK(settings->s3_secret_access_key.empty()); + + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint8); + + EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_None); + EXPECT_EQ( + int, "%d", settings->compression_codec, ZarrCompressionCodec_None); + EXPECT_EQ(int, "%d", settings->compression_level, 0); + EXPECT_EQ(int, "%d", settings->compression_shuffle, 0); + + CHECK(settings->dimensions.empty()); + + CHECK(!settings->multiscale); +} + +void +get_store_path(ZarrStreamSettings* settings) +{ + EXPECT_STR_EQ(ZarrStreamSettings_get_store_path(settings), ""); + + settings->store_path = TEST ".zarr"; + EXPECT_STR_EQ(ZarrStreamSettings_get_store_path(settings), + settings->store_path.c_str()); +} + +void +get_s3_settings(ZarrStreamSettings* settings) +{ + auto s3_settings = ZarrStreamSettings_get_s3_settings(settings); + EXPECT_STR_EQ(s3_settings.endpoint, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_endpoint, 1); + + EXPECT_STR_EQ(s3_settings.bucket_name, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_bucket_name, 1); + + EXPECT_STR_EQ(s3_settings.access_key_id, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_access_key_id, 1); + + EXPECT_STR_EQ(s3_settings.secret_access_key, ""); + EXPECT_EQ(int, "%d", s3_settings.bytes_of_secret_access_key, 1); + + settings->s3_endpoint = "https://s3.amazonaws.com"; + settings->s3_bucket_name = "bucket"; + settings->s3_access_key_id = "access_key"; + settings->s3_secret_access_key = "secret_access_key"; + + s3_settings = ZarrStreamSettings_get_s3_settings(settings); + EXPECT_STR_EQ(s3_settings.endpoint, settings->s3_endpoint.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_endpoint, + settings->s3_endpoint.size() + 1); + + EXPECT_STR_EQ(s3_settings.bucket_name, settings->s3_bucket_name.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_bucket_name, + settings->s3_bucket_name.size() + 1); + + EXPECT_STR_EQ(s3_settings.access_key_id, + settings->s3_access_key_id.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_access_key_id, + settings->s3_access_key_id.size() + 1); + + EXPECT_STR_EQ(s3_settings.secret_access_key, + settings->s3_secret_access_key.c_str()); + EXPECT_EQ(int, + "%d", + s3_settings.bytes_of_secret_access_key, + settings->s3_secret_access_key.size() + 1); +} + +void +get_compression(ZarrStreamSettings* settings) +{ + auto compression_settings = ZarrStreamSettings_get_compression(settings); + + EXPECT_EQ(int, "%d", compression_settings.compressor, ZarrCompressor_None); + EXPECT_EQ(int, "%d", compression_settings.codec, ZarrCompressionCodec_None); + EXPECT_EQ(int, "%d", compression_settings.level, 0); + EXPECT_EQ(int, "%d", compression_settings.shuffle, 0); + + settings->compressor = ZarrCompressor_Blosc1; + settings->compression_codec = ZarrCompressionCodec_BloscZstd; + settings->compression_level = 8; + settings->compression_shuffle = 2; + + compression_settings = ZarrStreamSettings_get_compression(settings); + EXPECT_EQ( + int, "%d", compression_settings.compressor, ZarrCompressor_Blosc1); + EXPECT_EQ( + int, "%d", compression_settings.codec, ZarrCompressionCodec_BloscZstd); + EXPECT_EQ(int, "%d", compression_settings.level, 8); + EXPECT_EQ(int, "%d", compression_settings.shuffle, 2); +} + +void +get_data_type(ZarrStreamSettings* settings) +{ + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_get_data_type(settings), + ZarrDataType_uint8); + + settings->dtype = ZarrDataType_float32; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_get_data_type(settings), + ZarrDataType_float32); +} + +void +get_dimensions(ZarrStreamSettings* settings) +{ + EXPECT_EQ(int, "%d", ZarrStreamSettings_get_dimension_count(settings), 0); + + settings->dimensions.resize(3); + EXPECT_EQ(int, "%d", ZarrStreamSettings_get_dimension_count(settings), 3); + + { + auto& dim = settings->dimensions[0]; + dim.name = "time"; + dim.type = ZarrDimensionType_Time; + dim.array_size_px = 100; + dim.chunk_size_px = 13; + dim.shard_size_chunks = 7; + } + { + auto& dim = settings->dimensions[1]; + dim.name = "height"; + dim.array_size_px = 300; + dim.chunk_size_px = 19; + dim.shard_size_chunks = 13; + } + { + auto& dim = settings->dimensions[2]; + dim.name = "width"; + dim.array_size_px = 200; + dim.chunk_size_px = 17; + dim.shard_size_chunks = 11; + } + + // can't get beyond the last dimension + auto dim = ZarrStreamSettings_get_dimension(settings, 3); + EXPECT_STR_EQ(dim.name, ""); + EXPECT_EQ(int, "%d", dim.bytes_of_name, 0); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", dim.array_size_px, 0); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 0); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 0); + + dim = ZarrStreamSettings_get_dimension(settings, 0); + EXPECT_STR_EQ(dim.name, "time"); + EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("time")); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", dim.array_size_px, 100); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 13); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 7); + + dim = ZarrStreamSettings_get_dimension(settings, 1); + EXPECT_STR_EQ(dim.name, "height"); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("height")); + EXPECT_EQ(int, "%d", dim.array_size_px, 300); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 19); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 13); + + dim = ZarrStreamSettings_get_dimension(settings, 2); + EXPECT_STR_EQ(dim.name, "width"); + EXPECT_EQ(int, "%d", dim.type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", dim.bytes_of_name, sizeof("width")); + EXPECT_EQ(int, "%d", dim.array_size_px, 200); + EXPECT_EQ(int, "%d", dim.chunk_size_px, 17); + EXPECT_EQ(int, "%d", dim.shard_size_chunks, 11); +} + +void +get_multiscale(ZarrStreamSettings* settings) +{ + CHECK(!ZarrStreamSettings_get_multiscale(settings)); + + settings->multiscale = true; + CHECK(ZarrStreamSettings_get_multiscale(settings)); +} + +void +get_custom_metadata(ZarrStreamSettings* settings) +{ + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + settings->custom_metadata = "this ain't even json"; // oops + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "this ain't even json"); +} + +int +main() +{ + int retval = 1; + + ZarrStreamSettings* settings = ZarrStreamSettings_create(); + try { + CHECK(settings); + check_preliminaries(settings); + get_store_path(settings); + get_s3_settings(settings); + get_compression(settings); + get_data_type(settings); + get_dimensions(settings); + get_multiscale(settings); + get_custom_metadata(settings); + + retval = 0; + } catch (const std::exception& exception) { + LOG_ERROR("%s", exception.what()); + } + ZarrStreamSettings_destroy(settings); + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/set-stream-parameters.cpp b/tests/unit-tests/set-stream-parameters.cpp new file mode 100644 index 00000000..d7ee16b7 --- /dev/null +++ b/tests/unit-tests/set-stream-parameters.cpp @@ -0,0 +1,274 @@ +#include "acquire.zarr.h" +#include "stream.settings.hh" +#include "unit.test.macros.hh" + +void +check_preliminaries(ZarrStreamSettings* settings) +{ + CHECK(settings); + + CHECK(settings->store_path.empty()); + + CHECK(settings->s3_endpoint.empty()); + CHECK(settings->s3_bucket_name.empty()); + CHECK(settings->s3_access_key_id.empty()); + CHECK(settings->s3_secret_access_key.empty()); + + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint8); + + EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_None); + EXPECT_EQ( + int, "%d", settings->compression_codec, ZarrCompressionCodec_None); + EXPECT_EQ(int, "%d", settings->compression_level, 0); + EXPECT_EQ(int, "%d", settings->compression_shuffle, 0); + + CHECK(settings->dimensions.empty()); + + CHECK(!settings->multiscale); +} + +void +set_store(ZarrStreamSettings* settings) +{ + std::string store_path = TEST ".zarr"; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_store( + settings, store_path.c_str(), store_path.size() + 1, nullptr), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->store_path.c_str(), store_path.c_str()); + settings->store_path = ""; // reset + + ZarrS3Settings s3_settings{ + .endpoint = "https://s3.amazonaws.com", + .bytes_of_endpoint = sizeof("https://s3.amazonaws.com"), + .bucket_name = "bucket", + .bytes_of_bucket_name = sizeof("bucket"), + .access_key_id = "access_key", + .bytes_of_access_key_id = sizeof("access_key"), + .secret_access_key = "secret_access_key", + .bytes_of_secret_access_key = sizeof("secret_access_key"), + }; + + EXPECT_EQ( + int, + "%d", + ZarrStreamSettings_set_store( + settings, store_path.c_str(), store_path.size() + 1, &s3_settings), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->store_path.c_str(), store_path.c_str()); + EXPECT_STR_EQ(settings->s3_endpoint.c_str(), s3_settings.endpoint); + EXPECT_STR_EQ(settings->s3_bucket_name.c_str(), s3_settings.bucket_name); + EXPECT_STR_EQ(settings->s3_access_key_id.c_str(), + s3_settings.access_key_id); + EXPECT_STR_EQ(settings->s3_secret_access_key.c_str(), + s3_settings.secret_access_key); +} + +void +set_compression(ZarrStreamSettings* settings) +{ + ZarrCompressionSettings compression_settings{ + .compressor = ZarrCompressor_Blosc1, + .codec = ZarrCompressionCodec_BloscLZ4, + .level = 5, + .shuffle = 1, + }; + + EXPECT_EQ( + int, + "%d", + ZarrStreamSettings_set_compression(settings, &compression_settings), + ZarrStatus_Success); + + EXPECT_EQ(int, "%d", settings->compressor, ZarrCompressor_Blosc1); + EXPECT_EQ( + int, "%d", settings->compression_codec, ZarrCompressionCodec_BloscLZ4); + EXPECT_EQ(int, "%d", settings->compression_level, 5); + EXPECT_EQ(int, "%d", settings->compression_shuffle, 1); +} + +void +set_data_type(ZarrStreamSettings* settings) +{ + ZarrDataType dtype = ZarrDataType_uint16; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_data_type(settings, dtype), + ZarrStatus_Success); + EXPECT_EQ(int, "%d", settings->dtype, ZarrDataType_uint16); +} + +void +set_dimensions(ZarrStreamSettings* settings) +{ + ZarrStreamSettings_reserve_dimensions(settings, 3); + EXPECT_EQ(int, "%d", settings->dimensions.size(), 3); + + ZarrDimensionProperties dim{ + .name = " time ", + .bytes_of_name = sizeof(" time "), + .type = ZarrDimensionType_Time, + .array_size_px = 100, + .chunk_size_px = 13, + .shard_size_chunks = 7, + }; + + // can't set a dimension that is out of bounds + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 3, &dim), + ZarrStatus_InvalidIndex); + + // set the first dimension + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 0, &dim), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); + EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); + EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); + EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); + + // other dimensions should still be unset + EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), ""); + EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 0); + + EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), ""); + EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 0); + + // set the 3rd dimension before the 2nd + dim.name = "width "; + dim.bytes_of_name = sizeof("width "); + dim.type = ZarrDimensionType_Space; + dim.array_size_px = 200; + dim.chunk_size_px = 17; + dim.shard_size_chunks = 11; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 2, &dim), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); + EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); + EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); + EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); + + // 2nd dimension should still be unset + EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), ""); + EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 0); + EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 0); + + EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), "width"); + EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 200); + EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 17); + EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 11); + + // set the 2nd dimension + dim.name = "height"; + dim.bytes_of_name = sizeof("height"); + dim.type = ZarrDimensionType_Space; + dim.array_size_px = 300; + dim.chunk_size_px = 19; + dim.shard_size_chunks = 13; + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_dimension(settings, 1, &dim), + ZarrStatus_Success); + + EXPECT_STR_EQ(settings->dimensions[0].name.c_str(), "time"); + EXPECT_EQ(int, "%d", settings->dimensions[0].type, ZarrDimensionType_Time); + EXPECT_EQ(int, "%d", settings->dimensions[0].array_size_px, 100); + EXPECT_EQ(int, "%d", settings->dimensions[0].chunk_size_px, 13); + EXPECT_EQ(int, "%d", settings->dimensions[0].shard_size_chunks, 7); + + EXPECT_STR_EQ(settings->dimensions[1].name.c_str(), "height"); + EXPECT_EQ(int, "%d", settings->dimensions[1].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[1].array_size_px, 300); + EXPECT_EQ(int, "%d", settings->dimensions[1].chunk_size_px, 19); + EXPECT_EQ(int, "%d", settings->dimensions[1].shard_size_chunks, 13); + + EXPECT_STR_EQ(settings->dimensions[2].name.c_str(), "width"); + EXPECT_EQ(int, "%d", settings->dimensions[2].type, ZarrDimensionType_Space); + EXPECT_EQ(int, "%d", settings->dimensions[2].array_size_px, 200); + EXPECT_EQ(int, "%d", settings->dimensions[2].chunk_size_px, 17); + EXPECT_EQ(int, "%d", settings->dimensions[2].shard_size_chunks, 11); +} + +void +set_multiscale(ZarrStreamSettings* settings) +{ + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_multiscale(settings, true), + ZarrStatus_Success); + CHECK(settings->multiscale); + + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_multiscale(settings, false), + ZarrStatus_Success); + CHECK(!settings->multiscale); +} + +void +set_custom_metadata(ZarrStreamSettings* settings) +{ + // fails when not JSON formatted + EXPECT_EQ(int, + "%d", + ZarrStreamSettings_set_custom_metadata( + settings, "this is not json", sizeof("this is not json")), + ZarrStatus_InvalidArgument); + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{}"); + + // succeeds when JSON formatted + EXPECT_EQ( + int, + "%d", + ZarrStreamSettings_set_custom_metadata( + settings, "{\"key\": \"value\"}", sizeof("{\"key\": \"value\"}")), + ZarrStatus_Success); + // whitespace is removed + EXPECT_STR_EQ(settings->custom_metadata.c_str(), "{\"key\":\"value\"}"); +} + +int +main() +{ + int retval = 1; + + ZarrStreamSettings* settings = ZarrStreamSettings_create(); + try { + CHECK(settings); + check_preliminaries(settings); + set_store(settings); + set_compression(settings); + set_data_type(settings); + set_dimensions(settings); + set_multiscale(settings); + set_custom_metadata(settings); + + retval = 0; + } catch (const std::exception& exception) { + LOG_ERROR("%s", exception.what()); + } + ZarrStreamSettings_destroy(settings); + return retval; +} \ No newline at end of file diff --git a/tests/unit-tests/unit.test.macros.hh b/tests/unit-tests/unit.test.macros.hh new file mode 100644 index 00000000..0100c112 --- /dev/null +++ b/tests/unit-tests/unit.test.macros.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "logger.hh" + +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + const std::string __err = LOG_ERROR(__VA_ARGS__); \ + throw std::runtime_error(__err); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) + +/// Check that a==b +/// example: `ASSERT_EQ(int,"%d",42,meaning_of_life())` +#define EXPECT_EQ(T, fmt, a, b) \ + do { \ + T a_ = (T)(a); \ + T b_ = (T)(b); \ + EXPECT(a_ == b_, "Expected %s==%s but " fmt "!=" fmt, #a, #b, a_, b_); \ + } while (0) + +#define EXPECT_STR_EQ(a, b) \ + do { \ + std::string a_ = (a) ? (a) : ""; \ + std::string b_ = (b) ? (b) : ""; \ + EXPECT(a_ == b_, \ + "Expected %s==%s but \"%s\"!=\"%s\"", \ + #a, \ + #b, \ + a_.c_str(), \ + b_.c_str()); \ + } while (0) + From 3cd2bb33ff2b08766df6e018d6430a82d7440785 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:10:43 -0400 Subject: [PATCH 2/3] Wrap C API functions in extern "C" {} --- src/streaming/stream.settings.cpp | 518 +++++++++++++++--------------- 1 file changed, 257 insertions(+), 261 deletions(-) diff --git a/src/streaming/stream.settings.cpp b/src/streaming/stream.settings.cpp index 09c8bdb8..ad36f7ba 100644 --- a/src/streaming/stream.settings.cpp +++ b/src/streaming/stream.settings.cpp @@ -196,318 +196,314 @@ ZarrStreamSettings_s::ZarrStreamSettings_s() { } -/* Lifecycle */ -ZarrStreamSettings* -ZarrStreamSettings_create() +extern "C" { - try { - return new ZarrStreamSettings(); - } catch (const std::bad_alloc&) { - return nullptr; + ZarrStreamSettings* ZarrStreamSettings_create() + { + try { + return new ZarrStreamSettings(); + } catch (const std::bad_alloc&) { + return nullptr; + } } -} - -void -ZarrStreamSettings_destroy(ZarrStreamSettings* settings) -{ - delete settings; -} -ZarrStreamSettings* -ZarrStreamSettings_copy(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return nullptr; + void ZarrStreamSettings_destroy(ZarrStreamSettings* settings) + { + delete settings; } - ZarrStreamSettings* copy = ZarrStreamSettings_create(); - if (!copy) { - LOG_ERROR("Failed to allocate memory for copy"); - return nullptr; - } + ZarrStreamSettings* ZarrStreamSettings_copy( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return nullptr; + } - *copy = *settings; + ZarrStreamSettings* copy = ZarrStreamSettings_create(); + if (!copy) { + LOG_ERROR("Failed to allocate memory for copy"); + return nullptr; + } - return copy; -} + *copy = *settings; -/* Setters */ -ZarrStatus -ZarrStreamSettings_set_store(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path, - const ZarrS3Settings* s3_settings) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); + return copy; + } - bytes_of_store_path = strnlen(store_path, bytes_of_store_path); - EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, - "Invalid store path. Must not be empty"); + /* Setters */ + ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); - std::string_view store_path_sv(store_path, bytes_of_store_path); - if (store_path_sv.empty()) { - LOG_ERROR("Invalid store path. Must not be empty"); - return ZarrStatus_InvalidArgument; - } + bytes_of_store_path = strnlen(store_path, bytes_of_store_path); + EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, + "Invalid store path. Must not be empty"); - if (nullptr != s3_settings) { - if (!validate_s3_settings(s3_settings)) { + std::string_view store_path_sv(store_path, bytes_of_store_path); + if (store_path_sv.empty()) { + LOG_ERROR("Invalid store path. Must not be empty"); return ZarrStatus_InvalidArgument; } - } else if (!validate_filesystem_store_path(store_path)) { - return ZarrStatus_InvalidArgument; - } - - if (nullptr != s3_settings) { - settings->s3_endpoint = s3_settings->endpoint; - settings->s3_bucket_name = s3_settings->bucket_name; - settings->s3_access_key_id = s3_settings->access_key_id; - settings->s3_secret_access_key = s3_settings->secret_access_key; - } - settings->store_path = store_path; + if (nullptr != s3_settings) { + if (!validate_s3_settings(s3_settings)) { + return ZarrStatus_InvalidArgument; + } + } else if (!validate_filesystem_store_path(store_path)) { + return ZarrStatus_InvalidArgument; + } - return ZarrStatus_Success; -} + if (nullptr != s3_settings) { + settings->s3_endpoint = s3_settings->endpoint; + settings->s3_bucket_name = s3_settings->bucket_name; + settings->s3_access_key_id = s3_settings->access_key_id; + settings->s3_secret_access_key = s3_settings->secret_access_key; + } -ZarrStatus -ZarrStreamSettings_set_compression( - ZarrStreamSettings* settings, - const ZarrCompressionSettings* compression_settings) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(compression_settings, - "Null pointer: compression_settings"); + settings->store_path = store_path; - if (!validate_compression_settings(compression_settings)) { - return ZarrStatus_InvalidArgument; + return ZarrStatus_Success; } - settings->compressor = compression_settings->compressor; - settings->compression_codec = compression_settings->codec; - settings->compression_level = compression_settings->level; - settings->compression_shuffle = compression_settings->shuffle; - - return ZarrStatus_Success; -} - -ZarrStatus -ZarrStreamSettings_set_custom_metadata(ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata) -{ - if (!settings) { - LOG_ERROR("Null pointer: settings"); - return ZarrStatus_InvalidArgument; - } + ZarrStatus ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(compression_settings, + "Null pointer: compression_settings"); - if (!external_metadata) { - LOG_ERROR("Null pointer: custom_metadata"); - return ZarrStatus_InvalidArgument; - } + if (!validate_compression_settings(compression_settings)) { + return ZarrStatus_InvalidArgument; + } - if (bytes_of_external_metadata == 0) { - LOG_ERROR("Invalid length: %zu. Must be greater than 0", - bytes_of_external_metadata); - return ZarrStatus_InvalidArgument; - } + settings->compressor = compression_settings->compressor; + settings->compression_codec = compression_settings->codec; + settings->compression_level = compression_settings->level; + settings->compression_shuffle = compression_settings->shuffle; - size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); - if (nbytes < 2) { - settings->custom_metadata = "{}"; return ZarrStatus_Success; } - auto val = nlohmann::json::parse(external_metadata, - external_metadata + nbytes, - nullptr, // callback - false, // allow exceptions - true // ignore comments - ); - - if (val.is_discarded()) { - LOG_ERROR("Invalid JSON: %s", external_metadata); - return ZarrStatus_InvalidArgument; - } - settings->custom_metadata = val.dump(); + ZarrStatus ZarrStreamSettings_set_custom_metadata( + ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata) + { + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return ZarrStatus_InvalidArgument; + } - return ZarrStatus_Success; -} + if (!external_metadata) { + LOG_ERROR("Null pointer: custom_metadata"); + return ZarrStatus_InvalidArgument; + } -ZarrStatus -ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, - ZarrDataType data_type) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT( - data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); + if (bytes_of_external_metadata == 0) { + LOG_ERROR("Invalid length: %zu. Must be greater than 0", + bytes_of_external_metadata); + return ZarrStatus_InvalidArgument; + } - settings->dtype = data_type; - return ZarrStatus_Success; -} + size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); + if (nbytes < 2) { + settings->custom_metadata = "{}"; + return ZarrStatus_Success; + } -ZarrStatus -ZarrStreamSettings_reserve_dimensions(ZarrStreamSettings* settings, - size_t count) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(count >= zarr_dimension_min && - count <= zarr_dimension_max, - "Invalid count: %zu. Count must be between %d and %d", - count, - zarr_dimension_min, - zarr_dimension_max); - - settings->dimensions.resize(count); - return ZarrStatus_Success; -} + auto val = nlohmann::json::parse(external_metadata, + external_metadata + nbytes, + nullptr, // callback + false, // allow exceptions + true // ignore comments + ); -ZarrStatus -ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, - size_t index, - const ZarrDimensionProperties* dimension) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); - EXPECT_VALID_INDEX(index < settings->dimensions.size(), - "Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); + if (val.is_discarded()) { + LOG_ERROR("Invalid JSON: %s", external_metadata); + return ZarrStatus_InvalidArgument; + } + settings->custom_metadata = val.dump(); - if (!validate_dimension(dimension)) { - return ZarrStatus_InvalidArgument; + return ZarrStatus_Success; } - struct ZarrDimension_s& dim = settings->dimensions[index]; + ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); - dim.name = trim(dimension->name, dimension->bytes_of_name - 1); - dim.type = dimension->type; - dim.array_size_px = dimension->array_size_px; - dim.chunk_size_px = dimension->chunk_size_px; - dim.shard_size_chunks = dimension->shard_size_chunks; + settings->dtype = data_type; + return ZarrStatus_Success; + } - return ZarrStatus_Success; -} + ZarrStatus ZarrStreamSettings_reserve_dimensions( + ZarrStreamSettings* settings, + size_t count) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + count >= zarr_dimension_min && count <= zarr_dimension_max, + "Invalid count: %zu. Count must be between %d and %d", + count, + zarr_dimension_min, + zarr_dimension_max); -ZarrStatus -ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, - uint8_t multiscale) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + settings->dimensions.resize(count); + return ZarrStatus_Success; + } - settings->multiscale = multiscale > 0; - return ZarrStatus_Success; -} + ZarrStatus ZarrStreamSettings_set_dimension( + ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); + EXPECT_VALID_INDEX(index < settings->dimensions.size(), + "Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); -/* Getters */ -const char* -ZarrStreamSettings_get_store_path(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, store_path); -} + if (!validate_dimension(dimension)) { + return ZarrStatus_InvalidArgument; + } -ZarrS3Settings -ZarrStreamSettings_get_s3_settings(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); - return {}; - } - - ZarrS3Settings s3_settings = { - settings->s3_endpoint.c_str(), - settings->s3_endpoint.length() + 1, - settings->s3_bucket_name.c_str(), - settings->s3_bucket_name.length() + 1, - settings->s3_access_key_id.c_str(), - settings->s3_access_key_id.length() + 1, - settings->s3_secret_access_key.c_str(), - settings->s3_secret_access_key.length() + 1, - }; - return s3_settings; -} + struct ZarrDimension_s& dim = settings->dimensions[index]; -const char* -ZarrStreamSettings_get_custom_metadata(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, custom_metadata); -} + dim.name = trim(dimension->name, dimension->bytes_of_name - 1); + dim.type = dimension->type; + dim.array_size_px = dimension->array_size_px; + dim.chunk_size_px = dimension->chunk_size_px; + dim.shard_size_chunks = dimension->shard_size_chunks; -ZarrDataType -ZarrStreamSettings_get_data_type(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); - return ZarrDataType_uint8; + return ZarrStatus_Success; } - return static_cast(settings->dtype); -} -ZarrCompressionSettings -ZarrStreamSettings_get_compression(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty compression."); - return {}; - } - - ZarrCompressionSettings compression = { - .compressor = settings->compressor, - .codec = static_cast(settings->compression_codec), - .level = settings->compression_level, - .shuffle = settings->compression_shuffle, - }; - return compression; -} + ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale) + { + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); -size_t -ZarrStreamSettings_get_dimension_count(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning 0."); - return 0; + settings->multiscale = multiscale > 0; + return ZarrStatus_Success; } - return settings->dimensions.size(); -} -ZarrDimensionProperties -ZarrStreamSettings_get_dimension(const ZarrStreamSettings* settings, - size_t index) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning empty dimension."); - return {}; + /* Getters */ + const char* ZarrStreamSettings_get_store_path( + const ZarrStreamSettings* settings) + { + SETTINGS_GET_STRING(settings, store_path); } - if (index >= settings->dimensions.size()) { - LOG_ERROR("Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); - return {}; + ZarrS3Settings ZarrStreamSettings_get_s3_settings( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); + return {}; + } + + ZarrS3Settings s3_settings = { + settings->s3_endpoint.c_str(), + settings->s3_endpoint.length() + 1, + settings->s3_bucket_name.c_str(), + settings->s3_bucket_name.length() + 1, + settings->s3_access_key_id.c_str(), + settings->s3_access_key_id.length() + 1, + settings->s3_secret_access_key.c_str(), + settings->s3_secret_access_key.length() + 1, + }; + return s3_settings; + } + + const char* ZarrStreamSettings_get_custom_metadata( + const ZarrStreamSettings* settings) + { + SETTINGS_GET_STRING(settings, custom_metadata); + } + + ZarrDataType ZarrStreamSettings_get_data_type( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); + return ZarrDataType_uint8; + } + return static_cast(settings->dtype); } - const auto& dim = settings->dimensions[index]; + ZarrCompressionSettings ZarrStreamSettings_get_compression( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty compression."); + return {}; + } - ZarrDimensionProperties dimension = { - .name = dim.name.c_str(), - .bytes_of_name = dim.name.size() + 1, - .type = dim.type, - .array_size_px = dim.array_size_px, - .chunk_size_px = dim.chunk_size_px, - .shard_size_chunks = dim.shard_size_chunks, - }; + ZarrCompressionSettings compression = { + .compressor = settings->compressor, + .codec = + static_cast(settings->compression_codec), + .level = settings->compression_level, + .shuffle = settings->compression_shuffle, + }; + return compression; + } + + size_t ZarrStreamSettings_get_dimension_count( + const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->dimensions.size(); + } - return dimension; -} + ZarrDimensionProperties ZarrStreamSettings_get_dimension( + const ZarrStreamSettings* settings, + size_t index) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty dimension."); + return {}; + } -bool -ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning false."); - return false; + if (index >= settings->dimensions.size()) { + LOG_ERROR("Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + return {}; + } + + const auto& dim = settings->dimensions[index]; + + ZarrDimensionProperties dimension = { + .name = dim.name.c_str(), + .bytes_of_name = dim.name.size() + 1, + .type = dim.type, + .array_size_px = dim.array_size_px, + .chunk_size_px = dim.chunk_size_px, + .shard_size_chunks = dim.shard_size_chunks, + }; + + return dimension; } - return settings->multiscale; -} + + bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) + { + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning false."); + return false; + } + return settings->multiscale; + } +} \ No newline at end of file From fdf5b086ee86e493fcdd128a44d5bef2aaf9b9da Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:49:15 -0400 Subject: [PATCH 3/3] No need to double CHECK the settings pointer. --- tests/unit-tests/get-stream-parameters.cpp | 1 - tests/unit-tests/set-stream-parameters.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/unit-tests/get-stream-parameters.cpp b/tests/unit-tests/get-stream-parameters.cpp index c3d287a4..5dea2cf6 100644 --- a/tests/unit-tests/get-stream-parameters.cpp +++ b/tests/unit-tests/get-stream-parameters.cpp @@ -217,7 +217,6 @@ main() ZarrStreamSettings* settings = ZarrStreamSettings_create(); try { - CHECK(settings); check_preliminaries(settings); get_store_path(settings); get_s3_settings(settings); diff --git a/tests/unit-tests/set-stream-parameters.cpp b/tests/unit-tests/set-stream-parameters.cpp index d7ee16b7..f8dedcda 100644 --- a/tests/unit-tests/set-stream-parameters.cpp +++ b/tests/unit-tests/set-stream-parameters.cpp @@ -256,7 +256,6 @@ main() ZarrStreamSettings* settings = ZarrStreamSettings_create(); try { - CHECK(settings); check_preliminaries(settings); set_store(settings); set_compression(settings);