diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a78bcbbc..30e1cc2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,37 @@ set(tgt acquire-zarr) add_library(${tgt} STATIC include/zarr.h + internal/logger.hh + internal/logger.cpp + internal/stream.settings.hh + internal/stream.settings.cpp +) + +target_include_directories(${tgt} + PUBLIC + $ + PRIVATE + $ +) + +target_link_libraries(${tgt} PRIVATE + blosc_static + miniocpp::miniocpp +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) + +# Install public header files +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ + DESTINATION include + FILES_MATCHING PATTERN "*.h" ) ####### Acquire Zarr Driver ####### diff --git a/src/internal/logger.cpp b/src/internal/logger.cpp new file mode 100644 index 00000000..4f177a1a --- /dev/null +++ b/src/internal/logger.cpp @@ -0,0 +1,81 @@ +#include "logger.hh" + +#include +#include +#include +#include +#include + +LogLevel Logger::current_level = LogLevel_Info; + +void +Logger::set_log_level(LogLevel level) +{ + current_level = level; +} + +LogLevel +Logger::get_log_level() +{ + return current_level; +} + +std::string +Logger::log(LogLevel level, + const char* file, + int line, + const char* func, + const char* format, + ...) +{ + if (current_level == LogLevel_None || level < current_level) { + return {}; // Suppress logs + } + + va_list args; + va_start(args, format); + + std::string prefix; + std::ostream* stream = &std::cout; + + switch (level) { + case LogLevel_Debug: + prefix = "[DEBUG] "; + break; + case LogLevel_Info: + prefix = "[INFO] "; + break; + case LogLevel_Warning: + prefix = "[WARNING] "; + stream = &std::cerr; + break; + case LogLevel_Error: + prefix = "[ERROR] "; + stream = &std::cerr; + break; + } + + // Get current time + auto now = std::chrono::system_clock::now(); + auto time = std::chrono::system_clock::to_time_t(now); + auto ms = std::chrono::duration_cast( + now.time_since_epoch()) % + 1000; + + // Get filename without path + std::filesystem::path filepath(file); + std::string filename = filepath.filename().string(); + + // Output timestamp, log level, filename + *stream << std::put_time(std::localtime(&time), "%Y-%m-%d %H:%M:%S") << '.' + << std::setfill('0') << std::setw(3) << ms.count() << " " << prefix + << filename << ":" << line << " " << func << ": "; + + char buffer[1024]; + vsnprintf(buffer, sizeof(buffer), format, args); + *stream << buffer << std::endl; + + va_end(args); + + return buffer; +} \ No newline at end of file diff --git a/src/internal/logger.hh b/src/internal/logger.hh new file mode 100644 index 00000000..c2417a91 --- /dev/null +++ b/src/internal/logger.hh @@ -0,0 +1,38 @@ +#include "zarr.h" + +#include + +class Logger +{ + public: + static void set_log_level(LogLevel level); + static LogLevel get_log_level(); + + static std::string log(LogLevel level, + const char* file, + int line, + const char* func, + const char* format, + ...); + + private: + static LogLevel current_level; +}; + +#define LOG_DEBUG(...) \ + Logger::log(LogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_INFO(...) \ + Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_WARNING(...) \ + Logger::log(LogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_ERROR(...) \ + Logger::log(LogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) + +#define EXPECT(e, ...) \ + do { \ + if (!(e)) { \ + const std::string __err = LOG_ERROR(__VA_ARGS__); \ + throw std::runtime_error(__err); \ + } \ + } while (0) +#define CHECK(e) EXPECT(e, "Expression evaluated as false:\n\t%s", #e) diff --git a/src/internal/stream.settings.cpp b/src/internal/stream.settings.cpp new file mode 100644 index 00000000..906209f1 --- /dev/null +++ b/src/internal/stream.settings.cpp @@ -0,0 +1,458 @@ +#include "stream.settings.hh" +#include "zarr.h" +#include "logger.hh" + +#include + +#include // memcpy + +#define EXPECT_VALID_ARGUMENT(e, ...) \ + do { \ + if (!(e)) { \ + LOG_ERROR(__VA_ARGS__); \ + return ZarrError_InvalidArgument; \ + } \ + } while (0) + +#define ZARR_DIMENSION_MIN 3 +#define ZARR_DIMENSION_MAX 32 + +#define SETTINGS_SET_STRING(settings, member, bytes_of_member) \ + do { \ + if (!(settings)) { \ + LOG_ERROR("Null pointer: %s", #settings); \ + return ZarrError_InvalidArgument; \ + } \ + if (!(member)) { \ + LOG_ERROR("Null pointer: %s", #member); \ + return ZarrError_InvalidArgument; \ + } \ + size_t nbytes = strnlen(member, bytes_of_member); \ + settings->member = { member, nbytes }; \ + return ZarrError_Success; \ + } while (0) + +#define SETTINGS_GET_STRING(settings, member) \ + do { \ + if (!settings) { \ + LOG_ERROR("Null pointer: %s", #settings); \ + return nullptr; \ + } \ + return settings->member.c_str(); \ + } while (0) + +namespace { +const char* +compressor_to_string(ZarrCompressor compressor) +{ + switch (compressor) { + case ZarrCompressor_None: + return "none"; + case ZarrCompressor_Blosc1: + return "blosc1"; + case ZarrCompressor_Blosc2: + return "blosc2"; + case ZarrCompressor_Zstd: + return "zstd"; + default: + return "(unknown)"; + } +} + +const char* +compression_codec_to_string(ZarrCompressionCodec codec) +{ + switch (codec) { + case ZarrCompressionCodec_None: + return "none"; + case ZarrCompressionCodec_BloscLZ4: + return "blosc-lz4"; + case ZarrCompressionCodec_BloscZstd: + return "blosc-zstd"; + default: + return "(unknown)"; + } +} + +inline std::string +trim(const char* s, size_t bytes_of_s) +{ + // trim left + std::string trimmed(s, bytes_of_s); + trimmed.erase(trimmed.begin(), + std::find_if(trimmed.begin(), trimmed.end(), [](char c) { + return !std::isspace(c); + })); + + // trim right + trimmed.erase(std::find_if(trimmed.rbegin(), + trimmed.rend(), + [](char c) { return !std::isspace(c); }) + .base(), + trimmed.end()); + + return trimmed; +} +} // namespace + +/* Create and destroy */ +ZarrStreamSettings* +ZarrStreamSettings_create() +{ + try { + return new ZarrStreamSettings(); + } catch (const std::bad_alloc&) { + return nullptr; + } +} + +void +ZarrStreamSettings_destroy(ZarrStreamSettings* settings) +{ + delete settings; +} + +/* Setters */ +ZarrError +ZarrStreamSettings_set_store_path(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path) +{ + SETTINGS_SET_STRING(settings, store_path, bytes_of_store_path); +} + +ZarrError +ZarrStreamSettings_set_s3_endpoint(ZarrStreamSettings* settings, + const char* s3_endpoint, + size_t bytes_of_s3_endpoint) +{ + SETTINGS_SET_STRING(settings, s3_endpoint, bytes_of_s3_endpoint); +} + +ZarrError +ZarrStreamSettings_set_s3_bucket_name(ZarrStreamSettings* settings, + const char* s3_bucket_name, + size_t bytes_of_s3_bucket_name) +{ + SETTINGS_SET_STRING(settings, s3_bucket_name, bytes_of_s3_bucket_name); +} + +ZarrError +ZarrStreamSettings_set_s3_access_key_id(ZarrStreamSettings* settings, + const char* s3_access_key_id, + size_t bytes_of_s3_access_key_id) +{ + SETTINGS_SET_STRING(settings, s3_access_key_id, bytes_of_s3_access_key_id); +} + +ZarrError +ZarrStreamSettings_set_s3_secret_access_key( + ZarrStreamSettings* settings, + const char* s3_secret_access_key, + size_t bytes_of_s3_secret_access_key) +{ + SETTINGS_SET_STRING( + settings, s3_secret_access_key, bytes_of_s3_secret_access_key); +} + +ZarrError +ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); + + settings->dtype = data_type; + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_set_compressor(ZarrStreamSettings* settings, + ZarrCompressor compressor) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT( + compressor < ZarrCompressorCount, "Invalid compressor: %d", compressor); + + if (compressor >= ZarrCompressor_Blosc2) { + LOG_ERROR("Compressor not yet implemented: %s", + compressor_to_string(compressor)); + return ZarrError_NotYetImplemented; + } + + settings->compressor = compressor; + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_set_compression_codec(ZarrStreamSettings* settings, + ZarrCompressionCodec codec) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(codec < ZarrCompressionCodecCount, + "Invalid codec: %s", + compression_codec_to_string(codec)); + + settings->compression_codec = codec; + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_set_compression_level(ZarrStreamSettings* settings, + uint8_t level) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(level >= 0 && level <= 9, + "Invalid level: %d. Must be between 0 (no " + "compression) and 9 (maximum compression).", + level); + + settings->compression_level = level; + + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_set_compression_shuffle(ZarrStreamSettings* settings, + uint8_t shuffle) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(shuffle == BLOSC_NOSHUFFLE || + shuffle == BLOSC_SHUFFLE || + shuffle == BLOSC_BITSHUFFLE, + "Invalid shuffle: %d. Must be %d (no shuffle), %d " + "(byte shuffle), or %d (bit shuffle)", + shuffle, + BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, + BLOSC_BITSHUFFLE); + + settings->compression_shuffle = shuffle; + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_reserve_dimensions(ZarrStreamSettings* settings, + size_t count) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(count >= ZARR_DIMENSION_MIN && + count <= ZARR_DIMENSION_MAX, + "Invalid count: %zu. Count must be between %d and %d", + count, + ZARR_DIMENSION_MIN, + ZARR_DIMENSION_MAX); + + settings->dimensions.resize(count); + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, + size_t index, + const char* name, + size_t bytes_of_name, + ZarrDimensionType kind, + uint32_t array_size_px, + uint32_t chunk_size_px, + uint32_t shard_size_chunks) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(name, "Null pointer: name"); + EXPECT_VALID_ARGUMENT( + bytes_of_name > 0, "Invalid name length: %zu", bytes_of_name); + EXPECT_VALID_ARGUMENT(strnlen(name, bytes_of_name) > 0, + "Invalid name. Must not be empty"); + EXPECT_VALID_ARGUMENT( + kind < ZarrDimensionTypeCount, "Invalid dimension type: %d", kind); + EXPECT_VALID_ARGUMENT( + chunk_size_px > 0, "Invalid chunk size: %zu", chunk_size_px); + + // Check that the index is within bounds + if (index >= settings->dimensions.size()) { + LOG_ERROR("Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + return ZarrError_InvalidIndex; + } + + std::string dim_name = trim(name, bytes_of_name); + if (dim_name.empty()) { + LOG_ERROR("Invalid name. Must not be empty"); + return ZarrError_InvalidArgument; + } + + struct ZarrDimension_s* dim = &settings->dimensions[index]; + + dim->name = dim_name; + dim->kind = kind; + dim->array_size_px = array_size_px; + dim->chunk_size_px = chunk_size_px; + dim->shard_size_chunks = shard_size_chunks; + + return ZarrError_Success; +} + +ZarrError +ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + + settings->multiscale = multiscale > 0; + return ZarrError_Success; +} + +/* Getters */ +const char* +ZarrStreamSettings_get_store_path(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, store_path); +} + +const char* +ZarrStreamSettings_get_s3_endpoint(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, s3_endpoint); +} + +const char* +ZarrStreamSettings_get_s3_bucket_name(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, s3_bucket_name); +} + +const char* +ZarrStreamSettings_get_s3_access_key_id(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, s3_access_key_id); +} + +const char* +ZarrStreamSettings_get_s3_secret_access_key(const ZarrStreamSettings* settings) +{ + SETTINGS_GET_STRING(settings, s3_secret_access_key); +} + +ZarrDataType +ZarrStreamSettings_get_data_type(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning DataType_uint8."); + return ZarrDataType_uint8; + } + return static_cast(settings->dtype); +} + +ZarrCompressor +ZarrStreamSettings_get_compressor(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning ZarrCompressor_None."); + return ZarrCompressor_None; + } + return static_cast(settings->compressor); +} + +ZarrCompressionCodec +ZarrStreamSettings_get_compression_codec(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING( + "Null pointer: settings. Returning ZarrCompressionCodec_None."); + return ZarrCompressionCodec_None; + } + return static_cast(settings->compression_codec); +} + +uint8_t +ZarrStreamSettings_get_compression_level(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->compression_level; +} + +uint8_t +ZarrStreamSettings_get_compression_shuffle(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->compression_shuffle; +} + +size_t +ZarrStreamSettings_get_dimension_count(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return settings->dimensions.size(); +} + +ZarrError +ZarrStreamSettings_get_dimension(const ZarrStreamSettings* settings, + size_t index, + char* name, + size_t bytes_of_name, + ZarrDimensionType* kind, + size_t* array_size_px, + size_t* chunk_size_px, + size_t* shard_size_chunks) +{ + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(name, "Null pointer: name"); + EXPECT_VALID_ARGUMENT(kind, "Null pointer: kind"); + EXPECT_VALID_ARGUMENT(array_size_px, "Null pointer: array_size_px"); + EXPECT_VALID_ARGUMENT(chunk_size_px, "Null pointer: chunk_size_px"); + EXPECT_VALID_ARGUMENT(shard_size_chunks, "Null pointer: shard_size_chunks"); + + if (index >= settings->dimensions.size()) { + LOG_ERROR("Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + return ZarrError_InvalidIndex; + } + + const struct ZarrDimension_s* dim = &settings->dimensions[index]; + + if (bytes_of_name < dim->name.length() + 1) { + LOG_ERROR("Insufficient buffer size: %zu. Need at least %zu", + bytes_of_name, + dim->name.length() + 1); + return ZarrError_Overflow; + } + + memcpy(name, dim->name.c_str(), dim->name.length() + 1); + *kind = static_cast(dim->kind); + *array_size_px = dim->array_size_px; + *chunk_size_px = dim->chunk_size_px; + *shard_size_chunks = dim->shard_size_chunks; + + return ZarrError_Success; +} + +uint8_t +ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) +{ + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning 0."); + return 0; + } + return static_cast(settings->multiscale); +} + +/* Internal functions */ + +bool +validate_dimension(const struct ZarrDimension_s& dimension) +{ + return !dimension.name.empty() && dimension.kind < ZarrDimensionTypeCount && + dimension.chunk_size_px > 0; +} diff --git a/src/internal/stream.settings.hh b/src/internal/stream.settings.hh new file mode 100644 index 00000000..0d08eba9 --- /dev/null +++ b/src/internal/stream.settings.hh @@ -0,0 +1,42 @@ +#pragma once + +#include // size_t +#include // uint8_t +#include +#include + +struct ZarrDimension_s +{ + std::string name; /* Name of the dimension */ + uint8_t kind; /* Type of dimension */ + + uint32_t array_size_px; /* Size of the array along this dimension */ + uint32_t chunk_size_px; /* Size of a chunk along this dimension */ + uint32_t shard_size_chunks; /* Number of chunks in a shard along this + dimension */ +}; + +struct ZarrStreamSettings_s +{ + std::string store_path; /* Path to the Zarr store on the local filesystem */ + + std::string s3_endpoint; /* Endpoint for the S3 service */ + std::string s3_bucket_name; /* Name of the S3 bucket */ + std::string s3_access_key_id; /* Access key ID for the S3 service */ + std::string s3_secret_access_key; /* Secret access key for the S3 service */ + + uint8_t dtype; /* Data type of the base array */ + + uint8_t compressor; /* Compression library to use */ + uint8_t compression_codec; /* Compression codec to use */ + uint8_t compression_level; /* Compression level to use */ + uint8_t compression_shuffle; /* Whether and how to shuffle the data before + compressing */ + + std::vector dimensions; /* Dimensions of the base array */ + + bool multiscale; /* Whether to stream to multiple resolutions */ +}; + +bool +validate_dimension(const struct ZarrDimension_s& dimension); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc4c2f4b..49fa4045 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,90 +1,6 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () - set(NOTEST "TRUE") - add_subdirectory(../acquire-common/acquire-driver-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-driver-common) - add_subdirectory(../acquire-common/acquire-video-runtime ${CMAKE_CURRENT_BINARY_DIR}/acquire-video-runtime) - set(NOTEST "FALSE") - - # - # PARAMETERS - # - set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - - # - # Tests - # - set(tests - list-devices - unit-tests - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 - ) - - foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") - endforeach () - - # - # Copy driver to tests - # - list(POP_FRONT tests onename) - - foreach (driver - acquire-driver-common - acquire-driver-zarr - ) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () - endforeach () -endif () + add_subdirectory(integration) + add_subdirectory(driver) +endif () \ No newline at end of file diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt new file mode 100644 index 00000000..46e52d77 --- /dev/null +++ b/tests/driver/CMakeLists.txt @@ -0,0 +1,87 @@ + +set(NOTEST "TRUE") +add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-driver-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-driver-common) +add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-video-runtime ${CMAKE_CURRENT_BINARY_DIR}/acquire-video-runtime) +set(NOTEST "FALSE") + +# +# PARAMETERS +# +set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project + +# +# Tests +# +set(tests + list-devices + unit-tests + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 +) + +foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") +endforeach () + +# +# Copy driver to tests +# +list(POP_FRONT tests onename) + +foreach (driver + acquire-driver-common + acquire-driver-zarr +) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" + ) + + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + endforeach () +endforeach () diff --git a/tests/README.md b/tests/driver/README.md similarity index 100% rename from tests/README.md rename to tests/driver/README.md diff --git a/tests/external-metadata-with-whitespace-ok.cpp b/tests/driver/external-metadata-with-whitespace-ok.cpp similarity index 100% rename from tests/external-metadata-with-whitespace-ok.cpp rename to tests/driver/external-metadata-with-whitespace-ok.cpp diff --git a/tests/get-meta.cpp b/tests/driver/get-meta.cpp similarity index 100% rename from tests/get-meta.cpp rename to tests/driver/get-meta.cpp diff --git a/tests/get-set-get.cpp b/tests/driver/get-set-get.cpp similarity index 100% rename from tests/get-set-get.cpp rename to tests/driver/get-set-get.cpp diff --git a/tests/get.cpp b/tests/driver/get.cpp similarity index 100% rename from tests/get.cpp rename to tests/driver/get.cpp diff --git a/tests/list-devices.cpp b/tests/driver/list-devices.cpp similarity index 100% rename from tests/list-devices.cpp rename to tests/driver/list-devices.cpp diff --git a/tests/metadata-dimension-sizes.cpp b/tests/driver/metadata-dimension-sizes.cpp similarity index 100% rename from tests/metadata-dimension-sizes.cpp rename to tests/driver/metadata-dimension-sizes.cpp diff --git a/tests/multiscales-metadata.cpp b/tests/driver/multiscales-metadata.cpp similarity index 100% rename from tests/multiscales-metadata.cpp rename to tests/driver/multiscales-metadata.cpp diff --git a/tests/repeat-start.cpp b/tests/driver/repeat-start.cpp similarity index 100% rename from tests/repeat-start.cpp rename to tests/driver/repeat-start.cpp diff --git a/tests/restart-stopped-zarr-resets-threadpool.cpp b/tests/driver/restart-stopped-zarr-resets-threadpool.cpp similarity index 100% rename from tests/restart-stopped-zarr-resets-threadpool.cpp rename to tests/driver/restart-stopped-zarr-resets-threadpool.cpp diff --git a/tests/unit-tests.cpp b/tests/driver/unit-tests.cpp similarity index 100% rename from tests/unit-tests.cpp rename to tests/driver/unit-tests.cpp diff --git a/tests/write-zarr-v2-compressed-multiscale.cpp b/tests/driver/write-zarr-v2-compressed-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-multiscale.cpp rename to tests/driver/write-zarr-v2-compressed-multiscale.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking.cpp diff --git a/tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp b/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp rename to tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp b/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp rename to tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale.cpp b/tests/driver/write-zarr-v2-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale.cpp rename to tests/driver/write-zarr-v2-raw-multiscale.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking.cpp diff --git a/tests/write-zarr-v2-raw-with-ragged-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-ragged-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp diff --git a/tests/write-zarr-v2-raw.cpp b/tests/driver/write-zarr-v2-raw.cpp similarity index 100% rename from tests/write-zarr-v2-raw.cpp rename to tests/driver/write-zarr-v2-raw.cpp diff --git a/tests/write-zarr-v2-to-s3.cpp b/tests/driver/write-zarr-v2-to-s3.cpp similarity index 100% rename from tests/write-zarr-v2-to-s3.cpp rename to tests/driver/write-zarr-v2-to-s3.cpp diff --git a/tests/write-zarr-v2-with-lz4-compression.cpp b/tests/driver/write-zarr-v2-with-lz4-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-lz4-compression.cpp rename to tests/driver/write-zarr-v2-with-lz4-compression.cpp diff --git a/tests/write-zarr-v2-with-zstd-compression.cpp b/tests/driver/write-zarr-v2-with-zstd-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-zstd-compression.cpp rename to tests/driver/write-zarr-v2-with-zstd-compression.cpp diff --git a/tests/write-zarr-v3-compressed.cpp b/tests/driver/write-zarr-v3-compressed.cpp similarity index 100% rename from tests/write-zarr-v3-compressed.cpp rename to tests/driver/write-zarr-v3-compressed.cpp diff --git a/tests/write-zarr-v3-raw-chunk-exceeds-array.cpp b/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp similarity index 100% rename from tests/write-zarr-v3-raw-chunk-exceeds-array.cpp rename to tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp diff --git a/tests/write-zarr-v3-raw-multiscale.cpp b/tests/driver/write-zarr-v3-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v3-raw-multiscale.cpp rename to tests/driver/write-zarr-v3-raw-multiscale.cpp diff --git a/tests/write-zarr-v3-raw-with-ragged-sharding.cpp b/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp similarity index 100% rename from tests/write-zarr-v3-raw-with-ragged-sharding.cpp rename to tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp diff --git a/tests/write-zarr-v3-raw.cpp b/tests/driver/write-zarr-v3-raw.cpp similarity index 100% rename from tests/write-zarr-v3-raw.cpp rename to tests/driver/write-zarr-v3-raw.cpp diff --git a/tests/write-zarr-v3-to-s3.cpp b/tests/driver/write-zarr-v3-to-s3.cpp similarity index 100% rename from tests/write-zarr-v3-to-s3.cpp rename to tests/driver/write-zarr-v3-to-s3.cpp diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt new file mode 100644 index 00000000..d17d0baa --- /dev/null +++ b/tests/integration/CMakeLists.txt @@ -0,0 +1,24 @@ +set(project acquire-zarr) + +set(tests + set-and-get-params +) + +foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp test.logger.hh test.logger.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE + "${CMAKE_SOURCE_DIR}/src/include") + target_link_libraries(${tgt} PRIVATE + acquire-zarr + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-zarr") +endforeach () \ No newline at end of file diff --git a/tests/integration/set-and-get-params.cpp b/tests/integration/set-and-get-params.cpp new file mode 100644 index 00000000..c8786754 --- /dev/null +++ b/tests/integration/set-and-get-params.cpp @@ -0,0 +1,253 @@ +#include "zarr.h" + +#include +#include +#include + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "Assertion failed: %s\n", #cond); \ + goto Error; \ + } \ + } while (0) + +#define CHECK_EQ(a, b) CHECK((a) == (b)) + +#define TRY_SET_STRING(stream, member, value) \ + do { \ + ZarrError err; \ + if (err = ZarrStreamSettings_set_##member( \ + stream, value, strlen(value) + 1); \ + err != ZarrError_Success) { \ + fprintf(stderr, \ + "Failed to set %s: %s\n", \ + #member, \ + Zarr_get_error_message(err)); \ + return false; \ + } \ + } while (0) + +#define SIZED(name) name, sizeof(name) + +bool +check_preconditions(ZarrStreamSettings* stream) +{ + std::string str_param_value; + + str_param_value = ZarrStreamSettings_get_store_path(stream); + CHECK(str_param_value.empty()); + + str_param_value = ZarrStreamSettings_get_s3_endpoint(stream); + CHECK(str_param_value.empty()); + + str_param_value = ZarrStreamSettings_get_s3_bucket_name(stream); + CHECK(str_param_value.empty()); + + str_param_value = ZarrStreamSettings_get_s3_access_key_id(stream); + CHECK(str_param_value.empty()); + + str_param_value = ZarrStreamSettings_get_s3_secret_access_key(stream); + CHECK(str_param_value.empty()); + + CHECK_EQ(ZarrStreamSettings_get_data_type(stream), ZarrDataType_uint8); + + CHECK_EQ(ZarrStreamSettings_get_compressor(stream), ZarrCompressor_None); + + CHECK_EQ(ZarrStreamSettings_get_compression_codec(stream), + ZarrCompressionCodec_None); + + CHECK_EQ(ZarrStreamSettings_get_dimension_count(stream), 0); + + CHECK_EQ(ZarrStreamSettings_get_multiscale(stream), 0); + + return true; + +Error: + return false; +} + +bool +set_and_get_parameters(ZarrStreamSettings* stream) +{ + std::string str_param_value; + + /* Set and get store path */ + TRY_SET_STRING(stream, store_path, "store_path"); + str_param_value = ZarrStreamSettings_get_store_path(stream); + CHECK_EQ(str_param_value, "store_path"); + + /* Set and get S3 endpoint */ + TRY_SET_STRING(stream, s3_endpoint, "s3_endpoint"); + str_param_value = ZarrStreamSettings_get_s3_endpoint(stream); + CHECK_EQ(str_param_value, "s3_endpoint"); + + /* Set and get S3 bucket name */ + TRY_SET_STRING(stream, s3_bucket_name, "s3_bucket_name"); + str_param_value = ZarrStreamSettings_get_s3_bucket_name(stream); + CHECK_EQ(str_param_value, "s3_bucket_name"); + + /* Set and get S3 access key ID */ + TRY_SET_STRING(stream, s3_access_key_id, "s3_access_key_id"); + str_param_value = ZarrStreamSettings_get_s3_access_key_id(stream); + CHECK_EQ(str_param_value, "s3_access_key_id"); + + /* Set and get S3 secret access key */ + TRY_SET_STRING(stream, s3_secret_access_key, "s3_secret_access_key"); + str_param_value = ZarrStreamSettings_get_s3_secret_access_key(stream); + CHECK_EQ(str_param_value, "s3_secret_access_key"); + + /* Set and get data type */ + CHECK_EQ(ZarrStreamSettings_set_data_type(stream, ZarrDataType_float16), + ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_get_data_type(stream), ZarrDataType_float16); + + /* Set and get compressor */ + CHECK_EQ(ZarrStreamSettings_set_compressor(stream, ZarrCompressor_Blosc1), + ZarrError_Success); + + // try to set an invalid compressor + CHECK_EQ(ZarrStreamSettings_set_compressor(stream, ZarrCompressor_Blosc2), + ZarrError_NotYetImplemented); + + // should still be the previous value + CHECK_EQ(ZarrStreamSettings_get_compressor(stream), ZarrCompressor_Blosc1); + + /* Set and get compression codec */ + CHECK_EQ(ZarrStreamSettings_set_compression_codec( + stream, ZarrCompressionCodec_BloscLZ4), + ZarrError_Success); + + /* Set and get some dimensions */ + CHECK_EQ(ZarrStreamSettings_reserve_dimensions(stream, 5), + ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_set_dimension( + stream, 4, SIZED("x"), ZarrDimensionType_Space, 10, 5, 2), + ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_set_dimension( + stream, 3, SIZED("y"), ZarrDimensionType_Space, 20, 10, 3), + ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_set_dimension( + stream, 2, SIZED("z"), ZarrDimensionType_Space, 30, 15, 4), + ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_set_dimension( + stream, 1, SIZED("c"), ZarrDimensionType_Channel, 40, 20, 5), + ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_set_dimension( + stream, 0, SIZED("t"), ZarrDimensionType_Time, 50, 25, 6), + ZarrError_Success); + + CHECK_EQ(ZarrStreamSettings_get_dimension_count(stream), 5); + + char name[64]; + ZarrDimensionType kind; + size_t array_size_px; + size_t chunk_size_px; + size_t shard_size_chunks; + + CHECK_EQ(ZarrStreamSettings_get_dimension(stream, + 0, + name, + 64, + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks), + ZarrError_Success); + CHECK_EQ(std::string(name), "t"); + CHECK_EQ(kind, ZarrDimensionType_Time); + CHECK_EQ(array_size_px, 50); + CHECK_EQ(chunk_size_px, 25); + CHECK_EQ(shard_size_chunks, 6); + + CHECK_EQ(ZarrStreamSettings_get_dimension(stream, + 1, + name, + 64, + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks), + ZarrError_Success); + CHECK_EQ(std::string(name), "c"); + CHECK_EQ(kind, ZarrDimensionType_Channel); + CHECK_EQ(array_size_px, 40); + CHECK_EQ(chunk_size_px, 20); + CHECK_EQ(shard_size_chunks, 5); + + CHECK_EQ(ZarrStreamSettings_get_dimension(stream, + 2, + name, + 64, + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks), + ZarrError_Success); + CHECK_EQ(std::string(name), "z"); + CHECK_EQ(kind, ZarrDimensionType_Space); + CHECK_EQ(array_size_px, 30); + CHECK_EQ(chunk_size_px, 15); + CHECK_EQ(shard_size_chunks, 4); + + CHECK_EQ(ZarrStreamSettings_get_dimension(stream, + 3, + name, + 64, + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks), + ZarrError_Success); + CHECK_EQ(std::string(name), "y"); + CHECK_EQ(kind, ZarrDimensionType_Space); + CHECK_EQ(array_size_px, 20); + CHECK_EQ(chunk_size_px, 10); + CHECK_EQ(shard_size_chunks, 3); + + CHECK_EQ(ZarrStreamSettings_get_dimension(stream, + 4, + name, + 64, + &kind, + &array_size_px, + &chunk_size_px, + &shard_size_chunks), + ZarrError_Success); + CHECK_EQ(std::string(name), "x"); + CHECK_EQ(kind, ZarrDimensionType_Space); + CHECK_EQ(array_size_px, 10); + CHECK_EQ(chunk_size_px, 5); + CHECK_EQ(shard_size_chunks, 2); + + /* Set and get multiscale */ + CHECK_EQ(ZarrStreamSettings_set_multiscale(stream, 10), ZarrError_Success); + CHECK_EQ(ZarrStreamSettings_get_multiscale(stream), 1); // normalized to 1 + + return true; + +Error: + return false; +} + +int +main() +{ + ZarrStreamSettings* stream = ZarrStreamSettings_create(); + if (!stream) { + fprintf(stderr, "Failed to create Zarr stream\n"); + return 1; + } + + int retval = 0; + + CHECK(check_preconditions(stream)); + CHECK(set_and_get_parameters(stream)); + +Finalize: + ZarrStreamSettings_destroy(stream); + return retval; +Error: + retval = 1; + goto Finalize; +} \ No newline at end of file