From 96b580ac0de1e0f875220b3305ca2afe77bdf180 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Mon, 16 Sep 2024 13:03:34 -0400 Subject: [PATCH] Finish responding to PR comments. --- include/zarr.h | 528 +++------------- include/zarr.types.h | 133 ++++ src/streaming/CMakeLists.txt | 7 +- src/streaming/logger.cpp | 18 +- src/streaming/logger.hh | 14 +- src/streaming/stream.settings.cpp | 578 +++++++++--------- src/streaming/stream.settings.hh | 21 +- src/streaming/zarr.stream.cpp | 387 ++++++------ src/streaming/zarr.stream.hh | 6 +- tests/integration/copy-settings.cpp | 202 +++--- .../integration/create-and-destroy-stream.cpp | 210 ++++--- tests/integration/set-and-get-params.cpp | 330 +++++----- tests/integration/test.logger.cpp | 18 +- tests/integration/test.logger.hh | 12 +- 14 files changed, 1131 insertions(+), 1333 deletions(-) create mode 100644 include/zarr.types.h diff --git a/include/zarr.h b/include/zarr.h index 37a5868b..eac50f35 100644 --- a/include/zarr.h +++ b/include/zarr.h @@ -1,302 +1,86 @@ #ifndef H_ACQUIRE_ZARR_V0 #define H_ACQUIRE_ZARR_V0 -#include // size_t -#include // uint8_t +#include "zarr.types.h" #ifdef __cplusplus extern "C" { #endif - /*************************************************************************** - * Log level - * - * The LogLevel enum lists the available log levels. Use LogLevel_None to - * suppress all log messages. - **************************************************************************/ - typedef enum - { - LogLevel_Debug, - LogLevel_Info, - LogLevel_Warning, - LogLevel_Error, - LogLevel_None, - LogLevelCount - } LogLevel; - - /*************************************************************************** - * Zarr stream settings - * - * The Zarr stream settings struct is created using - * ZarrStreamSettings_create and destroyed using ZarrStreamSettings_destroy. - * If the struct fails to be created, ZarrStreamSettings_create returns - * NULL. - **************************************************************************/ - typedef struct ZarrStreamSettings_s ZarrStreamSettings; - - /*************************************************************************** - * Data type - * - * The ZarrDataType enum lists the available pixel types. - **************************************************************************/ - - typedef enum - { - ZarrDataType_uint8, - ZarrDataType_uint16, - ZarrDataType_uint32, - ZarrDataType_uint64, - ZarrDataType_int8, - ZarrDataType_int16, - ZarrDataType_int32, - ZarrDataType_int64, - ZarrDataType_float32, - ZarrDataType_float64, - ZarrDataTypeCount - } ZarrDataType; - - /*************************************************************************** - * Compression - * - * Acquire Zarr uses libraries to compress data streams. The ZarrCompressor - * enum lists the available compressors. Note that a compressor is not - * the same as a codec. A codec is a specific implementation of a - * compression algorithm, while a compressor is a library that implements - * one or more codecs. The ZarrCompressor enum lists the available - * compressors. The ZarrCompressor_None value indicates that no compression - * is used. - * - * The ZarrCompressionCodec enum lists the available codecs. The - * ZarrCompressionCodec_None value should only be used when not compressing. - * If the compressor is set to ZarrCompressor_None, the codec is ignored. - **************************************************************************/ - - typedef enum - { - ZarrCompressor_None = 0, - ZarrCompressor_Blosc1, - ZarrCompressorCount - } ZarrCompressor; - - typedef enum - { - ZarrCompressionCodec_None = 0, - ZarrCompressionCodec_BloscLZ4, - ZarrCompressionCodec_BloscZstd, - ZarrCompressionCodecCount - } ZarrCompressionCodec; - - /*************************************************************************** - * Dimension types - * - * The ZarrDimensionType enum lists the available dimension types. The - * ZarrDimensionType_Space value indicates that the dimension is a spatial - * dimension. The ZarrDimensionType_Time value indicates that the dimension - * is a time dimension. The ZarrDimensionType_Channel value indicates that - * the dimension is a channel dimension. ZarrDimensionType_Other value - * indicates that the dimension is not a spatial, time, or channel - * dimension. - **************************************************************************/ - - typedef enum - { - ZarrDimensionType_Space = 0, - ZarrDimensionType_Channel, - ZarrDimensionType_Time, - ZarrDimensionType_Other, - ZarrDimensionTypeCount - } ZarrDimensionType; - - /*************************************************************************** - * Error handling - * - * The ZarrError enum lists the available error codes. The - * Zarr_get_error_message function returns a human-readable error message - * for the given error code. - **************************************************************************/ - - typedef enum - { - ZarrError_Success = 0, - ZarrError_InvalidArgument, - ZarrError_Overflow, - ZarrError_InvalidIndex, - ZarrError_InternalError, - ZarrErrorCount, - } ZarrError; + typedef struct ZarrStream_s ZarrStream; /** - * @brief Get the error message for the given error code. - * @param error The error code. - * @return A human-readable error message. + * @brief Get the version of the Zarr API. + * @return The version of the Zarr API. */ - const char* Zarr_get_error_message(ZarrError error); - - /*************************************************************************** - * Functions for creating and destroying a Zarr stream settings struct. - * - * ZarrStreamSettings is an opaque data structure that holds the parameters - * for the Zarr stream. The struct is created using - * ZarrStreamSettings_create and destroyed using ZarrStreamSettings_destroy. - * If the struct fails to be created, ZarrStreamSettings_create returns - * NULL. - **************************************************************************/ - - /** @brief Return a pointer to a Zarr stream settings struct. */ - ZarrStreamSettings* ZarrStreamSettings_create(); - - /** @brief Destroy a Zarr stream settings struct. Consumes the pointer */ - void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); - - /** @brief Make a copy of the Zarr stream settings struct. */ - ZarrStreamSettings* ZarrStreamSettings_copy( - const ZarrStreamSettings* settings); - - /*************************************************************************** - * Functions for setting parameters for a Zarr stream. - * - * These parameters are used to configure the Zarr stream. - * Each function returns a ZarrError to indicate success or failure, which - * can be converted to a human-readable error message using - * Zarr_get_error_message(). If the function fails, the settings struct is - * not modified. - **************************************************************************/ + uint32_t Zarr_get_api_version(); /** - * @brief Set the store path for the Zarr stream. - * @detail This parameter is required for all Zarr streams. When acquiring - * to the filesystem, the store path is a directory path. When acquiring to - * S3, the store path is a key prefix. - * @param[in, out] setting The Zarr stream settings struct. - * @param[in] store_path Path to the directory where the Zarr data is - * stored, e.g., "/path/to/dataset.zarr". - * @param[in] bytes_of_store_path The length of @p store_path in bytes, - * including the null terminator. - * @return ZarrError_Success on success, or an error code on failure. + * @brief Get the message for the given status code. + * @param status The status code. + * @return A human-readable status message. */ - ZarrError ZarrStreamSettings_set_store_path(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path); + const char* Zarr_get_error_message(ZarrStatus status); /** - * @brief Set the S3 endpoint for the Zarr stream, if streaming to S3. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] s3_endpoint The S3 endpoint, e.g., "https://s3.amazonaws.com" - * or "http://localhost:9000". Must begin with "http://" or "https://". - * @param[in] bytes_of_s3_endpoint The length of @p s3_endpoint in bytes, - * including the null terminator. - * @return ZarrError_Success on success, or an error code on failure. + * @brief Create a Zarr stream settings struct. + * @return A pointer to the Zarr stream settings struct, or NULL on failure. */ - ZarrError ZarrStreamSettings_set_s3_endpoint(ZarrStreamSettings* settings, - const char* s3_endpoint, - size_t bytes_of_s3_endpoint); + ZarrStreamSettings* ZarrStreamSettings_create(); /** - * @brief Set the S3 bucket name for the Zarr stream, if streaming to S3. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] s3_bucket_name The S3 bucket name, e.g., "my-bucket". - * @param[in] bytes_of_s3_bucket_name The length of @p s3_bucket_name in - * bytes, including the null terminator. - * @return ZarrError_Success on success, or an error code on failure. + * @brief Destroy a Zarr stream settings struct. + * @details This function frees the memory allocated for the Zarr stream + * settings struct. + * @param[in] settings The Zarr stream settings struct. */ - ZarrError ZarrStreamSettings_set_s3_bucket_name( - ZarrStreamSettings* settings, - const char* s3_bucket_name, - size_t bytes_of_s3_bucket_name); + void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); /** - * @brief Set the S3 access key ID for the Zarr stream, if streaming to S3. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] s3_access_key_id The access key ID. - * @param[in] bytes_of_s3_access_key_id The length of @p s3_access_key_id in - * bytes, including the null terminator. - * @return ZarrError_Success on success, or an error code on failure. + * @brief Copy a Zarr stream settings struct. + * @param[in] settings The Zarr stream settings struct to copy. + * @return A copy of the Zarr stream settings struct. */ - ZarrError ZarrStreamSettings_set_s3_access_key_id( - ZarrStreamSettings* settings, - const char* s3_access_key_id, - size_t bytes_of_s3_access_key_id); + ZarrStreamSettings* ZarrStreamSettings_copy( + const ZarrStreamSettings* settings); /** - * @brief Set the S3 secret access key for the Zarr stream, if streaming to - * S3. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] s3_secret_access_key The secret access key. - * @param[in] bytes_of_s3_secret_access_key The length of - * @p s3_secret_access_key in bytes, including the null terminator. - * @return + * @brief Set store path and S3 settings for the Zarr stream. + * @param[in, out] settings + * @param[in] store_path The store path for the Zarr stream. Directory path + * when acquiring to the filesystem, key prefix when acquiring to S3. + * @param[in] bytes_of_store_path The length of @p store_path in bytes, + * including the null terminator. + * @param[in] s3_settings Optional S3 settings. If NULL, the store path is + * assumed to be a directory path. + * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrError ZarrStreamSettings_set_s3_secret_access_key( - ZarrStreamSettings* settings, - const char* s3_secret_access_key, - size_t bytes_of_s3_secret_access_key); + ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings); /** - * @brief Set JSON-formatted external metadata for the Zarr stream. - * @details This metadata will be written to acquire-zarr.json in the - * metadata directory of the Zarr store. This parameter is optional. - * @param settings[in, out] settings The Zarr stream settings struct. - * @param external_metadata JSON-formatted external metadata. - * @param bytes_of_external_metadata The length of @p external_metadata in - * bytes, including the null terminator. - * @return ZarrError_Success on success, or an error code on failure. + * @brief Set the data type, compressor, codec, compression_settings level, + * and shuffle for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] compression_settings The compression_settings settings. + * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrError ZarrStreamSettings_set_external_metadata( + ZarrStatus ZarrStreamSettings_set_compression( ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata); + const ZarrCompressionSettings* compression_settings); /** * @brief Set the data type for the Zarr stream. * @param[in, out] settings The Zarr stream settings struct. * @param[in] data_type The data type. - * @return ZarrError_Success on success, or an error code on failure. - */ - ZarrError ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, - ZarrDataType data_type); - - /** - * @brief Set the compressor for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] compressor Enum value for the compressor. - * @return ZarrError_Success on success, or an error code on failure. + * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrError ZarrStreamSettings_set_compressor(ZarrStreamSettings* settings, - ZarrCompressor compressor); - - /** - * @brief Set the compression codec for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] codec Enum value for the compression codec. - * @return ZarrError_Success on success, or an error code on failure. - */ - ZarrError ZarrStreamSettings_set_compression_codec( - ZarrStreamSettings* settings, - ZarrCompressionCodec codec); - - /** - * @brief Set the compression level for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] compression_level A value between 0 and 9. Higher values - * indicate higher compression levels. Set to 0 for no compression. - * @return ZarrError_Success on success, or an error code on failure. - */ - ZarrError ZarrStreamSettings_set_compression_level( - ZarrStreamSettings* settings, - uint8_t compression_level); - - /** - * @brief Set the compression shuffle value for the Zarr stream - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] shuffle A value of 0, 1, or (if using Blosc) 2. 0 indicates - * no shuffle, 1 indicates shuffle, and 2 indicates bitshuffle. - * @return ZarrError_Success on success, or an error code on failure. - */ - ZarrError ZarrStreamSettings_set_compression_shuffle( - ZarrStreamSettings* settings, - uint8_t shuffle); + ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type); /** * @brief Reserve space for dimensions in the Zarr stream settings struct. @@ -305,9 +89,9 @@ extern "C" * 32 dimensions. * @param[in, out] settings The Zarr stream settings struct. * @param[in] count The number of dimensions to reserve space for. - * @return ZarrError_Success on success, or an error code on failure. + * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrError ZarrStreamSettings_reserve_dimensions( + ZarrStatus ZarrStreamSettings_reserve_dimensions( ZarrStreamSettings* settings, size_t count); @@ -322,139 +106,77 @@ extern "C" * @param[in, out] settings The Zarr stream settings struct. * @param[in] index The index of the dimension to set. Must be less than the * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. - * @param[in] name The name of the dimension. - * @param[in] bytes_of_name The length of @p name in bytes, including the - * null terminator. - * @param[in] kind The dimension type. - * @param[in] array_size_px The size of the entire array along this - * dimension, in pixels. This value must be positive for all dimensions - * except the first (i.e., the slowest varying dimension). - * @param[in] chunk_size_px The size of the chunks along this dimension, in - * pixels. This value must be positive for all dimensions. - * @param[in] shard_size_chunks The number of chunks in a shard. This value - * must be positive for all dimensions but is ignored for Zarr V2 streams. - * @return ZarrError_Success on success, or an error code on failure. + * @param[in] dimension The dimension's settings. */ - ZarrError ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, - size_t index, - const char* name, - size_t bytes_of_name, - ZarrDimensionType kind, - uint32_t array_size_px, - uint32_t chunk_size_px, - uint32_t shard_size_chunks); + ZarrStatus ZarrStreamSettings_set_dimension( + ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension); /** * @brief Set the multiscale flag for the Zarr stream. * @param[in, out] settings The Zarr stream settings struct. * @param[in] multiscale A flag indicating whether to stream to multiple * levels of detail. - * @return ZarrError_Success on success, or an error code on failure. + * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrError ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, - uint8_t multiscale); + ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale); - /*************************************************************************** - * Functions for getting parameters on the Zarr stream settings struct. - * - * These functions return the value of the specified parameter. - * If the struct is NULL, the functions return NULL or 0. - **************************************************************************/ + /** + * @brief Set JSON-formatted custom metadata for the Zarr stream. + * @details This metadata will be written to acquire-zarr.json in the + * metadata directory of the Zarr store. This parameter is optional. + * @param settings[in, out] settings The Zarr stream settings struct. + * @param external_metadata JSON-formatted external metadata. + * @param bytes_of_external_metadata The length of @p custom_metadata in + * bytes, including the null terminator. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_custom_metadata( + ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata); const char* ZarrStreamSettings_get_store_path( const ZarrStreamSettings* settings); - const char* ZarrStreamSettings_get_s3_endpoint( - const ZarrStreamSettings* settings); - const char* ZarrStreamSettings_get_s3_bucket_name( - const ZarrStreamSettings* settings); - const char* ZarrStreamSettings_get_s3_access_key_id( - const ZarrStreamSettings* settings); - const char* ZarrStreamSettings_get_s3_secret_access_key( - const ZarrStreamSettings* settings); - const char* ZarrStreamSettings_get_external_metadata( + ZarrS3Settings ZarrStreamSettings_get_s3_settings( const ZarrStreamSettings* settings); - ZarrDataType ZarrStreamSettings_get_data_type( + ZarrCompressionSettings ZarrStreamSettings_get_compression( const ZarrStreamSettings* settings); - ZarrCompressor ZarrStreamSettings_get_compressor( - const ZarrStreamSettings* settings); - ZarrCompressionCodec ZarrStreamSettings_get_compression_codec( - const ZarrStreamSettings* settings); - uint8_t ZarrStreamSettings_get_compression_level( - const ZarrStreamSettings* settings); - uint8_t ZarrStreamSettings_get_compression_shuffle( + ZarrDataType ZarrStreamSettings_get_data_type( const ZarrStreamSettings* settings); size_t ZarrStreamSettings_get_dimension_count( const ZarrStreamSettings* settings); - /** - * @brief Get the properties for an acquisition dimension. - * @param[in] settings The Zarr stream settings struct. - * @param[in] index The index of the dimension to get. Must be less than the - * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. - * @param[out] name The name of the dimension. - * @param[out] bytes_of_name The number of bytes in @p name. The function - * will fail if this value is less than the number of bytes in the stored - * name. - * @param[out] kind The dimension type. - * @param[out] array_size_px The size of the entire array along this - * dimension, in pixels. - * @param[out] chunk_size_px The size of a chunk along this dimension, in - * pixels. - * @param[out] shard_size_chunks The number of chunks in a shard. - * @return ZarrError_Success on success, or an error code on failure. - */ - ZarrError ZarrStreamSettings_get_dimension( + ZarrDimensionProperties ZarrStreamSettings_get_dimension( const ZarrStreamSettings* settings, - size_t index, - char* name, - size_t bytes_of_name, - ZarrDimensionType* kind, - size_t* array_size_px, - size_t* chunk_size_px, - size_t* shard_size_chunks); - - uint8_t ZarrStreamSettings_get_multiscale( - const ZarrStreamSettings* settings); - - /*************************************************************************** - * Zarr stream - * - * The Zarr stream struct is created using ZarrStream_create and destroyed - * using ZarrStream_destroy. If the struct fails to be created, - * ZarrStream_create returns NULL. - **************************************************************************/ + size_t index); - typedef enum - { - ZarrVersion_2 = 2, - ZarrVersion_3, - ZarrVersionCount - } ZarrVersion; + bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); - typedef struct ZarrStream_s ZarrStream; + const char* ZarrStreamSettings_get_custom_metadata( + const ZarrStreamSettings* settings); /** * @brief Create a Zarr stream. - * @param[in, out] settings The settings for the Zarr stream. This pointer - * is consumed by this function. + * @param[in, out] settings The settings for the Zarr stream. * @param[in] version The version of the Zarr stream. 2 or 3. * @return A pointer to the Zarr stream struct, or NULL on failure. */ ZarrStream* ZarrStream_create(ZarrStreamSettings* settings, ZarrVersion version); - void ZarrStream_destroy(ZarrStream* stream); - /*************************************************************************** - * Writing data to the Zarr stream. - * - * This function writes data to the Zarr stream. It returns a ZarrError to - * indicate success or failure, which can be converted to a human-readable - * error message using Zarr_get_error_message(). - **************************************************************************/ + /** + * @brief Destroy a Zarr stream. + * @details This function frees the memory allocated for the Zarr stream. + * @param stream The Zarr stream struct to destroy. + */ + void ZarrStream_destroy(ZarrStream* stream); /** * @brief Append data to the Zarr stream. @@ -463,81 +185,29 @@ extern "C" * @param[in] bytes_in The number of bytes in @p data. It should be at least * the size of a single frame. * @param[out] bytes_out The number of bytes written to the stream. - * @return ZarrError_Success on success, or an error code on failure. + * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrError ZarrStream_append(ZarrStream* stream, - const void* data, - size_t bytes_in, - size_t* bytes_out); - - /*************************************************************************** - * Functions for getting parameters on the Zarr stream. - * - * These functions return the value of the specified parameter. - * If the Zarr stream is NULL, the functions return NULL or 0. - **************************************************************************/ + ZarrStatus ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out); + /** + * @brief Get the version (i.e., 2 or 3) of the Zarr stream. + * @param stream The Zarr stream struct. + * @return The version of the Zarr stream. + */ ZarrVersion ZarrStream_get_version(const ZarrStream* stream); - const char* ZarrStream_get_store_path(const ZarrStream* stream); - const char* ZarrStream_get_s3_endpoint(const ZarrStream* stream); - const char* ZarrStream_get_s3_bucket_name(const ZarrStream* stream); - const char* ZarrStream_get_s3_access_key_id(const ZarrStream* stream); - const char* ZarrStream_get_s3_secret_access_key(const ZarrStream* stream); - - const char* ZarrStream_get_external_metadata(const ZarrStream* stream); - - ZarrCompressor ZarrStream_get_compressor(const ZarrStream* stream); - ZarrCompressionCodec ZarrStream_get_compression_codec( - const ZarrStream* stream); - uint8_t ZarrStream_get_compression_level(const ZarrStream* stream); - uint8_t ZarrStream_get_compression_shuffle(const ZarrStream* stream); - - size_t ZarrStream_get_dimension_count(const ZarrStream* stream); - /** - * @brief Get the properties for an acquisition dimension. - * @param[in] stream The Zarr stream struct. - * @param[in] index The index of the dimension to get. Must be less than the - * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. - * @param[out] name The name of the dimension. - * @param[out] bytes_of_name The number of bytes in @p name. The function - * will fail if this value is less than the number of bytes in the stored - * name. - * @param[out] kind The dimension type. - * @param[out] array_size_px The size of the entire array along this - * dimension, in pixels. - * @param[out] chunk_size_px The size of a chunk along this dimension, in - * pixels. - * @param[out] shard_size_chunks The number of chunks in a shard. - * @return ZarrError_Success on success, or an error code on failure. + * @brief Get a copy of the settings for the Zarr stream. + * @param stream The Zarr stream struct. + * @return A copy of the settings for the Zarr stream. */ - ZarrError ZarrStream_get_dimension(const ZarrStream* stream, - size_t index, - char* name, - size_t bytes_of_name, - ZarrDimensionType* kind, - size_t* array_size_px, - size_t* chunk_size_px, - size_t* shard_size_chunks); - - uint8_t ZarrStream_get_multiscale(const ZarrStream* stream); - - /** @brief Get a copy of the stream settings. */ ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); - /*************************************************************************** - * Logging - * - * The Zarr library uses a logging system to output messages. The log level - * can be set using Zarr_set_log_level. The log level can be retrieved using - * Zarr_get_log_level. The log level can be set to one of the values in the - * in the LogLevel enum. Use LogLevel_None to suppress all log messages. By - * default, the log level is set to LogLevel_Info. - **************************************************************************/ - - ZarrError Zarr_set_log_level(LogLevel level); - LogLevel Zarr_get_log_level(); + ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + ZarrLogLevel Zarr_get_log_level(); #ifdef __cplusplus } diff --git a/include/zarr.types.h b/include/zarr.types.h new file mode 100644 index 00000000..fdb6c715 --- /dev/null +++ b/include/zarr.types.h @@ -0,0 +1,133 @@ +#ifndef H_ACQUIRE_ZARR_TYPES_V0 +#define H_ACQUIRE_ZARR_TYPES_V0 + +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef enum + { + ZarrStatus_Success = 0, + ZarrStatus_InvalidArgument, + ZarrStatus_Overflow, + ZarrStatus_InvalidIndex, + ZarrStatus_NotYetImplemented, + ZarrStatus_InternalError, + ZarrStatus_OutOfMemory, + ZarrStatus_IOError, + ZarrStatus_CompressionError, + ZarrStatus_InvalidSettings, + ZarrStatusCount, + } ZarrStatus; + + typedef enum + { + ZarrVersion_2 = 2, + ZarrVersion_3, + ZarrVersionCount + } ZarrVersion; + + typedef enum + { + ZarrLogLevel_Debug, + ZarrLogLevel_Info, + ZarrLogLevel_Warning, + ZarrLogLevel_Error, + ZarrLogLevel_None, + ZarrLogLevelCount + } ZarrLogLevel; + + typedef enum + { + ZarrDataType_uint8, + ZarrDataType_uint16, + ZarrDataType_uint32, + ZarrDataType_uint64, + ZarrDataType_int8, + ZarrDataType_int16, + ZarrDataType_int32, + ZarrDataType_int64, + ZarrDataType_float32, + ZarrDataType_float64, + ZarrDataTypeCount + } ZarrDataType; + + typedef enum + { + ZarrCompressor_None = 0, + ZarrCompressor_Blosc1, + ZarrCompressorCount + } ZarrCompressor; + + typedef enum + { + ZarrCompressionCodec_None = 0, + ZarrCompressionCodec_BloscLZ4, + ZarrCompressionCodec_BloscZstd, + ZarrCompressionCodecCount + } ZarrCompressionCodec; + + typedef enum + { + ZarrDimensionType_Space = 0, + ZarrDimensionType_Channel, + ZarrDimensionType_Time, + ZarrDimensionType_Other, + ZarrDimensionTypeCount + } ZarrDimensionType; + + /** + * @brief S3 settings for streaming to Zarr. + */ + typedef struct + { + const char* endpoint; + size_t bytes_of_endpoint; + const char* bucket_name; + size_t bytes_of_bucket_name; + const char* access_key_id; + size_t bytes_of_access_key_id; + const char* secret_access_key; + size_t bytes_of_secret_access_key; + } ZarrS3Settings; + + /** + * @brief Compression settings for a Zarr array. + * @detail The compressor is not the same as the codec. A codec is + * a specific implementation of a compression algorithm, while a compressor + * is a library that implements one or more codecs. + */ + typedef struct + { + ZarrCompressor compressor; /**< Compressor to use */ + ZarrCompressionCodec codec; /**< Codec to use */ + uint8_t level; /**< Compression level */ + uint8_t shuffle; /**< Whether to shuffle the data before compressing */ + } ZarrCompressionSettings; + + /** + * @brief Properties of a dimension of the Zarr array. + */ + typedef struct + { + const char* name; /**< Name of the dimension */ + size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ + ZarrDimensionType kind; /**< Type of the dimension */ + uint32_t array_size_px; /**< Size of the array along this dimension in + pixels */ + uint32_t chunk_size_px; /**< Size of the chunks along this dimension in + pixels */ + uint32_t shard_size_chunks; /**< Number of chunks in a shard along this + dimension */ + } ZarrDimensionProperties; + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_TYPES_V0 \ No newline at end of file diff --git a/src/streaming/CMakeLists.txt b/src/streaming/CMakeLists.txt index 2ddb3f19..7a37d529 100644 --- a/src/streaming/CMakeLists.txt +++ b/src/streaming/CMakeLists.txt @@ -9,12 +9,11 @@ add_library(${tgt} zarr.stream.cpp ) -include(CMakePrintHelpers) -cmake_print_variables(CMAKE_SOURCE_DIR) +set(PUBLIC_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include/) target_include_directories(${tgt} PUBLIC - $ + $ PRIVATE $ ) @@ -34,7 +33,7 @@ install(TARGETS ${tgt} ) # Install public header files -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ +install(DIRECTORY ${PUBLIC_INCLUDE_DIR} DESTINATION include FILES_MATCHING PATTERN "*.h" ) \ No newline at end of file diff --git a/src/streaming/logger.cpp b/src/streaming/logger.cpp index 4f177a1a..406d49da 100644 --- a/src/streaming/logger.cpp +++ b/src/streaming/logger.cpp @@ -6,29 +6,29 @@ #include #include -LogLevel Logger::current_level = LogLevel_Info; +ZarrLogLevel Logger::current_level = ZarrLogLevel_Info; void -Logger::set_log_level(LogLevel level) +Logger::set_log_level(ZarrLogLevel level) { current_level = level; } -LogLevel +ZarrLogLevel Logger::get_log_level() { return current_level; } std::string -Logger::log(LogLevel level, +Logger::log(ZarrLogLevel level, const char* file, int line, const char* func, const char* format, ...) { - if (current_level == LogLevel_None || level < current_level) { + if (current_level == ZarrLogLevel_None || level < current_level) { return {}; // Suppress logs } @@ -39,17 +39,17 @@ Logger::log(LogLevel level, std::ostream* stream = &std::cout; switch (level) { - case LogLevel_Debug: + case ZarrLogLevel_Debug: prefix = "[DEBUG] "; break; - case LogLevel_Info: + case ZarrLogLevel_Info: prefix = "[INFO] "; break; - case LogLevel_Warning: + case ZarrLogLevel_Warning: prefix = "[WARNING] "; stream = &std::cerr; break; - case LogLevel_Error: + case ZarrLogLevel_Error: prefix = "[ERROR] "; stream = &std::cerr; break; diff --git a/src/streaming/logger.hh b/src/streaming/logger.hh index c2417a91..4521bbcc 100644 --- a/src/streaming/logger.hh +++ b/src/streaming/logger.hh @@ -5,10 +5,10 @@ class Logger { public: - static void set_log_level(LogLevel level); - static LogLevel get_log_level(); + static void set_log_level(ZarrLogLevel level); + static ZarrLogLevel get_log_level(); - static std::string log(LogLevel level, + static std::string log(ZarrLogLevel level, const char* file, int line, const char* func, @@ -16,17 +16,17 @@ class Logger ...); private: - static LogLevel current_level; + static ZarrLogLevel current_level; }; #define LOG_DEBUG(...) \ - Logger::log(LogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(ZarrLogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_INFO(...) \ Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_WARNING(...) \ - Logger::log(LogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(ZarrLogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_ERROR(...) \ - Logger::log(LogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(ZarrLogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) #define EXPECT(e, ...) \ do { \ diff --git a/src/streaming/stream.settings.cpp b/src/streaming/stream.settings.cpp index d1754113..19b78ca7 100644 --- a/src/streaming/stream.settings.cpp +++ b/src/streaming/stream.settings.cpp @@ -6,31 +6,22 @@ #include #include // memcpy +#include #define EXPECT_VALID_ARGUMENT(e, ...) \ do { \ if (!(e)) { \ LOG_ERROR(__VA_ARGS__); \ - return ZarrError_InvalidArgument; \ + return ZarrStatus_InvalidArgument; \ } \ } while (0) -#define ZARR_DIMENSION_MIN 3 -#define ZARR_DIMENSION_MAX 32 - -#define SETTINGS_SET_STRING(settings, member, bytes_of_member) \ +#define EXPECT_VALID_INDEX(e, ...) \ do { \ - if (!(settings)) { \ - LOG_ERROR("Null pointer: %s", #settings); \ - return ZarrError_InvalidArgument; \ - } \ - if (!(member)) { \ - LOG_ERROR("Null pointer: %s", #member); \ - return ZarrError_InvalidArgument; \ + if (!(e)) { \ + LOG_ERROR(__VA_ARGS__); \ + return ZarrStatus_InvalidIndex; \ } \ - size_t nbytes = strnlen(member, bytes_of_member); \ - settings->member = { member, nbytes }; \ - return ZarrError_Success; \ } while (0) #define SETTINGS_GET_STRING(settings, member) \ @@ -42,7 +33,12 @@ return settings->member.c_str(); \ } while (0) +namespace fs = std::filesystem; + namespace { +const size_t zarr_dimension_min = 3; +const size_t zarr_dimension_max = 32; + const char* compressor_to_string(ZarrCompressor compressor) { @@ -71,7 +67,7 @@ compression_codec_to_string(ZarrCompressionCodec codec) } } -inline std::string +std::string trim(const char* s, size_t bytes_of_s) { // trim left @@ -90,6 +86,140 @@ trim(const char* s, size_t bytes_of_s) return trimmed; } + +[[nodiscard]] +bool +validate_s3_settings(const ZarrS3Settings* settings) +{ + size_t len; + + if (len = strnlen_s(settings->endpoint, settings->bytes_of_endpoint); + len == 0) { + LOG_ERROR("S3 endpoint is empty"); + return false; + } + + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html + if (len = strnlen_s(settings->bucket_name, settings->bytes_of_bucket_name); + len < 4 || len > 64) { + LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " + "and 63 characters", + len); + return false; + } + + if (len = + strnlen_s(settings->access_key_id, settings->bytes_of_access_key_id); + len == 0) { + LOG_ERROR("S3 access key ID is empty"); + return false; + } + + if (len = strnlen_s(settings->secret_access_key, + settings->bytes_of_secret_access_key); + len == 0) { + LOG_ERROR("S3 secret access key is empty"); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_filesystem_store_path(std::string_view data_root) +{ + fs::path path(data_root); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) { + parent_path = "."; + } + + // parent path must exist and be a directory + if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { + LOG_ERROR("Parent path '%s' does not exist or is not a directory", + parent_path.c_str()); + return false; + } + + // parent path must be writable + const auto perms = fs::status(parent_path).permissions(); + const bool is_writable = + (perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none; + + if (!is_writable) { + LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_compression_settings(const ZarrCompressionSettings* settings) +{ + if (settings->compressor >= ZarrCompressorCount) { + LOG_ERROR("Invalid compressor: %d", settings->compressor); + return false; + } + + if (settings->codec >= ZarrCompressionCodecCount) { + LOG_ERROR("Invalid compression codec: %d", settings->codec); + return false; + } + + // if compressing, we require a compression codec + if (settings->compressor != ZarrCompressor_None && + settings->codec == ZarrCompressionCodec_None) { + LOG_ERROR("Compression codec must be set when using a compressor"); + return false; + } + + if (settings->level > 9) { + LOG_ERROR("Invalid compression level: %d. Must be between 0 and 9", + settings->level); + return false; + } + + if (settings->shuffle != BLOSC_NOSHUFFLE && + settings->shuffle != BLOSC_SHUFFLE && + settings->shuffle != BLOSC_BITSHUFFLE) { + LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " + "shuffle), or %d (bit shuffle)", + settings->shuffle, + BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, + BLOSC_BITSHUFFLE); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_dimension(const ZarrDimensionProperties* dimension) +{ + size_t len = strnlen(dimension->name, dimension->bytes_of_name); + if (len == 0) { + LOG_ERROR("Invalid name. Must not be empty"); + return false; + } + + if (dimension->kind >= ZarrDimensionTypeCount) { + LOG_ERROR("Invalid dimension type: %d", dimension->kind); + return false; + } + + if (dimension->chunk_size_px == 0) { + LOG_ERROR("Invalid chunk size: %zu", dimension->chunk_size_px); + return false; + } + + return true; +} } // namespace /* Create and destroy */ @@ -123,110 +253,116 @@ ZarrStreamSettings_copy(const ZarrStreamSettings* settings) return nullptr; } - copy->store_path = settings->store_path; - copy->s3_endpoint = settings->s3_endpoint; - copy->s3_bucket_name = settings->s3_bucket_name; - copy->s3_access_key_id = settings->s3_access_key_id; - copy->s3_secret_access_key = settings->s3_secret_access_key; - copy->external_metadata = settings->external_metadata; - copy->dtype = settings->dtype; - copy->compressor = settings->compressor; - copy->compression_codec = settings->compression_codec; - copy->compression_level = settings->compression_level; - copy->compression_shuffle = settings->compression_shuffle; - copy->dimensions = settings->dimensions; - copy->multiscale = settings->multiscale; + *copy = *settings; return copy; } /* Setters */ -ZarrError -ZarrStreamSettings_set_store_path(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path) +ZarrStatus +ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings) { - SETTINGS_SET_STRING(settings, store_path, bytes_of_store_path); -} + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(store_path, "Null pointer: store_path"); -ZarrError -ZarrStreamSettings_set_s3_endpoint(ZarrStreamSettings* settings, - const char* s3_endpoint, - size_t bytes_of_s3_endpoint) -{ - SETTINGS_SET_STRING(settings, s3_endpoint, bytes_of_s3_endpoint); -} + bytes_of_store_path = strnlen(store_path, bytes_of_store_path); + EXPECT_VALID_ARGUMENT(bytes_of_store_path > 1, + "Invalid store path. Must not be empty"); -ZarrError -ZarrStreamSettings_set_s3_bucket_name(ZarrStreamSettings* settings, - const char* s3_bucket_name, - size_t bytes_of_s3_bucket_name) -{ - SETTINGS_SET_STRING(settings, s3_bucket_name, bytes_of_s3_bucket_name); -} + std::string_view store_path_sv(store_path, bytes_of_store_path); + if (store_path_sv.empty()) { + LOG_ERROR("Invalid store path. Must not be empty"); + return ZarrStatus_InvalidArgument; + } -ZarrError -ZarrStreamSettings_set_s3_access_key_id(ZarrStreamSettings* settings, - const char* s3_access_key_id, - size_t bytes_of_s3_access_key_id) -{ - SETTINGS_SET_STRING(settings, s3_access_key_id, bytes_of_s3_access_key_id); + if (nullptr != s3_settings) { + if (!validate_s3_settings(s3_settings)) { + return ZarrStatus_InvalidArgument; + } + } else if (!validate_filesystem_store_path(store_path)) { + return ZarrStatus_InvalidArgument; + } + + if (nullptr != s3_settings) { + settings->s3_endpoint = s3_settings->endpoint; + settings->s3_bucket_name = s3_settings->bucket_name; + settings->s3_access_key_id = s3_settings->access_key_id; + settings->s3_secret_access_key = s3_settings->secret_access_key; + } + + settings->store_path = store_path; + + return ZarrStatus_Success; } -ZarrError -ZarrStreamSettings_set_s3_secret_access_key( +ZarrStatus +ZarrStreamSettings_set_compression( ZarrStreamSettings* settings, - const char* s3_secret_access_key, - size_t bytes_of_s3_secret_access_key) + const ZarrCompressionSettings* compression_settings) { - SETTINGS_SET_STRING( - settings, s3_secret_access_key, bytes_of_s3_secret_access_key); + EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); + EXPECT_VALID_ARGUMENT(compression_settings, + "Null pointer: compression_settings"); + + if (!validate_compression_settings(compression_settings)) { + return ZarrStatus_InvalidArgument; + } + + settings->compressor = compression_settings->compressor; + settings->compression_codec = compression_settings->codec; + settings->compression_level = compression_settings->level; + settings->compression_shuffle = compression_settings->shuffle; + + return ZarrStatus_Success; } -ZarrError -ZarrStreamSettings_set_external_metadata(ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata) +ZarrStatus +ZarrStreamSettings_set_custom_metadata(ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata) { if (!settings) { LOG_ERROR("Null pointer: settings"); - return ZarrError_InvalidArgument; + return ZarrStatus_InvalidArgument; } if (!external_metadata) { - LOG_ERROR("Null pointer: external_metadata"); - return ZarrError_InvalidArgument; + LOG_ERROR("Null pointer: custom_metadata"); + return ZarrStatus_InvalidArgument; } if (bytes_of_external_metadata == 0) { LOG_ERROR("Invalid length: %zu. Must be greater than 0", bytes_of_external_metadata); - return ZarrError_InvalidArgument; + return ZarrStatus_InvalidArgument; } - const size_t nbytes = - strnlen(external_metadata, bytes_of_external_metadata); + size_t nbytes = strnlen(external_metadata, bytes_of_external_metadata); if (nbytes < 2) { - return ZarrError_Success; + settings->custom_metadata = "{}"; + return ZarrStatus_Success; } - auto val = nlohmann::json::parse(external_metadata, - external_metadata + nbytes, - nullptr, // callback - false, // allow exceptions - true // ignore comments + auto val = nlohmann::json::parse(external_metadata, + external_metadata + nbytes, + nullptr, // callback + false, // allow exceptions + true // ignore comments ); if (val.is_discarded()) { LOG_ERROR("Invalid JSON: %s", external_metadata); - return ZarrError_InvalidArgument; + return ZarrStatus_InvalidArgument; } - settings->external_metadata = val.dump(); + settings->custom_metadata = val.dump(); - return ZarrError_Success; + return ZarrStatus_Success; } -ZarrError +ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, ZarrDataType data_type) { @@ -235,138 +371,60 @@ ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, data_type < ZarrDataTypeCount, "Invalid pixel type: %d", data_type); settings->dtype = data_type; - return ZarrError_Success; -} - -ZarrError -ZarrStreamSettings_set_compressor(ZarrStreamSettings* settings, - ZarrCompressor compressor) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT( - compressor < ZarrCompressorCount, "Invalid compressor: %d", compressor); - - settings->compressor = compressor; - return ZarrError_Success; -} - -ZarrError -ZarrStreamSettings_set_compression_codec(ZarrStreamSettings* settings, - ZarrCompressionCodec codec) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(codec < ZarrCompressionCodecCount, - "Invalid codec: %s", - compression_codec_to_string(codec)); - - settings->compression_codec = codec; - return ZarrError_Success; -} - -ZarrError -ZarrStreamSettings_set_compression_level(ZarrStreamSettings* settings, - uint8_t level) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(level >= 0 && level <= 9, - "Invalid level: %d. Must be between 0 (no " - "compression) and 9 (maximum compression).", - level); - - settings->compression_level = level; - - return ZarrError_Success; -} - -ZarrError -ZarrStreamSettings_set_compression_shuffle(ZarrStreamSettings* settings, - uint8_t shuffle) -{ - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(shuffle == BLOSC_NOSHUFFLE || - shuffle == BLOSC_SHUFFLE || - shuffle == BLOSC_BITSHUFFLE, - "Invalid shuffle: %d. Must be %d (no shuffle), %d " - "(byte shuffle), or %d (bit shuffle)", - shuffle, - BLOSC_NOSHUFFLE, - BLOSC_SHUFFLE, - BLOSC_BITSHUFFLE); - - settings->compression_shuffle = shuffle; - return ZarrError_Success; + return ZarrStatus_Success; } -ZarrError +ZarrStatus ZarrStreamSettings_reserve_dimensions(ZarrStreamSettings* settings, size_t count) { EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(count >= ZARR_DIMENSION_MIN && - count <= ZARR_DIMENSION_MAX, + EXPECT_VALID_ARGUMENT(count >= zarr_dimension_min && + count <= zarr_dimension_max, "Invalid count: %zu. Count must be between %d and %d", count, - ZARR_DIMENSION_MIN, - ZARR_DIMENSION_MAX); + zarr_dimension_min, + zarr_dimension_max); settings->dimensions.resize(count); - return ZarrError_Success; + return ZarrStatus_Success; } -ZarrError +ZarrStatus ZarrStreamSettings_set_dimension(ZarrStreamSettings* settings, size_t index, - const char* name, - size_t bytes_of_name, - ZarrDimensionType kind, - uint32_t array_size_px, - uint32_t chunk_size_px, - uint32_t shard_size_chunks) + const ZarrDimensionProperties* dimension) { EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(name, "Null pointer: name"); - EXPECT_VALID_ARGUMENT( - bytes_of_name > 0, "Invalid name length: %zu", bytes_of_name); - EXPECT_VALID_ARGUMENT(strnlen(name, bytes_of_name) > 0, - "Invalid name. Must not be empty"); - EXPECT_VALID_ARGUMENT( - kind < ZarrDimensionTypeCount, "Invalid dimension type: %d", kind); - EXPECT_VALID_ARGUMENT( - chunk_size_px > 0, "Invalid chunk size: %zu", chunk_size_px); - - // Check that the index is within bounds - if (index >= settings->dimensions.size()) { - LOG_ERROR("Invalid index: %zu. Must be less than %zu", - index, - settings->dimensions.size()); - return ZarrError_InvalidIndex; + EXPECT_VALID_ARGUMENT(dimension, "Null pointer: dimension"); + EXPECT_VALID_INDEX(index < settings->dimensions.size(), + "Invalid index: %zu. Must be less than %zu", + index, + settings->dimensions.size()); + + if (!validate_dimension(dimension)) { + return ZarrStatus_InvalidArgument; } - std::string dim_name = trim(name, bytes_of_name); - if (dim_name.empty()) { - LOG_ERROR("Invalid name. Must not be empty"); - return ZarrError_InvalidArgument; - } + struct ZarrDimension_s& dim = settings->dimensions[index]; - struct ZarrDimension_s* dim = &settings->dimensions[index]; + dim.name = dimension->name; + dim.kind = dimension->kind; + dim.array_size_px = dimension->array_size_px; + dim.chunk_size_px = dimension->chunk_size_px; + dim.shard_size_chunks = dimension->shard_size_chunks; - dim->name = dim_name; - dim->kind = kind; - dim->array_size_px = array_size_px; - dim->chunk_size_px = chunk_size_px; - dim->shard_size_chunks = shard_size_chunks; - - return ZarrError_Success; + return ZarrStatus_Success; } -ZarrError +ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, uint8_t multiscale) { EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); settings->multiscale = multiscale > 0; - return ZarrError_Success; + return ZarrStatus_Success; } /* Getters */ @@ -376,34 +434,31 @@ ZarrStreamSettings_get_store_path(const ZarrStreamSettings* settings) SETTINGS_GET_STRING(settings, store_path); } -const char* -ZarrStreamSettings_get_s3_endpoint(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, s3_endpoint); -} - -const char* -ZarrStreamSettings_get_s3_bucket_name(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, s3_bucket_name); -} - -const char* -ZarrStreamSettings_get_s3_access_key_id(const ZarrStreamSettings* settings) +ZarrS3Settings +ZarrStreamSettings_get_s3_settings(const ZarrStreamSettings* settings) { - SETTINGS_GET_STRING(settings, s3_access_key_id); -} + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty S3 settings."); + return {}; + } -const char* -ZarrStreamSettings_get_s3_secret_access_key(const ZarrStreamSettings* settings) -{ - SETTINGS_GET_STRING(settings, s3_secret_access_key); + ZarrS3Settings s3_settings = { + settings->s3_endpoint.c_str(), + settings->s3_endpoint.length(), + settings->s3_bucket_name.c_str(), + settings->s3_bucket_name.length(), + settings->s3_access_key_id.c_str(), + settings->s3_access_key_id.length(), + settings->s3_secret_access_key.c_str(), + settings->s3_secret_access_key.length(), + }; + return s3_settings; } const char* -ZarrStreamSettings_get_external_metadata(const ZarrStreamSettings* settings) +ZarrStreamSettings_get_custom_metadata(const ZarrStreamSettings* settings) { - SETTINGS_GET_STRING(settings, external_metadata); + SETTINGS_GET_STRING(settings, custom_metadata); } ZarrDataType @@ -416,45 +471,21 @@ ZarrStreamSettings_get_data_type(const ZarrStreamSettings* settings) return static_cast(settings->dtype); } -ZarrCompressor -ZarrStreamSettings_get_compressor(const ZarrStreamSettings* settings) +ZarrCompressionSettings +ZarrStreamSettings_get_compression(const ZarrStreamSettings* settings) { if (!settings) { - LOG_WARNING("Null pointer: settings. Returning ZarrCompressor_None."); - return ZarrCompressor_None; + LOG_WARNING("Null pointer: settings. Returning empty compression."); + return {}; } - return static_cast(settings->compressor); -} - -ZarrCompressionCodec -ZarrStreamSettings_get_compression_codec(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING( - "Null pointer: settings. Returning ZarrCompressionCodec_None."); - return ZarrCompressionCodec_None; - } - return static_cast(settings->compression_codec); -} - -uint8_t -ZarrStreamSettings_get_compression_level(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning 0."); - return 0; - } - return settings->compression_level; -} -uint8_t -ZarrStreamSettings_get_compression_shuffle(const ZarrStreamSettings* settings) -{ - if (!settings) { - LOG_WARNING("Null pointer: settings. Returning 0."); - return 0; - } - return settings->compression_shuffle; + ZarrCompressionSettings compression = { + .compressor = settings->compressor, + .codec = static_cast(settings->compression_codec), + .level = settings->compression_level, + .shuffle = settings->compression_shuffle, + }; + return compression; } size_t @@ -467,63 +498,42 @@ ZarrStreamSettings_get_dimension_count(const ZarrStreamSettings* settings) return settings->dimensions.size(); } -ZarrError +ZarrDimensionProperties ZarrStreamSettings_get_dimension(const ZarrStreamSettings* settings, - size_t index, - char* name, - size_t bytes_of_name, - ZarrDimensionType* kind, - size_t* array_size_px, - size_t* chunk_size_px, - size_t* shard_size_chunks) + size_t index) { - EXPECT_VALID_ARGUMENT(settings, "Null pointer: settings"); - EXPECT_VALID_ARGUMENT(name, "Null pointer: name"); - EXPECT_VALID_ARGUMENT(kind, "Null pointer: kind"); - EXPECT_VALID_ARGUMENT(array_size_px, "Null pointer: array_size_px"); - EXPECT_VALID_ARGUMENT(chunk_size_px, "Null pointer: chunk_size_px"); - EXPECT_VALID_ARGUMENT(shard_size_chunks, "Null pointer: shard_size_chunks"); + if (!settings) { + LOG_WARNING("Null pointer: settings. Returning empty dimension."); + return {}; + } if (index >= settings->dimensions.size()) { LOG_ERROR("Invalid index: %zu. Must be less than %zu", index, settings->dimensions.size()); - return ZarrError_InvalidIndex; + return {}; } - const struct ZarrDimension_s* dim = &settings->dimensions[index]; + const auto& dim = settings->dimensions[index]; - if (bytes_of_name < dim->name.length() + 1) { - LOG_ERROR("Insufficient buffer size: %zu. Need at least %zu", - bytes_of_name, - dim->name.length() + 1); - return ZarrError_Overflow; - } + ZarrDimensionProperties dimension = { + .name = dim.name.c_str(), + .bytes_of_name = dim.name.size() + 1, + .kind = dim.kind, + .array_size_px = dim.array_size_px, + .chunk_size_px = dim.chunk_size_px, + .shard_size_chunks = dim.shard_size_chunks, + }; - memcpy(name, dim->name.c_str(), dim->name.length() + 1); - *kind = static_cast(dim->kind); - *array_size_px = dim->array_size_px; - *chunk_size_px = dim->chunk_size_px; - *shard_size_chunks = dim->shard_size_chunks; - - return ZarrError_Success; + return dimension; } -uint8_t +bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings) { if (!settings) { - LOG_WARNING("Null pointer: settings. Returning 0."); - return 0; + LOG_WARNING("Null pointer: settings. Returning false."); + return false; } - return static_cast(settings->multiscale); -} - -/* Internal functions */ - -bool -validate_dimension(const struct ZarrDimension_s& dimension) -{ - return !dimension.name.empty() && dimension.kind < ZarrDimensionTypeCount && - dimension.chunk_size_px > 0; + return settings->multiscale; } diff --git a/src/streaming/stream.settings.hh b/src/streaming/stream.settings.hh index 0342b152..6f228469 100644 --- a/src/streaming/stream.settings.hh +++ b/src/streaming/stream.settings.hh @@ -1,5 +1,7 @@ #pragma once +#include "zarr.types.h" + #include // size_t #include // uint8_t #include @@ -7,8 +9,8 @@ struct ZarrDimension_s { - std::string name; /* Name of the dimension */ - uint8_t kind; /* Type of dimension */ + std::string name; /* Name of the dimension */ + ZarrDimensionType kind; /* Type of dimension */ uint32_t array_size_px; /* Size of the array along this dimension */ uint32_t chunk_size_px; /* Size of a chunk along this dimension */ @@ -25,21 +27,18 @@ struct ZarrStreamSettings_s std::string s3_access_key_id; /* Access key ID for the S3 service */ std::string s3_secret_access_key; /* Secret access key for the S3 service */ - std::string external_metadata; /* JSON formatted external metadata for the + std::string custom_metadata; /* JSON formatted external metadata for the base array */ - uint8_t dtype; /* Data type of the base array */ + ZarrDataType dtype; /* Data type of the base array */ - uint8_t compressor; /* Compression library to use */ - uint8_t compression_codec; /* Compression codec to use */ - uint8_t compression_level; /* Compression level to use */ + ZarrCompressor compressor; /* Compression library to use */ + ZarrCompressionCodec compression_codec; /* Compression codec to use */ + uint8_t compression_level; /* Compression level to use */ uint8_t compression_shuffle; /* Whether and how to shuffle the data before - compressing */ + compressing */ std::vector dimensions; /* Dimensions of the base array */ bool multiscale; /* Whether to stream to multiple resolutions */ }; - -bool -validate_dimension(const struct ZarrDimension_s& dimension); diff --git a/src/streaming/zarr.stream.cpp b/src/streaming/zarr.stream.cpp index 4cd54185..a8a834a8 100644 --- a/src/streaming/zarr.stream.cpp +++ b/src/streaming/zarr.stream.cpp @@ -1,7 +1,8 @@ #include "zarr.stream.hh" #include "logger.hh" +#include "zarr.types.h" -#include "zarr.h" +#include #include @@ -9,7 +10,7 @@ do { \ if (!(e)) { \ LOG_ERROR(__VA_ARGS__); \ - return ZarrError_InvalidArgument; \ + return ZarrStatus_InvalidArgument; \ } \ } while (0) @@ -35,73 +36,73 @@ is_s3_acquisition(const struct ZarrStreamSettings_s* settings) } bool -validate_settings(const struct ZarrStreamSettings_s* settings, - ZarrVersion version) +is_compressed_acquisition(const struct ZarrStreamSettings_s* settings) { - if (!settings) { - LOG_ERROR("Null pointer: settings"); + return settings->compressor != ZarrCompressor_None; +} + +[[nodiscard]] +bool +validate_s3_settings(const struct ZarrStreamSettings_s* settings) +{ + if (settings->s3_endpoint.empty()) { + LOG_ERROR("S3 endpoint is empty"); return false; } - if (version < ZarrVersion_2 || version >= ZarrVersionCount) { - LOG_ERROR("Invalid Zarr version: %d", version); + if (settings->s3_bucket_name.length() < 3 || + settings->s3_bucket_name.length() > 63) { + LOG_ERROR("Invalid length for S3 bucket name: %zu. Must be between 3 " + "and 63 characters", + settings->s3_bucket_name.length()); return false; } - - std::string store_path(settings->store_path); - std::string s3_endpoint(settings->s3_endpoint); - std::string s3_bucket_name(settings->s3_bucket_name); - std::string s3_access_key_id(settings->s3_access_key_id); - std::string s3_secret_access_key(settings->s3_secret_access_key); - - // we require the store_path to be nonempty - if (store_path.empty()) { - LOG_ERROR("Store path is empty"); + if (settings->s3_access_key_id.empty()) { + LOG_ERROR("S3 access key ID is empty"); + return false; + } + if (settings->s3_secret_access_key.empty()) { + LOG_ERROR("S3 secret access key is empty"); return false; } - // if all S3 settings are nonempty, we consider this an S3 store - if (is_s3_acquisition(settings)) { - // check that the S3 endpoint is a valid URL - if (s3_endpoint.find("http://") != 0 && - s3_endpoint.find("https://") != 0) { - LOG_ERROR("Invalid S3 endpoint: %s", s3_endpoint.c_str()); - return false; - } - - // test the S3 connection - // todo (aliddell): implement this - } else { - // if any S3 setting is nonempty, this is a filesystem store - fs::path path(store_path); - fs::path parent_path = path.parent_path(); - if (parent_path.empty()) { - parent_path = "."; - } - - // parent path must exist and be a directory - if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { - LOG_ERROR("Parent path '%s' does not exist or is not a directory", - parent_path.c_str()); - return false; - } + return true; +} - // parent path must be writable - const auto perms = fs::status(parent_path).permissions(); - const bool is_writable = - (perms & (fs::perms::owner_write | fs::perms::group_write | - fs::perms::others_write)) != fs::perms::none; +[[nodiscard]] +bool +validate_filesystem_store_path(std::string_view data_root) +{ + fs::path path(data_root); + fs::path parent_path = path.parent_path(); + if (parent_path.empty()) { + parent_path = "."; + } - if (!is_writable) { - LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); - return false; - } + // parent path must exist and be a directory + if (!fs::exists(parent_path) || !fs::is_directory(parent_path)) { + LOG_ERROR("Parent path '%s' does not exist or is not a directory", + parent_path.c_str()); + return false; } - if (settings->dtype >= ZarrDataTypeCount) { - LOG_ERROR("Invalid data type: %d", settings->dtype); + // parent path must be writable + const auto perms = fs::status(parent_path).permissions(); + const bool is_writable = + (perms & (fs::perms::owner_write | fs::perms::group_write | + fs::perms::others_write)) != fs::perms::none; + + if (!is_writable) { + LOG_ERROR("Parent path '%s' is not writable", parent_path.c_str()); return false; } + return true; +} + +[[nodiscard]] +bool +validate_compression_settings(const ZarrStreamSettings_s* settings) +{ if (settings->compressor >= ZarrCompressorCount) { LOG_ERROR("Invalid compressor: %d", settings->compressor); return false; @@ -119,26 +120,104 @@ validate_settings(const struct ZarrStreamSettings_s* settings, return false; } - // validate the dimensions individually - for (size_t i = 0; i < settings->dimensions.size(); ++i) { - if (!validate_dimension(settings->dimensions[i])) { - LOG_ERROR("Invalid dimension at index %d", i); - return false; - } + if (settings->compression_level > 9) { + LOG_ERROR("Invalid compression level: %d. Must be between 0 and 9", + settings->compression_level); + return false; + } - if (i > 0 && settings->dimensions[i].array_size_px == 0) { - LOG_ERROR("Only the first dimension can have an array size of 0"); - return false; - } + if (settings->compression_shuffle != BLOSC_NOSHUFFLE && + settings->compression_shuffle != BLOSC_SHUFFLE && + settings->compression_shuffle != BLOSC_BITSHUFFLE) { + LOG_ERROR("Invalid shuffle: %d. Must be %d (no shuffle), %d (byte " + "shuffle), or %d (bit shuffle)", + settings->compression_shuffle, + BLOSC_NOSHUFFLE, + BLOSC_SHUFFLE, + BLOSC_BITSHUFFLE); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_dimension(const struct ZarrDimension_s& dimension, + ZarrVersion version, + bool is_append) +{ + if (dimension.name.empty()) { + LOG_ERROR("Invalid name. Must not be empty"); + return false; + } + + if (dimension.kind >= ZarrDimensionTypeCount) { + LOG_ERROR("Invalid dimension type: %d", dimension.kind); + return false; + } + + if (!is_append && dimension.array_size_px == 0) { + LOG_ERROR("Array size must be nonzero"); + return false; + } + + if (dimension.chunk_size_px == 0) { + LOG_ERROR("Chunk size must be nonzero"); + return false; } - // if version 3, we require shard size to be positive - if (version == ZarrVersion_3) { - for (const auto& dim : settings->dimensions) { - if (dim.shard_size_chunks == 0) { - LOG_ERROR("Shard sizes must be positive"); - return false; - } + if (version == ZarrVersion_3 && dimension.shard_size_chunks == 0) { + LOG_ERROR("Shard size must be nonzero"); + return false; + } + + return true; +} + +[[nodiscard]] +bool +validate_settings(const struct ZarrStreamSettings_s* settings, + ZarrVersion version) +{ + if (!settings) { + LOG_ERROR("Null pointer: settings"); + return false; + } + if (version < ZarrVersion_2 || version >= ZarrVersionCount) { + LOG_ERROR("Invalid Zarr version: %d", version); + return false; + } + + std::string_view store_path(settings->store_path); + + // we require the store path (root of the dataset) to be nonempty + if (store_path.empty()) { + LOG_ERROR("Store path is empty"); + return false; + } + + if (is_s3_acquisition(settings) && !validate_s3_settings(settings)) { + return false; + } else if (!is_s3_acquisition(settings) && + !validate_filesystem_store_path(store_path)) { + return false; + } + + if (settings->dtype >= ZarrDataTypeCount) { + LOG_ERROR("Invalid data type: %d", settings->dtype); + return false; + } + + if (is_compressed_acquisition(settings) && + !validate_compression_settings(settings)) { + return false; + } + + // validate the dimensions individually + for (size_t i = 0; i < settings->dimensions.size(); ++i) { + if (!validate_dimension(settings->dimensions[i], version, i == 0)) { + return false; } } @@ -156,18 +235,15 @@ extern "C" } // initialize the stream - ZarrStream_s* stream; + ZarrStream_s* stream = nullptr; try { stream = new ZarrStream(settings, version); } catch (const std::bad_alloc&) { LOG_ERROR("Failed to allocate memory for Zarr stream"); - return nullptr; } catch (const std::exception& e) { LOG_ERROR("Error creating Zarr stream: %s", e.what()); - return nullptr; } - ZarrStreamSettings_destroy(settings); return stream; } @@ -179,10 +255,10 @@ extern "C" /* Appending data */ - ZarrError ZarrStream_append(ZarrStream* stream, - const void* data, - size_t bytes_in, - size_t* bytes_out) + ZarrStatus ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out) { EXPECT_VALID_ARGUMENT(stream, "Null pointer: stream"); EXPECT_VALID_ARGUMENT(data, "Null pointer: data"); @@ -192,127 +268,20 @@ extern "C" *bytes_out = stream->append(data, bytes_in); } catch (const std::exception& e) { LOG_ERROR("Error appending data: %s", e.what()); - return ZarrError_InternalError; + return ZarrStatus_InternalError; } - return ZarrError_Success; + return ZarrStatus_Success; } - /* Getters */ - ZarrVersion ZarrStream_get_version(const ZarrStream* stream) { if (!stream) { LOG_WARNING("Null pointer: stream. Returning ZarrVersion_2"); return ZarrVersion_2; } - return static_cast(stream->version()); - } - - const char* ZarrStream_get_store_path(const ZarrStream* stream) - { - STREAM_GET_STRING(stream, store_path); - } - - const char* ZarrStream_get_s3_endpoint(const ZarrStream* stream) - { - STREAM_GET_STRING(stream, s3_endpoint); - } - const char* ZarrStream_get_s3_bucket_name(const ZarrStream* stream) - { - STREAM_GET_STRING(stream, s3_bucket_name); - } - - const char* ZarrStream_get_s3_access_key_id(const ZarrStream* stream) - { - STREAM_GET_STRING(stream, s3_access_key_id); - } - - const char* ZarrStream_get_s3_secret_access_key(const ZarrStream* stream) - { - STREAM_GET_STRING(stream, s3_secret_access_key); - } - - const char* ZarrStream_get_external_metadata(const ZarrStream* stream) - { - STREAM_GET_STRING(stream, external_metadata); - } - - ZarrCompressor ZarrStream_get_compressor(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning ZarrCompressor_None"); - return ZarrCompressor_None; - } - return ZarrStreamSettings_get_compressor(&stream->settings()); - } - - ZarrCompressionCodec ZarrStream_get_compression_codec( - const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING( - "Null pointer: stream. Returning ZarrCompressionCodec_None"); - return ZarrCompressionCodec_None; - } - return ZarrStreamSettings_get_compression_codec(&stream->settings()); - } - - uint8_t ZarrStream_get_compression_level(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning 0"); - return 0; - } - return ZarrStreamSettings_get_compression_level(&stream->settings()); - } - - uint8_t ZarrStream_get_compression_shuffle(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning 0"); - return 0; - } - return ZarrStreamSettings_get_compression_shuffle(&stream->settings()); - } - - size_t ZarrStream_get_dimension_count(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning 0"); - return 0; - } - return ZarrStreamSettings_get_dimension_count(&stream->settings()); - } - - ZarrError ZarrStream_get_dimension(const ZarrStream* stream, - size_t index, - char* name, - size_t bytes_of_name, - ZarrDimensionType* kind, - size_t* array_size_px, - size_t* chunk_size_px, - size_t* shard_size_chunks) - { - EXPECT_VALID_ARGUMENT(stream, "Null pointer: stream"); - return ZarrStreamSettings_get_dimension(&stream->settings(), - index, - name, - bytes_of_name, - kind, - array_size_px, - chunk_size_px, - shard_size_chunks); - } - - uint8_t ZarrStream_get_multiscale(const ZarrStream* stream) - { - if (!stream) { - LOG_WARNING("Null pointer: stream. Returning 0"); - return 0; - } - return ZarrStreamSettings_get_multiscale(&stream->settings()); + return stream->version(); } ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream) @@ -327,38 +296,46 @@ extern "C" /* Logging */ - ZarrError Zarr_set_log_level(LogLevel level) + ZarrStatus Zarr_set_log_level(ZarrLogLevel level) { - if (level < LogLevel_Debug || level >= LogLevelCount) { - return ZarrError_InvalidArgument; + if (level < ZarrLogLevel_Debug || level >= ZarrLogLevelCount) { + return ZarrStatus_InvalidArgument; } Logger::set_log_level(level); - return ZarrError_Success; + return ZarrStatus_Success; } - LogLevel Zarr_get_log_level() + ZarrLogLevel Zarr_get_log_level() { return Logger::get_log_level(); } /* Error handling */ - const char* Zarr_get_error_message(ZarrError error) + const char* Zarr_get_error_message(ZarrStatus error) { switch (error) { - case ZarrError_Success: + case ZarrStatus_Success: return "Success"; - case ZarrError_InvalidArgument: + case ZarrStatus_InvalidArgument: return "Invalid argument"; - case ZarrError_Overflow: - return "Overflow"; - case ZarrError_InvalidIndex: + case ZarrStatus_Overflow: + return "Buffer overflow"; + case ZarrStatus_InvalidIndex: return "Invalid index"; - case ZarrError_NotYetImplemented: + case ZarrStatus_NotYetImplemented: return "Not yet implemented"; - case ZarrError_InternalError: + case ZarrStatus_InternalError: return "Internal error"; + case ZarrStatus_OutOfMemory: + return "Out of memory"; + case ZarrStatus_IOError: + return "I/O error"; + case ZarrStatus_CompressionError: + return "Compression error"; + case ZarrStatus_InvalidSettings: + return "Invalid settings"; default: return "Unknown error"; } @@ -367,16 +344,16 @@ extern "C" /* ZarrStream_s implementation */ -ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings, uint8_t version) +ZarrStream::ZarrStream_s(struct ZarrStreamSettings_s* settings, + ZarrVersion version) : settings_(*settings) , version_(version) , error_() { - settings_.dimensions = std::move(settings->dimensions); } size_t - ZarrStream::append(const void* data, size_t nbytes) +ZarrStream::append(const void* data, size_t nbytes) { // todo (aliddell): implement this return 0; diff --git a/src/streaming/zarr.stream.hh b/src/streaming/zarr.stream.hh index 18a7ff1b..3c9475ee 100644 --- a/src/streaming/zarr.stream.hh +++ b/src/streaming/zarr.stream.hh @@ -8,7 +8,7 @@ struct ZarrStream_s { public: - ZarrStream_s(struct ZarrStreamSettings_s* settings, uint8_t version); + ZarrStream_s(struct ZarrStreamSettings_s* settings, ZarrVersion version); ~ZarrStream_s() = default; /** @@ -19,11 +19,11 @@ struct ZarrStream_s */ size_t append(const void* data, size_t nbytes); - size_t version() const { return version_; } + ZarrVersion version() const { return version_; } const ZarrStreamSettings_s& settings() const { return settings_; } private: struct ZarrStreamSettings_s settings_; - uint8_t version_; // Zarr version. 2 or 3. + ZarrVersion version_; // Zarr version. 2 or 3. std::string error_; // error message. If nonempty, an error occurred. }; diff --git a/tests/integration/copy-settings.cpp b/tests/integration/copy-settings.cpp index 131709fe..eecaf1b9 100644 --- a/tests/integration/copy-settings.cpp +++ b/tests/integration/copy-settings.cpp @@ -17,86 +17,96 @@ test_ZarrStreamSettings_copy() const char* s3_bucket_name = "my-bucket"; const char* s3_access_key_id = "access_key_123"; const char* s3_secret_access_key = "secret_key_456"; - const char* external_metadata = "{\"key\":\"value\"}"; + const char* custom_metadata = "{\"key\":\"value\"}"; - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_store_path( - original, store_path, strlen(store_path) + 1)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_s3_endpoint( - original, s3_endpoint, strlen(s3_endpoint) + 1)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_s3_bucket_name( - original, s3_bucket_name, strlen(s3_bucket_name) + 1)); - EXPECT_EQ(ZarrError, + ZarrS3Settings s3_settings = { + .endpoint = s3_endpoint, + .bytes_of_endpoint = strlen(s3_endpoint) + 1, + .bucket_name = s3_bucket_name, + .bytes_of_bucket_name = strlen(s3_bucket_name) + 1, + .access_key_id = s3_access_key_id, + .bytes_of_access_key_id = strlen(s3_access_key_id) + 1, + .secret_access_key = s3_secret_access_key, + .bytes_of_secret_access_key = strlen(s3_secret_access_key) + 1, + }; + + EXPECT_EQ(ZarrStatus, "%d", - ZarrError_Success, - ZarrStreamSettings_set_s3_access_key_id( - original, s3_access_key_id, strlen(s3_access_key_id) + 1)); - EXPECT_EQ( - ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_s3_secret_access_key( - original, s3_secret_access_key, strlen(s3_secret_access_key) + 1)); - EXPECT_EQ(ZarrError, + ZarrStatus_Success, + ZarrStreamSettings_set_store( + original, store_path, strlen(store_path) + 1, &s3_settings)); + + EXPECT_EQ(ZarrStatus, "%d", - ZarrError_Success, - ZarrStreamSettings_set_external_metadata( - original, external_metadata, strlen(external_metadata) + 1)); + ZarrStatus_Success, + ZarrStreamSettings_set_custom_metadata( + original, custom_metadata, strlen(custom_metadata) + 1)); - EXPECT_EQ(ZarrError, + EXPECT_EQ(ZarrStatus, "%d", - ZarrError_Success, + ZarrStatus_Success, ZarrStreamSettings_set_data_type(original, ZarrDataType_float32)); + + ZarrCompressionSettings compression_settings = { + .compressor = ZarrCompressor_Blosc1, + .codec = ZarrCompressionCodec_BloscLZ4, + .level = 5, + .shuffle = 1, + }; + EXPECT_EQ( - ZarrError, + ZarrStatus, "%d", - ZarrError_Success, - ZarrStreamSettings_set_compressor(original, ZarrCompressor_Blosc1)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_compression_codec( - original, ZarrCompressionCodec_BloscLZ4)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_compression_level(original, 5)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_compression_shuffle(original, 1)); + ZarrStatus_Success, + ZarrStreamSettings_set_compression(original, &compression_settings)); - EXPECT_EQ(ZarrError, + EXPECT_EQ(ZarrStatus, "%d", - ZarrError_Success, + ZarrStatus_Success, ZarrStreamSettings_reserve_dimensions(original, 3)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_dimension( - original, 0, "z", 2, ZarrDimensionType_Space, 100, 10, 1)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_dimension( - original, 1, "y", 2, ZarrDimensionType_Space, 200, 20, 1)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_set_dimension( - original, 2, "x", 2, ZarrDimensionType_Space, 300, 30, 1)); - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, + ZarrDimensionProperties dimension = { + .name = "z", + .bytes_of_name = strlen("z") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 100, + .chunk_size_px = 10, + .shard_size_chunks = 1, + }; + EXPECT_EQ(ZarrStatus, + "%d", + ZarrStatus_Success, + ZarrStreamSettings_set_dimension(original, 0, &dimension)); + + dimension = { + .name = "y", + .bytes_of_name = strlen("y") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 200, + .chunk_size_px = 20, + .shard_size_chunks = 1, + }; + EXPECT_EQ(ZarrStatus, + "%d", + ZarrStatus_Success, + ZarrStreamSettings_set_dimension(original, 1, &dimension)); + + dimension = { + .name = "x", + .bytes_of_name = strlen("x") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 300, + .chunk_size_px = 30, + .shard_size_chunks = 1, + }; + EXPECT_EQ(ZarrStatus, + "%d", + ZarrStatus_Success, + ZarrStreamSettings_set_dimension(original, 2, &dimension)); + + EXPECT_EQ(ZarrStatus, + "%d", + ZarrStatus_Success, ZarrStreamSettings_set_multiscale(original, 1)); // Copy the settings @@ -105,58 +115,46 @@ test_ZarrStreamSettings_copy() // Verify copied settings EXPECT_STR_EQ(store_path, ZarrStreamSettings_get_store_path(copy)); - EXPECT_STR_EQ(s3_endpoint, ZarrStreamSettings_get_s3_endpoint(copy)); - EXPECT_STR_EQ(s3_bucket_name, ZarrStreamSettings_get_s3_bucket_name(copy)); - EXPECT_STR_EQ(s3_access_key_id, - ZarrStreamSettings_get_s3_access_key_id(copy)); - EXPECT_STR_EQ(s3_secret_access_key, - ZarrStreamSettings_get_s3_secret_access_key(copy)); - EXPECT_STR_EQ(external_metadata, - ZarrStreamSettings_get_external_metadata(copy)); + + ZarrS3Settings s3_settings_copy = ZarrStreamSettings_get_s3_settings(copy); + EXPECT_STR_EQ(s3_endpoint, s3_settings_copy.endpoint); + EXPECT_STR_EQ(s3_bucket_name, s3_settings_copy.bucket_name); + EXPECT_STR_EQ(s3_access_key_id, s3_settings_copy.access_key_id); + EXPECT_STR_EQ(s3_secret_access_key, s3_settings_copy.secret_access_key); + + EXPECT_STR_EQ(custom_metadata, + ZarrStreamSettings_get_custom_metadata(copy)); EXPECT_EQ(ZarrDataType, "%d", ZarrDataType_float32, ZarrStreamSettings_get_data_type(copy)); + + ZarrCompressionSettings compression_settings_copy = + ZarrStreamSettings_get_compression(copy); EXPECT_EQ(ZarrCompressor, "%d", ZarrCompressor_Blosc1, - ZarrStreamSettings_get_compressor(copy)); + compression_settings_copy.compressor); EXPECT_EQ(ZarrCompressionCodec, "%d", ZarrCompressionCodec_BloscLZ4, - ZarrStreamSettings_get_compression_codec(copy)); - EXPECT_EQ(uint8_t, "%u", 5, ZarrStreamSettings_get_compression_level(copy)); - EXPECT_EQ( - uint8_t, "%u", 1, ZarrStreamSettings_get_compression_shuffle(copy)); + compression_settings_copy.codec); + EXPECT_EQ(uint8_t, "%u", 5, compression_settings_copy.level); + EXPECT_EQ(uint8_t, "%u", 1, compression_settings_copy.shuffle); EXPECT_EQ(size_t, "%zu", 3, ZarrStreamSettings_get_dimension_count(copy)); // Check dimensions for (size_t i = 0; i < 3; ++i) { - char name[3]; - size_t bytes_of_name = sizeof(name); - ZarrDimensionType kind; - size_t array_size_px, chunk_size_px, shard_size_chunks; - - EXPECT_EQ(ZarrError, - "%d", - ZarrError_Success, - ZarrStreamSettings_get_dimension(copy, - i, - name, - bytes_of_name, - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks)); + auto dim = ZarrStreamSettings_get_dimension(copy, i); const char* expected_name = (i == 0) ? "z" : (i == 1) ? "y" : "x"; - EXPECT_STR_EQ(expected_name, name); - EXPECT_EQ(ZarrDimensionType, "%d", ZarrDimensionType_Space, kind); - EXPECT_EQ(size_t, "%zu", (i + 1) * 100, array_size_px); - EXPECT_EQ(size_t, "%zu", (i + 1) * 10, chunk_size_px); - EXPECT_EQ(size_t, "%zu", 1, shard_size_chunks); + EXPECT_STR_EQ(expected_name, dim.name); + EXPECT_EQ(ZarrDimensionType, "%d", ZarrDimensionType_Space, dim.kind); + EXPECT_EQ(size_t, "%zu", (i + 1) * 100, dim.array_size_px); + EXPECT_EQ(size_t, "%zu", (i + 1) * 10, dim.chunk_size_px); + EXPECT_EQ(size_t, "%zu", 1, dim.shard_size_chunks); } EXPECT_EQ(uint8_t, "%u", 1, ZarrStreamSettings_get_multiscale(copy)); diff --git a/tests/integration/create-and-destroy-stream.cpp b/tests/integration/create-and-destroy-stream.cpp index d79d581d..4ccbdd4c 100644 --- a/tests/integration/create-and-destroy-stream.cpp +++ b/tests/integration/create-and-destroy-stream.cpp @@ -25,25 +25,42 @@ try_with_invalid_settings() { ZarrStreamSettings* settings; ZarrStream* stream; + ZarrDimensionProperties dimension; settings = ZarrStreamSettings_create(); CHECK(settings); // reserve 3 dimensions, but only set 2 of them CHECK_EQ(ZarrStreamSettings_reserve_dimensions(settings, 3), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_set_dimension( - settings, 1, SIZED("y"), ZarrDimensionType_Space, 12, 3, 4), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_set_dimension( - settings, 2, SIZED("x"), ZarrDimensionType_Space, 1, 1, 1), - ZarrError_Success); + ZarrStatus_Success); + + dimension = { + .name = "y", + .bytes_of_name = strlen("y") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 12, + .chunk_size_px = 3, + .shard_size_chunks = 4, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 1, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "x", + .bytes_of_name = strlen("x") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 10, + .chunk_size_px = 5, + .shard_size_chunks = 1, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 2, &dimension), + ZarrStatus_Success); stream = ZarrStream_create(settings, ZarrVersion_2); CHECK(!stream); + ZarrStreamSettings_destroy(settings); + return true; Error: @@ -53,31 +70,55 @@ try_with_invalid_settings() bool try_with_valid_settings() { - ZarrStreamSettings* settings; + ZarrStreamSettings *settings, *stream_settings; ZarrStream* stream; + ZarrS3Settings s3_settings; + ZarrCompressionSettings compression_settings; + ZarrDimensionProperties dimension; const std::string store_path = TEST ".zarr"; settings = ZarrStreamSettings_create(); CHECK(settings); CHECK_EQ(ZarrStreamSettings_reserve_dimensions(settings, 3), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_set_dimension( - settings, 2, SIZED("x"), ZarrDimensionType_Space, 10, 5, 1), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_set_dimension( - settings, 1, SIZED("y"), ZarrDimensionType_Space, 12, 3, 4), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_set_dimension( - settings, 0, SIZED("t"), ZarrDimensionType_Time, 1, 1, 0), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_set_store_path( - settings, store_path.c_str(), store_path.size()), - ZarrError_Success); + ZarrStatus_Success); + + dimension = { + .name = "t", + .bytes_of_name = strlen("t") + 1, + .kind = ZarrDimensionType_Time, + .array_size_px = 1, + .chunk_size_px = 1, + .shard_size_chunks = 0, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 0, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "y", + .bytes_of_name = strlen("y") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 12, + .chunk_size_px = 3, + .shard_size_chunks = 4, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 1, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "x", + .bytes_of_name = strlen("x") + 1, + .kind = ZarrDimensionType_Space, + .array_size_px = 10, + .chunk_size_px = 5, + .shard_size_chunks = 1, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 2, &dimension), + ZarrStatus_Success); + + CHECK_EQ(ZarrStreamSettings_set_store( + settings, store_path.c_str(), store_path.size() + 1, nullptr), + ZarrStatus_Success); stream = ZarrStream_create(settings, ZarrVersion_2); CHECK(stream); @@ -85,78 +126,57 @@ try_with_valid_settings() // check the stream's settings are correct CHECK_EQ(ZarrStream_get_version(stream), ZarrVersion_2); - CHECK_EQ(std::string(ZarrStream_get_store_path(stream)), store_path); - CHECK_EQ(strlen(ZarrStream_get_s3_endpoint(stream)), 0); - CHECK_EQ(strlen(ZarrStream_get_s3_bucket_name(stream)), 0); - CHECK_EQ(strlen(ZarrStream_get_s3_access_key_id(stream)), 0); - CHECK_EQ(strlen(ZarrStream_get_s3_secret_access_key(stream)), 0); - - CHECK_EQ(ZarrStream_get_compressor(stream), ZarrCompressor_None); - CHECK_EQ(ZarrStream_get_compression_codec(stream), - ZarrCompressionCodec_None); - - CHECK_EQ(ZarrStream_get_dimension_count(stream), 3); - - char name[64]; - ZarrDimensionType kind; - size_t array_size_px, chunk_size_px, shard_size_chunks; - CHECK_EQ(ZarrStream_get_dimension(stream, - 0, - name, - sizeof(name), - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - CHECK_EQ(std::string(name), "t"); - CHECK_EQ(kind, ZarrDimensionType_Time); - CHECK_EQ(array_size_px, 1); - CHECK_EQ(chunk_size_px, 1); - CHECK_EQ(shard_size_chunks, 0); - - CHECK_EQ(ZarrStream_get_dimension(stream, - 1, - name, - sizeof(name), - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - - CHECK_EQ(std::string(name), "y"); - CHECK_EQ(kind, ZarrDimensionType_Space); - CHECK_EQ(array_size_px, 12); - CHECK_EQ(chunk_size_px, 3); - CHECK_EQ(shard_size_chunks, 4); - - CHECK_EQ(ZarrStream_get_dimension(stream, - 2, - name, - sizeof(name), - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - - CHECK_EQ(std::string(name), "x"); - CHECK_EQ(kind, ZarrDimensionType_Space); - CHECK_EQ(array_size_px, 10); - CHECK_EQ(chunk_size_px, 5); - CHECK_EQ(shard_size_chunks, 1); + stream_settings = ZarrStream_get_settings(stream); + + CHECK_EQ(std::string(ZarrStreamSettings_get_store_path(stream_settings)), + store_path); + + s3_settings = ZarrStreamSettings_get_s3_settings(stream_settings); + CHECK_EQ(strlen(s3_settings.endpoint), 0); + CHECK_EQ(s3_settings.bytes_of_endpoint, 0); + + CHECK_EQ(strlen(s3_settings.bucket_name), 0); + CHECK_EQ(s3_settings.bytes_of_bucket_name, 0); + + CHECK_EQ(strlen(s3_settings.access_key_id), 0); + CHECK_EQ(s3_settings.bytes_of_access_key_id, 0); + + CHECK_EQ(strlen(s3_settings.secret_access_key), 0); + CHECK_EQ(s3_settings.bytes_of_secret_access_key, 0); + + compression_settings = ZarrStreamSettings_get_compression(stream_settings); + + CHECK_EQ(compression_settings.compressor, ZarrCompressor_None); + CHECK_EQ(compression_settings.codec, ZarrCompressionCodec_None); + + CHECK_EQ(ZarrStreamSettings_get_dimension_count(stream_settings), 3); + + dimension = ZarrStreamSettings_get_dimension(stream_settings, 0); + CHECK_EQ(std::string(dimension.name), "t"); + CHECK_EQ(dimension.kind, ZarrDimensionType_Time); + CHECK_EQ(dimension.array_size_px, 1); + CHECK_EQ(dimension.chunk_size_px, 1); + CHECK_EQ(dimension.shard_size_chunks, 0); + + dimension = ZarrStreamSettings_get_dimension(stream_settings, 1); + CHECK_EQ(std::string(dimension.name), "y"); + CHECK_EQ(dimension.kind, ZarrDimensionType_Space); + CHECK_EQ(dimension.array_size_px, 12); + CHECK_EQ(dimension.chunk_size_px, 3); + CHECK_EQ(dimension.shard_size_chunks, 4); + + dimension = ZarrStreamSettings_get_dimension(stream_settings, 2); + CHECK_EQ(std::string(dimension.name), "x"); + CHECK_EQ(dimension.kind, ZarrDimensionType_Space); + CHECK_EQ(dimension.array_size_px, 10); + CHECK_EQ(dimension.chunk_size_px, 5); + CHECK_EQ(dimension.shard_size_chunks, 1); ZarrStream_destroy(stream); // cleanup - try { - fs::remove_all(store_path); - } catch (const fs::filesystem_error& e) { - fprintf( - stderr, "Failed to remove %s: %s\n", store_path.c_str(), e.what()); - return false; - } + ZarrStreamSettings_destroy(settings); + ZarrStreamSettings_destroy(stream_settings); return true; diff --git a/tests/integration/set-and-get-params.cpp b/tests/integration/set-and-get-params.cpp index 5c3bd7ec..a85b5c9d 100644 --- a/tests/integration/set-and-get-params.cpp +++ b/tests/integration/set-and-get-params.cpp @@ -1,7 +1,6 @@ #include "zarr.h" #include -#include #include #define CHECK(cond) \ @@ -14,52 +13,41 @@ #define CHECK_EQ(a, b) CHECK((a) == (b)) -#define TRY_SET_STRING(stream, member, value) \ - do { \ - ZarrError err; \ - if (err = ZarrStreamSettings_set_##member( \ - stream, value, strlen(value) + 1); \ - err != ZarrError_Success) { \ - fprintf(stderr, \ - "Failed to set %s: %s\n", \ - #member, \ - Zarr_get_error_message(err)); \ - return false; \ - } \ - } while (0) - #define SIZED(name) name, sizeof(name) bool -check_preconditions(ZarrStreamSettings* stream) +check_preconditions(ZarrStreamSettings* settings) { std::string str_param_value; + ZarrS3Settings s3_settings; + ZarrCompressionSettings compression_settings; - str_param_value = ZarrStreamSettings_get_store_path(stream); + str_param_value = ZarrStreamSettings_get_store_path(settings); CHECK(str_param_value.empty()); - str_param_value = ZarrStreamSettings_get_s3_endpoint(stream); + s3_settings = ZarrStreamSettings_get_s3_settings(settings); + str_param_value = s3_settings.endpoint; CHECK(str_param_value.empty()); - str_param_value = ZarrStreamSettings_get_s3_bucket_name(stream); + str_param_value = s3_settings.bucket_name; CHECK(str_param_value.empty()); - str_param_value = ZarrStreamSettings_get_s3_access_key_id(stream); + str_param_value = s3_settings.access_key_id; CHECK(str_param_value.empty()); - str_param_value = ZarrStreamSettings_get_s3_secret_access_key(stream); + str_param_value = s3_settings.secret_access_key; CHECK(str_param_value.empty()); - CHECK_EQ(ZarrStreamSettings_get_data_type(stream), ZarrDataType_uint8); + CHECK_EQ(ZarrStreamSettings_get_data_type(settings), ZarrDataType_uint8); - CHECK_EQ(ZarrStreamSettings_get_compressor(stream), ZarrCompressor_None); + compression_settings = ZarrStreamSettings_get_compression(settings); + CHECK_EQ(compression_settings.compressor, ZarrCompressor_None); - CHECK_EQ(ZarrStreamSettings_get_compression_codec(stream), - ZarrCompressionCodec_None); + CHECK_EQ(compression_settings.codec, ZarrCompressionCodec_None); - CHECK_EQ(ZarrStreamSettings_get_dimension_count(stream), 0); + CHECK_EQ(ZarrStreamSettings_get_dimension_count(settings), 0); - CHECK_EQ(ZarrStreamSettings_get_multiscale(stream), 0); + CHECK_EQ(ZarrStreamSettings_get_multiscale(settings), 0); return true; @@ -68,161 +56,165 @@ check_preconditions(ZarrStreamSettings* stream) } bool -set_and_get_parameters(ZarrStreamSettings* stream) +set_and_get_parameters(ZarrStreamSettings* settings) { std::string str_param_value; - /* Set and get store path */ - TRY_SET_STRING(stream, store_path, "store_path"); - str_param_value = ZarrStreamSettings_get_store_path(stream); + /* Set and get store */ + ZarrS3Settings s3_settings_in{ + .endpoint = "s3_endpoint", + .bytes_of_endpoint = sizeof("s3_endpoint"), + .bucket_name = "s3_bucket_name", + .bytes_of_bucket_name = sizeof("s3_bucket_name"), + .access_key_id = "s3_access_key_id", + .bytes_of_access_key_id = sizeof("s3_access_key_id"), + .secret_access_key = "s3_secret_access_key", + .bytes_of_secret_access_key = sizeof("s3_secret_access_key"), + }; + ZarrS3Settings s3_settings_out; + + ZarrCompressionSettings compression_settings_in{ + .compressor = ZarrCompressor_Blosc1, + .codec = ZarrCompressionCodec_BloscLZ4, + .level = 5, + .shuffle = 1, + }; + ZarrCompressionSettings compression_settings_out; + + ZarrDimensionProperties dimension; + + ZarrStatus status = ZarrStreamSettings_set_store( + settings, "store_path", sizeof("store_path"), &s3_settings_in); + CHECK_EQ(status, ZarrStatus_Success); + + str_param_value = ZarrStreamSettings_get_store_path(settings); CHECK_EQ(str_param_value, "store_path"); - /* Set and get S3 endpoint */ - TRY_SET_STRING(stream, s3_endpoint, "s3_endpoint"); - str_param_value = ZarrStreamSettings_get_s3_endpoint(stream); + s3_settings_out = ZarrStreamSettings_get_s3_settings(settings); + str_param_value = s3_settings_out.endpoint; CHECK_EQ(str_param_value, "s3_endpoint"); - /* Set and get S3 bucket name */ - TRY_SET_STRING(stream, s3_bucket_name, "s3_bucket_name"); - str_param_value = ZarrStreamSettings_get_s3_bucket_name(stream); + str_param_value = s3_settings_out.bucket_name; CHECK_EQ(str_param_value, "s3_bucket_name"); - /* Set and get S3 access key ID */ - TRY_SET_STRING(stream, s3_access_key_id, "s3_access_key_id"); - str_param_value = ZarrStreamSettings_get_s3_access_key_id(stream); + str_param_value = s3_settings_out.access_key_id; CHECK_EQ(str_param_value, "s3_access_key_id"); - /* Set and get S3 secret access key */ - TRY_SET_STRING(stream, s3_secret_access_key, "s3_secret_access_key"); - str_param_value = ZarrStreamSettings_get_s3_secret_access_key(stream); + str_param_value = s3_settings_out.secret_access_key; CHECK_EQ(str_param_value, "s3_secret_access_key"); /* Set and get data type */ - CHECK_EQ(ZarrStreamSettings_set_data_type(stream, ZarrDataType_float32), - ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_get_data_type(stream), ZarrDataType_float32); - - /* Set and get compressor */ - CHECK_EQ(ZarrStreamSettings_set_compressor(stream, ZarrCompressor_Blosc1), - ZarrError_Success); - - // try to set an invalid compressor - CHECK_EQ(ZarrStreamSettings_set_compressor(stream, ZarrCompressor_Blosc2), - ZarrError_NotYetImplemented); - - // should still be the previous value - CHECK_EQ(ZarrStreamSettings_get_compressor(stream), ZarrCompressor_Blosc1); + CHECK_EQ(ZarrStreamSettings_set_data_type(settings, ZarrDataType_float32), + ZarrStatus_Success); + CHECK_EQ(ZarrStreamSettings_get_data_type(settings), ZarrDataType_float32); - /* Set and get compression codec */ - CHECK_EQ(ZarrStreamSettings_set_compression_codec( - stream, ZarrCompressionCodec_BloscLZ4), - ZarrError_Success); + /* Set and get compression settings */ + CHECK_EQ( + ZarrStreamSettings_set_compression(settings, &compression_settings_in), + ZarrStatus_Success); /* Set and get some dimensions */ - CHECK_EQ(ZarrStreamSettings_reserve_dimensions(stream, 5), - ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_set_dimension( - stream, 4, SIZED("x"), ZarrDimensionType_Space, 10, 5, 2), - ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_set_dimension( - stream, 3, SIZED("y"), ZarrDimensionType_Space, 20, 10, 3), - ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_set_dimension( - stream, 2, SIZED("z"), ZarrDimensionType_Space, 30, 15, 4), - ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_set_dimension( - stream, 1, SIZED("c"), ZarrDimensionType_Channel, 40, 20, 5), - ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_set_dimension( - stream, 0, SIZED("t"), ZarrDimensionType_Time, 50, 25, 6), - ZarrError_Success); - - CHECK_EQ(ZarrStreamSettings_get_dimension_count(stream), 5); - - char name[64]; - ZarrDimensionType kind; - size_t array_size_px; - size_t chunk_size_px; - size_t shard_size_chunks; - - CHECK_EQ(ZarrStreamSettings_get_dimension(stream, - 0, - name, - 64, - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - CHECK_EQ(std::string(name), "t"); - CHECK_EQ(kind, ZarrDimensionType_Time); - CHECK_EQ(array_size_px, 50); - CHECK_EQ(chunk_size_px, 25); - CHECK_EQ(shard_size_chunks, 6); - - CHECK_EQ(ZarrStreamSettings_get_dimension(stream, - 1, - name, - 64, - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - CHECK_EQ(std::string(name), "c"); - CHECK_EQ(kind, ZarrDimensionType_Channel); - CHECK_EQ(array_size_px, 40); - CHECK_EQ(chunk_size_px, 20); - CHECK_EQ(shard_size_chunks, 5); - - CHECK_EQ(ZarrStreamSettings_get_dimension(stream, - 2, - name, - 64, - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - CHECK_EQ(std::string(name), "z"); - CHECK_EQ(kind, ZarrDimensionType_Space); - CHECK_EQ(array_size_px, 30); - CHECK_EQ(chunk_size_px, 15); - CHECK_EQ(shard_size_chunks, 4); - - CHECK_EQ(ZarrStreamSettings_get_dimension(stream, - 3, - name, - 64, - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - CHECK_EQ(std::string(name), "y"); - CHECK_EQ(kind, ZarrDimensionType_Space); - CHECK_EQ(array_size_px, 20); - CHECK_EQ(chunk_size_px, 10); - CHECK_EQ(shard_size_chunks, 3); - - CHECK_EQ(ZarrStreamSettings_get_dimension(stream, - 4, - name, - 64, - &kind, - &array_size_px, - &chunk_size_px, - &shard_size_chunks), - ZarrError_Success); - CHECK_EQ(std::string(name), "x"); - CHECK_EQ(kind, ZarrDimensionType_Space); - CHECK_EQ(array_size_px, 10); - CHECK_EQ(chunk_size_px, 5); - CHECK_EQ(shard_size_chunks, 2); + CHECK_EQ(ZarrStreamSettings_reserve_dimensions(settings, 5), + ZarrStatus_Success); + + dimension = { + .name = "t", + .bytes_of_name = sizeof("t"), + .kind = ZarrDimensionType_Time, + .array_size_px = 50, + .chunk_size_px = 25, + .shard_size_chunks = 6, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 0, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "c", + .bytes_of_name = sizeof("c"), + .kind = ZarrDimensionType_Channel, + .array_size_px = 40, + .chunk_size_px = 20, + .shard_size_chunks = 5, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 1, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "z", + .bytes_of_name = sizeof("z"), + .kind = ZarrDimensionType_Space, + .array_size_px = 30, + .chunk_size_px = 15, + .shard_size_chunks = 4, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 2, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "y", + .bytes_of_name = sizeof("y"), + .kind = ZarrDimensionType_Space, + .array_size_px = 20, + .chunk_size_px = 10, + .shard_size_chunks = 3, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 3, &dimension), + ZarrStatus_Success); + + dimension = { + .name = "x", + .bytes_of_name = sizeof("x"), + .kind = ZarrDimensionType_Space, + .array_size_px = 10, + .chunk_size_px = 5, + .shard_size_chunks = 2, + }; + CHECK_EQ(ZarrStreamSettings_set_dimension(settings, 4, &dimension), + ZarrStatus_Success); + + CHECK_EQ(ZarrStreamSettings_get_dimension_count(settings), 5); + + ZarrDimensionProperties dimension_out; + + dimension_out = ZarrStreamSettings_get_dimension(settings, 0); + CHECK_EQ(std::string(dimension_out.name), "t"); + CHECK_EQ(dimension_out.kind, ZarrDimensionType_Time); + CHECK_EQ(dimension_out.array_size_px, 50); + CHECK_EQ(dimension_out.chunk_size_px, 25); + CHECK_EQ(dimension_out.shard_size_chunks, 6); + + dimension_out = ZarrStreamSettings_get_dimension(settings, 1); + CHECK_EQ(std::string(dimension_out.name), "c"); + CHECK_EQ(dimension_out.kind, ZarrDimensionType_Channel); + CHECK_EQ(dimension_out.array_size_px, 40); + CHECK_EQ(dimension_out.chunk_size_px, 20); + CHECK_EQ(dimension_out.shard_size_chunks, 5); + + dimension_out = ZarrStreamSettings_get_dimension(settings, 2); + CHECK_EQ(std::string(dimension_out.name), "z"); + CHECK_EQ(dimension_out.kind, ZarrDimensionType_Space); + CHECK_EQ(dimension_out.array_size_px, 30); + CHECK_EQ(dimension_out.chunk_size_px, 15); + CHECK_EQ(dimension_out.shard_size_chunks, 4); + + dimension_out = ZarrStreamSettings_get_dimension(settings, 3); + CHECK_EQ(std::string(dimension_out.name), "y"); + CHECK_EQ(dimension_out.kind, ZarrDimensionType_Space); + CHECK_EQ(dimension_out.array_size_px, 20); + CHECK_EQ(dimension_out.chunk_size_px, 10); + CHECK_EQ(dimension_out.shard_size_chunks, 3); + + dimension_out = ZarrStreamSettings_get_dimension(settings, 4); + CHECK_EQ(std::string(dimension_out.name), "x"); + CHECK_EQ(dimension_out.kind, ZarrDimensionType_Space); + CHECK_EQ(dimension_out.array_size_px, 10); + CHECK_EQ(dimension_out.chunk_size_px, 5); + CHECK_EQ(dimension_out.shard_size_chunks, 2); /* Set and get multiscale */ - CHECK_EQ(ZarrStreamSettings_set_multiscale(stream, 10), ZarrError_Success); - CHECK_EQ(ZarrStreamSettings_get_multiscale(stream), 1); // normalized to 1 + CHECK_EQ(ZarrStreamSettings_set_multiscale(settings, 10), + ZarrStatus_Success); + CHECK_EQ(ZarrStreamSettings_get_multiscale(settings), 1); // normalized to 1 return true; @@ -233,19 +225,19 @@ set_and_get_parameters(ZarrStreamSettings* stream) int main() { - ZarrStreamSettings* stream = ZarrStreamSettings_create(); - if (!stream) { - fprintf(stderr, "Failed to create Zarr stream\n"); + ZarrStreamSettings* settings = ZarrStreamSettings_create(); + if (!settings) { + fprintf(stderr, "Failed to create Zarr settings\n"); return 1; } int retval = 0; - CHECK(check_preconditions(stream)); - CHECK(set_and_get_parameters(stream)); + CHECK(check_preconditions(settings)); + CHECK(set_and_get_parameters(settings)); Finalize: - ZarrStreamSettings_destroy(stream); + ZarrStreamSettings_destroy(settings); return retval; Error: retval = 1; diff --git a/tests/integration/test.logger.cpp b/tests/integration/test.logger.cpp index 16861601..701f7833 100644 --- a/tests/integration/test.logger.cpp +++ b/tests/integration/test.logger.cpp @@ -6,29 +6,29 @@ #include #include -LogLevel Logger::current_level = LogLevel_Info; +ZarrLogLevel Logger::current_level = ZarrLogLevel_Info; void -Logger::set_log_level(LogLevel level) +Logger::set_log_level(ZarrLogLevel level) { current_level = level; } -LogLevel +ZarrLogLevel Logger::get_log_level() { return current_level; } std::string -Logger::log(LogLevel level, +Logger::log(ZarrLogLevel level, const char* file, int line, const char* func, const char* format, ...) { - if (current_level == LogLevel_None || level < current_level) { + if (current_level == ZarrLogLevel_None || level < current_level) { return {}; // Suppress logs } @@ -39,17 +39,17 @@ Logger::log(LogLevel level, std::ostream* stream = &std::cout; switch (level) { - case LogLevel_Debug: + case ZarrLogLevel_Debug: prefix = "[DEBUG] "; break; - case LogLevel_Info: + case ZarrLogLevel_Info: prefix = "[INFO] "; break; - case LogLevel_Warning: + case ZarrLogLevel_Warning: prefix = "[WARNING] "; stream = &std::cerr; break; - case LogLevel_Error: + case ZarrLogLevel_Error: prefix = "[ERROR] "; stream = &std::cerr; break; diff --git a/tests/integration/test.logger.hh b/tests/integration/test.logger.hh index 72900445..093bd254 100644 --- a/tests/integration/test.logger.hh +++ b/tests/integration/test.logger.hh @@ -5,10 +5,10 @@ class Logger { public: - static void set_log_level(LogLevel level); - static LogLevel get_log_level(); + static void set_log_level(ZarrLogLevel level); + static ZarrLogLevel get_log_level(); - static std::string log(LogLevel level, + static std::string log(ZarrLogLevel level, const char* file, int line, const char* func, @@ -16,7 +16,7 @@ class Logger ...); private: - static LogLevel current_level; + static ZarrLogLevel current_level; }; #define LOG_DEBUG(...) \ @@ -24,9 +24,9 @@ class Logger #define LOG_INFO(...) \ Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_WARNING(...) \ - Logger::log(LogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(ZarrLogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_ERROR(...) \ - Logger::log(LogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(ZarrLogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) #define EXPECT(e, ...) \ do { \