From 9aeab62682483e5afd3f73b72be08383526822aa Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 24 Sep 2024 08:49:01 -0700 Subject: [PATCH] Standalone 2/N: Define the Zarr streaming API (#294) Defines but does not implement the API. Depends on #293. --- include/acquire.zarr.h | 112 ++++++++++++++++++++++++++++++++++++ include/zarr.types.h | 128 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+) create mode 100644 include/acquire.zarr.h create mode 100644 include/zarr.types.h diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h new file mode 100644 index 00000000..54a7288a --- /dev/null +++ b/include/acquire.zarr.h @@ -0,0 +1,112 @@ +#pragma once + +#include "zarr.types.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + /** + * @brief The settings for a Zarr stream. + * @details This struct contains the settings for a Zarr stream, including + * the store path, custom metadata, S3 settings, chunk compression settings, + * dimension properties, whether to stream to multiple levels of detail, the + * pixel data type, and the Zarr format version. + * @note The store path can be a filesystem path or an S3 key prefix. For example, + * supplying an endpoint "s3://my-endpoint.com" and a bucket "my-bucket" with a + * store_path of "my-dataset.zarr" will result in the store being written to + * "s3://my-endpoint.com/my-bucket/my-dataset.zarr". + * @note The dimensions array may be allocated with ZarrStreamSettings_create_dimension_array + * and freed with ZarrStreamSettings_destroy_dimension_array. The order in which you + * set the dimension properties in the array should match the order of the dimensions + * from slowest to fastest changing, for example, [Z, Y, X] for a 3D dataset. + */ + typedef struct ZarrStreamSettings_s + { + const char* store_path; /**< Path to the store. Filesystem path or S3 key prefix. */ + const char* custom_metadata; /**< JSON-formatted custom metadata to be stored with the dataset. */ + ZarrS3Settings* s3_settings; /**< Optional S3 settings for the store. */ + ZarrCompressionSettings* compression_settings; /**< Optional chunk compression settings for the store. */ + ZarrDimensionProperties* dimensions; /**< The properties of each dimension in the dataset. */ + size_t dimension_count; /**< The number of dimensions in the dataset. */ + bool multiscale; /**< Whether to stream to multiple levels of detail. */ + ZarrDataType data_type; /**< The pixel data type of the dataset. */ + ZarrVersion version; /**< The version of the Zarr format to use. 2 or 3. */ + } ZarrStreamSettings; + + typedef struct ZarrStream_s ZarrStream; + + /** + * @brief Get the version of the Zarr API. + * @return The version of the Zarr API. + */ + uint32_t Zarr_get_api_version(); + + /** + * @brief Set the log level for the Zarr API. + * @param level The log level. + * @return ZarrStatusCode_Success on success, or an error code on failure. + */ + ZarrStatusCode Zarr_set_log_level(ZarrLogLevel level); + + /** + * @brief Get the log level for the Zarr API. + * @return The log level for the Zarr API. + */ + ZarrLogLevel Zarr_get_log_level(); + + /** + * @brief Get the message for the given status code. + * @param status The status code. + * @return A human-readable status message. + */ + const char* Zarr_get_status_message(ZarrStatusCode status); + + /** + * @brief Allocate memory for the dimension array in the Zarr stream settings struct. + * @param[in, out] settings The Zarr stream settings struct. + * @param dimension_count The number of dimensions in the dataset to allocate memory for. + * @return ZarrStatusCode_Success on success, or an error code on failure. + */ + ZarrStatusCode ZarrStreamSettings_create_dimension_array(ZarrStreamSettings* settings, size_t dimension_count); + + /** + * @brief Free memory for the dimension array in the Zarr stream settings struct. + * @param[in, out] settings The Zarr stream settings struct containing the dimension array to free. + */ + void ZarrStreamSettings_destroy_dimension_array(ZarrStreamSettings* settings); + + /** + * @brief Create a Zarr stream. + * @param[in, out] settings The settings for the Zarr stream. + * @return A pointer to the Zarr stream struct, or NULL on failure. + */ + ZarrStream* ZarrStream_create(ZarrStreamSettings* settings); + + /** + * @brief Destroy a Zarr stream. + * @details This function frees the memory allocated for the Zarr stream. + * @param stream The Zarr stream struct to destroy. + */ + void ZarrStream_destroy(ZarrStream* stream); + + /** + * @brief Append data to the Zarr stream. + * @details This function will block while chunks are compressed and written + * to the store. It will return when all data has been written. + * @param[in, out] stream The Zarr stream struct. + * @param[in] data The data to append. + * @param[in] bytes_in The number of bytes in @p data. It should be at least + * the size of a single frame. + * @param[out] bytes_out The number of bytes written to the stream. + * @return ZarrStatusCode_Success on success, or an error code on failure. + */ + ZarrStatusCode ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out); + +#ifdef __cplusplus +} +#endif diff --git a/include/zarr.types.h b/include/zarr.types.h new file mode 100644 index 00000000..6e04c03d --- /dev/null +++ b/include/zarr.types.h @@ -0,0 +1,128 @@ +#ifndef H_ACQUIRE_ZARR_TYPES_V0 +#define H_ACQUIRE_ZARR_TYPES_V0 + +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef enum + { + ZarrStatusCode_Success = 0, + ZarrStatusCode_InvalidArgument, + ZarrStatusCode_Overflow, + ZarrStatusCode_InvalidIndex, + ZarrStatusCode_NotYetImplemented, + ZarrStatusCode_InternalError, + ZarrStatusCode_OutOfMemory, + ZarrStatusCode_IOError, + ZarrStatusCode_CompressionError, + ZarrStatusCode_InvalidSettings, + ZarrStatusCodeCount, + } ZarrStatusCode; + + typedef enum + { + ZarrVersion_2 = 2, + ZarrVersion_3, + ZarrVersionCount + } ZarrVersion; + + typedef enum + { + ZarrLogLevel_Debug = 0, + ZarrLogLevel_Info, + ZarrLogLevel_Warning, + ZarrLogLevel_Error, + ZarrLogLevel_None, + ZarrLogLevelCount + } ZarrLogLevel; + + typedef enum + { + ZarrDataType_uint8 = 0, + ZarrDataType_uint16, + ZarrDataType_uint32, + ZarrDataType_uint64, + ZarrDataType_int8, + ZarrDataType_int16, + ZarrDataType_int32, + ZarrDataType_int64, + ZarrDataType_float32, + ZarrDataType_float64, + ZarrDataTypeCount + } ZarrDataType; + + typedef enum + { + ZarrCompressor_None = 0, + ZarrCompressor_Blosc1, + ZarrCompressorCount + } ZarrCompressor; + + typedef enum + { + ZarrCompressionCodec_None = 0, + ZarrCompressionCodec_BloscLZ4, + ZarrCompressionCodec_BloscZstd, + ZarrCompressionCodecCount + } ZarrCompressionCodec; + + typedef enum + { + ZarrDimensionType_Space = 0, + ZarrDimensionType_Channel, + ZarrDimensionType_Time, + ZarrDimensionType_Other, + ZarrDimensionTypeCount + } ZarrDimensionType; + + /** + * @brief S3 settings for streaming to Zarr. + */ + typedef struct + { + const char* endpoint; + const char* bucket_name; + const char* access_key_id; + const char* secret_access_key; + } ZarrS3Settings; + + /** + * @brief Compression settings for a Zarr array. + * @detail The compressor is not the same as the codec. A codec is + * a specific implementation of a compression algorithm, while a compressor + * is a library that implements one or more codecs. + */ + typedef struct + { + ZarrCompressor compressor; /**< Compressor to use */ + ZarrCompressionCodec codec; /**< Codec to use */ + uint8_t level; /**< Compression level */ + uint8_t shuffle; /**< Whether to shuffle the data before compressing */ + } ZarrCompressionSettings; + + /** + * @brief Properties of a dimension of the Zarr array. + */ + typedef struct + { + const char* name; /**< Name of the dimension */ + ZarrDimensionType type; /**< Type of the dimension */ + uint32_t array_size_px; /**< Size of the array along this dimension in + pixels */ + uint32_t chunk_size_px; /**< Size of the chunks along this dimension in + pixels */ + uint32_t shard_size_chunks; /**< Number of chunks in a shard along this + dimension */ + } ZarrDimensionProperties; + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_TYPES_V0 \ No newline at end of file