From c7832d724950c7ce0ebec3797d716dd203ece9bc Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 09:59:22 -0400 Subject: [PATCH 1/2] Move driver source files and tests to a separate directory. --- src/CMakeLists.txt | 58 +------------ src/driver/CMakeLists.txt | 61 +++++++++++++ src/{ => driver}/README.md | 0 src/{ => driver}/common/dimension.cpp | 0 src/{ => driver}/common/dimension.hh | 0 src/{ => driver}/common/macros.hh | 0 src/{ => driver}/common/s3.connection.cpp | 0 src/{ => driver}/common/s3.connection.hh | 0 src/{ => driver}/common/thread.pool.cpp | 0 src/{ => driver}/common/thread.pool.hh | 0 src/{ => driver}/common/utilities.cpp | 0 src/{ => driver}/common/utilities.hh | 0 src/{ => driver}/writers/array.writer.cpp | 0 src/{ => driver}/writers/array.writer.hh | 0 src/{ => driver}/writers/blosc.compressor.cpp | 0 src/{ => driver}/writers/blosc.compressor.hh | 0 src/{ => driver}/writers/file.sink.cpp | 0 src/{ => driver}/writers/file.sink.hh | 0 src/{ => driver}/writers/s3.sink.cpp | 0 src/{ => driver}/writers/s3.sink.hh | 0 src/{ => driver}/writers/sink.creator.cpp | 0 src/{ => driver}/writers/sink.creator.hh | 0 src/{ => driver}/writers/sink.hh | 0 .../writers/zarrv2.array.writer.cpp | 0 .../writers/zarrv2.array.writer.hh | 0 .../writers/zarrv3.array.writer.cpp | 0 .../writers/zarrv3.array.writer.hh | 0 src/{ => driver}/zarr.cpp | 0 src/{ => driver}/zarr.driver.c | 0 src/{ => driver}/zarr.hh | 0 src/{ => driver}/zarr.v2.cpp | 0 src/{ => driver}/zarr.v2.hh | 0 src/{ => driver}/zarr.v3.cpp | 0 src/{ => driver}/zarr.v3.hh | 0 tests/CMakeLists.txt | 87 +------------------ tests/driver/CMakeLists.txt | 87 +++++++++++++++++++ tests/{ => driver}/README.md | 0 .../external-metadata-with-whitespace-ok.cpp | 0 tests/{ => driver}/get-meta.cpp | 0 tests/{ => driver}/get-set-get.cpp | 0 tests/{ => driver}/get.cpp | 0 tests/{ => driver}/list-devices.cpp | 0 .../{ => driver}/metadata-dimension-sizes.cpp | 0 tests/{ => driver}/multiscales-metadata.cpp | 0 tests/{ => driver}/repeat-start.cpp | 0 ...restart-stopped-zarr-resets-threadpool.cpp | 0 tests/{ => driver}/unit-tests.cpp | 0 .../write-zarr-v2-compressed-multiscale.cpp | 0 ...-compressed-with-chunking-and-rollover.cpp | 0 ...write-zarr-v2-compressed-with-chunking.cpp | 0 ...-raw-chunk-size-larger-than-frame-size.cpp | 0 ...-raw-multiscale-with-trivial-tile-size.cpp | 0 .../write-zarr-v2-raw-multiscale.cpp | 0 ...v2-raw-with-even-chunking-and-rollover.cpp | 0 .../write-zarr-v2-raw-with-even-chunking.cpp | 0 ...write-zarr-v2-raw-with-ragged-chunking.cpp | 0 tests/{ => driver}/write-zarr-v2-raw.cpp | 0 tests/{ => driver}/write-zarr-v2-to-s3.cpp | 0 .../write-zarr-v2-with-lz4-compression.cpp | 0 .../write-zarr-v2-with-zstd-compression.cpp | 0 .../{ => driver}/write-zarr-v3-compressed.cpp | 0 .../write-zarr-v3-raw-chunk-exceeds-array.cpp | 0 .../write-zarr-v3-raw-multiscale.cpp | 0 ...write-zarr-v3-raw-with-ragged-sharding.cpp | 0 tests/{ => driver}/write-zarr-v3-raw.cpp | 0 tests/{ => driver}/write-zarr-v3-to-s3.cpp | 0 66 files changed, 151 insertions(+), 142 deletions(-) create mode 100644 src/driver/CMakeLists.txt rename src/{ => driver}/README.md (100%) rename src/{ => driver}/common/dimension.cpp (100%) rename src/{ => driver}/common/dimension.hh (100%) rename src/{ => driver}/common/macros.hh (100%) rename src/{ => driver}/common/s3.connection.cpp (100%) rename src/{ => driver}/common/s3.connection.hh (100%) rename src/{ => driver}/common/thread.pool.cpp (100%) rename src/{ => driver}/common/thread.pool.hh (100%) rename src/{ => driver}/common/utilities.cpp (100%) rename src/{ => driver}/common/utilities.hh (100%) rename src/{ => driver}/writers/array.writer.cpp (100%) rename src/{ => driver}/writers/array.writer.hh (100%) rename src/{ => driver}/writers/blosc.compressor.cpp (100%) rename src/{ => driver}/writers/blosc.compressor.hh (100%) rename src/{ => driver}/writers/file.sink.cpp (100%) rename src/{ => driver}/writers/file.sink.hh (100%) rename src/{ => driver}/writers/s3.sink.cpp (100%) rename src/{ => driver}/writers/s3.sink.hh (100%) rename src/{ => driver}/writers/sink.creator.cpp (100%) rename src/{ => driver}/writers/sink.creator.hh (100%) rename src/{ => driver}/writers/sink.hh (100%) rename src/{ => driver}/writers/zarrv2.array.writer.cpp (100%) rename src/{ => driver}/writers/zarrv2.array.writer.hh (100%) rename src/{ => driver}/writers/zarrv3.array.writer.cpp (100%) rename src/{ => driver}/writers/zarrv3.array.writer.hh (100%) rename src/{ => driver}/zarr.cpp (100%) rename src/{ => driver}/zarr.driver.c (100%) rename src/{ => driver}/zarr.hh (100%) rename src/{ => driver}/zarr.v2.cpp (100%) rename src/{ => driver}/zarr.v2.hh (100%) rename src/{ => driver}/zarr.v3.cpp (100%) rename src/{ => driver}/zarr.v3.hh (100%) create mode 100644 tests/driver/CMakeLists.txt rename tests/{ => driver}/README.md (100%) rename tests/{ => driver}/external-metadata-with-whitespace-ok.cpp (100%) rename tests/{ => driver}/get-meta.cpp (100%) rename tests/{ => driver}/get-set-get.cpp (100%) rename tests/{ => driver}/get.cpp (100%) rename tests/{ => driver}/list-devices.cpp (100%) rename tests/{ => driver}/metadata-dimension-sizes.cpp (100%) rename tests/{ => driver}/multiscales-metadata.cpp (100%) rename tests/{ => driver}/repeat-start.cpp (100%) rename tests/{ => driver}/restart-stopped-zarr-resets-threadpool.cpp (100%) rename tests/{ => driver}/unit-tests.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-with-chunking-and-rollover.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-with-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-even-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-ragged-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw.cpp (100%) rename tests/{ => driver}/write-zarr-v2-to-s3.cpp (100%) rename tests/{ => driver}/write-zarr-v2-with-lz4-compression.cpp (100%) rename tests/{ => driver}/write-zarr-v2-with-zstd-compression.cpp (100%) rename tests/{ => driver}/write-zarr-v3-compressed.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-chunk-exceeds-array.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-with-ragged-sharding.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw.cpp (100%) rename tests/{ => driver}/write-zarr-v3-to-s3.cpp (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e6668d6..f9a3b42a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,57 +1,3 @@ -if (NOT TARGET acquire-core-logger) - add_subdirectory(../acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) -endif () +set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(tgt acquire-driver-zarr) -add_library(${tgt} MODULE - common/dimension.hh - common/dimension.cpp - common/thread.pool.hh - common/thread.pool.cpp - common/s3.connection.hh - common/s3.connection.cpp - common/utilities.hh - common/utilities.cpp - writers/sink.hh - writers/sink.creator.hh - writers/sink.creator.cpp - writers/file.sink.hh - writers/file.sink.cpp - writers/s3.sink.hh - writers/s3.sink.cpp - writers/array.writer.hh - writers/array.writer.cpp - writers/zarrv2.array.writer.hh - writers/zarrv2.array.writer.cpp - writers/zarrv3.array.writer.hh - writers/zarrv3.array.writer.cpp - writers/blosc.compressor.hh - writers/blosc.compressor.cpp - zarr.hh - zarr.cpp - zarr.v2.hh - zarr.v2.cpp - zarr.v3.hh - zarr.v3.cpp - zarr.driver.c -) - -target_include_directories(${tgt} PRIVATE - $ -) - -target_enable_simd(${tgt}) -target_link_libraries(${tgt} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp -) -set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS ${tgt} LIBRARY DESTINATION lib) +add_subdirectory(driver) diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt new file mode 100644 index 00000000..e223b5db --- /dev/null +++ b/src/driver/CMakeLists.txt @@ -0,0 +1,61 @@ +option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) + +if (BUILD_ACQUIRE_DRIVER_ZARR) + if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) + endif () + + set(tgt acquire-driver-zarr) + add_library(${tgt} MODULE + common/dimension.hh + common/dimension.cpp + common/thread.pool.hh + common/thread.pool.cpp + common/s3.connection.hh + common/s3.connection.cpp + common/utilities.hh + common/utilities.cpp + writers/sink.hh + writers/sink.creator.hh + writers/sink.creator.cpp + writers/file.sink.hh + writers/file.sink.cpp + writers/s3.sink.hh + writers/s3.sink.cpp + writers/array.writer.hh + writers/array.writer.cpp + writers/zarrv2.array.writer.hh + writers/zarrv2.array.writer.cpp + writers/zarrv3.array.writer.hh + writers/zarrv3.array.writer.cpp + writers/blosc.compressor.hh + writers/blosc.compressor.cpp + zarr.hh + zarr.cpp + zarr.v2.hh + zarr.v2.cpp + zarr.v3.hh + zarr.v3.cpp + zarr.driver.c + ) + + target_include_directories(${tgt} PRIVATE + $ + ) + + target_enable_simd(${tgt}) + target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + + install(TARGETS ${tgt} LIBRARY DESTINATION lib) +endif () \ No newline at end of file diff --git a/src/README.md b/src/driver/README.md similarity index 100% rename from src/README.md rename to src/driver/README.md diff --git a/src/common/dimension.cpp b/src/driver/common/dimension.cpp similarity index 100% rename from src/common/dimension.cpp rename to src/driver/common/dimension.cpp diff --git a/src/common/dimension.hh b/src/driver/common/dimension.hh similarity index 100% rename from src/common/dimension.hh rename to src/driver/common/dimension.hh diff --git a/src/common/macros.hh b/src/driver/common/macros.hh similarity index 100% rename from src/common/macros.hh rename to src/driver/common/macros.hh diff --git a/src/common/s3.connection.cpp b/src/driver/common/s3.connection.cpp similarity index 100% rename from src/common/s3.connection.cpp rename to src/driver/common/s3.connection.cpp diff --git a/src/common/s3.connection.hh b/src/driver/common/s3.connection.hh similarity index 100% rename from src/common/s3.connection.hh rename to src/driver/common/s3.connection.hh diff --git a/src/common/thread.pool.cpp b/src/driver/common/thread.pool.cpp similarity index 100% rename from src/common/thread.pool.cpp rename to src/driver/common/thread.pool.cpp diff --git a/src/common/thread.pool.hh b/src/driver/common/thread.pool.hh similarity index 100% rename from src/common/thread.pool.hh rename to src/driver/common/thread.pool.hh diff --git a/src/common/utilities.cpp b/src/driver/common/utilities.cpp similarity index 100% rename from src/common/utilities.cpp rename to src/driver/common/utilities.cpp diff --git a/src/common/utilities.hh b/src/driver/common/utilities.hh similarity index 100% rename from src/common/utilities.hh rename to src/driver/common/utilities.hh diff --git a/src/writers/array.writer.cpp b/src/driver/writers/array.writer.cpp similarity index 100% rename from src/writers/array.writer.cpp rename to src/driver/writers/array.writer.cpp diff --git a/src/writers/array.writer.hh b/src/driver/writers/array.writer.hh similarity index 100% rename from src/writers/array.writer.hh rename to src/driver/writers/array.writer.hh diff --git a/src/writers/blosc.compressor.cpp b/src/driver/writers/blosc.compressor.cpp similarity index 100% rename from src/writers/blosc.compressor.cpp rename to src/driver/writers/blosc.compressor.cpp diff --git a/src/writers/blosc.compressor.hh b/src/driver/writers/blosc.compressor.hh similarity index 100% rename from src/writers/blosc.compressor.hh rename to src/driver/writers/blosc.compressor.hh diff --git a/src/writers/file.sink.cpp b/src/driver/writers/file.sink.cpp similarity index 100% rename from src/writers/file.sink.cpp rename to src/driver/writers/file.sink.cpp diff --git a/src/writers/file.sink.hh b/src/driver/writers/file.sink.hh similarity index 100% rename from src/writers/file.sink.hh rename to src/driver/writers/file.sink.hh diff --git a/src/writers/s3.sink.cpp b/src/driver/writers/s3.sink.cpp similarity index 100% rename from src/writers/s3.sink.cpp rename to src/driver/writers/s3.sink.cpp diff --git a/src/writers/s3.sink.hh b/src/driver/writers/s3.sink.hh similarity index 100% rename from src/writers/s3.sink.hh rename to src/driver/writers/s3.sink.hh diff --git a/src/writers/sink.creator.cpp b/src/driver/writers/sink.creator.cpp similarity index 100% rename from src/writers/sink.creator.cpp rename to src/driver/writers/sink.creator.cpp diff --git a/src/writers/sink.creator.hh b/src/driver/writers/sink.creator.hh similarity index 100% rename from src/writers/sink.creator.hh rename to src/driver/writers/sink.creator.hh diff --git a/src/writers/sink.hh b/src/driver/writers/sink.hh similarity index 100% rename from src/writers/sink.hh rename to src/driver/writers/sink.hh diff --git a/src/writers/zarrv2.array.writer.cpp b/src/driver/writers/zarrv2.array.writer.cpp similarity index 100% rename from src/writers/zarrv2.array.writer.cpp rename to src/driver/writers/zarrv2.array.writer.cpp diff --git a/src/writers/zarrv2.array.writer.hh b/src/driver/writers/zarrv2.array.writer.hh similarity index 100% rename from src/writers/zarrv2.array.writer.hh rename to src/driver/writers/zarrv2.array.writer.hh diff --git a/src/writers/zarrv3.array.writer.cpp b/src/driver/writers/zarrv3.array.writer.cpp similarity index 100% rename from src/writers/zarrv3.array.writer.cpp rename to src/driver/writers/zarrv3.array.writer.cpp diff --git a/src/writers/zarrv3.array.writer.hh b/src/driver/writers/zarrv3.array.writer.hh similarity index 100% rename from src/writers/zarrv3.array.writer.hh rename to src/driver/writers/zarrv3.array.writer.hh diff --git a/src/zarr.cpp b/src/driver/zarr.cpp similarity index 100% rename from src/zarr.cpp rename to src/driver/zarr.cpp diff --git a/src/zarr.driver.c b/src/driver/zarr.driver.c similarity index 100% rename from src/zarr.driver.c rename to src/driver/zarr.driver.c diff --git a/src/zarr.hh b/src/driver/zarr.hh similarity index 100% rename from src/zarr.hh rename to src/driver/zarr.hh diff --git a/src/zarr.v2.cpp b/src/driver/zarr.v2.cpp similarity index 100% rename from src/zarr.v2.cpp rename to src/driver/zarr.v2.cpp diff --git a/src/zarr.v2.hh b/src/driver/zarr.v2.hh similarity index 100% rename from src/zarr.v2.hh rename to src/driver/zarr.v2.hh diff --git a/src/zarr.v3.cpp b/src/driver/zarr.v3.cpp similarity index 100% rename from src/zarr.v3.cpp rename to src/driver/zarr.v3.cpp diff --git a/src/zarr.v3.hh b/src/driver/zarr.v3.hh similarity index 100% rename from src/zarr.v3.hh rename to src/driver/zarr.v3.hh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc4c2f4b..87878b37 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,90 +1,5 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () - set(NOTEST "TRUE") - add_subdirectory(../acquire-common/acquire-driver-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-driver-common) - add_subdirectory(../acquire-common/acquire-video-runtime ${CMAKE_CURRENT_BINARY_DIR}/acquire-video-runtime) - set(NOTEST "FALSE") - - # - # PARAMETERS - # - set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - - # - # Tests - # - set(tests - list-devices - unit-tests - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 - ) - - foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") - endforeach () - - # - # Copy driver to tests - # - list(POP_FRONT tests onename) - - foreach (driver - acquire-driver-common - acquire-driver-zarr - ) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () - endforeach () + add_subdirectory(driver) endif () diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt new file mode 100644 index 00000000..1fd661a2 --- /dev/null +++ b/tests/driver/CMakeLists.txt @@ -0,0 +1,87 @@ +if (BUILD_ACQUIRE_DRIVER_ZARR) + set(NOTEST "TRUE") + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) + set(NOTEST "FALSE") + + # + # PARAMETERS + # + set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project + + # + # Tests + # + set(tests + list-devices + unit-tests + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 + ) + + foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") + endforeach () + + # + # Copy driver to tests + # + list(POP_FRONT tests onename) + + foreach (driver + acquire-driver-common + acquire-driver-zarr + ) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" + ) + + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + endforeach () + endforeach () +endif () \ No newline at end of file diff --git a/tests/README.md b/tests/driver/README.md similarity index 100% rename from tests/README.md rename to tests/driver/README.md diff --git a/tests/external-metadata-with-whitespace-ok.cpp b/tests/driver/external-metadata-with-whitespace-ok.cpp similarity index 100% rename from tests/external-metadata-with-whitespace-ok.cpp rename to tests/driver/external-metadata-with-whitespace-ok.cpp diff --git a/tests/get-meta.cpp b/tests/driver/get-meta.cpp similarity index 100% rename from tests/get-meta.cpp rename to tests/driver/get-meta.cpp diff --git a/tests/get-set-get.cpp b/tests/driver/get-set-get.cpp similarity index 100% rename from tests/get-set-get.cpp rename to tests/driver/get-set-get.cpp diff --git a/tests/get.cpp b/tests/driver/get.cpp similarity index 100% rename from tests/get.cpp rename to tests/driver/get.cpp diff --git a/tests/list-devices.cpp b/tests/driver/list-devices.cpp similarity index 100% rename from tests/list-devices.cpp rename to tests/driver/list-devices.cpp diff --git a/tests/metadata-dimension-sizes.cpp b/tests/driver/metadata-dimension-sizes.cpp similarity index 100% rename from tests/metadata-dimension-sizes.cpp rename to tests/driver/metadata-dimension-sizes.cpp diff --git a/tests/multiscales-metadata.cpp b/tests/driver/multiscales-metadata.cpp similarity index 100% rename from tests/multiscales-metadata.cpp rename to tests/driver/multiscales-metadata.cpp diff --git a/tests/repeat-start.cpp b/tests/driver/repeat-start.cpp similarity index 100% rename from tests/repeat-start.cpp rename to tests/driver/repeat-start.cpp diff --git a/tests/restart-stopped-zarr-resets-threadpool.cpp b/tests/driver/restart-stopped-zarr-resets-threadpool.cpp similarity index 100% rename from tests/restart-stopped-zarr-resets-threadpool.cpp rename to tests/driver/restart-stopped-zarr-resets-threadpool.cpp diff --git a/tests/unit-tests.cpp b/tests/driver/unit-tests.cpp similarity index 100% rename from tests/unit-tests.cpp rename to tests/driver/unit-tests.cpp diff --git a/tests/write-zarr-v2-compressed-multiscale.cpp b/tests/driver/write-zarr-v2-compressed-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-multiscale.cpp rename to tests/driver/write-zarr-v2-compressed-multiscale.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking.cpp diff --git a/tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp b/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp rename to tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp b/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp rename to tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale.cpp b/tests/driver/write-zarr-v2-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale.cpp rename to tests/driver/write-zarr-v2-raw-multiscale.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking.cpp diff --git a/tests/write-zarr-v2-raw-with-ragged-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-ragged-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp diff --git a/tests/write-zarr-v2-raw.cpp b/tests/driver/write-zarr-v2-raw.cpp similarity index 100% rename from tests/write-zarr-v2-raw.cpp rename to tests/driver/write-zarr-v2-raw.cpp diff --git a/tests/write-zarr-v2-to-s3.cpp b/tests/driver/write-zarr-v2-to-s3.cpp similarity index 100% rename from tests/write-zarr-v2-to-s3.cpp rename to tests/driver/write-zarr-v2-to-s3.cpp diff --git a/tests/write-zarr-v2-with-lz4-compression.cpp b/tests/driver/write-zarr-v2-with-lz4-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-lz4-compression.cpp rename to tests/driver/write-zarr-v2-with-lz4-compression.cpp diff --git a/tests/write-zarr-v2-with-zstd-compression.cpp b/tests/driver/write-zarr-v2-with-zstd-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-zstd-compression.cpp rename to tests/driver/write-zarr-v2-with-zstd-compression.cpp diff --git a/tests/write-zarr-v3-compressed.cpp b/tests/driver/write-zarr-v3-compressed.cpp similarity index 100% rename from tests/write-zarr-v3-compressed.cpp rename to tests/driver/write-zarr-v3-compressed.cpp diff --git a/tests/write-zarr-v3-raw-chunk-exceeds-array.cpp b/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp similarity index 100% rename from tests/write-zarr-v3-raw-chunk-exceeds-array.cpp rename to tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp diff --git a/tests/write-zarr-v3-raw-multiscale.cpp b/tests/driver/write-zarr-v3-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v3-raw-multiscale.cpp rename to tests/driver/write-zarr-v3-raw-multiscale.cpp diff --git a/tests/write-zarr-v3-raw-with-ragged-sharding.cpp b/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp similarity index 100% rename from tests/write-zarr-v3-raw-with-ragged-sharding.cpp rename to tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp diff --git a/tests/write-zarr-v3-raw.cpp b/tests/driver/write-zarr-v3-raw.cpp similarity index 100% rename from tests/write-zarr-v3-raw.cpp rename to tests/driver/write-zarr-v3-raw.cpp diff --git a/tests/write-zarr-v3-to-s3.cpp b/tests/driver/write-zarr-v3-to-s3.cpp similarity index 100% rename from tests/write-zarr-v3-to-s3.cpp rename to tests/driver/write-zarr-v3-to-s3.cpp From a811e221925bf2f4886c13a830bf038289788bfb Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:06:22 -0400 Subject: [PATCH 2/2] Define the Zarr streaming API. --- include/zarr.h | 216 +++++++++++++++++++++++++++++++++++++++++++ include/zarr.types.h | 133 ++++++++++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 include/zarr.h create mode 100644 include/zarr.types.h diff --git a/include/zarr.h b/include/zarr.h new file mode 100644 index 00000000..eac50f35 --- /dev/null +++ b/include/zarr.h @@ -0,0 +1,216 @@ +#ifndef H_ACQUIRE_ZARR_V0 +#define H_ACQUIRE_ZARR_V0 + +#include "zarr.types.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef struct ZarrStreamSettings_s ZarrStreamSettings; + typedef struct ZarrStream_s ZarrStream; + + /** + * @brief Get the version of the Zarr API. + * @return The version of the Zarr API. + */ + uint32_t Zarr_get_api_version(); + + /** + * @brief Get the message for the given status code. + * @param status The status code. + * @return A human-readable status message. + */ + const char* Zarr_get_error_message(ZarrStatus status); + + /** + * @brief Create a Zarr stream settings struct. + * @return A pointer to the Zarr stream settings struct, or NULL on failure. + */ + ZarrStreamSettings* ZarrStreamSettings_create(); + + /** + * @brief Destroy a Zarr stream settings struct. + * @details This function frees the memory allocated for the Zarr stream + * settings struct. + * @param[in] settings The Zarr stream settings struct. + */ + void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); + + /** + * @brief Copy a Zarr stream settings struct. + * @param[in] settings The Zarr stream settings struct to copy. + * @return A copy of the Zarr stream settings struct. + */ + ZarrStreamSettings* ZarrStreamSettings_copy( + const ZarrStreamSettings* settings); + + /** + * @brief Set store path and S3 settings for the Zarr stream. + * @param[in, out] settings + * @param[in] store_path The store path for the Zarr stream. Directory path + * when acquiring to the filesystem, key prefix when acquiring to S3. + * @param[in] bytes_of_store_path The length of @p store_path in bytes, + * including the null terminator. + * @param[in] s3_settings Optional S3 settings. If NULL, the store path is + * assumed to be a directory path. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings); + + /** + * @brief Set the data type, compressor, codec, compression_settings level, + * and shuffle for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] compression_settings The compression_settings settings. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings); + + /** + * @brief Set the data type for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] data_type The data type. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type); + + /** + * @brief Reserve space for dimensions in the Zarr stream settings struct. + * @detail *Must* precede calls to ZarrStreamSettings_set_dimension. We + * require at least 3 dimensions to validate settings, but you may set up to + * 32 dimensions. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] count The number of dimensions to reserve space for. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_reserve_dimensions( + ZarrStreamSettings* settings, + size_t count); + + /** + * @brief Set properties for an acquisition dimension. + * @detail The order of the dimensions in the Zarr stream is the order in + * which they are set. The first dimension set is the slowest varying + * dimension, and the last dimension set is the fastest varying dimension. + * For example, if the dimensions are set in the order z, y, x, the fastest + * varying dimension is x, the next fastest varying dimension is y, and the + * slowest varying dimension is z. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] index The index of the dimension to set. Must be less than the + * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. + * @param[in] dimension The dimension's settings. + */ + ZarrStatus ZarrStreamSettings_set_dimension( + ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension); + + /** + * @brief Set the multiscale flag for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] multiscale A flag indicating whether to stream to multiple + * levels of detail. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale); + + /** + * @brief Set JSON-formatted custom metadata for the Zarr stream. + * @details This metadata will be written to acquire-zarr.json in the + * metadata directory of the Zarr store. This parameter is optional. + * @param settings[in, out] settings The Zarr stream settings struct. + * @param external_metadata JSON-formatted external metadata. + * @param bytes_of_external_metadata The length of @p custom_metadata in + * bytes, including the null terminator. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_custom_metadata( + ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata); + + const char* ZarrStreamSettings_get_store_path( + const ZarrStreamSettings* settings); + + ZarrS3Settings ZarrStreamSettings_get_s3_settings( + const ZarrStreamSettings* settings); + + ZarrCompressionSettings ZarrStreamSettings_get_compression( + const ZarrStreamSettings* settings); + + ZarrDataType ZarrStreamSettings_get_data_type( + const ZarrStreamSettings* settings); + + size_t ZarrStreamSettings_get_dimension_count( + const ZarrStreamSettings* settings); + + ZarrDimensionProperties ZarrStreamSettings_get_dimension( + const ZarrStreamSettings* settings, + size_t index); + + bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); + + const char* ZarrStreamSettings_get_custom_metadata( + const ZarrStreamSettings* settings); + + /** + * @brief Create a Zarr stream. + * @param[in, out] settings The settings for the Zarr stream. + * @param[in] version The version of the Zarr stream. 2 or 3. + * @return A pointer to the Zarr stream struct, or NULL on failure. + */ + ZarrStream* ZarrStream_create(ZarrStreamSettings* settings, + ZarrVersion version); + + /** + * @brief Destroy a Zarr stream. + * @details This function frees the memory allocated for the Zarr stream. + * @param stream The Zarr stream struct to destroy. + */ + void ZarrStream_destroy(ZarrStream* stream); + + /** + * @brief Append data to the Zarr stream. + * @param[in, out] stream The Zarr stream struct. + * @param[in] data The data to append. + * @param[in] bytes_in The number of bytes in @p data. It should be at least + * the size of a single frame. + * @param[out] bytes_out The number of bytes written to the stream. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out); + + /** + * @brief Get the version (i.e., 2 or 3) of the Zarr stream. + * @param stream The Zarr stream struct. + * @return The version of the Zarr stream. + */ + ZarrVersion ZarrStream_get_version(const ZarrStream* stream); + + /** + * @brief Get a copy of the settings for the Zarr stream. + * @param stream The Zarr stream struct. + * @return A copy of the settings for the Zarr stream. + */ + ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); + + ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + ZarrLogLevel Zarr_get_log_level(); + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_V0 diff --git a/include/zarr.types.h b/include/zarr.types.h new file mode 100644 index 00000000..fdb6c715 --- /dev/null +++ b/include/zarr.types.h @@ -0,0 +1,133 @@ +#ifndef H_ACQUIRE_ZARR_TYPES_V0 +#define H_ACQUIRE_ZARR_TYPES_V0 + +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef enum + { + ZarrStatus_Success = 0, + ZarrStatus_InvalidArgument, + ZarrStatus_Overflow, + ZarrStatus_InvalidIndex, + ZarrStatus_NotYetImplemented, + ZarrStatus_InternalError, + ZarrStatus_OutOfMemory, + ZarrStatus_IOError, + ZarrStatus_CompressionError, + ZarrStatus_InvalidSettings, + ZarrStatusCount, + } ZarrStatus; + + typedef enum + { + ZarrVersion_2 = 2, + ZarrVersion_3, + ZarrVersionCount + } ZarrVersion; + + typedef enum + { + ZarrLogLevel_Debug, + ZarrLogLevel_Info, + ZarrLogLevel_Warning, + ZarrLogLevel_Error, + ZarrLogLevel_None, + ZarrLogLevelCount + } ZarrLogLevel; + + typedef enum + { + ZarrDataType_uint8, + ZarrDataType_uint16, + ZarrDataType_uint32, + ZarrDataType_uint64, + ZarrDataType_int8, + ZarrDataType_int16, + ZarrDataType_int32, + ZarrDataType_int64, + ZarrDataType_float32, + ZarrDataType_float64, + ZarrDataTypeCount + } ZarrDataType; + + typedef enum + { + ZarrCompressor_None = 0, + ZarrCompressor_Blosc1, + ZarrCompressorCount + } ZarrCompressor; + + typedef enum + { + ZarrCompressionCodec_None = 0, + ZarrCompressionCodec_BloscLZ4, + ZarrCompressionCodec_BloscZstd, + ZarrCompressionCodecCount + } ZarrCompressionCodec; + + typedef enum + { + ZarrDimensionType_Space = 0, + ZarrDimensionType_Channel, + ZarrDimensionType_Time, + ZarrDimensionType_Other, + ZarrDimensionTypeCount + } ZarrDimensionType; + + /** + * @brief S3 settings for streaming to Zarr. + */ + typedef struct + { + const char* endpoint; + size_t bytes_of_endpoint; + const char* bucket_name; + size_t bytes_of_bucket_name; + const char* access_key_id; + size_t bytes_of_access_key_id; + const char* secret_access_key; + size_t bytes_of_secret_access_key; + } ZarrS3Settings; + + /** + * @brief Compression settings for a Zarr array. + * @detail The compressor is not the same as the codec. A codec is + * a specific implementation of a compression algorithm, while a compressor + * is a library that implements one or more codecs. + */ + typedef struct + { + ZarrCompressor compressor; /**< Compressor to use */ + ZarrCompressionCodec codec; /**< Codec to use */ + uint8_t level; /**< Compression level */ + uint8_t shuffle; /**< Whether to shuffle the data before compressing */ + } ZarrCompressionSettings; + + /** + * @brief Properties of a dimension of the Zarr array. + */ + typedef struct + { + const char* name; /**< Name of the dimension */ + size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ + ZarrDimensionType kind; /**< Type of the dimension */ + uint32_t array_size_px; /**< Size of the array along this dimension in + pixels */ + uint32_t chunk_size_px; /**< Size of the chunks along this dimension in + pixels */ + uint32_t shard_size_chunks; /**< Number of chunks in a shard along this + dimension */ + } ZarrDimensionProperties; + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_TYPES_V0 \ No newline at end of file