From c7832d724950c7ce0ebec3797d716dd203ece9bc Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 09:59:22 -0400 Subject: [PATCH 01/11] Move driver source files and tests to a separate directory. --- src/CMakeLists.txt | 58 +------------ src/driver/CMakeLists.txt | 61 +++++++++++++ src/{ => driver}/README.md | 0 src/{ => driver}/common/dimension.cpp | 0 src/{ => driver}/common/dimension.hh | 0 src/{ => driver}/common/macros.hh | 0 src/{ => driver}/common/s3.connection.cpp | 0 src/{ => driver}/common/s3.connection.hh | 0 src/{ => driver}/common/thread.pool.cpp | 0 src/{ => driver}/common/thread.pool.hh | 0 src/{ => driver}/common/utilities.cpp | 0 src/{ => driver}/common/utilities.hh | 0 src/{ => driver}/writers/array.writer.cpp | 0 src/{ => driver}/writers/array.writer.hh | 0 src/{ => driver}/writers/blosc.compressor.cpp | 0 src/{ => driver}/writers/blosc.compressor.hh | 0 src/{ => driver}/writers/file.sink.cpp | 0 src/{ => driver}/writers/file.sink.hh | 0 src/{ => driver}/writers/s3.sink.cpp | 0 src/{ => driver}/writers/s3.sink.hh | 0 src/{ => driver}/writers/sink.creator.cpp | 0 src/{ => driver}/writers/sink.creator.hh | 0 src/{ => driver}/writers/sink.hh | 0 .../writers/zarrv2.array.writer.cpp | 0 .../writers/zarrv2.array.writer.hh | 0 .../writers/zarrv3.array.writer.cpp | 0 .../writers/zarrv3.array.writer.hh | 0 src/{ => driver}/zarr.cpp | 0 src/{ => driver}/zarr.driver.c | 0 src/{ => driver}/zarr.hh | 0 src/{ => driver}/zarr.v2.cpp | 0 src/{ => driver}/zarr.v2.hh | 0 src/{ => driver}/zarr.v3.cpp | 0 src/{ => driver}/zarr.v3.hh | 0 tests/CMakeLists.txt | 87 +------------------ tests/driver/CMakeLists.txt | 87 +++++++++++++++++++ tests/{ => driver}/README.md | 0 .../external-metadata-with-whitespace-ok.cpp | 0 tests/{ => driver}/get-meta.cpp | 0 tests/{ => driver}/get-set-get.cpp | 0 tests/{ => driver}/get.cpp | 0 tests/{ => driver}/list-devices.cpp | 0 .../{ => driver}/metadata-dimension-sizes.cpp | 0 tests/{ => driver}/multiscales-metadata.cpp | 0 tests/{ => driver}/repeat-start.cpp | 0 ...restart-stopped-zarr-resets-threadpool.cpp | 0 tests/{ => driver}/unit-tests.cpp | 0 .../write-zarr-v2-compressed-multiscale.cpp | 0 ...-compressed-with-chunking-and-rollover.cpp | 0 ...write-zarr-v2-compressed-with-chunking.cpp | 0 ...-raw-chunk-size-larger-than-frame-size.cpp | 0 ...-raw-multiscale-with-trivial-tile-size.cpp | 0 .../write-zarr-v2-raw-multiscale.cpp | 0 ...v2-raw-with-even-chunking-and-rollover.cpp | 0 .../write-zarr-v2-raw-with-even-chunking.cpp | 0 ...write-zarr-v2-raw-with-ragged-chunking.cpp | 0 tests/{ => driver}/write-zarr-v2-raw.cpp | 0 tests/{ => driver}/write-zarr-v2-to-s3.cpp | 0 .../write-zarr-v2-with-lz4-compression.cpp | 0 .../write-zarr-v2-with-zstd-compression.cpp | 0 .../{ => driver}/write-zarr-v3-compressed.cpp | 0 .../write-zarr-v3-raw-chunk-exceeds-array.cpp | 0 .../write-zarr-v3-raw-multiscale.cpp | 0 ...write-zarr-v3-raw-with-ragged-sharding.cpp | 0 tests/{ => driver}/write-zarr-v3-raw.cpp | 0 tests/{ => driver}/write-zarr-v3-to-s3.cpp | 0 66 files changed, 151 insertions(+), 142 deletions(-) create mode 100644 src/driver/CMakeLists.txt rename src/{ => driver}/README.md (100%) rename src/{ => driver}/common/dimension.cpp (100%) rename src/{ => driver}/common/dimension.hh (100%) rename src/{ => driver}/common/macros.hh (100%) rename src/{ => driver}/common/s3.connection.cpp (100%) rename src/{ => driver}/common/s3.connection.hh (100%) rename src/{ => driver}/common/thread.pool.cpp (100%) rename src/{ => driver}/common/thread.pool.hh (100%) rename src/{ => driver}/common/utilities.cpp (100%) rename src/{ => driver}/common/utilities.hh (100%) rename src/{ => driver}/writers/array.writer.cpp (100%) rename src/{ => driver}/writers/array.writer.hh (100%) rename src/{ => driver}/writers/blosc.compressor.cpp (100%) rename src/{ => driver}/writers/blosc.compressor.hh (100%) rename src/{ => driver}/writers/file.sink.cpp (100%) rename src/{ => driver}/writers/file.sink.hh (100%) rename src/{ => driver}/writers/s3.sink.cpp (100%) rename src/{ => driver}/writers/s3.sink.hh (100%) rename src/{ => driver}/writers/sink.creator.cpp (100%) rename src/{ => driver}/writers/sink.creator.hh (100%) rename src/{ => driver}/writers/sink.hh (100%) rename src/{ => driver}/writers/zarrv2.array.writer.cpp (100%) rename src/{ => driver}/writers/zarrv2.array.writer.hh (100%) rename src/{ => driver}/writers/zarrv3.array.writer.cpp (100%) rename src/{ => driver}/writers/zarrv3.array.writer.hh (100%) rename src/{ => driver}/zarr.cpp (100%) rename src/{ => driver}/zarr.driver.c (100%) rename src/{ => driver}/zarr.hh (100%) rename src/{ => driver}/zarr.v2.cpp (100%) rename src/{ => driver}/zarr.v2.hh (100%) rename src/{ => driver}/zarr.v3.cpp (100%) rename src/{ => driver}/zarr.v3.hh (100%) create mode 100644 tests/driver/CMakeLists.txt rename tests/{ => driver}/README.md (100%) rename tests/{ => driver}/external-metadata-with-whitespace-ok.cpp (100%) rename tests/{ => driver}/get-meta.cpp (100%) rename tests/{ => driver}/get-set-get.cpp (100%) rename tests/{ => driver}/get.cpp (100%) rename tests/{ => driver}/list-devices.cpp (100%) rename tests/{ => driver}/metadata-dimension-sizes.cpp (100%) rename tests/{ => driver}/multiscales-metadata.cpp (100%) rename tests/{ => driver}/repeat-start.cpp (100%) rename tests/{ => driver}/restart-stopped-zarr-resets-threadpool.cpp (100%) rename tests/{ => driver}/unit-tests.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-with-chunking-and-rollover.cpp (100%) rename tests/{ => driver}/write-zarr-v2-compressed-with-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-even-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw-with-ragged-chunking.cpp (100%) rename tests/{ => driver}/write-zarr-v2-raw.cpp (100%) rename tests/{ => driver}/write-zarr-v2-to-s3.cpp (100%) rename tests/{ => driver}/write-zarr-v2-with-lz4-compression.cpp (100%) rename tests/{ => driver}/write-zarr-v2-with-zstd-compression.cpp (100%) rename tests/{ => driver}/write-zarr-v3-compressed.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-chunk-exceeds-array.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-multiscale.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw-with-ragged-sharding.cpp (100%) rename tests/{ => driver}/write-zarr-v3-raw.cpp (100%) rename tests/{ => driver}/write-zarr-v3-to-s3.cpp (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e6668d6..f9a3b42a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,57 +1,3 @@ -if (NOT TARGET acquire-core-logger) - add_subdirectory(../acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) -endif () +set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(tgt acquire-driver-zarr) -add_library(${tgt} MODULE - common/dimension.hh - common/dimension.cpp - common/thread.pool.hh - common/thread.pool.cpp - common/s3.connection.hh - common/s3.connection.cpp - common/utilities.hh - common/utilities.cpp - writers/sink.hh - writers/sink.creator.hh - writers/sink.creator.cpp - writers/file.sink.hh - writers/file.sink.cpp - writers/s3.sink.hh - writers/s3.sink.cpp - writers/array.writer.hh - writers/array.writer.cpp - writers/zarrv2.array.writer.hh - writers/zarrv2.array.writer.cpp - writers/zarrv3.array.writer.hh - writers/zarrv3.array.writer.cpp - writers/blosc.compressor.hh - writers/blosc.compressor.cpp - zarr.hh - zarr.cpp - zarr.v2.hh - zarr.v2.cpp - zarr.v3.hh - zarr.v3.cpp - zarr.driver.c -) - -target_include_directories(${tgt} PRIVATE - $ -) - -target_enable_simd(${tgt}) -target_link_libraries(${tgt} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp -) -set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" -) - -install(TARGETS ${tgt} LIBRARY DESTINATION lib) +add_subdirectory(driver) diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt new file mode 100644 index 00000000..e223b5db --- /dev/null +++ b/src/driver/CMakeLists.txt @@ -0,0 +1,61 @@ +option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) + +if (BUILD_ACQUIRE_DRIVER_ZARR) + if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) + endif () + + set(tgt acquire-driver-zarr) + add_library(${tgt} MODULE + common/dimension.hh + common/dimension.cpp + common/thread.pool.hh + common/thread.pool.cpp + common/s3.connection.hh + common/s3.connection.cpp + common/utilities.hh + common/utilities.cpp + writers/sink.hh + writers/sink.creator.hh + writers/sink.creator.cpp + writers/file.sink.hh + writers/file.sink.cpp + writers/s3.sink.hh + writers/s3.sink.cpp + writers/array.writer.hh + writers/array.writer.cpp + writers/zarrv2.array.writer.hh + writers/zarrv2.array.writer.cpp + writers/zarrv3.array.writer.hh + writers/zarrv3.array.writer.cpp + writers/blosc.compressor.hh + writers/blosc.compressor.cpp + zarr.hh + zarr.cpp + zarr.v2.hh + zarr.v2.cpp + zarr.v3.hh + zarr.v3.cpp + zarr.driver.c + ) + + target_include_directories(${tgt} PRIVATE + $ + ) + + target_enable_simd(${tgt}) + target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + + install(TARGETS ${tgt} LIBRARY DESTINATION lib) +endif () \ No newline at end of file diff --git a/src/README.md b/src/driver/README.md similarity index 100% rename from src/README.md rename to src/driver/README.md diff --git a/src/common/dimension.cpp b/src/driver/common/dimension.cpp similarity index 100% rename from src/common/dimension.cpp rename to src/driver/common/dimension.cpp diff --git a/src/common/dimension.hh b/src/driver/common/dimension.hh similarity index 100% rename from src/common/dimension.hh rename to src/driver/common/dimension.hh diff --git a/src/common/macros.hh b/src/driver/common/macros.hh similarity index 100% rename from src/common/macros.hh rename to src/driver/common/macros.hh diff --git a/src/common/s3.connection.cpp b/src/driver/common/s3.connection.cpp similarity index 100% rename from src/common/s3.connection.cpp rename to src/driver/common/s3.connection.cpp diff --git a/src/common/s3.connection.hh b/src/driver/common/s3.connection.hh similarity index 100% rename from src/common/s3.connection.hh rename to src/driver/common/s3.connection.hh diff --git a/src/common/thread.pool.cpp b/src/driver/common/thread.pool.cpp similarity index 100% rename from src/common/thread.pool.cpp rename to src/driver/common/thread.pool.cpp diff --git a/src/common/thread.pool.hh b/src/driver/common/thread.pool.hh similarity index 100% rename from src/common/thread.pool.hh rename to src/driver/common/thread.pool.hh diff --git a/src/common/utilities.cpp b/src/driver/common/utilities.cpp similarity index 100% rename from src/common/utilities.cpp rename to src/driver/common/utilities.cpp diff --git a/src/common/utilities.hh b/src/driver/common/utilities.hh similarity index 100% rename from src/common/utilities.hh rename to src/driver/common/utilities.hh diff --git a/src/writers/array.writer.cpp b/src/driver/writers/array.writer.cpp similarity index 100% rename from src/writers/array.writer.cpp rename to src/driver/writers/array.writer.cpp diff --git a/src/writers/array.writer.hh b/src/driver/writers/array.writer.hh similarity index 100% rename from src/writers/array.writer.hh rename to src/driver/writers/array.writer.hh diff --git a/src/writers/blosc.compressor.cpp b/src/driver/writers/blosc.compressor.cpp similarity index 100% rename from src/writers/blosc.compressor.cpp rename to src/driver/writers/blosc.compressor.cpp diff --git a/src/writers/blosc.compressor.hh b/src/driver/writers/blosc.compressor.hh similarity index 100% rename from src/writers/blosc.compressor.hh rename to src/driver/writers/blosc.compressor.hh diff --git a/src/writers/file.sink.cpp b/src/driver/writers/file.sink.cpp similarity index 100% rename from src/writers/file.sink.cpp rename to src/driver/writers/file.sink.cpp diff --git a/src/writers/file.sink.hh b/src/driver/writers/file.sink.hh similarity index 100% rename from src/writers/file.sink.hh rename to src/driver/writers/file.sink.hh diff --git a/src/writers/s3.sink.cpp b/src/driver/writers/s3.sink.cpp similarity index 100% rename from src/writers/s3.sink.cpp rename to src/driver/writers/s3.sink.cpp diff --git a/src/writers/s3.sink.hh b/src/driver/writers/s3.sink.hh similarity index 100% rename from src/writers/s3.sink.hh rename to src/driver/writers/s3.sink.hh diff --git a/src/writers/sink.creator.cpp b/src/driver/writers/sink.creator.cpp similarity index 100% rename from src/writers/sink.creator.cpp rename to src/driver/writers/sink.creator.cpp diff --git a/src/writers/sink.creator.hh b/src/driver/writers/sink.creator.hh similarity index 100% rename from src/writers/sink.creator.hh rename to src/driver/writers/sink.creator.hh diff --git a/src/writers/sink.hh b/src/driver/writers/sink.hh similarity index 100% rename from src/writers/sink.hh rename to src/driver/writers/sink.hh diff --git a/src/writers/zarrv2.array.writer.cpp b/src/driver/writers/zarrv2.array.writer.cpp similarity index 100% rename from src/writers/zarrv2.array.writer.cpp rename to src/driver/writers/zarrv2.array.writer.cpp diff --git a/src/writers/zarrv2.array.writer.hh b/src/driver/writers/zarrv2.array.writer.hh similarity index 100% rename from src/writers/zarrv2.array.writer.hh rename to src/driver/writers/zarrv2.array.writer.hh diff --git a/src/writers/zarrv3.array.writer.cpp b/src/driver/writers/zarrv3.array.writer.cpp similarity index 100% rename from src/writers/zarrv3.array.writer.cpp rename to src/driver/writers/zarrv3.array.writer.cpp diff --git a/src/writers/zarrv3.array.writer.hh b/src/driver/writers/zarrv3.array.writer.hh similarity index 100% rename from src/writers/zarrv3.array.writer.hh rename to src/driver/writers/zarrv3.array.writer.hh diff --git a/src/zarr.cpp b/src/driver/zarr.cpp similarity index 100% rename from src/zarr.cpp rename to src/driver/zarr.cpp diff --git a/src/zarr.driver.c b/src/driver/zarr.driver.c similarity index 100% rename from src/zarr.driver.c rename to src/driver/zarr.driver.c diff --git a/src/zarr.hh b/src/driver/zarr.hh similarity index 100% rename from src/zarr.hh rename to src/driver/zarr.hh diff --git a/src/zarr.v2.cpp b/src/driver/zarr.v2.cpp similarity index 100% rename from src/zarr.v2.cpp rename to src/driver/zarr.v2.cpp diff --git a/src/zarr.v2.hh b/src/driver/zarr.v2.hh similarity index 100% rename from src/zarr.v2.hh rename to src/driver/zarr.v2.hh diff --git a/src/zarr.v3.cpp b/src/driver/zarr.v3.cpp similarity index 100% rename from src/zarr.v3.cpp rename to src/driver/zarr.v3.cpp diff --git a/src/zarr.v3.hh b/src/driver/zarr.v3.hh similarity index 100% rename from src/zarr.v3.hh rename to src/driver/zarr.v3.hh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cc4c2f4b..87878b37 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,90 +1,5 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () - set(NOTEST "TRUE") - add_subdirectory(../acquire-common/acquire-driver-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-driver-common) - add_subdirectory(../acquire-common/acquire-video-runtime ${CMAKE_CURRENT_BINARY_DIR}/acquire-video-runtime) - set(NOTEST "FALSE") - - # - # PARAMETERS - # - set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - - # - # Tests - # - set(tests - list-devices - unit-tests - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 - ) - - foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") - endforeach () - - # - # Copy driver to tests - # - list(POP_FRONT tests onename) - - foreach (driver - acquire-driver-common - acquire-driver-zarr - ) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () - endforeach () + add_subdirectory(driver) endif () diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt new file mode 100644 index 00000000..1fd661a2 --- /dev/null +++ b/tests/driver/CMakeLists.txt @@ -0,0 +1,87 @@ +if (BUILD_ACQUIRE_DRIVER_ZARR) + set(NOTEST "TRUE") + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) + set(NOTEST "FALSE") + + # + # PARAMETERS + # + set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project + + # + # Tests + # + set(tests + list-devices + unit-tests + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 + ) + + foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) + + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") + endforeach () + + # + # Copy driver to tests + # + list(POP_FRONT tests onename) + + foreach (driver + acquire-driver-common + acquire-driver-zarr + ) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" + ) + + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) + endforeach () + endforeach () +endif () \ No newline at end of file diff --git a/tests/README.md b/tests/driver/README.md similarity index 100% rename from tests/README.md rename to tests/driver/README.md diff --git a/tests/external-metadata-with-whitespace-ok.cpp b/tests/driver/external-metadata-with-whitespace-ok.cpp similarity index 100% rename from tests/external-metadata-with-whitespace-ok.cpp rename to tests/driver/external-metadata-with-whitespace-ok.cpp diff --git a/tests/get-meta.cpp b/tests/driver/get-meta.cpp similarity index 100% rename from tests/get-meta.cpp rename to tests/driver/get-meta.cpp diff --git a/tests/get-set-get.cpp b/tests/driver/get-set-get.cpp similarity index 100% rename from tests/get-set-get.cpp rename to tests/driver/get-set-get.cpp diff --git a/tests/get.cpp b/tests/driver/get.cpp similarity index 100% rename from tests/get.cpp rename to tests/driver/get.cpp diff --git a/tests/list-devices.cpp b/tests/driver/list-devices.cpp similarity index 100% rename from tests/list-devices.cpp rename to tests/driver/list-devices.cpp diff --git a/tests/metadata-dimension-sizes.cpp b/tests/driver/metadata-dimension-sizes.cpp similarity index 100% rename from tests/metadata-dimension-sizes.cpp rename to tests/driver/metadata-dimension-sizes.cpp diff --git a/tests/multiscales-metadata.cpp b/tests/driver/multiscales-metadata.cpp similarity index 100% rename from tests/multiscales-metadata.cpp rename to tests/driver/multiscales-metadata.cpp diff --git a/tests/repeat-start.cpp b/tests/driver/repeat-start.cpp similarity index 100% rename from tests/repeat-start.cpp rename to tests/driver/repeat-start.cpp diff --git a/tests/restart-stopped-zarr-resets-threadpool.cpp b/tests/driver/restart-stopped-zarr-resets-threadpool.cpp similarity index 100% rename from tests/restart-stopped-zarr-resets-threadpool.cpp rename to tests/driver/restart-stopped-zarr-resets-threadpool.cpp diff --git a/tests/unit-tests.cpp b/tests/driver/unit-tests.cpp similarity index 100% rename from tests/unit-tests.cpp rename to tests/driver/unit-tests.cpp diff --git a/tests/write-zarr-v2-compressed-multiscale.cpp b/tests/driver/write-zarr-v2-compressed-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-multiscale.cpp rename to tests/driver/write-zarr-v2-compressed-multiscale.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-compressed-with-chunking.cpp b/tests/driver/write-zarr-v2-compressed-with-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-compressed-with-chunking.cpp rename to tests/driver/write-zarr-v2-compressed-with-chunking.cpp diff --git a/tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp b/tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp rename to tests/driver/write-zarr-v2-raw-chunk-size-larger-than-frame-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp b/tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp rename to tests/driver/write-zarr-v2-raw-multiscale-with-trivial-tile-size.cpp diff --git a/tests/write-zarr-v2-raw-multiscale.cpp b/tests/driver/write-zarr-v2-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v2-raw-multiscale.cpp rename to tests/driver/write-zarr-v2-raw-multiscale.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking-and-rollover.cpp diff --git a/tests/write-zarr-v2-raw-with-even-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-even-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-even-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-even-chunking.cpp diff --git a/tests/write-zarr-v2-raw-with-ragged-chunking.cpp b/tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp similarity index 100% rename from tests/write-zarr-v2-raw-with-ragged-chunking.cpp rename to tests/driver/write-zarr-v2-raw-with-ragged-chunking.cpp diff --git a/tests/write-zarr-v2-raw.cpp b/tests/driver/write-zarr-v2-raw.cpp similarity index 100% rename from tests/write-zarr-v2-raw.cpp rename to tests/driver/write-zarr-v2-raw.cpp diff --git a/tests/write-zarr-v2-to-s3.cpp b/tests/driver/write-zarr-v2-to-s3.cpp similarity index 100% rename from tests/write-zarr-v2-to-s3.cpp rename to tests/driver/write-zarr-v2-to-s3.cpp diff --git a/tests/write-zarr-v2-with-lz4-compression.cpp b/tests/driver/write-zarr-v2-with-lz4-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-lz4-compression.cpp rename to tests/driver/write-zarr-v2-with-lz4-compression.cpp diff --git a/tests/write-zarr-v2-with-zstd-compression.cpp b/tests/driver/write-zarr-v2-with-zstd-compression.cpp similarity index 100% rename from tests/write-zarr-v2-with-zstd-compression.cpp rename to tests/driver/write-zarr-v2-with-zstd-compression.cpp diff --git a/tests/write-zarr-v3-compressed.cpp b/tests/driver/write-zarr-v3-compressed.cpp similarity index 100% rename from tests/write-zarr-v3-compressed.cpp rename to tests/driver/write-zarr-v3-compressed.cpp diff --git a/tests/write-zarr-v3-raw-chunk-exceeds-array.cpp b/tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp similarity index 100% rename from tests/write-zarr-v3-raw-chunk-exceeds-array.cpp rename to tests/driver/write-zarr-v3-raw-chunk-exceeds-array.cpp diff --git a/tests/write-zarr-v3-raw-multiscale.cpp b/tests/driver/write-zarr-v3-raw-multiscale.cpp similarity index 100% rename from tests/write-zarr-v3-raw-multiscale.cpp rename to tests/driver/write-zarr-v3-raw-multiscale.cpp diff --git a/tests/write-zarr-v3-raw-with-ragged-sharding.cpp b/tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp similarity index 100% rename from tests/write-zarr-v3-raw-with-ragged-sharding.cpp rename to tests/driver/write-zarr-v3-raw-with-ragged-sharding.cpp diff --git a/tests/write-zarr-v3-raw.cpp b/tests/driver/write-zarr-v3-raw.cpp similarity index 100% rename from tests/write-zarr-v3-raw.cpp rename to tests/driver/write-zarr-v3-raw.cpp diff --git a/tests/write-zarr-v3-to-s3.cpp b/tests/driver/write-zarr-v3-to-s3.cpp similarity index 100% rename from tests/write-zarr-v3-to-s3.cpp rename to tests/driver/write-zarr-v3-to-s3.cpp From a811e221925bf2f4886c13a830bf038289788bfb Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:06:22 -0400 Subject: [PATCH 02/11] Define the Zarr streaming API. --- include/zarr.h | 216 +++++++++++++++++++++++++++++++++++++++++++ include/zarr.types.h | 133 ++++++++++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 include/zarr.h create mode 100644 include/zarr.types.h diff --git a/include/zarr.h b/include/zarr.h new file mode 100644 index 00000000..eac50f35 --- /dev/null +++ b/include/zarr.h @@ -0,0 +1,216 @@ +#ifndef H_ACQUIRE_ZARR_V0 +#define H_ACQUIRE_ZARR_V0 + +#include "zarr.types.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef struct ZarrStreamSettings_s ZarrStreamSettings; + typedef struct ZarrStream_s ZarrStream; + + /** + * @brief Get the version of the Zarr API. + * @return The version of the Zarr API. + */ + uint32_t Zarr_get_api_version(); + + /** + * @brief Get the message for the given status code. + * @param status The status code. + * @return A human-readable status message. + */ + const char* Zarr_get_error_message(ZarrStatus status); + + /** + * @brief Create a Zarr stream settings struct. + * @return A pointer to the Zarr stream settings struct, or NULL on failure. + */ + ZarrStreamSettings* ZarrStreamSettings_create(); + + /** + * @brief Destroy a Zarr stream settings struct. + * @details This function frees the memory allocated for the Zarr stream + * settings struct. + * @param[in] settings The Zarr stream settings struct. + */ + void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); + + /** + * @brief Copy a Zarr stream settings struct. + * @param[in] settings The Zarr stream settings struct to copy. + * @return A copy of the Zarr stream settings struct. + */ + ZarrStreamSettings* ZarrStreamSettings_copy( + const ZarrStreamSettings* settings); + + /** + * @brief Set store path and S3 settings for the Zarr stream. + * @param[in, out] settings + * @param[in] store_path The store path for the Zarr stream. Directory path + * when acquiring to the filesystem, key prefix when acquiring to S3. + * @param[in] bytes_of_store_path The length of @p store_path in bytes, + * including the null terminator. + * @param[in] s3_settings Optional S3 settings. If NULL, the store path is + * assumed to be a directory path. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, + const char* store_path, + size_t bytes_of_store_path, + const ZarrS3Settings* s3_settings); + + /** + * @brief Set the data type, compressor, codec, compression_settings level, + * and shuffle for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] compression_settings The compression_settings settings. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_compression( + ZarrStreamSettings* settings, + const ZarrCompressionSettings* compression_settings); + + /** + * @brief Set the data type for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] data_type The data type. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, + ZarrDataType data_type); + + /** + * @brief Reserve space for dimensions in the Zarr stream settings struct. + * @detail *Must* precede calls to ZarrStreamSettings_set_dimension. We + * require at least 3 dimensions to validate settings, but you may set up to + * 32 dimensions. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] count The number of dimensions to reserve space for. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_reserve_dimensions( + ZarrStreamSettings* settings, + size_t count); + + /** + * @brief Set properties for an acquisition dimension. + * @detail The order of the dimensions in the Zarr stream is the order in + * which they are set. The first dimension set is the slowest varying + * dimension, and the last dimension set is the fastest varying dimension. + * For example, if the dimensions are set in the order z, y, x, the fastest + * varying dimension is x, the next fastest varying dimension is y, and the + * slowest varying dimension is z. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] index The index of the dimension to set. Must be less than the + * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. + * @param[in] dimension The dimension's settings. + */ + ZarrStatus ZarrStreamSettings_set_dimension( + ZarrStreamSettings* settings, + size_t index, + const ZarrDimensionProperties* dimension); + + /** + * @brief Set the multiscale flag for the Zarr stream. + * @param[in, out] settings The Zarr stream settings struct. + * @param[in] multiscale A flag indicating whether to stream to multiple + * levels of detail. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, + uint8_t multiscale); + + /** + * @brief Set JSON-formatted custom metadata for the Zarr stream. + * @details This metadata will be written to acquire-zarr.json in the + * metadata directory of the Zarr store. This parameter is optional. + * @param settings[in, out] settings The Zarr stream settings struct. + * @param external_metadata JSON-formatted external metadata. + * @param bytes_of_external_metadata The length of @p custom_metadata in + * bytes, including the null terminator. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStreamSettings_set_custom_metadata( + ZarrStreamSettings* settings, + const char* external_metadata, + size_t bytes_of_external_metadata); + + const char* ZarrStreamSettings_get_store_path( + const ZarrStreamSettings* settings); + + ZarrS3Settings ZarrStreamSettings_get_s3_settings( + const ZarrStreamSettings* settings); + + ZarrCompressionSettings ZarrStreamSettings_get_compression( + const ZarrStreamSettings* settings); + + ZarrDataType ZarrStreamSettings_get_data_type( + const ZarrStreamSettings* settings); + + size_t ZarrStreamSettings_get_dimension_count( + const ZarrStreamSettings* settings); + + ZarrDimensionProperties ZarrStreamSettings_get_dimension( + const ZarrStreamSettings* settings, + size_t index); + + bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); + + const char* ZarrStreamSettings_get_custom_metadata( + const ZarrStreamSettings* settings); + + /** + * @brief Create a Zarr stream. + * @param[in, out] settings The settings for the Zarr stream. + * @param[in] version The version of the Zarr stream. 2 or 3. + * @return A pointer to the Zarr stream struct, or NULL on failure. + */ + ZarrStream* ZarrStream_create(ZarrStreamSettings* settings, + ZarrVersion version); + + /** + * @brief Destroy a Zarr stream. + * @details This function frees the memory allocated for the Zarr stream. + * @param stream The Zarr stream struct to destroy. + */ + void ZarrStream_destroy(ZarrStream* stream); + + /** + * @brief Append data to the Zarr stream. + * @param[in, out] stream The Zarr stream struct. + * @param[in] data The data to append. + * @param[in] bytes_in The number of bytes in @p data. It should be at least + * the size of a single frame. + * @param[out] bytes_out The number of bytes written to the stream. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus ZarrStream_append(ZarrStream* stream, + const void* data, + size_t bytes_in, + size_t* bytes_out); + + /** + * @brief Get the version (i.e., 2 or 3) of the Zarr stream. + * @param stream The Zarr stream struct. + * @return The version of the Zarr stream. + */ + ZarrVersion ZarrStream_get_version(const ZarrStream* stream); + + /** + * @brief Get a copy of the settings for the Zarr stream. + * @param stream The Zarr stream struct. + * @return A copy of the settings for the Zarr stream. + */ + ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); + + ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + ZarrLogLevel Zarr_get_log_level(); + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_V0 diff --git a/include/zarr.types.h b/include/zarr.types.h new file mode 100644 index 00000000..fdb6c715 --- /dev/null +++ b/include/zarr.types.h @@ -0,0 +1,133 @@ +#ifndef H_ACQUIRE_ZARR_TYPES_V0 +#define H_ACQUIRE_ZARR_TYPES_V0 + +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef enum + { + ZarrStatus_Success = 0, + ZarrStatus_InvalidArgument, + ZarrStatus_Overflow, + ZarrStatus_InvalidIndex, + ZarrStatus_NotYetImplemented, + ZarrStatus_InternalError, + ZarrStatus_OutOfMemory, + ZarrStatus_IOError, + ZarrStatus_CompressionError, + ZarrStatus_InvalidSettings, + ZarrStatusCount, + } ZarrStatus; + + typedef enum + { + ZarrVersion_2 = 2, + ZarrVersion_3, + ZarrVersionCount + } ZarrVersion; + + typedef enum + { + ZarrLogLevel_Debug, + ZarrLogLevel_Info, + ZarrLogLevel_Warning, + ZarrLogLevel_Error, + ZarrLogLevel_None, + ZarrLogLevelCount + } ZarrLogLevel; + + typedef enum + { + ZarrDataType_uint8, + ZarrDataType_uint16, + ZarrDataType_uint32, + ZarrDataType_uint64, + ZarrDataType_int8, + ZarrDataType_int16, + ZarrDataType_int32, + ZarrDataType_int64, + ZarrDataType_float32, + ZarrDataType_float64, + ZarrDataTypeCount + } ZarrDataType; + + typedef enum + { + ZarrCompressor_None = 0, + ZarrCompressor_Blosc1, + ZarrCompressorCount + } ZarrCompressor; + + typedef enum + { + ZarrCompressionCodec_None = 0, + ZarrCompressionCodec_BloscLZ4, + ZarrCompressionCodec_BloscZstd, + ZarrCompressionCodecCount + } ZarrCompressionCodec; + + typedef enum + { + ZarrDimensionType_Space = 0, + ZarrDimensionType_Channel, + ZarrDimensionType_Time, + ZarrDimensionType_Other, + ZarrDimensionTypeCount + } ZarrDimensionType; + + /** + * @brief S3 settings for streaming to Zarr. + */ + typedef struct + { + const char* endpoint; + size_t bytes_of_endpoint; + const char* bucket_name; + size_t bytes_of_bucket_name; + const char* access_key_id; + size_t bytes_of_access_key_id; + const char* secret_access_key; + size_t bytes_of_secret_access_key; + } ZarrS3Settings; + + /** + * @brief Compression settings for a Zarr array. + * @detail The compressor is not the same as the codec. A codec is + * a specific implementation of a compression algorithm, while a compressor + * is a library that implements one or more codecs. + */ + typedef struct + { + ZarrCompressor compressor; /**< Compressor to use */ + ZarrCompressionCodec codec; /**< Codec to use */ + uint8_t level; /**< Compression level */ + uint8_t shuffle; /**< Whether to shuffle the data before compressing */ + } ZarrCompressionSettings; + + /** + * @brief Properties of a dimension of the Zarr array. + */ + typedef struct + { + const char* name; /**< Name of the dimension */ + size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ + ZarrDimensionType kind; /**< Type of the dimension */ + uint32_t array_size_px; /**< Size of the array along this dimension in + pixels */ + uint32_t chunk_size_px; /**< Size of the chunks along this dimension in + pixels */ + uint32_t shard_size_chunks; /**< Number of chunks in a shard along this + dimension */ + } ZarrDimensionProperties; + +#ifdef __cplusplus +} +#endif + +#endif // H_ACQUIRE_ZARR_TYPES_V0 \ No newline at end of file From cf89e13d638b2a445815bd4222e0bce9a2045456 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:50:56 -0400 Subject: [PATCH 03/11] Define the Zarr logger. --- src/CMakeLists.txt | 1 + src/logger/CMakeLists.txt | 23 +++++++++++ src/logger/logger.cpp | 83 +++++++++++++++++++++++++++++++++++++++ src/logger/logger.hh | 30 ++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 src/logger/CMakeLists.txt create mode 100644 src/logger/logger.cpp create mode 100644 src/logger/logger.hh diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9a3b42a..1323f310 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,4 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) +add_subdirectory(logger) add_subdirectory(driver) diff --git a/src/logger/CMakeLists.txt b/src/logger/CMakeLists.txt new file mode 100644 index 00000000..37b28416 --- /dev/null +++ b/src/logger/CMakeLists.txt @@ -0,0 +1,23 @@ +set(tgt acquire-zarr-logger) + +add_library(${tgt} + logger.hh + logger.cpp +) + +set(PUBLIC_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include/) + +target_include_directories(${tgt} + PUBLIC + $ + PRIVATE + $ +) + +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) + +install(TARGETS ${tgt} + LIBRARY DESTINATION lib +) \ No newline at end of file diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp new file mode 100644 index 00000000..c316fcad --- /dev/null +++ b/src/logger/logger.cpp @@ -0,0 +1,83 @@ +#include "logger.hh" + +#include +#include +#include +#include +#include +#include + +ZarrLogLevel Logger::current_level_ = ZarrLogLevel_Info; + +void +Logger::set_log_level(ZarrLogLevel level) +{ + current_level_ = level; +} + +ZarrLogLevel +Logger::get_log_level() +{ + return current_level_; +} + +std::string +Logger::log(ZarrLogLevel level, + const char* file, + int line, + const char* func, + const char* format, + ...) +{ + std::scoped_lock lock(log_mutex_); + if (current_level_ == ZarrLogLevel_None || level < current_level_) { + return {}; // Suppress logs + } + + va_list args; + va_start(args, format); + + std::string prefix; + std::ostream* stream = &std::cout; + + switch (level) { + case ZarrLogLevel_Debug: + prefix = "[DEBUG] "; + break; + case ZarrLogLevel_Info: + prefix = "[INFO] "; + break; + case ZarrLogLevel_Warning: + prefix = "[WARNING] "; + stream = &std::cerr; + break; + case ZarrLogLevel_Error: + prefix = "[ERROR] "; + stream = &std::cerr; + break; + } + + // Get current time + auto now = std::chrono::system_clock::now(); + auto time = std::chrono::system_clock::to_time_t(now); + auto ms = std::chrono::duration_cast( + now.time_since_epoch()) % + 1000; + + // Get filename without path + std::filesystem::path filepath(file); + std::string filename = filepath.filename().string(); + + // Output timestamp, log level, filename + *stream << std::put_time(std::localtime(&time), "%Y-%m-%d %H:%M:%S") << '.' + << std::setfill('0') << std::setw(3) << ms.count() << " " << prefix + << filename << ":" << line << " " << func << ": "; + + char buffer[1024]; + vsnprintf(buffer, sizeof(buffer), format, args); + *stream << buffer << std::endl; + + va_end(args); + + return buffer; +} \ No newline at end of file diff --git a/src/logger/logger.hh b/src/logger/logger.hh new file mode 100644 index 00000000..1f1f339f --- /dev/null +++ b/src/logger/logger.hh @@ -0,0 +1,30 @@ +#include "zarr.types.h" + +#include + +class Logger +{ + public: + static void set_log_level(ZarrLogLevel level); + static ZarrLogLevel get_log_level(); + + static std::string log(ZarrLogLevel level, + const char* file, + int line, + const char* func, + const char* format, + ...); + + private: + static ZarrLogLevel current_level_; + static std::mutex log_mutex_; +}; + +#define LOG_DEBUG(...) \ + Logger::log(ZarrLogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_INFO(...) \ + Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_WARNING(...) \ + Logger::log(ZarrLogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) +#define LOG_ERROR(...) \ + Logger::log(ZarrLogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) From fff86c61fb3798f6894b60d251e420fea30d57af Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 10:52:59 -0400 Subject: [PATCH 04/11] Rename zarr.h to acquire.zarr.h. --- include/{zarr.h => acquire.zarr.h} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename include/{zarr.h => acquire.zarr.h} (100%) diff --git a/include/zarr.h b/include/acquire.zarr.h similarity index 100% rename from include/zarr.h rename to include/acquire.zarr.h From bdd0c51dba2756f8907c023a7459b84d9dfc1c50 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 11:23:38 -0400 Subject: [PATCH 05/11] Instantiate the logger's mutex. --- src/logger/logger.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp index c316fcad..2dfa5e2d 100644 --- a/src/logger/logger.cpp +++ b/src/logger/logger.cpp @@ -8,6 +8,7 @@ #include ZarrLogLevel Logger::current_level_ = ZarrLogLevel_Info; +std::mutex Logger::log_mutex_{}; void Logger::set_log_level(ZarrLogLevel level) From d099795ef23bb538bbb31b8e82c6ab9f4e9d30da Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:38:19 -0400 Subject: [PATCH 06/11] Document the StreamSettings getters. --- include/acquire.zarr.h | 62 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index eac50f35..e2a33051 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -17,6 +17,19 @@ extern "C" */ uint32_t Zarr_get_api_version(); + /** + * @brief Set the log level for the Zarr API. + * @param level The log level. + * @return ZarrStatus_Success on success, or an error code on failure. + */ + ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + + /** + * @brief Get the log level for the Zarr API. + * @return The log level for the Zarr API. + */ + ZarrLogLevel Zarr_get_log_level(); + /** * @brief Get the message for the given status code. * @param status The status code. @@ -138,27 +151,73 @@ extern "C" const char* external_metadata, size_t bytes_of_external_metadata); + /** + * @brief Get the store path for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The store path for the Zarr stream, or NULL on failure. + */ const char* ZarrStreamSettings_get_store_path( const ZarrStreamSettings* settings); + /** + * @brief Get the S3 settings for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The S3 settings for the Zarr stream, or an uninitialized struct on + * failure. + */ ZarrS3Settings ZarrStreamSettings_get_s3_settings( const ZarrStreamSettings* settings); + /** + * @brief Get the compression settings for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The compression settings for the Zarr stream, or an uninitialized + * struct on failure. + */ ZarrCompressionSettings ZarrStreamSettings_get_compression( const ZarrStreamSettings* settings); + /** + * @brief Get the data type for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The data type for the Zarr stream, or ZarrDataType_uint8 on failure. + */ ZarrDataType ZarrStreamSettings_get_data_type( const ZarrStreamSettings* settings); + /** + * @brief Get the number of dimensions in the Zarr stream settings struct. + * @param settings The Zarr stream settings struct. + * @return The number of dimensions in the Zarr stream settings struct, or 0 on + * failure. + */ size_t ZarrStreamSettings_get_dimension_count( const ZarrStreamSettings* settings); + /** + * @brief Get the properties for an acquisition dimension. + * @param settings The Zarr stream settings struct. + * @param index The index of the dimension to get. + * @return The properties for the @p index th dimension, or an uninitialized struct on + * failure. + */ ZarrDimensionProperties ZarrStreamSettings_get_dimension( const ZarrStreamSettings* settings, size_t index); + /** + * @brief Get the multiscale flag for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The multiscale flag for the Zarr stream, or false on failure. + */ bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); + /** + * @brief Get the JSON-formatted custom metadata for the Zarr stream. + * @param settings The Zarr stream settings struct. + * @return The JSON-formatted custom metadata for the Zarr stream, or NULL on + * failure. + */ const char* ZarrStreamSettings_get_custom_metadata( const ZarrStreamSettings* settings); @@ -206,9 +265,6 @@ extern "C" */ ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); - ZarrStatus Zarr_set_log_level(ZarrLogLevel level); - ZarrLogLevel Zarr_get_log_level(); - #ifdef __cplusplus } #endif From da98e57ab3b359ccb56cdcd9a911f42a9bcb403f Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 17 Sep 2024 14:41:40 -0400 Subject: [PATCH 07/11] call it 'type' --- include/zarr.types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/zarr.types.h b/include/zarr.types.h index fdb6c715..56a54a56 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -117,7 +117,7 @@ extern "C" { const char* name; /**< Name of the dimension */ size_t bytes_of_name; /**< Bytes in @p name, including null terminator */ - ZarrDimensionType kind; /**< Type of the dimension */ + ZarrDimensionType type; /**< Type of the dimension */ uint32_t array_size_px; /**< Size of the array along this dimension in pixels */ uint32_t chunk_size_px; /**< Size of the chunks along this dimension in From 996d22e9e276f0225082a7304c54adf935cc0cf9 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 18 Sep 2024 11:34:33 -0400 Subject: [PATCH 08/11] Document that ZarrStream_append will block for compression and flushing --- include/acquire.zarr.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index e2a33051..eb287663 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -239,6 +239,8 @@ extern "C" /** * @brief Append data to the Zarr stream. + * @details This function will block while chunks are compressed and written + * to the store. It will return when all data has been written. * @param[in, out] stream The Zarr stream struct. * @param[in] data The data to append. * @param[in] bytes_in The number of bytes in @p data. It should be at least From e546f7d23e849d94e45ea03b8895421d89fb5abd Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 18 Sep 2024 15:38:03 -0400 Subject: [PATCH 09/11] Respond to PR comments. --- CMakeLists.txt | 3 + src/CMakeLists.txt | 6 +- src/driver/CMakeLists.txt | 110 +++++++++++++------------- tests/CMakeLists.txt | 4 +- tests/driver/CMakeLists.txt | 154 ++++++++++++++++++------------------ 5 files changed, 138 insertions(+), 139 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2925e3f8..0419a48b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,9 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +option(NOTEST "Disable all tests" OFF) +option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) + add_subdirectory(src) add_subdirectory(tests) add_subdirectory(examples) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9a3b42a..31635d84 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,3 @@ -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -add_subdirectory(driver) +if (BUILD_ACQUIRE_DRIVER_ZARR) + add_subdirectory(driver) +endif () diff --git a/src/driver/CMakeLists.txt b/src/driver/CMakeLists.txt index e223b5db..5b72798a 100644 --- a/src/driver/CMakeLists.txt +++ b/src/driver/CMakeLists.txt @@ -1,61 +1,57 @@ -option(BUILD_ACQUIRE_DRIVER_ZARR "Build the Acquire Zarr driver" ON) +if (NOT TARGET acquire-core-logger) + add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) +endif () -if (BUILD_ACQUIRE_DRIVER_ZARR) - if (NOT TARGET acquire-core-logger) - add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) - endif () +set(tgt acquire-driver-zarr) +add_library(${tgt} MODULE + common/dimension.hh + common/dimension.cpp + common/thread.pool.hh + common/thread.pool.cpp + common/s3.connection.hh + common/s3.connection.cpp + common/utilities.hh + common/utilities.cpp + writers/sink.hh + writers/sink.creator.hh + writers/sink.creator.cpp + writers/file.sink.hh + writers/file.sink.cpp + writers/s3.sink.hh + writers/s3.sink.cpp + writers/array.writer.hh + writers/array.writer.cpp + writers/zarrv2.array.writer.hh + writers/zarrv2.array.writer.cpp + writers/zarrv3.array.writer.hh + writers/zarrv3.array.writer.cpp + writers/blosc.compressor.hh + writers/blosc.compressor.cpp + zarr.hh + zarr.cpp + zarr.v2.hh + zarr.v2.cpp + zarr.v3.hh + zarr.v3.cpp + zarr.driver.c +) - set(tgt acquire-driver-zarr) - add_library(${tgt} MODULE - common/dimension.hh - common/dimension.cpp - common/thread.pool.hh - common/thread.pool.cpp - common/s3.connection.hh - common/s3.connection.cpp - common/utilities.hh - common/utilities.cpp - writers/sink.hh - writers/sink.creator.hh - writers/sink.creator.cpp - writers/file.sink.hh - writers/file.sink.cpp - writers/s3.sink.hh - writers/s3.sink.cpp - writers/array.writer.hh - writers/array.writer.cpp - writers/zarrv2.array.writer.hh - writers/zarrv2.array.writer.cpp - writers/zarrv3.array.writer.hh - writers/zarrv3.array.writer.cpp - writers/blosc.compressor.hh - writers/blosc.compressor.cpp - zarr.hh - zarr.cpp - zarr.v2.hh - zarr.v2.cpp - zarr.v3.hh - zarr.v3.cpp - zarr.driver.c - ) +target_include_directories(${tgt} PRIVATE + $ +) - target_include_directories(${tgt} PRIVATE - $ - ) +target_enable_simd(${tgt}) +target_link_libraries(${tgt} PRIVATE + acquire-core-logger + acquire-core-platform + acquire-device-kit + acquire-device-properties + blosc_static + nlohmann_json::nlohmann_json + miniocpp::miniocpp +) +set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" +) - target_enable_simd(${tgt}) - target_link_libraries(${tgt} PRIVATE - acquire-core-logger - acquire-core-platform - acquire-device-kit - acquire-device-properties - blosc_static - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - - install(TARGETS ${tgt} LIBRARY DESTINATION lib) -endif () \ No newline at end of file +install(TARGETS ${tgt} LIBRARY DESTINATION lib) \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 87878b37..d10a9f44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,7 @@ if (${NOTEST}) message(STATUS "Skipping test targets") else () - add_subdirectory(driver) + if (BUILD_ACQUIRE_DRIVER_ZARR) + add_subdirectory(driver) + endif () endif () diff --git a/tests/driver/CMakeLists.txt b/tests/driver/CMakeLists.txt index 1fd661a2..b2ea8266 100644 --- a/tests/driver/CMakeLists.txt +++ b/tests/driver/CMakeLists.txt @@ -1,87 +1,85 @@ -if (BUILD_ACQUIRE_DRIVER_ZARR) - set(NOTEST "TRUE") - add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) - set(NOTEST "FALSE") +set(NOTEST "TRUE") +add_subdirectory(${CMAKE_SOURCE_DIR}/acquire-common ${CMAKE_CURRENT_BINARY_DIR}/acquire-common) +set(NOTEST "FALSE") - # - # PARAMETERS - # - set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project +# +# PARAMETERS +# +set(project acquire-driver-zarr) # CMAKE_PROJECT_NAME gets overridden if this is a subtree of another project - # - # Tests - # - set(tests - list-devices - unit-tests - get - get-meta - get-set-get - external-metadata-with-whitespace-ok - restart-stopped-zarr-resets-threadpool - repeat-start - metadata-dimension-sizes - write-zarr-v2-raw - write-zarr-v2-raw-chunk-size-larger-than-frame-size - write-zarr-v2-raw-with-even-chunking - write-zarr-v2-raw-with-even-chunking-and-rollover - write-zarr-v2-raw-with-ragged-chunking - write-zarr-v2-with-lz4-compression - write-zarr-v2-with-zstd-compression - write-zarr-v2-compressed-with-chunking - write-zarr-v2-compressed-with-chunking-and-rollover - write-zarr-v2-raw-multiscale - write-zarr-v2-raw-multiscale-with-trivial-tile-size - write-zarr-v2-compressed-multiscale - write-zarr-v2-to-s3 - multiscales-metadata - write-zarr-v3-raw - write-zarr-v3-raw-with-ragged-sharding - write-zarr-v3-raw-chunk-exceeds-array - write-zarr-v3-compressed - write-zarr-v3-raw-multiscale - write-zarr-v3-to-s3 - ) +# +# Tests +# +set(tests + list-devices + unit-tests + get + get-meta + get-set-get + external-metadata-with-whitespace-ok + restart-stopped-zarr-resets-threadpool + repeat-start + metadata-dimension-sizes + write-zarr-v2-raw + write-zarr-v2-raw-chunk-size-larger-than-frame-size + write-zarr-v2-raw-with-even-chunking + write-zarr-v2-raw-with-even-chunking-and-rollover + write-zarr-v2-raw-with-ragged-chunking + write-zarr-v2-with-lz4-compression + write-zarr-v2-with-zstd-compression + write-zarr-v2-compressed-with-chunking + write-zarr-v2-compressed-with-chunking-and-rollover + write-zarr-v2-raw-multiscale + write-zarr-v2-raw-multiscale-with-trivial-tile-size + write-zarr-v2-compressed-multiscale + write-zarr-v2-to-s3 + multiscales-metadata + write-zarr-v3-raw + write-zarr-v3-raw-with-ragged-sharding + write-zarr-v3-raw-chunk-exceeds-array + write-zarr-v3-compressed + write-zarr-v3-raw-multiscale + write-zarr-v3-to-s3 +) - foreach (name ${tests}) - set(tgt "${project}-${name}") - add_executable(${tgt} ${name}.cpp) - target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") - set_target_properties(${tgt} PROPERTIES - MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" - ) - target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") - target_link_libraries(${tgt} - acquire-core-logger - acquire-core-platform - acquire-video-runtime - nlohmann_json::nlohmann_json - miniocpp::miniocpp - ) +foreach (name ${tests}) + set(tgt "${project}-${name}") + add_executable(${tgt} ${name}.cpp) + target_compile_definitions(${tgt} PUBLIC "TEST=\"${tgt}\"") + set_target_properties(${tgt} PROPERTIES + MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" + ) + target_include_directories(${tgt} PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../") + target_link_libraries(${tgt} + acquire-core-logger + acquire-core-platform + acquire-video-runtime + nlohmann_json::nlohmann_json + miniocpp::miniocpp + ) - add_test(NAME test-${tgt} COMMAND ${tgt}) - set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") - endforeach () + add_test(NAME test-${tgt} COMMAND ${tgt}) + set_tests_properties(test-${tgt} PROPERTIES LABELS "anyplatform;acquire-driver-zarr") +endforeach () - # - # Copy driver to tests - # - list(POP_FRONT tests onename) +# +# Copy driver to tests +# +list(POP_FRONT tests onename) - foreach (driver - acquire-driver-common - acquire-driver-zarr +foreach (driver + acquire-driver-common + acquire-driver-zarr +) + add_custom_target(${project}-copy-${driver}-for-tests + COMMAND ${CMAKE_COMMAND} -E copy + $ + $ + DEPENDS ${driver} + COMMENT "Copying ${driver} to $" ) - add_custom_target(${project}-copy-${driver}-for-tests - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - DEPENDS ${driver} - COMMENT "Copying ${driver} to $" - ) - foreach (name ${tests}) - add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) - endforeach () + foreach (name ${tests}) + add_dependencies(${tgt} ${project}-copy-${driver}-for-tests) endforeach () -endif () \ No newline at end of file +endforeach () \ No newline at end of file From 42a494055bf9a835d12dd944b7bfca3135cb1134 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 20 Sep 2024 09:45:17 -0400 Subject: [PATCH 10/11] Respond to PR comments. --- include/acquire.zarr.h | 223 ++++------------------------------------- include/zarr.types.h | 10 +- 2 files changed, 20 insertions(+), 213 deletions(-) diff --git a/include/acquire.zarr.h b/include/acquire.zarr.h index eb287663..cbbbaf86 100644 --- a/include/acquire.zarr.h +++ b/include/acquire.zarr.h @@ -1,5 +1,4 @@ -#ifndef H_ACQUIRE_ZARR_V0 -#define H_ACQUIRE_ZARR_V0 +#pragma once #include "zarr.types.h" @@ -8,7 +7,19 @@ extern "C" { #endif - typedef struct ZarrStreamSettings_s ZarrStreamSettings; + typedef struct ZarrStreamSettings_s + { + ZarrS3Settings s3_settings; + ZarrCompressionSettings compression_settings; + ZarrDimensionProperties* dimensions; + size_t dimension_count; + char* store_path; + char* custom_metadata; + ZarrDataType data_type; + ZarrVersion version; + bool multiscale; + } ZarrStreamSettings; + typedef struct ZarrStream_s ZarrStream; /** @@ -22,7 +33,7 @@ extern "C" * @param level The log level. * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrStatus Zarr_set_log_level(ZarrLogLevel level); + ZarrStatusCode Zarr_set_log_level(ZarrLogLevel level); /** * @brief Get the log level for the Zarr API. @@ -35,191 +46,7 @@ extern "C" * @param status The status code. * @return A human-readable status message. */ - const char* Zarr_get_error_message(ZarrStatus status); - - /** - * @brief Create a Zarr stream settings struct. - * @return A pointer to the Zarr stream settings struct, or NULL on failure. - */ - ZarrStreamSettings* ZarrStreamSettings_create(); - - /** - * @brief Destroy a Zarr stream settings struct. - * @details This function frees the memory allocated for the Zarr stream - * settings struct. - * @param[in] settings The Zarr stream settings struct. - */ - void ZarrStreamSettings_destroy(ZarrStreamSettings* settings); - - /** - * @brief Copy a Zarr stream settings struct. - * @param[in] settings The Zarr stream settings struct to copy. - * @return A copy of the Zarr stream settings struct. - */ - ZarrStreamSettings* ZarrStreamSettings_copy( - const ZarrStreamSettings* settings); - - /** - * @brief Set store path and S3 settings for the Zarr stream. - * @param[in, out] settings - * @param[in] store_path The store path for the Zarr stream. Directory path - * when acquiring to the filesystem, key prefix when acquiring to S3. - * @param[in] bytes_of_store_path The length of @p store_path in bytes, - * including the null terminator. - * @param[in] s3_settings Optional S3 settings. If NULL, the store path is - * assumed to be a directory path. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_store(ZarrStreamSettings* settings, - const char* store_path, - size_t bytes_of_store_path, - const ZarrS3Settings* s3_settings); - - /** - * @brief Set the data type, compressor, codec, compression_settings level, - * and shuffle for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] compression_settings The compression_settings settings. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_compression( - ZarrStreamSettings* settings, - const ZarrCompressionSettings* compression_settings); - - /** - * @brief Set the data type for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] data_type The data type. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_data_type(ZarrStreamSettings* settings, - ZarrDataType data_type); - - /** - * @brief Reserve space for dimensions in the Zarr stream settings struct. - * @detail *Must* precede calls to ZarrStreamSettings_set_dimension. We - * require at least 3 dimensions to validate settings, but you may set up to - * 32 dimensions. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] count The number of dimensions to reserve space for. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_reserve_dimensions( - ZarrStreamSettings* settings, - size_t count); - - /** - * @brief Set properties for an acquisition dimension. - * @detail The order of the dimensions in the Zarr stream is the order in - * which they are set. The first dimension set is the slowest varying - * dimension, and the last dimension set is the fastest varying dimension. - * For example, if the dimensions are set in the order z, y, x, the fastest - * varying dimension is x, the next fastest varying dimension is y, and the - * slowest varying dimension is z. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] index The index of the dimension to set. Must be less than the - * number of dimensions reserved with ZarrStreamSettings_reserve_dimensions. - * @param[in] dimension The dimension's settings. - */ - ZarrStatus ZarrStreamSettings_set_dimension( - ZarrStreamSettings* settings, - size_t index, - const ZarrDimensionProperties* dimension); - - /** - * @brief Set the multiscale flag for the Zarr stream. - * @param[in, out] settings The Zarr stream settings struct. - * @param[in] multiscale A flag indicating whether to stream to multiple - * levels of detail. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_multiscale(ZarrStreamSettings* settings, - uint8_t multiscale); - - /** - * @brief Set JSON-formatted custom metadata for the Zarr stream. - * @details This metadata will be written to acquire-zarr.json in the - * metadata directory of the Zarr store. This parameter is optional. - * @param settings[in, out] settings The Zarr stream settings struct. - * @param external_metadata JSON-formatted external metadata. - * @param bytes_of_external_metadata The length of @p custom_metadata in - * bytes, including the null terminator. - * @return ZarrStatus_Success on success, or an error code on failure. - */ - ZarrStatus ZarrStreamSettings_set_custom_metadata( - ZarrStreamSettings* settings, - const char* external_metadata, - size_t bytes_of_external_metadata); - - /** - * @brief Get the store path for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The store path for the Zarr stream, or NULL on failure. - */ - const char* ZarrStreamSettings_get_store_path( - const ZarrStreamSettings* settings); - - /** - * @brief Get the S3 settings for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The S3 settings for the Zarr stream, or an uninitialized struct on - * failure. - */ - ZarrS3Settings ZarrStreamSettings_get_s3_settings( - const ZarrStreamSettings* settings); - - /** - * @brief Get the compression settings for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The compression settings for the Zarr stream, or an uninitialized - * struct on failure. - */ - ZarrCompressionSettings ZarrStreamSettings_get_compression( - const ZarrStreamSettings* settings); - - /** - * @brief Get the data type for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The data type for the Zarr stream, or ZarrDataType_uint8 on failure. - */ - ZarrDataType ZarrStreamSettings_get_data_type( - const ZarrStreamSettings* settings); - - /** - * @brief Get the number of dimensions in the Zarr stream settings struct. - * @param settings The Zarr stream settings struct. - * @return The number of dimensions in the Zarr stream settings struct, or 0 on - * failure. - */ - size_t ZarrStreamSettings_get_dimension_count( - const ZarrStreamSettings* settings); - - /** - * @brief Get the properties for an acquisition dimension. - * @param settings The Zarr stream settings struct. - * @param index The index of the dimension to get. - * @return The properties for the @p index th dimension, or an uninitialized struct on - * failure. - */ - ZarrDimensionProperties ZarrStreamSettings_get_dimension( - const ZarrStreamSettings* settings, - size_t index); - - /** - * @brief Get the multiscale flag for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The multiscale flag for the Zarr stream, or false on failure. - */ - bool ZarrStreamSettings_get_multiscale(const ZarrStreamSettings* settings); - - /** - * @brief Get the JSON-formatted custom metadata for the Zarr stream. - * @param settings The Zarr stream settings struct. - * @return The JSON-formatted custom metadata for the Zarr stream, or NULL on - * failure. - */ - const char* ZarrStreamSettings_get_custom_metadata( - const ZarrStreamSettings* settings); + const char* Zarr_get_status_message(ZarrStatusCode status); /** * @brief Create a Zarr stream. @@ -248,27 +75,11 @@ extern "C" * @param[out] bytes_out The number of bytes written to the stream. * @return ZarrStatus_Success on success, or an error code on failure. */ - ZarrStatus ZarrStream_append(ZarrStream* stream, + ZarrStatusCode ZarrStream_append(ZarrStream* stream, const void* data, size_t bytes_in, size_t* bytes_out); - /** - * @brief Get the version (i.e., 2 or 3) of the Zarr stream. - * @param stream The Zarr stream struct. - * @return The version of the Zarr stream. - */ - ZarrVersion ZarrStream_get_version(const ZarrStream* stream); - - /** - * @brief Get a copy of the settings for the Zarr stream. - * @param stream The Zarr stream struct. - * @return A copy of the settings for the Zarr stream. - */ - ZarrStreamSettings* ZarrStream_get_settings(const ZarrStream* stream); - #ifdef __cplusplus } #endif - -#endif // H_ACQUIRE_ZARR_V0 diff --git a/include/zarr.types.h b/include/zarr.types.h index 56a54a56..2dfd0dd1 100644 --- a/include/zarr.types.h +++ b/include/zarr.types.h @@ -23,7 +23,7 @@ extern "C" ZarrStatus_CompressionError, ZarrStatus_InvalidSettings, ZarrStatusCount, - } ZarrStatus; + } ZarrStatusCode; typedef enum { @@ -34,7 +34,7 @@ extern "C" typedef enum { - ZarrLogLevel_Debug, + ZarrLogLevel_Debug = 0, ZarrLogLevel_Info, ZarrLogLevel_Warning, ZarrLogLevel_Error, @@ -44,7 +44,7 @@ extern "C" typedef enum { - ZarrDataType_uint8, + ZarrDataType_uint8 = 0, ZarrDataType_uint16, ZarrDataType_uint32, ZarrDataType_uint64, @@ -87,13 +87,9 @@ extern "C" typedef struct { const char* endpoint; - size_t bytes_of_endpoint; const char* bucket_name; - size_t bytes_of_bucket_name; const char* access_key_id; - size_t bytes_of_access_key_id; const char* secret_access_key; - size_t bytes_of_secret_access_key; } ZarrS3Settings; /** From bda14d7572d03c0b0b29fb6f93d65c347ce89389 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 24 Sep 2024 14:54:37 -0400 Subject: [PATCH 11/11] Respond to PR comments --- src/logger/CMakeLists.txt | 14 +++----- src/logger/logger.cpp | 68 ++++++++----------------------------- src/logger/logger.hh | 70 +++++++++++++++++++++++++++++++++------ src/logger/logger.types.h | 8 +++++ 4 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 src/logger/logger.types.h diff --git a/src/logger/CMakeLists.txt b/src/logger/CMakeLists.txt index d3d96555..9d7c9998 100644 --- a/src/logger/CMakeLists.txt +++ b/src/logger/CMakeLists.txt @@ -1,25 +1,19 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(tgt acquire-zarr-logger) - -add_library(${tgt} +add_library(acquire-logger logger.hh logger.cpp ) -set(PUBLIC_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include/) - -target_include_directories(${tgt} - PUBLIC - $ +target_include_directories(acquire-logger PRIVATE $ ) -set_target_properties(${tgt} PROPERTIES +set_target_properties(acquire-logger PROPERTIES MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" ) -install(TARGETS ${tgt} +install(TARGETS acquire-logger LIBRARY DESTINATION lib ) \ No newline at end of file diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp index 2dfa5e2d..77e92435 100644 --- a/src/logger/logger.cpp +++ b/src/logger/logger.cpp @@ -1,84 +1,44 @@ #include "logger.hh" -#include #include -#include -#include #include #include -ZarrLogLevel Logger::current_level_ = ZarrLogLevel_Info; +LogLevel Logger::current_level_ = LogLevel_Info; std::mutex Logger::log_mutex_{}; void -Logger::set_log_level(ZarrLogLevel level) +Logger::set_log_level(LogLevel level) { current_level_ = level; } -ZarrLogLevel +LogLevel Logger::get_log_level() { return current_level_; } std::string -Logger::log(ZarrLogLevel level, - const char* file, - int line, - const char* func, - const char* format, - ...) +Logger::get_timestamp_() { - std::scoped_lock lock(log_mutex_); - if (current_level_ == ZarrLogLevel_None || level < current_level_) { - return {}; // Suppress logs - } - va_list args; - va_start(args, format); - - std::string prefix; - std::ostream* stream = &std::cout; - - switch (level) { - case ZarrLogLevel_Debug: - prefix = "[DEBUG] "; - break; - case ZarrLogLevel_Info: - prefix = "[INFO] "; - break; - case ZarrLogLevel_Warning: - prefix = "[WARNING] "; - stream = &std::cerr; - break; - case ZarrLogLevel_Error: - prefix = "[ERROR] "; - stream = &std::cerr; - break; - } - - // Get current time auto now = std::chrono::system_clock::now(); auto time = std::chrono::system_clock::to_time_t(now); auto ms = std::chrono::duration_cast( now.time_since_epoch()) % 1000; - // Get filename without path - std::filesystem::path filepath(file); - std::string filename = filepath.filename().string(); - - // Output timestamp, log level, filename - *stream << std::put_time(std::localtime(&time), "%Y-%m-%d %H:%M:%S") << '.' - << std::setfill('0') << std::setw(3) << ms.count() << " " << prefix - << filename << ":" << line << " " << func << ": "; - - char buffer[1024]; - vsnprintf(buffer, sizeof(buffer), format, args); - *stream << buffer << std::endl; + std::tm tm{}; +#if defined(_WIN32) + localtime_s(&tm, &time); +#else + localtime_r(&time, &tm); +#endif - va_end(args); + std::ostringstream ss; + ss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S") << '.' << std::setfill('0') + << std::setw(3) << ms.count(); - return buffer; + return ss.str(); } \ No newline at end of file diff --git a/src/logger/logger.hh b/src/logger/logger.hh index 1f1f339f..c5046990 100644 --- a/src/logger/logger.hh +++ b/src/logger/logger.hh @@ -1,30 +1,80 @@ -#include "zarr.types.h" +#include "logger.types.h" +#include +#include #include +#include class Logger { public: - static void set_log_level(ZarrLogLevel level); - static ZarrLogLevel get_log_level(); + static void set_log_level(LogLevel level); + static LogLevel get_log_level(); - static std::string log(ZarrLogLevel level, + template + static std::string log(LogLevel level, const char* file, int line, const char* func, - const char* format, - ...); + Args&&... args) + { + namespace fs = std::filesystem; + + std::scoped_lock lock(log_mutex_); + + std::string prefix; + auto stream = &std::cout; + + switch (level) { + case LogLevel_Debug: + prefix = "[DEBUG] "; + break; + case LogLevel_Info: + prefix = "[INFO] "; + break; + case LogLevel_Warning: + prefix = "[WARNING] "; + break; + default: + prefix = "[ERROR] "; + stream = &std::cerr; + break; + } + + fs::path filepath(file); + std::string filename = filepath.filename().string(); + + std::ostringstream ss; + ss << get_timestamp_() << " " << prefix << filename << ":" << line + << " " << func << ": "; + + format_arg_(ss, std::forward(args)...); + + std::string message = ss.str(); + *stream << message << std::endl; + + return message; + } private: - static ZarrLogLevel current_level_; + static LogLevel current_level_; static std::mutex log_mutex_; + + static void format_arg_(std::ostream& ss) {}; // base case + template + static void format_arg_(std::ostream& ss, T&& arg, Args&&... args) { + ss << std::forward(arg); + format_arg_(ss, std::forward(args)...); + } + + static std::string get_timestamp_(); }; #define LOG_DEBUG(...) \ - Logger::log(ZarrLogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(LogLevel_Debug, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_INFO(...) \ Logger::log(LogLevel_Info, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_WARNING(...) \ - Logger::log(ZarrLogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(LogLevel_Warning, __FILE__, __LINE__, __func__, __VA_ARGS__) #define LOG_ERROR(...) \ - Logger::log(ZarrLogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) + Logger::log(LogLevel_Error, __FILE__, __LINE__, __func__, __VA_ARGS__) diff --git a/src/logger/logger.types.h b/src/logger/logger.types.h new file mode 100644 index 00000000..86d9c0b1 --- /dev/null +++ b/src/logger/logger.types.h @@ -0,0 +1,8 @@ +#pragma once + +typedef enum { + LogLevel_Debug, + LogLevel_Info, + LogLevel_Warning, + LogLevel_Error +} LogLevel;