Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add some examples. #21

Merged
merged 8 commits into from
Dec 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,7 @@ env/
venv/
ENV/
env.bak/
venv.bak/
venv.bak/

# Test/example output
*.zarr
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

option(BUILD_PYTHON "Build Python bindings" OFF)
option(BUILD_EXAMPLES "Build examples" OFF)
option(BUILD_BENCHMARK "Build benchmarks" OFF)

if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
Expand All @@ -34,6 +35,12 @@ else ()
message(STATUS "Skipping test targets")
endif ()

if (${BUILD_EXAMPLES})
add_subdirectory(examples)
else ()
message(STATUS "Skipping examples")
endif ()

if (BUILD_BENCHMARK)
add_subdirectory(benchmarks)
else ()
Expand Down
35 changes: 35 additions & 0 deletions examples/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
set(project acquire-zarr)

set(examples
zarrv2-raw-filesystem
zarrv2-compressed-s3
zarrv3-compressed-filesystem
zarrv3-raw-multiscale-filesystem
zarrv3-raw-s3
)

foreach (name ${examples})
set(tgt "${project}-${name}")
add_executable(${tgt} ${name}.c)
set_target_properties(${tgt} PROPERTIES
MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>"
)
target_include_directories(${tgt} PRIVATE
${PROJECT_SOURCE_DIR}/include
)
target_link_libraries(${tgt} PRIVATE
acquire-logger
acquire-zarr
nlohmann_json::nlohmann_json
miniocpp::miniocpp
)

add_test(NAME example-${tgt} COMMAND ${tgt})

set(test_labels "examples")
if (name MATCHES ".*s3.*")
list(APPEND test_labels "s3")
endif ()

set_tests_properties(example-${tgt} PROPERTIES LABELS "${test_labels}")
endforeach ()
68 changes: 68 additions & 0 deletions examples/python/zarrv2-compressed-multiscale-filesystem.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# Basic Zarr V2 to filesystem
import numpy as np
from acquire_zarr import (
StreamSettings, ZarrStream, Dimension, DimensionType, ZarrVersion, DataType, Compressor, CompressionCodec,
CompressionSettings
)


def make_sample_data():
return np.random.randint(
0, 65535,
(32, 48, 64), # Shape matches chunk size for time dimension
dtype=np.int32
)


def main():
# Configure stream settings
settings = StreamSettings()

# Configure compression
settings.compression = CompressionSettings(
compressor=Compressor.BLOSC1,
codec=CompressionCodec.BLOSC_LZ4,
level=1,
shuffle=2, # bitshuffle
)

# Configure dimensions (t, y, x)
settings.dimensions.extend([
Dimension(
name="t",
kind=DimensionType.TIME,
array_size_px=0, # Unlimited
chunk_size_px=32,
shard_size_chunks=1,
),
Dimension(
name="y",
kind=DimensionType.SPACE,
array_size_px=48,
chunk_size_px=16,
shard_size_chunks=1,
),
Dimension(
name="x",
kind=DimensionType.SPACE,
array_size_px=64,
chunk_size_px=32,
shard_size_chunks=1,
),
])

settings.store_path = "output_v2_multiscale.zarr"
settings.version = ZarrVersion.V2
settings.data_type = DataType.INT32
settings.multiscale = True

# Create stream
stream = ZarrStream(settings)

# Create and write sample data
for i in range(10):
stream.append(make_sample_data())


if __name__ == "__main__":
main()
80 changes: 80 additions & 0 deletions examples/python/zarrv2-compressed-s3.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Zarr V2 with ZSTD compression to S3
import numpy as np
from acquire_zarr import (
StreamSettings, ZarrStream, Dimension, DimensionType, ZarrVersion,
DataType, Compressor, CompressionCodec, CompressionSettings, S3Settings
)


def make_sample_data():
return np.random.randint(
0, 65535,
(32, 3, 48, 64), # Shape matches chunk sizes
dtype=np.int32
)

def main():
settings = StreamSettings()

# Configure S3
settings.s3 = S3Settings(
endpoint="http://localhost:9000",
bucket_name="mybucket",
access_key_id="myaccesskey",
secret_access_key="mysecretkey"
)

# Configure compression
settings.compression = CompressionSettings(
compressor=Compressor.BLOSC1,
codec=CompressionCodec.BLOSC_ZSTD,
level=1,
shuffle=1,
)

# Configure 4D array (t, c, y, x)
settings.dimensions.extend([
Dimension(
name="t",
kind=DimensionType.TIME,
array_size_px=0, # Unlimited
chunk_size_px=32,
shard_size_chunks=1,
),
Dimension(
name="c",
kind=DimensionType.CHANNEL,
array_size_px=3,
chunk_size_px=3,
shard_size_chunks=1,
),
Dimension(
name="y",
kind=DimensionType.SPACE,
array_size_px=48,
chunk_size_px=16,
shard_size_chunks=1,
),
Dimension(
name="x",
kind=DimensionType.SPACE,
array_size_px=64,
chunk_size_px=32,
shard_size_chunks=1,
),
])

settings.store_path = "output_v2_s3.zarr"
settings.version = ZarrVersion.V2
settings.data_type = DataType.INT32

# Create stream
stream = ZarrStream(settings)

# Create and write sample data
for i in range(10):
stream.append(make_sample_data())


if __name__ == "__main__":
main()
57 changes: 57 additions & 0 deletions examples/python/zarrv2-raw-filesystem.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Basic Zarr V2 to filesystem
import numpy as np
from acquire_zarr import (
StreamSettings, ZarrStream, Dimension, DimensionType, ZarrVersion, DataType
)


def make_sample_data():
return np.random.randint(
0, 65535,
(32, 48, 64), # Shape matches chunk size for time dimension
dtype=np.int32
)

def main():
# Configure stream settings
settings = StreamSettings()

# Configure dimensions (t, y, x)
settings.dimensions.extend([
Dimension(
name="t",
kind=DimensionType.TIME,
array_size_px=0, # Unlimited
chunk_size_px=32,
shard_size_chunks=1,
),
Dimension(
name="y",
kind=DimensionType.SPACE,
array_size_px=48,
chunk_size_px=16,
shard_size_chunks=1,
),
Dimension(
name="x",
kind=DimensionType.SPACE,
array_size_px=64,
chunk_size_px=32,
shard_size_chunks=1,
),
])

settings.store_path = "output_v2.zarr"
settings.version = ZarrVersion.V2
settings.data_type = DataType.INT32

# Create stream
stream = ZarrStream(settings)

# Create and write sample data
for i in range(10):
jeskesen marked this conversation as resolved.
Show resolved Hide resolved
stream.append(make_sample_data())


if __name__ == "__main__":
main()
78 changes: 78 additions & 0 deletions examples/python/zarrv3-compressed-filesystem.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Zarr V3 with LZ4 compression to filesystem
import numpy as np
from acquire_zarr import (
StreamSettings, ZarrStream, Dimension, DimensionType, ZarrVersion,
DataType, Compressor, CompressionCodec, CompressionSettings
)


def make_sample_data():
return np.random.randint(
0, 65535,
(5, 4, 2, 48, 64), # Shape matches chunk sizes
dtype=np.uint16
)

def main():
settings = StreamSettings()

# Configure compression
settings.compression = CompressionSettings(
compressor=Compressor.BLOSC1,
codec=CompressionCodec.BLOSC_LZ4,
level=1,
shuffle=1,
)

# Configure 5D array (t, c, z, y, x)
settings.dimensions.extend([
Dimension(
name="t",
kind=DimensionType.TIME,
array_size_px=10,
chunk_size_px=5,
shard_size_chunks=2,
),
Dimension(
name="c",
kind=DimensionType.CHANNEL,
array_size_px=8,
chunk_size_px=4,
shard_size_chunks=2,
),
Dimension(
name="z",
kind=DimensionType.SPACE,
array_size_px=6,
chunk_size_px=2,
shard_size_chunks=1,
),
Dimension(
name="y",
kind=DimensionType.SPACE,
array_size_px=48,
chunk_size_px=16,
shard_size_chunks=1,
),
Dimension(
name="x",
kind=DimensionType.SPACE,
array_size_px=64,
chunk_size_px=16,
shard_size_chunks=2,
),
])

settings.store_path = "output_v3_compressed.zarr"
settings.version = ZarrVersion.V3
settings.data_type = DataType.UINT16

# Create stream
stream = ZarrStream(settings)

# Write sample data
stream.append(make_sample_data())


if __name__ == "__main__":
main()
Loading
Loading