From 9dc961cbbca6741b165746037ec491118f20d417 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 19 Oct 2023 11:30:36 -0400 Subject: [PATCH] Make filters (de-)serialize to the correct thing --- .../janelia/saalfeldlab/n5/zarr/DType.java | 40 +++++++------ .../janelia/saalfeldlab/n5/zarr/Filter.java | 56 ++++++++++++++----- .../zarr/ZarrCompatibleStringDataBlock.java | 4 -- .../n5/zarr/ZarrKeyValueWriter.java | 5 +- 4 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java index 13146f3..3741172 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java @@ -28,13 +28,11 @@ */ package org.janelia.saalfeldlab.n5.zarr; -import java.lang.reflect.Type; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collection; import java.util.EnumMap; -import java.util.List; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; @@ -45,13 +43,7 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; +import static org.janelia.saalfeldlab.n5.zarr.Filter.VLEN_UTF8; /** * Enumerates available zarr data types as defined at @@ -219,8 +211,15 @@ public DType(final String typestr, final Collection filters) { case OBJECT: nBytes = 1; nBits = 0; - dataBlockFactory = null; - byteBlockFactory = null; + if (filters.contains(VLEN_UTF8)) { + dataBlockFactory = (blockSize, gridPosition, numElements) -> + new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); + byteBlockFactory = (blockSize, gridPosition, numElements) -> + new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); + } else { + dataBlockFactory = null; + byteBlockFactory = null; + } break; // case BOOLEAN: // case OTHER: // not sure about this @@ -237,7 +236,7 @@ public DType(final String typestr, final Collection filters) { new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]); } - dataType = getDataType(primitive, nBytes); + dataType = getDataType(primitive, nBytes, filters); } public DType(final DataType dataType, final int nPrimitives) { @@ -279,6 +278,11 @@ public DType(final DataType dataType, final int nPrimitives) { break; // case INT8: // case UINT8: + case STRING: + nBytes = 1; + dataBlockFactory = (blockSize, gridPosition, numElements) -> + new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]); + break; default: nBytes = nPrimitives; dataBlockFactory = (blockSize, gridPosition, numElements) -> @@ -300,7 +304,8 @@ public DataType getDataType() { protected final static DataType getDataType( final Primitive primitive, - final int nBytes) { + final int nBytes, + final Collection filters) { switch (primitive) { case INT: @@ -344,7 +349,10 @@ protected final static DataType getDataType( return DataType.UINT8; // fallback } case OBJECT: - return DataType.OBJECT; + if (filters.contains(VLEN_UTF8)) + return DataType.STRING; + else + return DataType.OBJECT; default: return DataType.UINT8; // fallback } @@ -362,10 +370,10 @@ public String toString() { * * @return list of filters */ - public List getFilters() { + public Collection getFilters() { if (dataType == DataType.STRING) { ArrayList filterSet = new ArrayList<>(); - filterSet.add(new ZarrCompatibleStringDataBlock.VLenStringFilter()); + filterSet.add(VLEN_UTF8); return filterSet; } else diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java index 9428d41..988f243 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java @@ -33,21 +33,45 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; import java.lang.reflect.Type; /** - * Place holder interface for filters + * Filter types * * TODO implement some * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> + * @author Michael Innerberger */ -public interface Filter { +public enum Filter { + // Note: the JSON (de-)serializer below is very much tailored to this filter, which serializes to "{"id":"vlen-utf8"}" + // If additional filters are implemented, consider also changing the type adapter below + VLEN_UTF8("vlen-utf8"); - public static JsonAdapter jsonAdapter = new Filter.JsonAdapter(); + private final String id; - class JsonAdapter implements JsonDeserializer { + Filter(final String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public static Filter fromString(final String id) { + for (final Filter filter : values()) + if (filter.getId().equals(id)) + return filter; + return null; + } + + public static final JsonAdapter jsonAdapter = new JsonAdapter(); + + public static class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override public Filter deserialize( @@ -55,17 +79,23 @@ public Filter deserialize( final Type typeOfT, final JsonDeserializationContext context) throws JsonParseException { - final JsonObject jsonObject = json.getAsJsonObject(); - final JsonElement jsonId = jsonObject.get("id"); + final JsonElement jsonId = json.getAsJsonObject().get("id"); if (jsonId == null) return null; - final String id = jsonId.getAsString(); - switch (id) { - case "vlen-utf8": - return new ZarrCompatibleStringDataBlock.VLenStringFilter(); - default: - return null; - } + + final String stringId = jsonId.getAsString(); + return Filter.fromString(stringId); + } + + @Override + public JsonElement serialize( + final Filter filter, + final Type typeOfSrc, + final JsonSerializationContext context) { + + final JsonObject serialization = new JsonObject(); + serialization.add("id", new JsonPrimitive(filter.getId())); + return serialization; } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java index ce72586..9b166c1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompatibleStringDataBlock.java @@ -67,8 +67,4 @@ protected String[] deserialize(byte[] rawBytes) { return strings; } - - public static class VLenStringFilter implements Filter { - public final String id = "vlen-utf8"; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java index 6b2bf47..b922879 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java @@ -368,17 +368,18 @@ protected ZArrayAttributes createZArrayAttributes(final DatasetAttributes datase reorder(shape); final int[] chunks = datasetAttributes.getBlockSize().clone(); reorder(chunks); + final DType dType = new DType(datasetAttributes.getDataType()); final ZArrayAttributes zArrayAttributes = new ZArrayAttributes( N5ZarrReader.VERSION.getMajor(), shape, chunks, - new DType(datasetAttributes.getDataType()), + dType, ZarrCompressor.fromCompression(datasetAttributes.getCompression()), "0", 'C', dimensionSeparator, - null); + dType.getFilters()); return zArrayAttributes; }