From 45467725973937c0c2bda33a5a3c36975e9aa116 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Jun 2024 11:52:33 -0400 Subject: [PATCH 1/4] fix: don't enforce `serializeNulls`, but DO still ensure `null` is still serialized regardless of `serializeNulls` for Raw compression --- .../saalfeldlab/n5/zarr/ZarrCompressor.java | 21 ++++++++++++++++--- .../n5/zarr/ZarrKeyValueReader.java | 1 - .../n5/zarr/ZarrKeyValueWriter.java | 14 +++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompressor.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompressor.java index 815a9df..8f24163 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompressor.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompressor.java @@ -28,6 +28,7 @@ */ package org.janelia.saalfeldlab.n5.zarr; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Type; import java.util.AbstractMap.SimpleImmutableEntry; @@ -35,8 +36,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.gson.JsonNull; -import com.google.gson.JsonSerializer; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; import org.janelia.saalfeldlab.n5.Bzip2Compression; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.GzipCompression; @@ -297,5 +299,18 @@ public ZarrCompressor deserialize(final JsonElement json, final Type typeOfT, fi } } - JsonSerializer rawNullAdapter = (src, typeOfSrc, context) -> JsonNull.INSTANCE; + TypeAdapter rawNullAdapter = new TypeAdapter() { + + @Override public void write(JsonWriter out, Raw value) throws IOException { + final boolean serializeNull = out.getSerializeNulls(); + out.setSerializeNulls(true); + out.nullValue(); + out.setSerializeNulls(serializeNull); + } + + @Override public Raw read(JsonReader in) { + + return new Raw(); + } + }; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index c43104e..dc5d832 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -863,7 +863,6 @@ protected static GsonBuilder addTypeAdapters(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(ZArrayAttributes.class, ZArrayAttributes.jsonAdapter); gsonBuilder.registerTypeHierarchyAdapter(Filter.class, Filter.jsonAdapter); gsonBuilder.disableHtmlEscaping(); - gsonBuilder.serializeNulls(); return gsonBuilder; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java index 78721f2..740ad2a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java @@ -27,7 +27,9 @@ import java.io.IOException; import java.io.OutputStream; +import java.io.Writer; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -258,8 +260,10 @@ else if (getCache().isGroup(normalPath, ZGROUP_FILE)) { // These three lines are preferable to setDatasetAttributes because they // are more efficient wrt caching final ZArrayAttributes zarray = createZArrayAttributes(datasetAttributes); - final JsonElement attributes = gson.toJsonTree(zarray.asMap()); - writeJsonResource(normalPath, ZARRAY_FILE, attributes); + final HashMap zarrayMap = zarray.asMap(); + final JsonElement attributes = gson.toJsonTree(zarrayMap); + writeJsonResource(normalPath, ZARRAY_FILE, zarrayMap); + if( wasGroup ) deleteJsonResource(normalPath, ZGROUP_FILE ); @@ -416,14 +420,16 @@ protected void deleteJsonResource(final String normalPath, final String jsonName protected void writeJsonResource( final String normalPath, final String jsonName, - final JsonElement attributes) throws N5Exception { + final Object attributes) throws N5Exception { if (attributes == null) return; final String absolutePath = keyValueAccess.compose(uri, normalPath, jsonName); try (final LockedChannel lock = keyValueAccess.lockForWriting(absolutePath)) { - GsonUtils.writeAttributes(lock.newWriter(), attributes, gson); + final Writer writer = lock.newWriter(); + gson.toJson(attributes, writer); + writer.flush(); } catch (final Throwable e) { throw new N5IOException("Failed to write " + absolutePath, e); } From 550782d48865cff714d0b2c59b4cc09e446c1152 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Jun 2024 16:42:23 -0400 Subject: [PATCH 2/4] fix: don't require gsonBuilder to `serializeNull` - added a serializer for ZArrayAttributes - writeZArray via ZArrayAttributes object, not JsonElement - expected compression can be `JsonNull` OR `null` depending on whether `serializeNull` or not test: `serializeNulls` not set by default, so no `null` value here, therefore it doesn't parse as `JsonNull` but instead just returns `null` --- .../saalfeldlab/n5/zarr/ZArrayAttributes.java | 22 ++++++++++++++++++- .../n5/zarr/ZarrKeyValueReader.java | 2 +- .../n5/zarr/ZarrKeyValueWriter.java | 2 +- .../saalfeldlab/n5/zarr/N5ZarrTest.java | 3 +-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java index 10b4253..514b6b6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java @@ -34,6 +34,8 @@ import java.util.HashMap; import java.util.List; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.RawCompression; @@ -222,7 +224,7 @@ public Collection getFilters() { public static JsonAdapter jsonAdapter = new JsonAdapter(); - public static class JsonAdapter implements JsonDeserializer { + public static class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { @@ -249,5 +251,23 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial } } + @Override + public JsonElement serialize(ZArrayAttributes src, Type typeOfSrc, JsonSerializationContext context) { + + final JsonObject jsonObject = new JsonObject(); + + jsonObject.addProperty("zarr_format", src.getZarrFormat()); + jsonObject.add("shape", context.serialize(src.getShape())); + jsonObject.add("chunks", context.serialize(src.getChunks())); + + jsonObject.add("dtype", context.serialize(src.getDType().toString())); + jsonObject.add("compressor", context.serialize(src.getCompressor())); + jsonObject.addProperty("fill_value", src.getFillValue()); + jsonObject.addProperty("order", src.getOrder()); + jsonObject.addProperty("dimension_separator", src.getDimensionSeparator()); + jsonObject.add("filters", context.serialize(src.getFilters())); + + return jsonObject; + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java index dc5d832..959a88f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java @@ -525,7 +525,7 @@ protected JsonElement zarrToN5DatasetAttributes(final JsonElement elem ) { attrs.addProperty(DatasetAttributes.DATA_TYPE_KEY, zattrs.getDType().getDataType().toString()); final JsonElement e = attrs.get(ZArrayAttributes.compressorKey); - if (e == JsonNull.INSTANCE) { + if (e == JsonNull.INSTANCE || e == null) { attrs.add(DatasetAttributes.COMPRESSION_KEY, gson.toJsonTree(new RawCompression())); } else { attrs.add(DatasetAttributes.COMPRESSION_KEY, gson.toJsonTree( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java index 740ad2a..766c585 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java @@ -442,7 +442,7 @@ protected void writeZArray( if (attributes == null) return; - writeJsonResource(normalGroupPath, ZARRAY_FILE, attributes); + writeJsonResource(normalGroupPath, ZARRAY_FILE, gson.fromJson(attributes, ZArrayAttributes.class)); if (cacheMeta()) cache.updateCacheInfo(normalGroupPath, ZARRAY_FILE, attributes); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 251daca..518d1ea 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -831,8 +831,7 @@ public void testAttributeMapping() { n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, rawCompression); n5Compression = n5.getAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, Compression.class); assertEquals(rawCompression, n5Compression); - assertThrows(N5Exception.N5ClassCastException.class, () -> n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class)); - + assertNull(n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class)); final GzipCompression gzipCompression = new GzipCompression(); n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, gzipCompression); zarrCompression = n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class); From 124c75d8b248d5aaf621628b4a2ec9af345ea17b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Jun 2024 16:54:30 -0400 Subject: [PATCH 3/4] fix(test): don't ignore input GsonBuilder --- src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 518d1ea..c7f950a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -146,7 +146,7 @@ protected N5Writer createTempN5Writer( final String dimensionSeparator, final boolean mapN5DatasetAttributes) throws IOException { - return createTempN5Writer(location, new GsonBuilder(), dimensionSeparator, mapN5DatasetAttributes, false); + return createTempN5Writer(location, gsonBuilder, dimensionSeparator, mapN5DatasetAttributes, false); } protected N5Writer createTempN5Writer( @@ -831,7 +831,7 @@ public void testAttributeMapping() { n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, rawCompression); n5Compression = n5.getAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, Compression.class); assertEquals(rawCompression, n5Compression); - assertNull(n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class)); + assertThrows(N5Exception.N5ClassCastException.class, () -> n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class)); final GzipCompression gzipCompression = new GzipCompression(); n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, gzipCompression); zarrCompression = n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class); From 88a88ffa5e270504fd815aea8025ba8839e0ee60 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Jun 2024 17:00:44 -0400 Subject: [PATCH 4/4] fix(test): serializeNulls for consistency between Cached and non-cache versions This is because when reading from the container with no cache, you get `null` but when writing/reading with the cache, the value is `JsonNull`. --- src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index c7f950a..cd0f74b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -778,7 +778,7 @@ public void testAttributes() { public void testAttributeMapping() { // attribute mapping on by default - try (final N5Writer n5 = createTempN5Writer()) { + try (final N5Writer n5 = createTempN5Writer(tempN5Location(), new GsonBuilder().serializeNulls())) { n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, getCompressions()[0]);