Merge pull request #42 from cmhulbert/serializeNulls

* fix: don't enforce `serializeNulls`, but DO still ensure `null` is still serialized regardless of `serializeNulls` for Raw compression

* fix: don't require gsonBuilder to `serializeNull`

- added a serializer for ZArrayAttributes
- writeZArray via ZArrayAttributes object, not JsonElement
- expected compression can be `JsonNull` OR `null` depending on whether `serializeNull` or not

test: `serializeNulls` not set by default, so no `null` value here, therefore it doesn't parse as `JsonNull` but instead just returns `null`

* fix(test): don't ignore input GsonBuilder

* fix(test): serializeNulls for consistency between Cached and non-cache versions

This is because when reading from the container with no cache, you get `null` but when writing/reading with the cache, the value is `JsonNull`.

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java

@@ -34,6 +34,8 @@
 import java.util.HashMap;
 import java.util.List;
 
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.RawCompression;
 
@@ -222,7 +224,7 @@ public Collection<Filter> getFilters() {
 
 	public static JsonAdapter jsonAdapter = new JsonAdapter();
 
-	public static class JsonAdapter implements JsonDeserializer<ZArrayAttributes> {
+	public static class JsonAdapter implements JsonDeserializer<ZArrayAttributes>, JsonSerializer<ZArrayAttributes> {
 
 		@Override
 		public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
@@ -249,5 +251,23 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial
 			}
 		}
 
+		@Override
+		public JsonElement serialize(ZArrayAttributes src, Type typeOfSrc, JsonSerializationContext context) {
+
+			final JsonObject jsonObject = new JsonObject();
+
+			jsonObject.addProperty("zarr_format", src.getZarrFormat());
+			jsonObject.add("shape", context.serialize(src.getShape()));
+			jsonObject.add("chunks", context.serialize(src.getChunks()));
+
+			jsonObject.add("dtype", context.serialize(src.getDType().toString()));
+			jsonObject.add("compressor", context.serialize(src.getCompressor()));
+			jsonObject.addProperty("fill_value", src.getFillValue());
+			jsonObject.addProperty("order", src.getOrder());
+			jsonObject.addProperty("dimension_separator", src.getDimensionSeparator());
+			jsonObject.add("filters", context.serialize(src.getFilters()));
+
+			return jsonObject;
+		}
 	}
 }

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrCompressor.java

@@ -28,15 +28,17 @@
  */
 package org.janelia.saalfeldlab.n5.zarr;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Type;
 import java.util.AbstractMap.SimpleImmutableEntry;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.google.gson.JsonNull;
-import com.google.gson.JsonSerializer;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonWriter;
 import org.janelia.saalfeldlab.n5.Bzip2Compression;
 import org.janelia.saalfeldlab.n5.Compression;
 import org.janelia.saalfeldlab.n5.GzipCompression;
@@ -297,5 +299,18 @@ public ZarrCompressor deserialize(final JsonElement json, final Type typeOfT, fi
 		}
 	}
 
-	JsonSerializer<Raw> rawNullAdapter = (src, typeOfSrc, context) -> JsonNull.INSTANCE;
+	TypeAdapter<Raw> rawNullAdapter = new TypeAdapter<Raw>() {
+
+		@Override public void write(JsonWriter out, Raw value) throws IOException {
+			final boolean serializeNull = out.getSerializeNulls();
+			out.setSerializeNulls(true);
+			out.nullValue();
+			out.setSerializeNulls(serializeNull);
+		}
+
+		@Override public Raw read(JsonReader in) {
+
+			return new Raw();
+		}
+	};
 }

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueReader.java

@@ -525,7 +525,7 @@ protected JsonElement zarrToN5DatasetAttributes(final JsonElement elem ) {
 		attrs.addProperty(DatasetAttributes.DATA_TYPE_KEY, zattrs.getDType().getDataType().toString());
 
 		final JsonElement e = attrs.get(ZArrayAttributes.compressorKey);
-		if (e == JsonNull.INSTANCE) {
+		if (e == JsonNull.INSTANCE || e == null) {
 			attrs.add(DatasetAttributes.COMPRESSION_KEY, gson.toJsonTree(new RawCompression()));
 		} else {
 			attrs.add(DatasetAttributes.COMPRESSION_KEY, gson.toJsonTree(
@@ -863,7 +863,6 @@ protected static GsonBuilder addTypeAdapters(final GsonBuilder gsonBuilder) {
 		gsonBuilder.registerTypeAdapter(ZArrayAttributes.class, ZArrayAttributes.jsonAdapter);
 		gsonBuilder.registerTypeHierarchyAdapter(Filter.class, Filter.jsonAdapter);
 		gsonBuilder.disableHtmlEscaping();
-		gsonBuilder.serializeNulls();
 
 		return gsonBuilder;
 	}

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZarrKeyValueWriter.java

@@ -27,7 +27,9 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.Writer;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -258,8 +260,10 @@ else if (getCache().isGroup(normalPath, ZGROUP_FILE)) {
 		// These three lines are preferable to setDatasetAttributes because they
 		// are more efficient wrt caching
 		final ZArrayAttributes zarray = createZArrayAttributes(datasetAttributes);
-		final JsonElement attributes = gson.toJsonTree(zarray.asMap());
-		writeJsonResource(normalPath, ZARRAY_FILE, attributes);
+		final HashMap<String, Object> zarrayMap = zarray.asMap();
+		final JsonElement attributes = gson.toJsonTree(zarrayMap);
+		writeJsonResource(normalPath, ZARRAY_FILE, zarrayMap);
+
 		if( wasGroup )
 			deleteJsonResource(normalPath, ZGROUP_FILE );
 
@@ -416,14 +420,16 @@ protected void deleteJsonResource(final String normalPath, final String jsonName
 	protected void writeJsonResource(
 			final String normalPath,
 			final String jsonName,
-			final JsonElement attributes) throws N5Exception {
+			final Object attributes) throws N5Exception {
 
 		if (attributes == null)
 			return;
 
 		final String absolutePath = keyValueAccess.compose(uri, normalPath, jsonName);
 		try (final LockedChannel lock = keyValueAccess.lockForWriting(absolutePath)) {
-			GsonUtils.writeAttributes(lock.newWriter(), attributes, gson);
+			final Writer writer = lock.newWriter();
+			gson.toJson(attributes, writer);
+			writer.flush();
 		} catch (final Throwable e) {
 			throw new N5IOException("Failed to write " + absolutePath, e);
 		}
@@ -436,7 +442,7 @@ protected void writeZArray(
 		if (attributes == null)
 			return;
 
-		writeJsonResource(normalGroupPath, ZARRAY_FILE, attributes);
+		writeJsonResource(normalGroupPath, ZARRAY_FILE, gson.fromJson(attributes, ZArrayAttributes.class));
 		if (cacheMeta())
 			cache.updateCacheInfo(normalGroupPath, ZARRAY_FILE, attributes);
 	}

src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java

@@ -146,7 +146,7 @@ protected N5Writer createTempN5Writer(
 			final String dimensionSeparator,
 			final boolean mapN5DatasetAttributes) throws IOException {
 
-		return createTempN5Writer(location, new GsonBuilder(), dimensionSeparator, mapN5DatasetAttributes, false);
+		return createTempN5Writer(location, gsonBuilder, dimensionSeparator, mapN5DatasetAttributes, false);
 	}
 
 	protected N5Writer createTempN5Writer(
@@ -778,7 +778,7 @@ public void testAttributes()  {
 	public void testAttributeMapping()  {
 
 		// attribute mapping on by default
-		try (final N5Writer n5 = createTempN5Writer()) {
+		try (final N5Writer n5 = createTempN5Writer(tempN5Location(), new GsonBuilder().serializeNulls())) {
 
 			n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, getCompressions()[0]);
 
@@ -832,7 +832,6 @@ public void testAttributeMapping()  {
 			n5Compression = n5.getAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, Compression.class);
 			assertEquals(rawCompression, n5Compression);
 			assertThrows(N5Exception.N5ClassCastException.class, () -> n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class));
-
 			final GzipCompression gzipCompression = new GzipCompression();
 			n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, gzipCompression);
 			zarrCompression = n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class);