From fb886ceaca752416828056b08a99c2b417e70a5e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 15 May 2024 10:21:21 -0400 Subject: [PATCH] Write fill_value as json number not string (#40) * perf: null or empty fillValue map to JsonNull * test: fill_value should be json number * style: N5ZarrTest --- .../saalfeldlab/n5/zarr/ZArrayAttributes.java | 32 +++++- .../saalfeldlab/n5/zarr/N5ZarrTest.java | 101 +++++++++--------- 2 files changed, 80 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java index 62ffa2c..4887cc5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.List; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.RawCompression; import com.google.gson.JsonDeserializationContext; @@ -42,6 +43,7 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; import com.google.gson.reflect.TypeToken; @@ -74,7 +76,7 @@ public class ZArrayAttributes { private final int[] chunks; private final DType dtype; private final ZarrCompressor compressor; - private final String fill_value; + private final JsonElement fill_value; private final char order; private final String dimensionSeparator; private final List filters = new ArrayList<>(); @@ -95,7 +97,7 @@ public ZArrayAttributes( this.chunks = chunks; this.dtype = dtype; this.compressor = compressor == null ? new ZarrCompressor.Raw() : compressor; - this.fill_value = fill_value; + this.fill_value = parseFillValue(fill_value, dtype.getDataType()); this.order = order; this.dimensionSeparator = dimensionSeparator; if (filters != null) @@ -133,10 +135,30 @@ public ZarrDatasetAttributes getDatasetAttributes() { dtype, compressor.getCompression(), isRowMajor, - fill_value, + fill_value.getAsString(), dimensionSeparator); } + private static JsonElement parseFillValue(String fillValue, DataType dtype) { + + if (fillValue == null || fillValue.isEmpty()) + return JsonNull.INSTANCE; + + switch (dtype) { + case INT8: + case UINT8: + case INT16: + case UINT16: + case INT32: + case UINT32: + case INT64: + case UINT64: + return new JsonPrimitive(Long.parseLong(fillValue)); + default: + return new JsonPrimitive(Double.parseDouble(fillValue)); + } + } + public long[] getShape() { return shape; @@ -178,7 +200,7 @@ public String getDimensionSeparator() { public String getFillValue() { - return fill_value; + return fill_value.getAsString(); } public HashMap asMap() { @@ -229,7 +251,7 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial obj.get("order").getAsCharacter(), sepElem != null ? sepElem.getAsString() : ".", filters); - } catch (Exception e) { + } catch (final Exception e) { return null; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 3381927..251daca 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -28,23 +28,27 @@ */ package org.janelia.saalfeldlab.n5.zarr; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; -import net.imglib2.RandomAccess; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.type.numeric.IntegerType; -import net.imglib2.type.numeric.RealType; -import net.imglib2.type.numeric.integer.LongType; -import net.imglib2.type.numeric.integer.UnsignedByteType; -import net.imglib2.type.numeric.integer.UnsignedIntType; -import net.imglib2.type.numeric.integer.UnsignedLongType; -import net.imglib2.type.numeric.real.DoubleType; -import net.imglib2.type.numeric.real.FloatType; -import net.imglib2.view.Views; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URISyntaxException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + import org.janelia.saalfeldlab.n5.AbstractN5Test; import org.janelia.saalfeldlab.n5.Bzip2Compression; import org.janelia.saalfeldlab.n5.Compression; @@ -72,26 +76,24 @@ import org.junit.Ignore; import org.junit.Test; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URISyntaxException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import net.imglib2.RandomAccess; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.type.numeric.IntegerType; +import net.imglib2.type.numeric.RealType; +import net.imglib2.type.numeric.integer.LongType; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedIntType; +import net.imglib2.type.numeric.integer.UnsignedLongType; +import net.imglib2.type.numeric.real.DoubleType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.view.Views; /** * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> @@ -109,7 +111,7 @@ protected String tempN5Location() { try { return Files.createTempDirectory("n5-zarr-test").toUri().getPath(); - } catch (IOException e) { + } catch (final IOException e) { throw new RuntimeException(e); } } @@ -197,6 +199,9 @@ public void testCreateDataset() { assertArrayEquals(blockSize, info.getBlockSize()); assertEquals(DataType.UINT64, info.getDataType()); assertEquals(getCompressions()[0].getClass(), info.getCompression().getClass()); + + final JsonElement elem = n5.getAttribute(datasetName, "/", JsonElement.class); + assertTrue(elem.getAsJsonObject().get("fill_value").getAsJsonPrimitive().isNumber()); } } @@ -396,19 +401,19 @@ public void testWriteReadSerializableBlock() { @Test @Override public void testWriteReadStringBlock() { - DataType dataType = DataType.STRING; - int[] blockSize = new int[]{3, 2, 1}; - String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; - Compression[] compressions = this.getCompressions(); + final DataType dataType = DataType.STRING; + final int[] blockSize = new int[]{3, 2, 1}; + final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; + final Compression[] compressions = this.getCompressions(); - for (Compression compression : compressions) { + for (final Compression compression : compressions) { try (final N5Writer n5 = createTempN5Writer()) { n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression); - DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); - StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); + final DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); + final StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); n5.writeBlock("/test/group/dataset", attributes, dataBlock); - DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); + final DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); assertTrue(n5.remove("/test/group/dataset")); } @@ -418,12 +423,12 @@ public void testWriteReadStringBlock() { private boolean runPythonTest(final String script, final String containerPath) throws InterruptedException { try { - Process process = Runtime.getRuntime().exec("poetry run python src/test/python/" + script + " " + containerPath); + final Process process = Runtime.getRuntime().exec("poetry run python src/test/python/" + script + " " + containerPath); final int exitCode = process.waitFor(); new BufferedReader(new InputStreamReader(process.getErrorStream())).lines().forEach(System.out::println); process.destroy(); return (exitCode == 0); - } catch (IOException e) { + } catch (final IOException e) { return false; } @@ -828,7 +833,7 @@ public void testAttributeMapping() { assertEquals(rawCompression, n5Compression); assertThrows(N5Exception.N5ClassCastException.class, () -> n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class)); - GzipCompression gzipCompression = new GzipCompression(); + final GzipCompression gzipCompression = new GzipCompression(); n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, gzipCompression); zarrCompression = n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class); n5Compression = n5.getAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, Compression.class);