Skip to content

Commit

Permalink
Write fill_value as json number not string (#40)
Browse files Browse the repository at this point in the history 
* perf: null or empty fillValue map to JsonNull
* test: fill_value should be json number
* style: N5ZarrTest
  • Loading branch information
@bogovicj
bogovicj authored May 15, 2024
1 parent 8ede858 commit fb886ce
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 53 deletions.
32 changes: 27 additions & 5 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.util.HashMap;
import java.util.List;

import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.RawCompression;

import com.google.gson.JsonDeserializationContext;
Expand All @@ -42,6 +43,7 @@
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.reflect.TypeToken;


Expand Down Expand Up @@ -74,7 +76,7 @@ public class ZArrayAttributes {
private final int[] chunks;
private final DType dtype;
private final ZarrCompressor compressor;
private final String fill_value;
private final JsonElement fill_value;
private final char order;
private final String dimensionSeparator;
private final List<Filter> filters = new ArrayList<>();
Expand All @@ -95,7 +97,7 @@ public ZArrayAttributes(
this.chunks = chunks;
this.dtype = dtype;
this.compressor = compressor == null ? new ZarrCompressor.Raw() : compressor;
this.fill_value = fill_value;
this.fill_value = parseFillValue(fill_value, dtype.getDataType());
this.order = order;
this.dimensionSeparator = dimensionSeparator;
if (filters != null)
Expand Down Expand Up @@ -133,10 +135,30 @@ public ZarrDatasetAttributes getDatasetAttributes() {
dtype,
compressor.getCompression(),
isRowMajor,
fill_value,
fill_value.getAsString(),
dimensionSeparator);
}

private static JsonElement parseFillValue(String fillValue, DataType dtype) {

if (fillValue == null || fillValue.isEmpty())
return JsonNull.INSTANCE;

switch (dtype) {
case INT8:
case UINT8:
case INT16:
case UINT16:
case INT32:
case UINT32:
case INT64:
case UINT64:
return new JsonPrimitive(Long.parseLong(fillValue));
default:
return new JsonPrimitive(Double.parseDouble(fillValue));
}
}

public long[] getShape() {

return shape;
Expand Down Expand Up @@ -178,7 +200,7 @@ public String getDimensionSeparator() {

public String getFillValue() {

return fill_value;
return fill_value.getAsString();
}

public HashMap<String, Object> asMap() {
Expand Down Expand Up @@ -229,7 +251,7 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial
obj.get("order").getAsCharacter(),
sepElem != null ? sepElem.getAsString() : ".",
filters);
} catch (Exception e) {
} catch (final Exception e) {
return null;
}
}
Expand Down
101 changes: 53 additions & 48 deletions src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,27 @@
*/
package org.janelia.saalfeldlab.n5.zarr;

import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.gson.reflect.TypeToken;
import net.imglib2.RandomAccess;
import net.imglib2.RandomAccessibleInterval;
import net.imglib2.type.numeric.IntegerType;
import net.imglib2.type.numeric.RealType;
import net.imglib2.type.numeric.integer.LongType;
import net.imglib2.type.numeric.integer.UnsignedByteType;
import net.imglib2.type.numeric.integer.UnsignedIntType;
import net.imglib2.type.numeric.integer.UnsignedLongType;
import net.imglib2.type.numeric.real.DoubleType;
import net.imglib2.type.numeric.real.FloatType;
import net.imglib2.view.Views;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;

import org.janelia.saalfeldlab.n5.AbstractN5Test;
import org.janelia.saalfeldlab.n5.Bzip2Compression;
import org.janelia.saalfeldlab.n5.Compression;
Expand Down Expand Up @@ -72,26 +76,24 @@
import org.junit.Ignore;
import org.junit.Test;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.gson.reflect.TypeToken;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import net.imglib2.RandomAccess;
import net.imglib2.RandomAccessibleInterval;
import net.imglib2.type.numeric.IntegerType;
import net.imglib2.type.numeric.RealType;
import net.imglib2.type.numeric.integer.LongType;
import net.imglib2.type.numeric.integer.UnsignedByteType;
import net.imglib2.type.numeric.integer.UnsignedIntType;
import net.imglib2.type.numeric.integer.UnsignedLongType;
import net.imglib2.type.numeric.real.DoubleType;
import net.imglib2.type.numeric.real.FloatType;
import net.imglib2.view.Views;

/**
* @author Stephan Saalfeld &lt;[email protected]&gt;
Expand All @@ -109,7 +111,7 @@ protected String tempN5Location() {

try {
return Files.createTempDirectory("n5-zarr-test").toUri().getPath();
} catch (IOException e) {
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
Expand Down Expand Up @@ -197,6 +199,9 @@ public void testCreateDataset() {
assertArrayEquals(blockSize, info.getBlockSize());
assertEquals(DataType.UINT64, info.getDataType());
assertEquals(getCompressions()[0].getClass(), info.getCompression().getClass());

final JsonElement elem = n5.getAttribute(datasetName, "/", JsonElement.class);
assertTrue(elem.getAsJsonObject().get("fill_value").getAsJsonPrimitive().isNumber());
}
}

Expand Down Expand Up @@ -396,19 +401,19 @@ public void testWriteReadSerializableBlock() {
@Test
@Override
public void testWriteReadStringBlock() {
DataType dataType = DataType.STRING;
int[] blockSize = new int[]{3, 2, 1};
String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"};
Compression[] compressions = this.getCompressions();
final DataType dataType = DataType.STRING;
final int[] blockSize = new int[]{3, 2, 1};
final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"};
final Compression[] compressions = this.getCompressions();

for (Compression compression : compressions) {
for (final Compression compression : compressions) {

try (final N5Writer n5 = createTempN5Writer()) {
n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression);
DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset");
StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock);
final DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset");
final StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock);
n5.writeBlock("/test/group/dataset", attributes, dataBlock);
DataBlock<?> loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L);
final DataBlock<?> loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L);
assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData());
assertTrue(n5.remove("/test/group/dataset"));
}
Expand All @@ -418,12 +423,12 @@ public void testWriteReadStringBlock() {
private boolean runPythonTest(final String script, final String containerPath) throws InterruptedException {

try {
Process process = Runtime.getRuntime().exec("poetry run python src/test/python/" + script + " " + containerPath);
final Process process = Runtime.getRuntime().exec("poetry run python src/test/python/" + script + " " + containerPath);
final int exitCode = process.waitFor();
new BufferedReader(new InputStreamReader(process.getErrorStream())).lines().forEach(System.out::println);
process.destroy();
return (exitCode == 0);
} catch (IOException e) {
} catch (final IOException e) {
return false;
}

Expand Down Expand Up @@ -828,7 +833,7 @@ public void testAttributeMapping() {
assertEquals(rawCompression, n5Compression);
assertThrows(N5Exception.N5ClassCastException.class, () -> n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class));

GzipCompression gzipCompression = new GzipCompression();
final GzipCompression gzipCompression = new GzipCompression();
n5.setAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, gzipCompression);
zarrCompression = n5.getAttribute(datasetName, ZArrayAttributes.compressorKey, ZarrCompressor.class);
n5Compression = n5.getAttribute(datasetName, DatasetAttributes.COMPRESSION_KEY, Compression.class);
Expand Down

0 comments on commit fb886ce

Please sign in to comment.