Skip to content

Commit

Permalink
Change construction of DType
Browse files Browse the repository at this point in the history
DType is constructed by a typestring and a collection of filters. In
particular, it's not directly de-serialized anymore.
  • Loading branch information
minnerbe committed Oct 19, 2023
1 parent 74b535f commit edccdaa
Show file tree
Hide file tree
Showing 5 changed files with 20 additions and 34 deletions.
35 changes: 7 additions & 28 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumMap;
import java.util.List;

Expand Down Expand Up @@ -76,6 +77,7 @@ public class DType {
typestrs.put(DataType.FLOAT32, ">f4");
typestrs.put(DataType.FLOAT64, ">f8");
typestrs.put(DataType.STRING, "|O");
typestrs.put(DataType.OBJECT, "|O");
}

public static enum Primitive {
Expand Down Expand Up @@ -131,7 +133,7 @@ public static Primitive fromCode(final char code) {
/* the closest possible N5 DataType */
protected final DataType dataType;

public DType(final String typestr) {
public DType(final String typestr, final Collection<Filter> filters) {

this.typestr = typestr;

Expand Down Expand Up @@ -217,10 +219,8 @@ public DType(final String typestr) {
case OBJECT:
nBytes = 1;
nBits = 0;
dataBlockFactory = (blockSize, gridPosition, numElements) ->
new ZarrCompatibleStringDataBlock(blockSize, gridPosition, new String[0]);
byteBlockFactory = (blockSize, gridPosition, numElements) ->
new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]);
dataBlockFactory = null;
byteBlockFactory = null;
break;
// case BOOLEAN:
// case OTHER: // not sure about this
Expand Down Expand Up @@ -343,8 +343,8 @@ protected final static DataType getDataType(
default:
return DataType.UINT8; // fallback
}
case OBJECT: // todo: this should also depend on filters!
return DataType.STRING;
case OBJECT:
return DataType.OBJECT;
default:
return DataType.UINT8; // fallback
}
Expand Down Expand Up @@ -433,27 +433,6 @@ private static interface ByteBlockFactory {
public ByteArrayDataBlock createByteBlock(final int[] blockSize, final long[] gridPosition, final int numElements);
}

static public class JsonAdapter implements JsonDeserializer<DType>, JsonSerializer<DType> {

@Override
public DType deserialize(
final JsonElement json,
final Type typeOfT,
final JsonDeserializationContext context) throws JsonParseException {

return new DType(json.getAsString());
}

@Override
public JsonElement serialize(
final DType src,
final Type typeOfSrc,
final JsonSerializationContext context) {

return new JsonPrimitive(src.toString());
}
}

public ByteOrder getOrder() {

return order;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ public N5ZarrReader(final String basePath,
new FileSystemKeyValueAccess(FileSystems.getDefault()),
basePath,
gsonBuilder
.registerTypeAdapter(DType.class, new DType.JsonAdapter())
.registerTypeAdapter(ZarrCompressor.class, ZarrCompressor.jsonAdapter),
mapN5DatasetAttributes,
mergeAttributes,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,16 +215,20 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial
final JsonObject obj = json.getAsJsonObject();
final JsonElement sepElem = obj.get("dimension_separator");
try {
final Collection<Filter> filters = context.deserialize(obj.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType());
final String typestr = context.deserialize(obj.get("dtype"), String.class);
final DType dType = new DType(typestr, filters);

return new ZArrayAttributes(
obj.get("zarr_format").getAsInt(),
context.deserialize( obj.get("shape"), long[].class),
context.deserialize( obj.get("chunks"), int[].class),
context.deserialize( obj.get("dtype"), DType.class), // fix
dType, // fix
context.deserialize( obj.get("compressor"), ZarrCompressor.class), // fix
obj.get("fill_value").getAsString(),
obj.get("order").getAsCharacter(),
sepElem != null ? sepElem.getAsString() : ".",
context.deserialize( obj.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType()));
filters);
} catch (Exception e) {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -836,7 +836,6 @@ static Gson registerGson(final GsonBuilder gsonBuilder) {

protected static GsonBuilder addTypeAdapters(final GsonBuilder gsonBuilder) {

gsonBuilder.registerTypeAdapter(DType.class, new DType.JsonAdapter());
gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter());
gsonBuilder.registerTypeAdapter(ZarrCompressor.class, ZarrCompressor.jsonAdapter);
gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter());
Expand Down
9 changes: 7 additions & 2 deletions src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;

Expand Down Expand Up @@ -800,7 +801,9 @@ public void testAttributeMapping() throws IOException {
int[] blkN5 = n5.getAttribute(datasetName, DatasetAttributes.BLOCK_SIZE_KEY, int[].class);
assertArrayEquals(blkZarr, blkN5);

DType dtype = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, DType.class);
String typestr = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, String.class);
Collection<Filter> filters = n5.getAttribute(datasetName, ZArrayAttributes.filtersKey, TypeToken.getParameterized(Collection.class, Filter.class).getType());
DType dtype = new DType(typestr, filters);
// read to a string because zarr may not have the N5 DataType deserializer
DataType n5DataType = DataType.fromString(n5.getAttribute(datasetName, DatasetAttributes.DATA_TYPE_KEY, String.class));
assertEquals(dtype.getDataType(), n5DataType);
Expand Down Expand Up @@ -828,7 +831,9 @@ public void testAttributeMapping() throws IOException {

n5.setAttribute(datasetName, DatasetAttributes.DATA_TYPE_KEY, newDtype.toString());

dtype = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, DType.class);
typestr = n5.getAttribute(datasetName, ZArrayAttributes.dTypeKey, String.class);
filters = n5.getAttribute(datasetName, ZArrayAttributes.filtersKey, TypeToken.getParameterized(Collection.class, Filter.class).getType());
dtype = new DType(typestr, filters);
n5DataType = DataType.fromString(n5.getAttribute(datasetName, DatasetAttributes.DATA_TYPE_KEY, String.class));
assertEquals(newDtype, dtype.getDataType());
assertEquals(newDtype, n5DataType);
Expand Down

0 comments on commit edccdaa

Please sign in to comment.