Skip to content

Commit

Permalink
Merge pull request #34 from saalfeldlab/dev-1.1.0
Browse files Browse the repository at this point in the history 
Dev 1.1.0
  • Loading branch information
@bogovicj
bogovicj authored Nov 14, 2023
2 parents be82f35 + 97edf1d commit c9144aa
Show file tree
Hide file tree
Showing 11 changed files with 337 additions and 72 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.scijava</groupId>
<artifactId>pom-scijava</artifactId>
<version>36.0.0</version>
<version>37.0.0</version>
<relativePath />
</parent>

Expand Down Expand Up @@ -138,7 +138,7 @@

<json-simple.version>1.1.1</json-simple.version>

<n5.version>3.0.2</n5.version>
<n5.version>3.1.1</n5.version>
</properties>

<dependencies>
Expand Down
82 changes: 48 additions & 34 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,10 @@
*/
package org.janelia.saalfeldlab.n5.zarr;

import java.lang.reflect.Type;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumMap;

import org.janelia.saalfeldlab.n5.ByteArrayDataBlock;
Expand All @@ -42,13 +43,7 @@
import org.janelia.saalfeldlab.n5.LongArrayDataBlock;
import org.janelia.saalfeldlab.n5.ShortArrayDataBlock;

import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import static org.janelia.saalfeldlab.n5.zarr.Filter.VLEN_UTF8;

/**
* Enumerates available zarr data types as defined at
Expand All @@ -73,6 +68,8 @@ public class DType {
typestrs.put(DataType.UINT64, ">u8");
typestrs.put(DataType.FLOAT32, ">f4");
typestrs.put(DataType.FLOAT64, ">f8");
typestrs.put(DataType.STRING, "|O");
typestrs.put(DataType.OBJECT, "|O");
}

public static enum Primitive {
Expand Down Expand Up @@ -128,13 +125,13 @@ public static Primitive fromCode(final char code) {
/* the closest possible N5 DataType */
protected final DataType dataType;

public DType(final String typestr) {
public DType(final String typestr, final Collection<Filter> filters) {

this.typestr = typestr;

order = typestr.charAt(0) == '<' ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN;
final Primitive primitive = Primitive.fromCode(typestr.charAt(1));
final int nB = Integer.parseInt(typestr.substring(2));
final int nB = (primitive == Primitive.OBJECT) ? 0 : Integer.parseInt(typestr.substring(2));

switch (primitive) {
case BIT:
Expand Down Expand Up @@ -211,8 +208,20 @@ public DType(final String typestr) {
byteBlockFactory = (blockSize, gridPosition, numElements) ->
new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]);
break;
case OBJECT:
nBytes = 1;
nBits = 0;
if (filters.contains(VLEN_UTF8)) {
dataBlockFactory = (blockSize, gridPosition, numElements) ->
new ZarrStringDataBlock(blockSize, gridPosition, new String[0]);
byteBlockFactory = (blockSize, gridPosition, numElements) ->
new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]);
} else {
dataBlockFactory = null;
byteBlockFactory = null;
}
break;
// case BOOLEAN:
// case OBJECT: // not sure about this
// case OTHER: // not sure about this
// case STRING: // not sure about this
// case UNICODE: // not sure about this
Expand All @@ -227,7 +236,7 @@ public DType(final String typestr) {
new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements * nBytes]);
}

dataType = getDataType(primitive, nBytes);
dataType = getDataType(primitive, nBytes, filters);
}

public DType(final DataType dataType, final int nPrimitives) {
Expand Down Expand Up @@ -269,6 +278,11 @@ public DType(final DataType dataType, final int nPrimitives) {
break;
// case INT8:
// case UINT8:
case STRING:
nBytes = 1;
dataBlockFactory = (blockSize, gridPosition, numElements) ->
new ZarrStringDataBlock(blockSize, gridPosition, new String[0]);
break;
default:
nBytes = nPrimitives;
dataBlockFactory = (blockSize, gridPosition, numElements) ->
Expand All @@ -290,7 +304,8 @@ public DataType getDataType() {

protected final static DataType getDataType(
final Primitive primitive,
final int nBytes) {
final int nBytes,
final Collection<Filter> filters) {

switch (primitive) {
case INT:
Expand Down Expand Up @@ -333,6 +348,11 @@ protected final static DataType getDataType(
default:
return DataType.UINT8; // fallback
}
case OBJECT:
if (filters.contains(VLEN_UTF8))
return DataType.STRING;
else
return DataType.OBJECT;
default:
return DataType.UINT8; // fallback
}
Expand All @@ -345,6 +365,21 @@ public String toString() {
return typestr;
}

/**
* Returns a list of {@link Filter filters} for the corresponding {@link DType}.
*
* @return list of filters
*/
public Collection<Filter> getFilters() {
if (dataType == DataType.STRING) {
ArrayList<Filter> filterSet = new ArrayList<>();
filterSet.add(VLEN_UTF8);
return filterSet;
}
else
return null;
}

/**
* Factory for {@link DataBlock DataBlocks}.
*
Expand Down Expand Up @@ -406,27 +441,6 @@ private static interface ByteBlockFactory {
public ByteArrayDataBlock createByteBlock(final int[] blockSize, final long[] gridPosition, final int numElements);
}

static public class JsonAdapter implements JsonDeserializer<DType>, JsonSerializer<DType> {

@Override
public DType deserialize(
final JsonElement json,
final Type typeOfT,
final JsonDeserializationContext context) throws JsonParseException {

return new DType(json.getAsString());
}

@Override
public JsonElement serialize(
final DType src,
final Type typeOfSrc,
final JsonSerializationContext context) {

return new JsonPrimitive(src.toString());
}
}

public ByteOrder getOrder() {

return order;
Expand Down
63 changes: 62 additions & 1 deletion src/main/java/org/janelia/saalfeldlab/n5/zarr/Filter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,74 @@
*/
package org.janelia.saalfeldlab.n5.zarr;

import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;

import java.lang.reflect.Type;

/**
* Place holder interface for filters
* Filter types
*
* TODO implement some
*
* @author Stephan Saalfeld &lt;[email protected]&gt;
* @author Michael Innerberger
*/
public interface Filter {

String getId();

// Note: the JSON (de-)serializer below is very much tailored to this filter, which serializes to "{"id":"vlen-utf8"}"
// If additional filters are implemented, consider also changing the type adapter below
Filter VLEN_UTF8 = new VLenStringFilter();

class VLenStringFilter implements Filter {
private static final String id = "vlen-utf8";
@Override
public String getId() {
return id;
}
};

static Filter fromString(final String id) {
if (VLEN_UTF8.getId().equals(id))
return VLEN_UTF8;
return null;
}

JsonAdapter jsonAdapter = new JsonAdapter();

class JsonAdapter implements JsonDeserializer<Filter>, JsonSerializer<Filter> {

@Override
public Filter deserialize(
final JsonElement json,
final Type typeOfT,
final JsonDeserializationContext context) throws JsonParseException {

final JsonElement jsonId = json.getAsJsonObject().get("id");
if (jsonId == null)
return null;

final String stringId = jsonId.getAsString();
return Filter.fromString(stringId);
}

@Override
public JsonElement serialize(
final Filter filter,
final Type typeOfSrc,
final JsonSerializationContext context) {

final JsonObject serialization = new JsonObject();
serialization.add("id", new JsonPrimitive(filter.getId()));
return serialization;
}
}
}
2 changes: 0 additions & 2 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ public class N5ZarrReader extends ZarrKeyValueReader {
* If true, fields from .zgroup, .zarray, and .zattrs will be merged
* when calling getAttributes, and variants of getAttribute
* @param cacheMeta cache attributes and meta data
* @param cacheMeta cache attributes and meta data
* Setting this to true avoids frequent reading and parsing of JSON
* encoded attributes and other meta data that requires accessing the
* store. This is most interesting for high latency backends. Changes
Expand All @@ -81,7 +80,6 @@ public N5ZarrReader(final String basePath,
new FileSystemKeyValueAccess(FileSystems.getDefault()),
basePath,
gsonBuilder
.registerTypeAdapter(DType.class, new DType.JsonAdapter())
.registerTypeAdapter(ZarrCompressor.class, ZarrCompressor.jsonAdapter),
mapN5DatasetAttributes,
mergeAttributes,
Expand Down
1 change: 0 additions & 1 deletion src/main/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,5 +208,4 @@ public N5ZarrWriter(final String basePath) throws N5Exception {
this(basePath, new GsonBuilder());
}


}
8 changes: 6 additions & 2 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,16 +215,20 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial
final JsonObject obj = json.getAsJsonObject();
final JsonElement sepElem = obj.get("dimension_separator");
try {
final Collection<Filter> filters = context.deserialize(obj.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType());
final String typestr = context.deserialize(obj.get("dtype"), String.class);
final DType dType = new DType(typestr, filters);

return new ZArrayAttributes(
obj.get("zarr_format").getAsInt(),
context.deserialize( obj.get("shape"), long[].class),
context.deserialize( obj.get("chunks"), int[].class),
context.deserialize( obj.get("dtype"), DType.class), // fix
dType, // fix
context.deserialize( obj.get("compressor"), ZarrCompressor.class), // fix
obj.get("fill_value").getAsString(),
obj.get("order").getAsCharacter(),
sepElem != null ? sepElem.getAsString() : ".",
context.deserialize( obj.get("filters"), TypeToken.getParameterized(Collection.class, Filter.class).getType()));
filters);
} catch (Exception e) {
return null;
}
Expand Down
Loading

0 comments on commit c9144aa

Please sign in to comment.