Wip/codec shards/tensorstore

pom.xml

@@ -147,6 +147,7 @@
 		<dependency>
 			<groupId>org.janelia.saalfeldlab</groupId>
 			<artifactId>n5</artifactId>
+			<version>3.3.1-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>net.imglib2</groupId>
@@ -162,6 +163,10 @@
 			<version>${n5-zstandard.version}</version>
 		</dependency>
 		<dependency>
+  	  <groupId>commons-codec</groupId>
+  	  <artifactId>commons-codec</artifactId>
+    </dependency>
+    <dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-compress</artifactId>
 		</dependency>

pyproject.toml

@@ -12,6 +12,7 @@ zarr = "^2.15.0"
 numcodecs = "^0.11.0"
 pathlib = "^1.0.1"
 tensorstore = "^0.1.64"
+crc32c = "^2.7.1"
 
 
 [build-system]

src/main/java/org/janelia/saalfeldlab/n5/zarr/DType.java

@@ -478,6 +478,9 @@ public int getNBits() {
 	}
 
 	public byte[] createFillBytes(final String fill_value) {
+		if (fill_value == null) {
+			return createFillBytes("0");
+		}
 
 		final byte[] fillBytes = new byte[nBytes];
 		final ByteBuffer fillBuffer = ByteBuffer.wrap(fillBytes);

src/main/java/org/janelia/saalfeldlab/n5/zarr/ZArrayAttributes.java

@@ -138,7 +138,7 @@ public ZarrDatasetAttributes getDatasetAttributes() {
 				dtype,
 				compressor.getCompression(),
 				isRowMajor,
-				fill_value.getAsString(),
+				fill_value.isJsonNull() ? null : fill_value.getAsString(),
 				dimensionSeparator);
 	}
 
@@ -243,7 +243,7 @@ public ZArrayAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserial
 						context.deserialize( obj.get("chunks"), int[].class),
 						dType, // fix
 						context.deserialize( obj.get("compressor"), ZarrCompressor.class), // fix
-						obj.get("fill_value").getAsString(),
+						obj.get("fill_value").isJsonNull() ? null : obj.get("fill_value").getAsString(),
 						obj.get("order").getAsCharacter(),
 						sepElem != null ? sepElem.getAsString() : ".",
 						filters);

src/main/java/org/janelia/saalfeldlab/n5/zarr/v3/ZarrV3DatasetAttributes.java

@@ -153,7 +153,7 @@ else if (codecs == null)
 		return out;
 	}
 
-	private static Codec[] prependArrayToBytes(Codec.ArrayToBytes arrayToBytes, Codec[] codecs) {
+	private static Codec[] prependArrayToBytes(Codec.ArrayCodec arrayToBytes, Codec[] codecs) {
 
 		final Codec[] out = new Codec[codecs.length + 1];
 		out[0] = arrayToBytes;
@@ -238,7 +238,7 @@ public HashMap<String, Object> asMap() {
 
 		map.put(FILL_VALUE_KEY, fillValue);
 		// map.put(CODECS_KEY, codecsToZarrCompressors(getCodecs()));
-		map.put(CODECS_KEY, prependArrayToBytes(getArrayToBytesCodec(), getCodecs()));
+		map.put(CODECS_KEY, prependArrayToBytes(getArrayCodec(), getCodecs()));
 
 		return map;
 	}

src/main/java/org/janelia/saalfeldlab/n5/zarr/v3/ZarrV3KeyValueWriter.java

@@ -190,12 +190,12 @@ protected ZarrV3DatasetAttributes createZArrayAttributes(final DatasetAttributes
 				dType,
 				"0",
 				datasetAttributes.getCompression(),
-				prependArrayToBytes(datasetAttributes.getArrayToBytesCodec(), datasetAttributes.getCodecs()));
+				prependArrayToBytes(datasetAttributes.getArrayCodec(), datasetAttributes.getCodecs()));
 
 		return zArrayAttributes;
 	}
 
-	private static Codec[] prependArrayToBytes(Codec.ArrayToBytes arrayToBytes, Codec[] codecs) {
+	private static Codec[] prependArrayToBytes(Codec.ArrayCodec arrayToBytes, Codec[] codecs) {
 
 		final Codec[] out = new Codec[codecs.length + 1];
 		out[0] = arrayToBytes;

src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java

@@ -464,10 +464,19 @@ private static <T extends RealType<T>> void assertIsSequence(
 	@Test
 	public void testReadZarrPython() throws IOException, InterruptedException {
 
-		final String testZarrDirPath = tempN5Location();
-
+		final String testZarrDirPath = tempN5Location();	
+		//TODO: decided what to do with it for windows
+		String testZarrDirPathForPython;
+
+		if (System.getProperty("os.name").startsWith("Windows"))
+			testZarrDirPathForPython = testZarrDirPath.substring(1);
+		else
+			testZarrDirPathForPython = testZarrDirPath;
+
+		System.err.println("For Python: " + testZarrDirPathForPython);
+
 		/* create test data with python */
-		if (!runPythonTest("zarr-test.py", testZarrDirPath)) {
+		if (!runPythonTest("zarr-test.py", testZarrDirPathForPython)) {
 			System.out.println("Couldn't run Python test, skipping compatibility test with Python.");
 			return;
 		}