From a8763792c16326283a7980d63f6ba1187a901aad Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 Feb 2024 16:58:38 -0500 Subject: [PATCH] feat(test): replace direct file access attempts with KVA calls, so test class can be reused with different backends --- .../saalfeldlab/n5/zarr/N5ZarrTest.java | 112 +++++++++--------- .../saalfeldlab/n5/zarr/ZarrCachedFSTest.java | 2 +- 2 files changed, 55 insertions(+), 59 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java index 95f4f51..8e4eac2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/N5ZarrTest.java @@ -28,29 +28,23 @@ */ package org.janelia.saalfeldlab.n5.zarr; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URISyntaxException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; +import net.imglib2.RandomAccess; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.type.numeric.IntegerType; +import net.imglib2.type.numeric.RealType; +import net.imglib2.type.numeric.integer.LongType; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedIntType; +import net.imglib2.type.numeric.integer.UnsignedLongType; +import net.imglib2.type.numeric.real.DoubleType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.view.Views; import org.janelia.saalfeldlab.n5.AbstractN5Test; import org.janelia.saalfeldlab.n5.Bzip2Compression; import org.janelia.saalfeldlab.n5.Compression; @@ -59,12 +53,14 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.blosc.BloscCompression; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; @@ -76,24 +72,26 @@ import org.junit.Ignore; import org.junit.Test; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URISyntaxException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; -import net.imglib2.RandomAccess; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.type.numeric.IntegerType; -import net.imglib2.type.numeric.RealType; -import net.imglib2.type.numeric.integer.LongType; -import net.imglib2.type.numeric.integer.UnsignedByteType; -import net.imglib2.type.numeric.integer.UnsignedIntType; -import net.imglib2.type.numeric.integer.UnsignedLongType; -import net.imglib2.type.numeric.real.DoubleType; -import net.imglib2.type.numeric.real.FloatType; -import net.imglib2.view.Views; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; /** * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> @@ -102,6 +100,10 @@ public class N5ZarrTest extends AbstractN5Test { static private final String testZarrDatasetName = "/test/data"; + public static KeyValueAccess createKeyValueAccess() { + return new FileSystemKeyValueAccess(FileSystems.getDefault()); + } + @Override protected String tempN5Location() { @@ -113,10 +115,10 @@ protected String tempN5Location() { } @Override - protected N5Writer createN5Writer() throws IOException { + protected N5Writer createN5Writer() { final String testDirPath = tempN5Location(); - return new ZarrKeyValueWriter(new FileSystemKeyValueAccess(FileSystems.getDefault()), testDirPath, new GsonBuilder(), true, true, ".",false) { + return new ZarrKeyValueWriter(createKeyValueAccess(), testDirPath, new GsonBuilder(), true, true, ".",false) { @Override public void close() { @@ -141,7 +143,7 @@ protected N5Writer createN5Writer(final String location, final String dimensionS protected N5Writer createN5Writer(final String location, final String dimensionSeparator, final boolean cacheAttributes) throws IOException { - return new ZarrKeyValueWriter(new FileSystemKeyValueAccess(FileSystems.getDefault()), location, new GsonBuilder(), true, true, dimensionSeparator, cacheAttributes); + return new ZarrKeyValueWriter(createKeyValueAccess(), location, new GsonBuilder(), true, true, dimensionSeparator, cacheAttributes); } protected N5Writer createN5Writer( @@ -151,13 +153,13 @@ protected N5Writer createN5Writer( final boolean mapN5DatasetAttributes) throws IOException { - return new ZarrKeyValueWriter(new FileSystemKeyValueAccess(FileSystems.getDefault()), location, gsonBuilder, mapN5DatasetAttributes, true, dimensionSeparator, false); + return new ZarrKeyValueWriter(createKeyValueAccess(), location, gsonBuilder, mapN5DatasetAttributes, true, dimensionSeparator, false); } @Override protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException { - return new ZarrKeyValueReader(new FileSystemKeyValueAccess(FileSystems.getDefault()), location, gson, true, true, false); + return new ZarrKeyValueReader(createKeyValueAccess(), location, gson, true, true, false); } @Override @@ -283,9 +285,7 @@ public void testVersion() throws NumberFormatException, IOException { @Override public void testReaderCreation() throws IOException, URISyntaxException { - final File tmpFile = Files.createTempDirectory("reader-create-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); + final String canonicalPath = tempN5Location(); try (N5Writer writer = createN5Writer(canonicalPath)) { final N5Reader n5r = createN5Reader(canonicalPath); @@ -397,6 +397,7 @@ public void testWriteReadSerializableBlock() { } @Test + @Override public void testWriteReadStringBlock() { DataType dataType = DataType.STRING; int[] blockSize = new int[]{3, 2, 1}; @@ -414,12 +415,8 @@ public void testWriteReadStringBlock() { DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); assertTrue(n5.remove("/test/group/dataset")); - } catch (IOException e) { - e.printStackTrace(); - Assert.fail("Block cannot be written."); } } - } private boolean runPythonTest(final String script, final String containerPath) throws InterruptedException { @@ -699,17 +696,17 @@ public void testReadZarrNestedPython() throws IOException, InterruptedException public void testRawCompressorNullInZarray() throws IOException, ParseException { final ZarrKeyValueWriter n5 = (ZarrKeyValueWriter) createN5Writer(); - final String testZarrDirPath = n5.getURI().getPath(); n5.createDataset( testZarrDatasetName, new long[]{1, 2, 3}, new int[]{1, 2, 3}, DataType.UINT16, new RawCompression()); + final String zarrayLocation = n5.keyValueAccess.compose(n5.uri, testZarrDatasetName, ".zarray"); + final LockedChannel zarrayChannel = n5.keyValueAccess.lockForReading(zarrayLocation); final JSONParser jsonParser = new JSONParser(); - final File zArrayFile = Paths.get(testZarrDirPath, testZarrDatasetName, ".zarray").toFile(); - try (FileReader freader = new FileReader(zArrayFile)) { - final JSONObject zarray = (JSONObject)jsonParser.parse(freader); + try (Reader reader = zarrayChannel.newReader()) { + final JSONObject zarray = (JSONObject)jsonParser.parse(reader); final JSONObject compressor = (JSONObject)zarray.get("compressor"); assertNull(compressor); } finally { @@ -756,7 +753,6 @@ public void testAttributes() throws IOException { }.getType())); - // test the case where the resulting file becomes shorter n5.setAttribute(groupName, "key1", 1); n5.setAttribute(groupName, "key2", 2); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java index 8801093..47f67d3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java @@ -41,7 +41,7 @@ protected String tempN5Location() { } @Override - protected N5ZarrWriter createN5Writer() throws IOException { + protected N5ZarrWriter createN5Writer() { final String testDirPath = tempN5Location(); return new N5ZarrWriter(testDirPath, new GsonBuilder(), ".", true, true) {