diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 7b4bac50..0df4f283 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -102,6 +102,12 @@ public enum DataType { blockSize, gridPosition, new double[numElements])), + VLENSTRING( + "String(-1)", + (blockSize, gridPosition, numElements) -> new VLenStringDataBlock( + blockSize, + gridPosition, + new byte[numElements])), OBJECT( "object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( @@ -111,9 +117,9 @@ public enum DataType { private final String label; - private DataBlockFactory dataBlockFactory; + private final DataBlockFactory dataBlockFactory; - private DataType(final String label, final DataBlockFactory dataBlockFactory) { + DataType(final String label, final DataBlockFactory dataBlockFactory) { this.label = label; this.dataBlockFactory = dataBlockFactory; @@ -165,9 +171,9 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize)); } - private static interface DataBlockFactory { + private interface DataBlockFactory { - public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); + DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); } static public class JsonAdapter implements JsonDeserializer, JsonSerializer { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java new file mode 100644 index 00000000..deebe65f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +public class VLenStringDataBlock extends AbstractDataBlock { + + protected static final Charset ENCODING = StandardCharsets.UTF_8; + protected static final String NULLCHAR = "\0"; + protected byte[] serializedData = null; + protected String[] actualData = null; + + public VLenStringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + super(size, gridPosition, new String[0]); + actualData = data; + } + + public VLenStringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { + super(size, gridPosition, new String[0]); + serializedData = data; + } + + @Override + public ByteBuffer toByteBuffer() { + if (serializedData == null) + serializedData = serialize(actualData); + return ByteBuffer.wrap(serializedData); + } + + @Override + public void readData(final ByteBuffer buffer) { + if (buffer.array() != serializedData) + buffer.get(serializedData); + actualData = deserialize(buffer.array()); + } + + protected byte[] serialize(String[] strings) { + final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; + return flattenedArray.getBytes(ENCODING); + } + + protected String[] deserialize(byte[] rawBytes) { + final String rawChars = new String(rawBytes, ENCODING); + return rawChars.split(NULLCHAR); + } + + @Override + public int getNumElements() { + if (serializedData == null) + serializedData = serialize(actualData); + return serializedData.length; + } + + @Override + public String[] getData() { + if (actualData == null) + actualData = deserialize(serializedData); + return actualData; + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 0e6fb608..b1c171e9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -76,7 +76,7 @@ public abstract class AbstractN5Test { static protected final String datasetName = "/test/group/dataset"; static protected final long[] dimensions = new long[]{100, 200, 300}; static protected final int[] blockSize = new int[]{44, 33, 22}; - static protected final int blockNumElements = 44 * 33 * 22; + static protected final int blockNumElements = blockSize[0] * blockSize[1] * blockSize[2]; static protected byte[] byteBlock; static protected short[] shortBlock; @@ -119,21 +119,18 @@ protected Compression[] getCompressions() { }; } - /** - * @throws IOException - */ @Before - public void setUpOnce() throws IOException, URISyntaxException { + public void setUpOnce() { final Random rnd = new Random(); - byteBlock = new byte[blockSize[0] * blockSize[1] * blockSize[2]]; - shortBlock = new short[blockSize[0] * blockSize[1] * blockSize[2]]; - intBlock = new int[blockSize[0] * blockSize[1] * blockSize[2]]; - longBlock = new long[blockSize[0] * blockSize[1] * blockSize[2]]; - floatBlock = new float[blockSize[0] * blockSize[1] * blockSize[2]]; - doubleBlock = new double[blockSize[0] * blockSize[1] * blockSize[2]]; + byteBlock = new byte[blockNumElements]; + shortBlock = new short[blockNumElements]; + intBlock = new int[blockNumElements]; + longBlock = new long[blockNumElements]; + floatBlock = new float[blockNumElements]; + doubleBlock = new double[blockNumElements]; rnd.nextBytes(byteBlock); - for (int i = 0; i < floatBlock.length; ++i) { + for (int i = 0; i < blockNumElements; ++i) { shortBlock[i] = (short)rnd.nextInt(); intBlock[i] = rnd.nextInt(); longBlock[i] = rnd.nextLong(); @@ -151,8 +148,7 @@ public void testCreateGroup() throws IOException, URISyntaxException { String subGroup = ""; for (int i = 0; i < groupPath.getNameCount(); ++i) { subGroup = subGroup + "/" + groupPath.getName(i); - if (!n5.exists(subGroup)) - fail("Group does not exist: " + subGroup); + assertTrue("Group does not exist: " + subGroup, n5.exists(subGroup)); } } } @@ -175,8 +171,7 @@ public void testCreateDataset() throws IOException, URISyntaxException { try (N5Writer writer = createN5Writer()) { writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); - if (!writer.exists(datasetName)) - fail("Dataset does not exist"); + assertTrue("Dataset does not exist", writer.exists(datasetName)); info = writer.getDatasetAttributes(datasetName); } @@ -199,7 +194,7 @@ public void testWriteReadByteBlock() throws URISyntaxException { final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, new long[]{0, 0, 0}, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); assertTrue(n5.remove(datasetName)); @@ -211,6 +206,36 @@ public void testWriteReadByteBlock() throws URISyntaxException { } } + @Test + public void testWriteReadStringBlock() throws URISyntaxException { + + // test dataset; all characters are valid UTF8 but may have different numbers of bytes! + final DataType dataType = DataType.VLENSTRING; + final int[] blockSize = new int[]{3, 2, 1}; + final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-รพ"}; + + for (final Compression compression : getCompressions()) { + + System.out.println("Testing " + compression.getType() + " " + dataType); + try (final N5Writer n5 = createN5Writer()) { + n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final VLenStringDataBlock dataBlock = new VLenStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); + n5.writeBlock(datasetName, attributes, dataBlock); + + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0L, 0L, 0L); + + assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); + + assertTrue(n5.remove(datasetName)); + + } catch (final IOException e) { + e.printStackTrace(); + fail("Block cannot be written."); + } + } + } + @Test public void testWriteReadShortBlock() throws URISyntaxException { @@ -225,7 +250,7 @@ public void testWriteReadShortBlock() throws URISyntaxException { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); @@ -253,7 +278,7 @@ public void testWriteReadIntBlock() throws URISyntaxException { final IntArrayDataBlock dataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedDataBlock.getData()); @@ -281,7 +306,7 @@ public void testWriteReadLongBlock() throws URISyntaxException { final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, longBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(longBlock, (long[])loadedDataBlock.getData()); @@ -305,7 +330,7 @@ public void testWriteReadFloatBlock() throws URISyntaxException { final FloatArrayDataBlock dataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(floatBlock, (float[])loadedDataBlock.getData(), 0.001f); @@ -328,7 +353,7 @@ public void testWriteReadDoubleBlock() throws URISyntaxException { final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[]{0, 0, 0}, doubleBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(doubleBlock, (double[])loadedDataBlock.getData(), 0.001); @@ -357,7 +382,7 @@ public void testMode1WriteReadByteBlock() throws URISyntaxException { final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(differentBlockSize, new long[]{0, 0, 0}, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); @@ -387,11 +412,11 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException, URIS object.get("one").add(new double[]{1, 2, 3}); object.get("two").add(new double[]{4, 5, 6, 7, 8}); - n5.writeSerializedBlock(object, datasetName, attributes, new long[]{0, 0, 0}); + n5.writeSerializedBlock(object, datasetName, attributes, 0, 0, 0); final HashMap> loadedObject = n5.readSerializedBlock(datasetName, attributes, new long[]{0, 0, 0}); - object.entrySet().stream().forEach(e -> assertArrayEquals(e.getValue().get(0), loadedObject.get(e.getKey()).get(0), 0.01)); + object.forEach((key, value) -> assertArrayEquals(value.get(0), loadedObject.get(key).get(0), 0.01)); assertTrue(n5.remove(datasetName)); @@ -411,13 +436,13 @@ public void testOverwriteBlock() throws URISyntaxException { final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, randomDataBlock); - final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); // test the case where the resulting file becomes shorter final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, new int[DataBlock.getNumElements(blockSize)]); n5.writeBlock(datasetName, attributes, emptyDataBlock); - final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData()); assertTrue(n5.remove(datasetName)); @@ -508,10 +533,10 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti public void testAttributes() throws IOException, URISyntaxException { try (final N5Writer n5 = createN5Writer()) { - assertEquals(null, n5.getAttribute(groupName, "test", String.class)); + assertNull(n5.getAttribute(groupName, "test", String.class)); assertEquals(0, n5.listAttributes(groupName).size()); n5.createGroup(groupName); - assertEquals(null, n5.getAttribute(groupName, "test", String.class)); + assertNull(n5.getAttribute(groupName, "test", String.class)); assertEquals(0, n5.listAttributes(groupName).size()); @@ -546,8 +571,8 @@ public void testAttributes() throws IOException, URISyntaxException { }.getType())); // test the case where the resulting file becomes shorter - n5.setAttribute(groupName, "key1", Integer.valueOf(1)); - n5.setAttribute(groupName, "key2", Integer.valueOf(2)); + n5.setAttribute(groupName, "key1", 1); + n5.setAttribute(groupName, "key2", 2); assertEquals(3, n5.listAttributes(groupName).size()); /* class interface */ assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", Integer.class)); @@ -805,15 +830,13 @@ public void testRemoveGroup() throws IOException, URISyntaxException { try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); n5.remove(groupName); - if (n5.exists(groupName)) { - fail("Group still exists"); - } + assertFalse("Group still exists", n5.exists(groupName)); } } @Test - public void testList() throws IOException, URISyntaxException { + public void testList() throws URISyntaxException { try (final N5Writer listN5 = createN5Writer()) { listN5.createGroup(groupName); @@ -873,7 +896,6 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); final String prefix = "/test"; - final String datasetSuffix = "group/dataset"; final List datasetList3 = Arrays.asList(n5.deepList(prefix)); for (final String subGroup : subGroupNames) assertTrue("deepList contents", datasetList3.contains("group/" + subGroup)); @@ -934,7 +956,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce n5.deepList(prefix, n5::datasetExists)); final List datasetListFilterDandBC = Arrays.asList(n5.deepListDatasets(prefix, isBorC)); - assertTrue("deepListDatasetFilter", datasetListFilterDandBC.size() == 0); + assertEquals("deepListDatasetFilter", 0, datasetListFilterDandBC.size()); assertArrayEquals( datasetListFilterDandBC.toArray(), n5.deepList(prefix, a -> n5.datasetExists(a) && isBorC.test(a))); @@ -950,7 +972,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List datasetListFilterDandBCP = Arrays.asList(n5.deepListDatasets(prefix, isBorC, Executors.newFixedThreadPool(2))); - assertTrue("deepListDatasetFilter Parallel", datasetListFilterDandBCP.size() == 0); + assertEquals("deepListDatasetFilter Parallel", 0, datasetListFilterDandBCP.size()); assertArrayEquals( datasetListFilterDandBCP.toArray(), n5.deepList(prefix, a -> n5.datasetExists(a) && isBorC.test(a), Executors.newFixedThreadPool(2))); @@ -995,14 +1017,14 @@ public void testListAttributes() throws IOException, URISyntaxException { n5.setAttribute(datasetName2, "attr8", new Object[]{"1", 2, 3.1}); Map> attributesMap = n5.listAttributes(datasetName2); - assertTrue(attributesMap.get("attr1") == double[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == double.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == long[].class); - assertTrue(attributesMap.get("attr6") == long.class); - assertTrue(attributesMap.get("attr7") == double[].class); - assertTrue(attributesMap.get("attr8") == Object[].class); + assertEquals(attributesMap.get("attr1"), double[].class); + assertEquals(attributesMap.get("attr2"), String[].class); + assertEquals(attributesMap.get("attr3"), double.class); + assertEquals(attributesMap.get("attr4"), String.class); + assertEquals(attributesMap.get("attr5"), long[].class); + assertEquals(attributesMap.get("attr6"), long.class); + assertEquals(attributesMap.get("attr7"), double[].class); + assertEquals(attributesMap.get("attr8"), Object[].class); n5.createGroup(groupName2); n5.setAttribute(groupName2, "attr1", new double[]{1.1, 2.1, 3.1}); @@ -1015,15 +1037,14 @@ public void testListAttributes() throws IOException, URISyntaxException { n5.setAttribute(groupName2, "attr8", new Object[]{"1", 2, 3.1}); attributesMap = n5.listAttributes(groupName2); - assertTrue(attributesMap.get("attr1") == double[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == double.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == long[].class); - assertTrue(attributesMap.get("attr6") == long.class); - assertTrue(attributesMap.get("attr7") == double[].class); - assertTrue(attributesMap.get("attr8") == Object[].class); - + assertEquals(attributesMap.get("attr1"), double[].class); + assertEquals(attributesMap.get("attr2"), String[].class); + assertEquals(attributesMap.get("attr3"), double.class); + assertEquals(attributesMap.get("attr4"), String.class); + assertEquals(attributesMap.get("attr5"), long[].class); + assertEquals(attributesMap.get("attr6"), long.class); + assertEquals(attributesMap.get("attr7"), double[].class); + assertEquals(attributesMap.get("attr8"), Object[].class); } } @@ -1176,7 +1197,7 @@ protected static void addAndTest(final N5Writer writer, final ArrayList> existingTests) throws IOException { + protected static void runTests(final N5Writer writer, final ArrayList> existingTests) { for (final TestData test : existingTests) { assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); @@ -1394,7 +1415,7 @@ private String jsonKeyVal(final String key, final String val) { n5.setAttribute(groupName, "/", true); final JsonElement booleanPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(booleanPrimitive.isJsonPrimitive()); - assertEquals(true, booleanPrimitive.getAsBoolean()); + assertTrue(booleanPrimitive.getAsBoolean()); n5.setAttribute(groupName, "/", null); final JsonElement jsonNull = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(jsonNull.isJsonNull()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 41abb9f7..7ba57bff 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -36,8 +36,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,9 +43,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.commons.io.FileUtils; import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; -import org.junit.AfterClass; import org.junit.Test; import com.google.gson.GsonBuilder; @@ -143,7 +139,7 @@ public void customObjectTest() throws IOException, URISyntaxException { } // @Test - public void testReadLock() throws IOException, InterruptedException { + public void testReadLock() throws IOException { final Path path = Paths.get(tempN5PathName(), "lock"); LockedChannel lock = access.lockForWriting(path);