From 08412f5d9c7c8e23e9e595c12e8d22e9258a3b08 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 7 Feb 2023 17:13:25 -0500 Subject: [PATCH 01/10] Clean up syntax of unit tests --- .../saalfeldlab/n5/AbstractN5Test.java | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 3add6d28..d29d219b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -53,7 +53,7 @@ public abstract class AbstractN5Test { static protected final String datasetName = "/test/group/dataset"; static protected final long[] dimensions = new long[]{100, 200, 300}; static protected final int[] blockSize = new int[]{44, 33, 22}; - static protected final int blockNumElements = 44 * 33 * 22; + static protected final int blockNumElements = blockSize[0] * blockSize[1] * blockSize[2]; static protected byte[] byteBlock; static protected short[] shortBlock; @@ -90,14 +90,14 @@ public void setUpOnce() throws IOException { n5 = createN5Writer(); final Random rnd = new Random(); - byteBlock = new byte[blockSize[0] * blockSize[1] * blockSize[2]]; - shortBlock = new short[blockSize[0] * blockSize[1] * blockSize[2]]; - intBlock = new int[blockSize[0] * blockSize[1] * blockSize[2]]; - longBlock = new long[blockSize[0] * blockSize[1] * blockSize[2]]; - floatBlock = new float[blockSize[0] * blockSize[1] * blockSize[2]]; - doubleBlock = new double[blockSize[0] * blockSize[1] * blockSize[2]]; + byteBlock = new byte[blockNumElements]; + shortBlock = new short[blockNumElements]; + intBlock = new int[blockNumElements]; + longBlock = new long[blockNumElements]; + floatBlock = new float[blockNumElements]; + doubleBlock = new double[blockNumElements]; rnd.nextBytes(byteBlock); - for(int i = 0; i < floatBlock.length; ++i) { + for(int i = 0; i < blockNumElements; ++i) { shortBlock[i] = (short)rnd.nextInt(); intBlock[i] = rnd.nextInt(); longBlock[i] = rnd.nextLong(); @@ -129,8 +129,7 @@ public void testCreateGroup() { final Path groupPath = Paths.get(groupName); for (int i = 0; i < groupPath.getNameCount(); ++i) - if (!n5.exists(groupPath.subpath(0, i + 1).toString())) - fail("Group does not exist"); + Assert.assertTrue("Group does not exist", n5.exists(groupPath.subpath(0, i + 1).toString())); } @Test @@ -142,8 +141,7 @@ public void testCreateDataset() { fail(e.getMessage()); } - if (!n5.exists(datasetName)) - fail("Dataset does not exist"); + Assert.assertTrue("Dataset does not exist", n5.exists(datasetName)); try { final DatasetAttributes info = n5.getDatasetAttributes(datasetName); @@ -467,8 +465,7 @@ public void testRemove() { fail(e.getMessage()); } - if (n5.exists(groupName)) - fail("Group still exists"); + Assert.assertFalse("Group still exists", n5.exists(groupName)); } @Test @@ -682,14 +679,14 @@ public void testListAttributes() { n5.setAttribute(datasetName2, "attr8", new Object[] {"1", 2, 3.1}); Map> attributesMap = n5.listAttributes(datasetName2); - Assert.assertTrue(attributesMap.get("attr1") == double[].class); - Assert.assertTrue(attributesMap.get("attr2") == String[].class); - Assert.assertTrue(attributesMap.get("attr3") == double.class); - Assert.assertTrue(attributesMap.get("attr4") == String.class); - Assert.assertTrue(attributesMap.get("attr5") == long[].class); - Assert.assertTrue(attributesMap.get("attr6") == long.class); - Assert.assertTrue(attributesMap.get("attr7") == double[].class); - Assert.assertTrue(attributesMap.get("attr8") == Object[].class); + Assert.assertEquals(attributesMap.get("attr1"), double[].class); + Assert.assertEquals(attributesMap.get("attr2"), String[].class); + Assert.assertEquals(attributesMap.get("attr3"), double.class); + Assert.assertEquals(attributesMap.get("attr4"), String.class); + Assert.assertEquals(attributesMap.get("attr5"), long[].class); + Assert.assertEquals(attributesMap.get("attr6"), long.class); + Assert.assertEquals(attributesMap.get("attr7"), double[].class); + Assert.assertEquals(attributesMap.get("attr8"), Object[].class); n5.createGroup(groupName2); n5.setAttribute(groupName2, "attr1", new double[] {1.1, 2.1, 3.1}); @@ -702,14 +699,14 @@ public void testListAttributes() { n5.setAttribute(groupName2, "attr8", new Object[] {"1", 2, 3.1}); attributesMap = n5.listAttributes(groupName2); - Assert.assertTrue(attributesMap.get("attr1") == double[].class); - Assert.assertTrue(attributesMap.get("attr2") == String[].class); - Assert.assertTrue(attributesMap.get("attr3") == double.class); - Assert.assertTrue(attributesMap.get("attr4") == String.class); - Assert.assertTrue(attributesMap.get("attr5") == long[].class); - Assert.assertTrue(attributesMap.get("attr6") == long.class); - Assert.assertTrue(attributesMap.get("attr7") == double[].class); - Assert.assertTrue(attributesMap.get("attr8") == Object[].class); + Assert.assertEquals(attributesMap.get("attr1"), double[].class); + Assert.assertEquals(attributesMap.get("attr2"), String[].class); + Assert.assertEquals(attributesMap.get("attr3"), double.class); + Assert.assertEquals(attributesMap.get("attr4"), String.class); + Assert.assertEquals(attributesMap.get("attr5"), long[].class); + Assert.assertEquals(attributesMap.get("attr6"), long.class); + Assert.assertEquals(attributesMap.get("attr7"), double[].class); + Assert.assertEquals(attributesMap.get("attr8"), Object[].class); } catch (final IOException e) { fail(e.getMessage()); } From d499119877e7af8fbfac1404f903ce3051e12402 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 16 Feb 2023 18:10:57 -0500 Subject: [PATCH 02/10] Create DataBlock for reading and writing variable length strings This is a bad hack and should only serve as a proof of (bad) concept --- .../org/janelia/saalfeldlab/n5/DataType.java | 1 + .../saalfeldlab/n5/VLenStringDataBlock.java | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 6a87e390..d1105225 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -52,6 +52,7 @@ public enum DataType { INT64("int64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock(blockSize, gridPosition, new long[numElements])), FLOAT32("float32", (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock(blockSize, gridPosition, new float[numElements])), FLOAT64("float64", (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock(blockSize, gridPosition, new double[numElements])), + VLENSTRING("String(-1)", (blockSize, gridPosition, numElements) -> new VLenStringDataBlock(blockSize, gridPosition, new byte[numElements])), OBJECT("object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements])); private final String label; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java new file mode 100644 index 00000000..30211d88 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +public class VLenStringDataBlock extends AbstractDataBlock { + + private static final Charset ENCODING = StandardCharsets.UTF_8; + private static final String NULLCHAR = "\0"; + private byte[] serializedData; + private String[] actualData; + + public VLenStringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + super(size, gridPosition, new String[0]); + actualData = data; + + final byte[] nullSequence = NULLCHAR.getBytes(ENCODING); + final int nullSequenceLength = nullSequence.length; + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + for (String str : data) { + final byte[] strSequence = str.getBytes(ENCODING); + stream.write(strSequence, 0, strSequence.length); + stream.write(nullSequence, 0, nullSequenceLength); + } + serializedData = stream.toByteArray(); + } + + public VLenStringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { + super(size, gridPosition, new String[0]); + serializedData = data; + } + + @Override + public ByteBuffer toByteBuffer() { + return ByteBuffer.wrap(serializedData); + } + + @Override + public void readData(final ByteBuffer buffer) { + if (buffer.array() != serializedData) + buffer.get(serializedData); + + final String rawChars = new String(buffer.array(), ENCODING); + actualData = rawChars.split(NULLCHAR); + } + + @Override + public int getNumElements() { + return serializedData.length; + } + + @Override + public String[] getData() { + return actualData; + } +} From d473fbf80dc4ab7582fab3b48e6a8af6ed2042fe Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 16 Feb 2023 18:13:19 -0500 Subject: [PATCH 03/10] Add standard test showing that the hack works --- .../saalfeldlab/n5/AbstractN5Test.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index d29d219b..05b52acf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -184,6 +184,38 @@ public void testWriteReadByteBlock() { } } + @Test + public void testWriteReadStringBlock() { + + final DataType dataType = DataType.VLENSTRING; + final long[] dimensions = new long[]{3, 2}; + final int[] blockSize = new int[]{2, 2}; + + // test dataset; all characters are valid UTF8! + final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; + + for (final Compression compression : getCompressions()) { + + System.out.println("Testing " + compression.getType() + " " + dataType); + try { + n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final VLenStringDataBlock dataBlock = new VLenStringDataBlock(blockSize, new long[]{0, 0, 0}, stringBlock); + n5.writeBlock(datasetName, attributes, dataBlock); + + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + + Assert.assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); + + Assert.assertTrue(n5.remove(datasetName)); + + } catch (final IOException e) { + e.printStackTrace(); + fail("Block cannot be written."); + } + } + } + @Test public void testWriteReadShortBlock() { From 7a05a5e612c4a884980e54a62dac7ad53374dc86 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 24 Feb 2023 13:37:29 -0500 Subject: [PATCH 04/10] Clean up a little --- .../java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java | 3 +-- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java index 30211d88..f34d61f7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -42,13 +42,12 @@ public VLenStringDataBlock(final int[] size, final long[] gridPosition, final St actualData = data; final byte[] nullSequence = NULLCHAR.getBytes(ENCODING); - final int nullSequenceLength = nullSequence.length; final ByteArrayOutputStream stream = new ByteArrayOutputStream(); for (String str : data) { final byte[] strSequence = str.getBytes(ENCODING); stream.write(strSequence, 0, strSequence.length); - stream.write(nullSequence, 0, nullSequenceLength); + stream.write(nullSequence, 0, nullSequence.length); } serializedData = stream.toByteArray(); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 05b52acf..eb307af0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -191,7 +191,7 @@ public void testWriteReadStringBlock() { final long[] dimensions = new long[]{3, 2}; final int[] blockSize = new int[]{2, 2}; - // test dataset; all characters are valid UTF8! + // test dataset; all characters are valid UTF8 but may have different numbers of bytes! final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; for (final Compression compression : getCompressions()) { From bfa5dbf97d9228a08a4b588afebdd7ea8187f3c4 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 7 Mar 2023 15:18:42 -0500 Subject: [PATCH 05/10] Fix dimensions of String block for testing --- .../org/janelia/saalfeldlab/n5/AbstractN5Test.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index eb307af0..8069dc7d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -187,11 +187,9 @@ public void testWriteReadByteBlock() { @Test public void testWriteReadStringBlock() { - final DataType dataType = DataType.VLENSTRING; - final long[] dimensions = new long[]{3, 2}; - final int[] blockSize = new int[]{2, 2}; - // test dataset; all characters are valid UTF8 but may have different numbers of bytes! + final DataType dataType = DataType.VLENSTRING; + final int[] blockSize = new int[]{3, 2, 1}; final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; for (final Compression compression : getCompressions()) { @@ -200,10 +198,10 @@ public void testWriteReadStringBlock() { try { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - final VLenStringDataBlock dataBlock = new VLenStringDataBlock(blockSize, new long[]{0, 0, 0}, stringBlock); + final VLenStringDataBlock dataBlock = new VLenStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0L, 0L, 0L}); Assert.assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); From 5171d38d7724024c6fb093be74a55ef664b33cde Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 8 Mar 2023 11:58:29 -0500 Subject: [PATCH 06/10] Simplify serialization method --- .../janelia/saalfeldlab/n5/VLenStringDataBlock.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java index f34d61f7..41090fc6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -41,15 +41,8 @@ public VLenStringDataBlock(final int[] size, final long[] gridPosition, final St super(size, gridPosition, new String[0]); actualData = data; - final byte[] nullSequence = NULLCHAR.getBytes(ENCODING); - final ByteArrayOutputStream stream = new ByteArrayOutputStream(); - - for (String str : data) { - final byte[] strSequence = str.getBytes(ENCODING); - stream.write(strSequence, 0, strSequence.length); - stream.write(nullSequence, 0, nullSequence.length); - } - serializedData = stream.toByteArray(); + final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; + serializedData = flattenedArray.getBytes(ENCODING); } public VLenStringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { From 41184cbdb1da8a7c81949bae95d377c73182c5e0 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 8 Mar 2023 12:46:52 -0500 Subject: [PATCH 07/10] Clean up (de-)serialization method for string arrays --- .../saalfeldlab/n5/VLenStringDataBlock.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java index 41090fc6..84baa720 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -40,14 +40,13 @@ public class VLenStringDataBlock extends AbstractDataBlock { public VLenStringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { super(size, gridPosition, new String[0]); actualData = data; - - final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; - serializedData = flattenedArray.getBytes(ENCODING); + serializedData = serialize(data); } public VLenStringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { super(size, gridPosition, new String[0]); serializedData = data; + actualData = deserialize(data); } @Override @@ -59,9 +58,17 @@ public ByteBuffer toByteBuffer() { public void readData(final ByteBuffer buffer) { if (buffer.array() != serializedData) buffer.get(serializedData); + actualData = deserialize(buffer.array()); + } + + static protected byte[] serialize(String[] strings) { + final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; + return flattenedArray.getBytes(ENCODING); + } - final String rawChars = new String(buffer.array(), ENCODING); - actualData = rawChars.split(NULLCHAR); + static protected String[] deserialize(byte[] rawBytes) { + final String rawChars = new String(rawBytes, ENCODING); + return rawChars.split(NULLCHAR); } @Override From 09f3f16897b66901352d0cd158acaea8954688f1 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Thu, 9 Mar 2023 13:21:34 -0500 Subject: [PATCH 08/10] Make VLenStringDataBlock inheritable --- .../org/janelia/saalfeldlab/n5/VLenStringDataBlock.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java index 84baa720..5fb7d64f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -32,10 +32,10 @@ public class VLenStringDataBlock extends AbstractDataBlock { - private static final Charset ENCODING = StandardCharsets.UTF_8; - private static final String NULLCHAR = "\0"; - private byte[] serializedData; - private String[] actualData; + protected static final Charset ENCODING = StandardCharsets.UTF_8; + protected static final String NULLCHAR = "\0"; + protected byte[] serializedData; + protected String[] actualData; public VLenStringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { super(size, gridPosition, new String[0]); From d234d37de0b55bf45a3a1772ecbe852836a52584 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 10 Mar 2023 14:26:23 -0500 Subject: [PATCH 09/10] Defer (de-)serialization until data is needed --- .../saalfeldlab/n5/VLenStringDataBlock.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java index 5fb7d64f..deebe65f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/VLenStringDataBlock.java @@ -34,23 +34,23 @@ public class VLenStringDataBlock extends AbstractDataBlock { protected static final Charset ENCODING = StandardCharsets.UTF_8; protected static final String NULLCHAR = "\0"; - protected byte[] serializedData; - protected String[] actualData; + protected byte[] serializedData = null; + protected String[] actualData = null; public VLenStringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { super(size, gridPosition, new String[0]); actualData = data; - serializedData = serialize(data); } public VLenStringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { super(size, gridPosition, new String[0]); serializedData = data; - actualData = deserialize(data); } @Override public ByteBuffer toByteBuffer() { + if (serializedData == null) + serializedData = serialize(actualData); return ByteBuffer.wrap(serializedData); } @@ -61,23 +61,27 @@ public void readData(final ByteBuffer buffer) { actualData = deserialize(buffer.array()); } - static protected byte[] serialize(String[] strings) { + protected byte[] serialize(String[] strings) { final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; return flattenedArray.getBytes(ENCODING); } - static protected String[] deserialize(byte[] rawBytes) { + protected String[] deserialize(byte[] rawBytes) { final String rawChars = new String(rawBytes, ENCODING); return rawChars.split(NULLCHAR); } @Override public int getNumElements() { + if (serializedData == null) + serializedData = serialize(actualData); return serializedData.length; } @Override public String[] getData() { + if (actualData == null) + actualData = deserialize(serializedData); return actualData; } } From 62b99387b57aa249fa3b8b778ec2abe1df3436bd Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sun, 15 Oct 2023 13:43:14 -0400 Subject: [PATCH 10/10] Fix some IDE warnings --- .../org/janelia/saalfeldlab/n5/DataType.java | 8 ++-- .../saalfeldlab/n5/AbstractN5Test.java | 48 +++++++++---------- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 6 +-- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index fa51aa20..0df4f283 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -117,9 +117,9 @@ public enum DataType { private final String label; - private DataBlockFactory dataBlockFactory; + private final DataBlockFactory dataBlockFactory; - private DataType(final String label, final DataBlockFactory dataBlockFactory) { + DataType(final String label, final DataBlockFactory dataBlockFactory) { this.label = label; this.dataBlockFactory = dataBlockFactory; @@ -171,9 +171,9 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize)); } - private static interface DataBlockFactory { + private interface DataBlockFactory { - public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); + DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); } static public class JsonAdapter implements JsonDeserializer, JsonSerializer { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 4cf05a17..b1c171e9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -119,11 +119,8 @@ protected Compression[] getCompressions() { }; } - /** - * @throws IOException - */ @Before - public void setUpOnce() throws IOException, URISyntaxException { + public void setUpOnce() { final Random rnd = new Random(); byteBlock = new byte[blockNumElements]; @@ -197,7 +194,7 @@ public void testWriteReadByteBlock() throws URISyntaxException { final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, new long[]{0, 0, 0}, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); assertTrue(n5.remove(datasetName)); @@ -226,7 +223,7 @@ public void testWriteReadStringBlock() throws URISyntaxException { final VLenStringDataBlock dataBlock = new VLenStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0L, 0L, 0L}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0L, 0L, 0L); assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); @@ -253,7 +250,7 @@ public void testWriteReadShortBlock() throws URISyntaxException { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); @@ -281,7 +278,7 @@ public void testWriteReadIntBlock() throws URISyntaxException { final IntArrayDataBlock dataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedDataBlock.getData()); @@ -309,7 +306,7 @@ public void testWriteReadLongBlock() throws URISyntaxException { final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, longBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(longBlock, (long[])loadedDataBlock.getData()); @@ -333,7 +330,7 @@ public void testWriteReadFloatBlock() throws URISyntaxException { final FloatArrayDataBlock dataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(floatBlock, (float[])loadedDataBlock.getData(), 0.001f); @@ -356,7 +353,7 @@ public void testWriteReadDoubleBlock() throws URISyntaxException { final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[]{0, 0, 0}, doubleBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(doubleBlock, (double[])loadedDataBlock.getData(), 0.001); @@ -385,7 +382,7 @@ public void testMode1WriteReadByteBlock() throws URISyntaxException { final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(differentBlockSize, new long[]{0, 0, 0}, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); @@ -415,11 +412,11 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException, URIS object.get("one").add(new double[]{1, 2, 3}); object.get("two").add(new double[]{4, 5, 6, 7, 8}); - n5.writeSerializedBlock(object, datasetName, attributes, new long[]{0, 0, 0}); + n5.writeSerializedBlock(object, datasetName, attributes, 0, 0, 0); final HashMap> loadedObject = n5.readSerializedBlock(datasetName, attributes, new long[]{0, 0, 0}); - object.entrySet().stream().forEach(e -> assertArrayEquals(e.getValue().get(0), loadedObject.get(e.getKey()).get(0), 0.01)); + object.forEach((key, value) -> assertArrayEquals(value.get(0), loadedObject.get(key).get(0), 0.01)); assertTrue(n5.remove(datasetName)); @@ -439,13 +436,13 @@ public void testOverwriteBlock() throws URISyntaxException { final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, randomDataBlock); - final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); // test the case where the resulting file becomes shorter final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, new int[DataBlock.getNumElements(blockSize)]); n5.writeBlock(datasetName, attributes, emptyDataBlock); - final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); + final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData()); assertTrue(n5.remove(datasetName)); @@ -536,10 +533,10 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti public void testAttributes() throws IOException, URISyntaxException { try (final N5Writer n5 = createN5Writer()) { - assertEquals(null, n5.getAttribute(groupName, "test", String.class)); + assertNull(n5.getAttribute(groupName, "test", String.class)); assertEquals(0, n5.listAttributes(groupName).size()); n5.createGroup(groupName); - assertEquals(null, n5.getAttribute(groupName, "test", String.class)); + assertNull(n5.getAttribute(groupName, "test", String.class)); assertEquals(0, n5.listAttributes(groupName).size()); @@ -574,8 +571,8 @@ public void testAttributes() throws IOException, URISyntaxException { }.getType())); // test the case where the resulting file becomes shorter - n5.setAttribute(groupName, "key1", Integer.valueOf(1)); - n5.setAttribute(groupName, "key2", Integer.valueOf(2)); + n5.setAttribute(groupName, "key1", 1); + n5.setAttribute(groupName, "key2", 2); assertEquals(3, n5.listAttributes(groupName).size()); /* class interface */ assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", Integer.class)); @@ -839,7 +836,7 @@ public void testRemoveGroup() throws IOException, URISyntaxException { } @Test - public void testList() throws IOException, URISyntaxException { + public void testList() throws URISyntaxException { try (final N5Writer listN5 = createN5Writer()) { listN5.createGroup(groupName); @@ -899,7 +896,6 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); final String prefix = "/test"; - final String datasetSuffix = "group/dataset"; final List datasetList3 = Arrays.asList(n5.deepList(prefix)); for (final String subGroup : subGroupNames) assertTrue("deepList contents", datasetList3.contains("group/" + subGroup)); @@ -960,7 +956,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce n5.deepList(prefix, n5::datasetExists)); final List datasetListFilterDandBC = Arrays.asList(n5.deepListDatasets(prefix, isBorC)); - assertTrue("deepListDatasetFilter", datasetListFilterDandBC.size() == 0); + assertEquals("deepListDatasetFilter", 0, datasetListFilterDandBC.size()); assertArrayEquals( datasetListFilterDandBC.toArray(), n5.deepList(prefix, a -> n5.datasetExists(a) && isBorC.test(a))); @@ -976,7 +972,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List datasetListFilterDandBCP = Arrays.asList(n5.deepListDatasets(prefix, isBorC, Executors.newFixedThreadPool(2))); - assertTrue("deepListDatasetFilter Parallel", datasetListFilterDandBCP.size() == 0); + assertEquals("deepListDatasetFilter Parallel", 0, datasetListFilterDandBCP.size()); assertArrayEquals( datasetListFilterDandBCP.toArray(), n5.deepList(prefix, a -> n5.datasetExists(a) && isBorC.test(a), Executors.newFixedThreadPool(2))); @@ -1201,7 +1197,7 @@ protected static void addAndTest(final N5Writer writer, final ArrayList> existingTests) throws IOException { + protected static void runTests(final N5Writer writer, final ArrayList> existingTests) { for (final TestData test : existingTests) { assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); @@ -1419,7 +1415,7 @@ private String jsonKeyVal(final String key, final String val) { n5.setAttribute(groupName, "/", true); final JsonElement booleanPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(booleanPrimitive.isJsonPrimitive()); - assertEquals(true, booleanPrimitive.getAsBoolean()); + assertTrue(booleanPrimitive.getAsBoolean()); n5.setAttribute(groupName, "/", null); final JsonElement jsonNull = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(jsonNull.isJsonNull()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 41abb9f7..7ba57bff 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -36,8 +36,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,9 +43,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.commons.io.FileUtils; import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; -import org.junit.AfterClass; import org.junit.Test; import com.google.gson.GsonBuilder; @@ -143,7 +139,7 @@ public void customObjectTest() throws IOException, URISyntaxException { } // @Test - public void testReadLock() throws IOException, InterruptedException { + public void testReadLock() throws IOException { final Path path = Paths.get(tempN5PathName(), "lock"); LockedChannel lock = access.lockForWriting(path);