From b5006be010c0dc6fb20d4808b960158de509ed07 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 9 Oct 2024 12:33:09 -0400 Subject: [PATCH] test: tryWaitRepeat throws InterruptedException * use tryWaitRepeat in TestExportImports * no longer ignore some tests --- .../saalfeldlab/n5/TestExportImports.java | 98 ++++++++++++++----- .../janelia/saalfeldlab/n5/TestRunners.java | 22 ++--- .../n5/metadata/MetadataTests.java | 17 +++- 3 files changed, 95 insertions(+), 42 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/TestExportImports.java b/src/test/java/org/janelia/saalfeldlab/n5/TestExportImports.java index 293f2336..f9512388 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/TestExportImports.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/TestExportImports.java @@ -99,7 +99,6 @@ public void testEmptyMeta() throws InterruptedException } @Test - @Ignore // TODO intermittent failures on GH actions public void test4dN5v() { final int nChannels = 3; @@ -121,7 +120,19 @@ public void test4dN5v() for( int i = 0; i < nChannels; i++) { final String n5PathAndDataset = String.format("%s/%s/c%d/s0", n5RootPath, dataset, i); - final List< ImagePlus > impList = reader.process( n5PathAndDataset, false ); + + final Optional> impListOpt = TestRunners.tryWaitRepeat(() -> { + return reader.process(n5PathAndDataset, false); + }); + + List impList; + if (impListOpt.isPresent()) { + impList = impListOpt.get(); + } else { + System.err.println(String.format("Skipping test for [ %s : %s ] due to intermittent error ", n5RootPath, dataset)); + return; + } + Assert.assertEquals("n5v load channel", 1, impList.size()); Assert.assertTrue("n5v channel equals", equalChannel(imp, i, impList.get(0))); } @@ -136,7 +147,6 @@ public void test4dN5v() } @Test - @Ignore // TODO intermittent failures on GH actions public void testReadWriteParse() throws InterruptedException { final HashMap typeToExtension = new HashMap<>(); @@ -281,6 +291,7 @@ public static void singleReadWriteParseTest( // wait writer.getExecutorService().awaitTermination(1000, TimeUnit.MILLISECONDS); + readParseTest( imp, outputPath, dataset, blockSizeString, metadataType, compressionType, testMeta, testData, 5); deleteContainer(outputPath); } @@ -294,7 +305,7 @@ private static void readParseTest( final String compressionType, final boolean testMeta, final boolean testData, - final int nTries) { + final int nTries) throws InterruptedException { final String readerDataset; if (metadataType.equals(N5Importer.MetadataN5ViewerKey) || (metadataType.equals(N5Importer.MetadataN5CosemKey) && imp.getNChannels() > 1)) @@ -372,7 +383,6 @@ public void testRgb() throws InterruptedException * */ @Test - @Ignore // TODO intermittent failures on GH actions public void testMultiChannel() { for( final String suffix : new String[] { ".h5", ".n5", ".zarr" }) @@ -389,8 +399,7 @@ public void testMultiChannel() } @Test - @Ignore // TODO intermittent failures on GH actions - public void testOverwrite() { + public void testOverwrite() throws InterruptedException { final String n5Root = baseDir + "/overwriteTest.n5"; final String dataset = "dataset"; @@ -410,33 +419,72 @@ public void testOverwrite() { writer.setOverwrite(true); writer.run(); - final N5Writer n5 = new N5FSWriter(n5Root); - assertTrue(n5.datasetExists(dataset)); + try (final N5Writer n5 = new N5FSWriter(n5Root)) { - assertArrayEquals("size orig", szBig, n5.getDatasetAttributes(dataset).getDimensions()); + Optional dsetAttrsOpt = TestRunners.tryWaitRepeat(() -> { + return n5.getDatasetAttributes(dataset); + }); - final N5ScalePyramidExporter writerNoOverride = new N5ScalePyramidExporter(); - writerNoOverride.setOptions(impSmall, n5Root, dataset, N5ScalePyramidExporter.AUTO_FORMAT, blockSizeString, false, - N5ScalePyramidExporter.DOWN_SAMPLE, metadataType, compressionString); - writerNoOverride.setOverwrite(false); - writerNoOverride.run(); + DatasetAttributes dsetAttrs; + if (dsetAttrsOpt.isPresent()) { + dsetAttrs = dsetAttrsOpt.get(); + assertArrayEquals("size orig", szBig, dsetAttrs.getDimensions()); + } else { + System.err.println(String.format("Skipping test for [ %s : %s ] due to intermittent error ", n5Root, dataset)); + n5.remove(); + n5.close(); + return; + } + dsetAttrsOpt = Optional.empty(); - assertArrayEquals("size after no overwrite", szBig, n5.getDatasetAttributes(dataset).getDimensions()); + final N5ScalePyramidExporter writerNoOverride = new N5ScalePyramidExporter(); + writerNoOverride.setOptions(impSmall, n5Root, dataset, N5ScalePyramidExporter.AUTO_FORMAT, blockSizeString, false, + N5ScalePyramidExporter.DOWN_SAMPLE, metadataType, compressionString); + writerNoOverride.setOverwrite(false); + writerNoOverride.run(); + + dsetAttrsOpt = TestRunners.tryWaitRepeat(() -> { + return n5.getDatasetAttributes(dataset); + }); + + if (dsetAttrsOpt.isPresent()) { + dsetAttrs = dsetAttrsOpt.get(); + assertArrayEquals("size after no overwrite", szBig, dsetAttrs.getDimensions()); + } else { + System.err.println(String.format("Skipping test for [ %s : %s ] due to intermittent error ", n5Root, dataset)); + n5.remove(); + n5.close(); + return; + } + dsetAttrsOpt = Optional.empty(); - final N5ScalePyramidExporter writerOverride = new N5ScalePyramidExporter(); - writerOverride.setOptions(impSmall, n5Root, dataset, N5ScalePyramidExporter.AUTO_FORMAT, blockSizeString, false, - N5ScalePyramidExporter.DOWN_SAMPLE, metadataType, compressionString); - writerOverride.setOverwrite(true); - writerOverride.run(); + final N5ScalePyramidExporter writerOverride = new N5ScalePyramidExporter(); + writerOverride.setOptions(impSmall, n5Root, dataset, N5ScalePyramidExporter.AUTO_FORMAT, blockSizeString, false, + N5ScalePyramidExporter.DOWN_SAMPLE, metadataType, compressionString); + writerOverride.setOverwrite(true); + writerOverride.run(); + + dsetAttrsOpt = TestRunners.tryWaitRepeat(() -> { + return n5.getDatasetAttributes(dataset); + }); + + if (dsetAttrsOpt.isPresent()) { + dsetAttrs = dsetAttrsOpt.get(); + assertArrayEquals("size after overwrite", szSmall, dsetAttrs.getDimensions()); + } else { + System.err.println(String.format("Skipping test for [ %s : %s ] due to intermittent error ", n5Root, dataset)); + n5.remove(); + n5.close(); + return; + } - assertArrayEquals("size after overwrite", szSmall, n5.getDatasetAttributes(dataset).getDimensions()); + n5.remove(); + n5.close(); + } - n5.remove(); - n5.close(); } @Test - @Ignore // TODO intermittent failures on GH actions public void testFormatOptions() { final String n5Root = baseDir + "/root_of_some_container"; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/TestRunners.java b/src/test/java/org/janelia/saalfeldlab/n5/TestRunners.java index 2682a126..991f92b0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/TestRunners.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/TestRunners.java @@ -5,17 +5,17 @@ public class TestRunners { - public static Optional tryWaitRepeat(Supplier supplier) { + public static Optional tryWaitRepeat(Supplier supplier) throws InterruptedException { return tryWaitRepeat(supplier, 5, 50, 2); } - public static Optional tryWaitRepeat(Supplier supplier, int nTries) { + public static Optional tryWaitRepeat(Supplier supplier, int nTries) throws InterruptedException { return tryWaitRepeat(supplier, nTries, 50, 2); } - public static Optional tryWaitRepeat(Supplier supplier, int nTries, long waitTimeMillis) { + public static Optional tryWaitRepeat(Supplier supplier, int nTries, long waitTimeMillis) throws InterruptedException { return tryWaitRepeat(supplier, nTries, waitTimeMillis, 2); } @@ -37,13 +37,14 @@ public static Optional tryWaitRepeat(Supplier supplier, int nTries, lo * @return an {@link Optional} containing the result from the supplier if a non-null result is returned * before the maximum number of tries, or an empty {@code Optional} if all attempts fail or * return null + * @throws InterruptedException thrown if interrupted while waiting */ public static Optional tryWaitRepeat( - final Supplier supplier, - final int nTries, - final long initialWaitTimeMillis, - final int waitTimeMultiplier ) { - + final Supplier supplier, + final int nTries, + final long initialWaitTimeMillis, + final int waitTimeMultiplier) throws InterruptedException { + int i = 0; long waitTime = initialWaitTimeMillis; while (i < nTries) { @@ -57,10 +58,7 @@ public static Optional tryWaitRepeat( return Optional.of(result); } catch (RuntimeException e) {} - try { - Thread.sleep(waitTime); - } catch (InterruptedException e) {} - + Thread.sleep(waitTime); waitTime *= waitTimeMultiplier; i++; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/metadata/MetadataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/metadata/MetadataTests.java index 5954e43c..a27b8844 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/metadata/MetadataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/metadata/MetadataTests.java @@ -44,7 +44,7 @@ public static void setUp() throws IOException { } @Test - public void testCosemMetadataMultiscale() { + public void testCosemMetadataMultiscale() throws InterruptedException { final N5MetadataParser[] parsers = new N5MetadataParser[]{new N5CosemMetadataParser()}; final N5MetadataParser[] grpparsers = new N5MetadataParser[]{new N5CosemMultiScaleMetadata.CosemMultiScaleParser()}; @@ -85,7 +85,7 @@ public void testCosemMetadataMultiscale() { } @Test - public void testCosemMetadata() { + public void testCosemMetadata() throws InterruptedException { final double eps = 1e-6; @@ -96,7 +96,12 @@ public void testCosemMetadata() { TestRunners.tryWaitRepeat(() -> { try { - return discoverer.discoverAndParseRecursive("/"); + N5TreeNode node = discoverer.discoverAndParseRecursive("/"); + if( node.childrenList().size() < 3 ) + return null; + else + return node; + } catch (IOException e) { return null; // so that the runner tries again } @@ -125,7 +130,7 @@ public void testCosemMetadata() { } @Test - public void testN5ViewerMetadata() { + public void testN5ViewerMetadata() throws InterruptedException { final double eps = 1e-6; @@ -147,9 +152,11 @@ public void testN5ViewerMetadata() { List childrenWithMetadata = n5root.childrenList().stream() .filter(x -> Objects.nonNull(x.getMetadata())) .collect(Collectors.toList()); + long childrenNoMetadataCount = n5root.childrenList().stream() .filter(x -> Objects.isNull(x.getMetadata())) .count(); + Assert.assertEquals("discovery node count with single scale metadata", 4, childrenWithMetadata.size()); Assert.assertEquals("discovery node count without single scale metadata", 1, childrenNoMetadataCount); @@ -182,7 +189,7 @@ public void testN5ViewerMetadata() { } @Test - public void testGenericMetadata() { + public void testGenericMetadata() throws InterruptedException { final double eps = 1e-6;