From 6d6d16ddd609d5034ae6ccf2dbe2ff7d8a5f2b70 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 9 May 2024 11:06:37 -0400 Subject: [PATCH 1/4] chore(pom): update and organize deps * use snapshot versions of n5-ij and n5-universe --- pom.xml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 08552b8..5120247 100644 --- a/pom.xml +++ b/pom.xml @@ -131,14 +131,18 @@ 1.4.1 + 10.4.13 + 1.0.0-beta-34 + 3.2.0 + 4.1.2 + 1.1.1 + 4.1.0 2.2.0 - 4.1.3 - 1.4.3 - 1.0.2 + 4.1.4-SNAPSHOT + 1.4.4-SNAPSHOT 1.3.1 - 10.4.13 - 1.0.0-beta-34 + 1.0.2 @@ -195,12 +199,24 @@ org.janelia.saalfeldlab n5-google-cloud - 4.1.0 org.janelia.saalfeldlab n5-aws-s3 - 4.1.2 + + + + + junit + junit + test + + + org.janelia.saalfeldlab + n5-universe + ${n5-universe.version} + tests + test From 515ecb03be061920d789a6aeed8d45eb35729c1b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 9 May 2024 11:14:41 -0400 Subject: [PATCH 2/4] fix: better handle loading of arbitrarily axis-ordered data --- .../janelia/saalfeldlab/n5/bdv/N5Viewer.java | 92 ++++++--- .../n5/bdv/N5vAxisPermutationTests.java | 180 ++++++++++++++++++ 2 files changed, 243 insertions(+), 29 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/bdv/N5vAxisPermutationTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/bdv/N5Viewer.java b/src/main/java/org/janelia/saalfeldlab/n5/bdv/N5Viewer.java index 283ad69..55f8781 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/bdv/N5Viewer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/bdv/N5Viewer.java @@ -118,6 +118,7 @@ import net.imglib2.converter.Converter; import net.imglib2.converter.Converters; import net.imglib2.img.basictypeaccess.AccessFlags; +import net.imglib2.realtransform.AffineGet; import net.imglib2.realtransform.AffineTransform3D; import net.imglib2.type.NativeType; import net.imglib2.type.label.LabelMultisetType; @@ -190,14 +191,6 @@ public & NativeType, V extends Volatile & Numeri Prefs.showScaleBar(true); this.sharedQueue = new SharedQueue(Math.max(1, Runtime.getRuntime().availableProcessors() / 2)); - - // TODO: These setups are not used anymore, because BdvFunctions creates - // its own. - // They either need to be deleted from here or integrated somehow. - final List converterSetups = new ArrayList<>(); - - final List> sourcesAndConverters = new ArrayList<>(); - final List selected = new ArrayList<>(); for (final N5Metadata meta : dataSelection.metadata) { if (meta instanceof N5ViewerMultichannelMetadata) { @@ -271,7 +264,7 @@ public static BdvHandle show(final String uri) { try { return show(new N5URI(uri)); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { e.printStackTrace(); } return null; @@ -321,12 +314,12 @@ public static & NativeType> BdvHandle show(final St final HashMap> selectionsByContainer = new HashMap<>(); final N5ViewerReaderFun n5fun = new N5ViewerReaderFun(); - for( String uri : uris ) + for( final String uri : uris ) { N5URI n5uri; try { n5uri = new N5URI(uri); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { System.err.println("Could not parse url: " + uri); continue; } @@ -347,7 +340,7 @@ public static & NativeType> BdvHandle show(final St } // if this is called, can assume metadata have not been parsed yet. so parse now - once for each container. - for( N5Reader n5 : selectionsByContainer.keySet()) + for( final N5Reader n5 : selectionsByContainer.keySet()) { final N5TreeNode containerRoot = N5DatasetDiscoverer.discover(n5, Arrays.asList(N5ViewerCreator.n5vParsers), @@ -365,7 +358,7 @@ public static & NativeType> BdvHandle show(final St try { numTimepoints = Math.max(numTimepoints, buildN5Sources(n5, selection, sharedQueue, converterSetups, sourcesAndConverters, options)); - } catch (IOException e) { + } catch (final IOException e) { System.err.println("Could not load from: " + n5.getURI().toString()); } } @@ -391,7 +384,7 @@ public static & NativeType> BdvHandle show(N5Reader sourcesAndConverters, options); - } catch (IOException e1) { + } catch (final IOException e1) { e1.printStackTrace(); return null; } @@ -427,7 +420,7 @@ public static & NativeType> BdvHandle show(final Li } } - BdvHandle bdv = bdvHandle; + final BdvHandle bdv = bdvHandle; if (bdv != null) { final ViewerPanel viewerPanel = bdv.getViewerPanel(); if (viewerPanel != null) { @@ -499,7 +492,7 @@ public static & NativeType, V extends Volatile & sharedQueue, converterSetups, sourcesAndConverters, options); } - public static & NativeType, V extends Volatile & NumericType> int buildN5Sources( + public static & NativeType, V extends Volatile & NumericType, M extends AxisMetadata & N5Metadata> int buildN5Sources( final N5Reader n5, final List selectedMetadata, final SharedQueue sharedQueue, @@ -521,7 +514,8 @@ public static & NativeType, V extends Volatile & final N5Metadata metadata = selectedMetadata.get(i); final String srcName = metadata.getName(); - // TODO: simplify this if/elseif block: much of these if cases can be combined + + // TODO: simplify this if/elseif block: much of these ifwall cases can be combined if (metadata instanceof N5SingleScaleMetadata) { final N5SingleScaleMetadata singleScaleDataset = (N5SingleScaleMetadata)metadata; final String[] tmpDatasets = new String[]{singleScaleDataset.getPath()}; @@ -589,7 +583,7 @@ public static & NativeType, V extends Volatile & final RandomAccessibleInterval< ? > imagejImg; if (metadata instanceof AxisMetadata) { - imagejImg = AxisUtils.permuteForImagePlus( img, (AxisMetadata)metadata ); + imagejImg = AxisUtils.permuteForImagePlus(img, (M)metadata); unit = unitFromAxes(((AxisMetadata)metadata).getAxes()); } else if( metadata instanceof N5SingleScaleMetadata ) @@ -608,14 +602,14 @@ else if( isCosemMultiscale(metadata)) { final N5CosemMultiScaleMetadata cosemMulti = ((N5CosemMultiScaleMetadata)metadata); final N5CosemMetadata cosemMeta = cosemMulti.getChildrenMetadata()[0]; - imagejImg = AxisUtils.permuteForImagePlus(img, cosemMeta); + imagejImg = permuteForImagePlus(img, transforms[s], cosemMeta); unit = cosemMeta.unit(); } else { final NgffSingleScaleAxesMetadata ngffMeta = isNgffMultiscale(metadata); if( ngffMeta != null ) { - imagejImg = AxisUtils.permuteForImagePlus(img, ngffMeta); + imagejImg = permuteForImagePlus(img, transforms[s], ngffMeta); unit = ngffMeta.unit(); } else @@ -650,8 +644,7 @@ else if( isCosemMultiscale(metadata)) @SuppressWarnings("unchecked") final T type = (T)Util.getTypeFromInterval(images[0]); - // TODO this could / should be generalized - // resolutions + // this could / should be generalized final double rx = transforms[0].get(0, 0); final double ry = transforms[0].get(1, 1); final double rz = transforms[0].get(2, 2); @@ -684,6 +677,44 @@ else if( isCosemMultiscale(metadata)) return numTimepoints; } + /** + * Returns an image with dimensions in a canonical order XYCZY. Also + * permutes the given pixel to physical transform in-place. + * + * @param + * the type + * @param img + * the image + * @param transform + * the pixel to physical transfom + * @param meta + * axis metadata + * @return a possibly permuted image + */ + protected static RandomAccessibleInterval permuteForImagePlus( + final RandomAccessibleInterval img, + AffineTransform3D transform, + final A meta) { + + final int[] p = AxisUtils.findImagePlusPermutation(meta); + AxisUtils.fillPermutation(p); + + RandomAccessibleInterval imgTmp = img; + while (imgTmp.numDimensions() < 5) + imgTmp = Views.addDimension(imgTmp, 0, 0); + + if (AxisUtils.isIdentityPermutation(p)) + return imgTmp; + + // update spatial transformation + // exchange rows and columns of permutation matrix appropriately + final int[] spatialPermutation = new int[]{p[0], p[1], p[3]}; + final AffineGet permTform = AxisUtils.axisPermutationTransform(spatialPermutation); + transform.concatenate(permTform.inverse()).preConcatenate(permTform); + + return AxisUtils.permute(imgTmp, AxisUtils.invertPermutation(p)); + } + /* * If the image is of type {@link LabelMultisetType} to {@link UnsignedLongType}. */ @@ -704,16 +735,19 @@ protected static & NativeType> RandomAccessibleInte // return (RandomAccessibleInterval)convertLabelMultisetVolatile( // (CachedCellImg)img); } - return (RandomAccessibleInterval)img; + + if (OmeNgffMultiScaleMetadata.fOrder(n5.getDatasetAttributes(dataset))) + return AxisUtils.reverseDimensions(img); + else + return (RandomAccessibleInterval)img; } private static RandomAccessibleInterval convertLabelMultisetVolatile( final CachedCellImg lmsImg ) { // TODO this isn't working (VolatileViews throws a NPE), but have not yet investigated why - System.out.println( "convert volatile"); // see ViewCosem in n5-utils for something similar - RandomAccessibleInterval> vimg = VolatileViews.wrapAsVolatile( lmsImg ); + final RandomAccessibleInterval> vimg = VolatileViews.wrapAsVolatile( lmsImg ); return Converters.convert2(vimg, (a, b) -> { b.set(a.get().argMax()); @@ -793,10 +827,10 @@ private static boolean isCosemMultiscale( final N5Metadata metadata ) return false; } - private static NgffSingleScaleAxesMetadata isNgffMultiscale( final N5Metadata metadata ) - { - if(metadata instanceof OmeNgffMetadata ) - { + private static NgffSingleScaleAxesMetadata isNgffMultiscale(final N5Metadata metadata) { + + if (metadata instanceof OmeNgffMetadata) { + final OmeNgffMetadata ngff = (OmeNgffMetadata)metadata; final OmeNgffMultiScaleMetadata[] ms = ngff.multiscales; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/bdv/N5vAxisPermutationTests.java b/src/test/java/org/janelia/saalfeldlab/n5/bdv/N5vAxisPermutationTests.java new file mode 100644 index 0000000..9122560 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/bdv/N5vAxisPermutationTests.java @@ -0,0 +1,180 @@ +package org.janelia.saalfeldlab.n5.bdv; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.lang3.ArrayUtils; +import org.janelia.saalfeldlab.n5.N5URI; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.ui.DataSelection; +import org.janelia.saalfeldlab.n5.universe.N5DatasetDiscoverer; +import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.N5TreeNode; +import org.janelia.saalfeldlab.n5.universe.metadata.N5Metadata; +import org.janelia.saalfeldlab.n5.universe.metadata.NgffTests; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import bdv.cache.SharedQueue; +import bdv.tools.brightness.ConverterSetup; +import bdv.util.BdvOptions; +import bdv.viewer.Source; +import bdv.viewer.SourceAndConverter; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.realtransform.AffineTransform3D; +import net.imglib2.type.NativeType; +import net.imglib2.type.numeric.NumericType; + +public class N5vAxisPermutationTests { + + public static final double EPS = 1e-6; + + public static final String[] defaultAxes = new String[]{"x", "y", "c", "z", "t"}; + + private URI containerUri; + + @Before + public void before() { + + System.setProperty("java.awt.headless", "true"); + + try { + containerUri = new File(tempN5PathName()).getCanonicalFile().toURI(); + } catch (final IOException e) {} + } + + @After + public void after() { + + final N5Writer n5 = new N5Factory().openWriter(containerUri.toString()); + n5.remove(); + } + + private static String tempN5PathName() { + + try { + final File tmpFile = Files.createTempDirectory("n5-viewer-ngff-test-").toFile(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + protected String tempN5Location() throws URISyntaxException { + + final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); + return new URI("file", null, basePath, null).toString(); + } + + @Test + public void testPermutations() throws IOException { + + final N5Writer zarr = new N5Factory().openWriter(containerUri.toString()); + // don't check every axis permutation, but some relevant ones, and some strange ones + final String[] names = new String[]{ + + // TODO five still don't work + "xyz", "zyx", "yzx", + "xyc", "xcy", "cyx", + "xyt", "xty", "tyx", + "xyzt", "xtyz", "zxty", "tyzx", + "xyczt", "xyzct", "xytcz", "tzcyx", "ctzxy" + }; + + // check both c- and f-order storage + for (final String axes : names) { + writeAndTest(zarr, axes + "_c"); + writeAndTest(zarr, axes + "_f"); + } + + } + + protected void writeAndTest(final N5Writer zarr, final String dset) throws IOException { + + writeAndTest(zarr, dset, NgffTests.isCOrderFromName(dset), NgffTests.permutationFromName(dset)); + } + + protected & NativeType> void writeAndTest(final N5Writer zarr, final String dset, final boolean cOrder, final int[] p) + throws IOException { + + // write + NgffTests.writePermutedAxes(zarr, baseName(p, cOrder), cOrder, p); + final String dstNrm = N5URI.normalizeGroupPath(dset); + + // read + final N5TreeNode node = N5DatasetDiscoverer.discover(zarr); + final Stream metaStream = N5TreeNode.flattenN5Tree(node) + .filter(x -> N5URI.normalizeGroupPath(x.getPath()).equals(dstNrm)) + .map(x -> { + return (N5Metadata)x.getMetadata(); + }); + + final DataSelection selection = new DataSelection(zarr, Collections.singletonList(metaStream.findFirst().get())); + final SharedQueue sharedQueue = new SharedQueue(1); + final List converterSetups = new ArrayList<>(); + final List> sourcesAndConverters = new ArrayList<>(); + + final BdvOptions options = BdvOptions.options().frameTitle("N5 Viewer"); + final int numTimepoints = N5Viewer.buildN5Sources( + zarr, + selection, + sharedQueue, + converterSetups, + sourcesAndConverters, + options); + + assertTrue("sources found", sourcesAndConverters.size() > 0); + final Source src = sourcesAndConverters.get(0).getSpimSource(); + final RandomAccessibleInterval rai = src.getSource(0, 0); + final long[] dims = rai.dimensionsAsLongArray(); + + final AffineTransform3D tform = new AffineTransform3D(); + src.getSourceTransform(0, 0, tform); + + // System.out.println(""); + // System.out.println("" + sourcesAndConverters.size()); + // System.out.println(""); + + // test + assertEquals(dset + "size x", NgffTests.NX, dims[0]); + assertEquals(dset + "size y", NgffTests.NY, dims[1]); + + assertEquals(dset + "res x", NgffTests.RX, tform.get(0, 0), EPS); + assertEquals(dset + "res y", NgffTests.RY, tform.get(1, 1), EPS); + + final char[] axes = dset.split("_")[0].toCharArray(); + if (ArrayUtils.contains(axes, NgffTests.Z)) { + assertEquals(dset + "n slices", NgffTests.NZ, dims[2]); + assertEquals(dset + "res z", NgffTests.RZ, tform.get(2, 2), EPS); + } + + if (ArrayUtils.contains(axes, NgffTests.C)) { + assertEquals(dset + "n channels", NgffTests.NC, sourcesAndConverters.size()); + } + + if (ArrayUtils.contains(axes, NgffTests.T)) { + assertEquals(dset + "n timepoints", NgffTests.NT, numTimepoints); + } + } + + private static String baseName(final int[] p, final boolean cOrder) { + + final String suffix = cOrder ? "_c" : "_f"; + return Arrays.stream(p).mapToObj(i -> defaultAxes[i]).collect(Collectors.joining()) + suffix; + } + +} From 170686a6ab93efe859c63a1dab2253a6625d5613 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 15 May 2024 17:59:23 -0400 Subject: [PATCH 3/4] chore(pom): bump n5-ij, -universe, -zarr versions --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 5120247..03a6aa7 100644 --- a/pom.xml +++ b/pom.xml @@ -139,9 +139,9 @@ 1.1.1 4.1.0 2.2.0 - 4.1.4-SNAPSHOT - 1.4.4-SNAPSHOT - 1.3.1 + 4.2.0 + 1.5.0 + 1.3.2 1.0.2 From 43a70e022f9112281b1702023728495fe091aed8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 15 May 2024 18:04:17 -0400 Subject: [PATCH 4/4] chore: add platform-test gh actions workflow --- .github/workflows/platform-test.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/platform-test.yml diff --git a/.github/workflows/platform-test.yml b/.github/workflows/platform-test.yml new file mode 100644 index 0000000..019f001 --- /dev/null +++ b/.github/workflows/platform-test.yml @@ -0,0 +1,44 @@ +name: test + +on: + push: + branches: + - master + tags: + - "*-[0-9]+.*" + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install blosc (Windows) + if: matrix.os == 'windows-latest' + run: | + pip install blosc --no-input --target src/test/resources + mv src/test/resources/bin/* src/test/resources + - name: Install blosc (Ubuntu) + if: matrix.os == 'ubuntu-latest' + run: | + pip install blosc --no-input --target src/test/resources + mv src/test/resources/lib64/* src/test/resources + - name: Set up Java + uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'zulu' + cache: 'maven' + - name: Maven Test + run: mvn -B clean test --file pom.xml