From 8cfde84046f9263ed77e22ec614676101f7ed51c Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 29 Oct 2024 16:30:16 -0400
Subject: [PATCH 01/24] registering CoordinateTransform GSON serializer for
 OME-ZARR

---
 src/main/java/util/URITools.java | 64 ++++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 15 deletions(-)

diff --git a/src/main/java/util/URITools.java b/src/main/java/util/URITools.java
index ec3171ac..4cab21b7 100644
--- a/src/main/java/util/URITools.java
+++ b/src/main/java/util/URITools.java
@@ -35,7 +35,6 @@
 import java.io.PrintWriter;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Date;
 import java.util.regex.Pattern;
@@ -51,6 +50,8 @@
 import org.janelia.saalfeldlab.n5.s3.AmazonS3Utils;
 import org.janelia.saalfeldlab.n5.universe.N5Factory;
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformationAdapter;
 import org.janelia.saalfeldlab.n5.zarr.N5ZarrReader;
 import org.janelia.saalfeldlab.n5.zarr.N5ZarrWriter;
 import org.jdom2.Document;
@@ -59,6 +60,8 @@
 import org.jdom2.output.Format;
 import org.jdom2.output.XMLOutputter;
 
+import com.google.gson.GsonBuilder;
+
 import bdv.ViewerImgLoader;
 import mpicbg.spim.data.SpimDataException;
 import mpicbg.spim.data.SpimDataIOException;
@@ -261,10 +264,15 @@ else if ( format.equals( StorageFormat.HDF5 ))
 		{
 			N5Writer n5w;
 
+			final GsonBuilder builder = new GsonBuilder().registerTypeAdapter(
+					CoordinateTransformation.class,
+					new CoordinateTransformationAdapter() );
+
 			try
 			{
 				//System.out.println( "Trying writing with credentials ..." );
-				N5Factory factory = new N5Factory();
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
 				factory.s3UseCredentials();
 				n5w = factory.openWriter( format, uri );
 			}
@@ -272,11 +280,12 @@ else if ( format.equals( StorageFormat.HDF5 ))
 			{
 				System.out.println( "With credentials failed; trying anonymous ..." );
 
-				n5w = new N5Factory().openWriter( format, uri );
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
+				n5w = factory.openWriter( format, uri );
 			}
 
 			return n5w;
-			//return new N5Factory().openWriter( format, uri ); // cloud support, avoid dependency hell if it is a local file
 		}
 	}
 
@@ -295,21 +304,28 @@ else if ( format.equals( StorageFormat.ZARR ))
 		{
 			N5Reader n5r;
 
+			final GsonBuilder builder = new GsonBuilder().registerTypeAdapter(
+					CoordinateTransformation.class,
+					new CoordinateTransformationAdapter() );
+
 			try
 			{
 				//System.out.println( "Trying reading with credentials ..." );
-				N5Factory factory = new N5Factory();
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
 				factory.s3UseCredentials();
 				n5r = factory.openReader( format, uri );
 			}
 			catch ( Exception e )
 			{
-				System.out.println( "With credentials failed; trying anonymous ..." );
-				n5r = new N5Factory().openReader( format, uri );
+				System.out.println( "With credentials failed; trying anonymous with gson builder ..." );
+
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
+				n5r = factory.openReader( format, uri );
 			}
 
 			return n5r;
-			//return new N5Factory().openReader( format, uri ); // cloud support, avoid dependency hell if it is a local file
 		}
 	}
 
@@ -328,21 +344,27 @@ else if ( uri.toString().toLowerCase().endsWith( ".n5" ) )
 		{
 			N5Reader n5r;
 
+			final GsonBuilder builder = new GsonBuilder().registerTypeAdapter(
+					CoordinateTransformation.class,
+					new CoordinateTransformationAdapter() );
+
 			try
 			{
 				//System.out.println( "Trying reading with credentials ..." );
-				N5Factory factory = new N5Factory();
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
 				factory.s3UseCredentials();
 				n5r = factory.openReader( uri.toString() );
 			}
 			catch ( Exception e )
 			{
 				System.out.println( "With credentials failed; trying anonymous ..." );
-				n5r = new N5Factory().openReader( uri.toString() );
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
+				n5r = factory.openReader( uri.toString() );
 			}
 
 			return n5r;
-			//return new N5Factory().openReader( uri.toString() ); // cloud support, avoid dependency hell if it is a local file
 		}
 	}
 
@@ -361,22 +383,27 @@ else if ( uri.toString().toLowerCase().endsWith( ".n5" ) )
 		{
 			N5Writer n5w;
 
+			final GsonBuilder builder = new GsonBuilder().registerTypeAdapter(
+					CoordinateTransformation.class,
+					new CoordinateTransformationAdapter() );
+
 			try
 			{
 				//System.out.println( "Trying writing with credentials ..." );
-				N5Factory factory = new N5Factory();
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
 				factory.s3UseCredentials();
 				n5w = factory.openWriter( uri.toString() );
 			}
 			catch ( Exception e )
 			{
 				System.out.println( "Writing with credentials failed; trying anonymous writing ..." );
-				n5w = new N5Factory().openWriter( uri.toString() );
+				final N5Factory factory = new N5Factory();
+				factory.gsonBuilder( builder );
+				n5w = factory.openWriter( uri.toString() );
 			}
 
 			return n5w;
-
-			//return new N5Factory().openWriter( uri.toString() ); // cloud support, avoid dependency hell if it is a local file
 		}
 	}
 
@@ -411,6 +438,7 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
 			final SpimData2 data = io.fromXml( docRoot, xmlURI );
 
 			// more threads for cloud-based fetching
+			System.out.println( "Setting num fetcher threads to " + cloudThreads + " for cloud access." );
 			( (ViewerImgLoader) data.getSequenceDescription().getImgLoader() ).setNumFetcherThreads( cloudThreads );
 
 			return data;
@@ -686,6 +714,12 @@ public static void minimalExampleTobiS3GS() throws URISyntaxException, IOExcepti
 
 	public static void main( String[] args ) throws SpimDataException, IOException, URISyntaxException
 	{
+		URI uri1 = URITools.toURI( "s3://aind-open-data/exaSPIM_708373_2024-04-02_19-49-38/SPIM.ome.zarr" );
+
+		System.out.println( uri1.getHost() );
+		System.out.println( uri1.getPath() );
+		System.exit( 0 );
+
 		minimalExampleTobiS3GS();
 
 		System.exit( 0 );

From 42695bdb4fb9a8fb8c3d5b5a2b206472d1de7594 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 29 Oct 2024 16:30:52 -0400
Subject: [PATCH 02/24] use a SNAPSHOT version of bigdataviewer-core that
 contains generalizations of N5ImageLoader

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 195f4f1d..65535a02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,9 +112,9 @@ like Selective Plane Illumination Microscopy (SPIM) Data.</description>
 		<imglib2-algorithm.version>0.16.0</imglib2-algorithm.version>
 		<imglib2-roi.version>0.15.1</imglib2-roi.version>
 		<n5-imglib2.version>7.0.2</n5-imglib2.version>
-		<bigdataviewer-core.version>10.6.1</bigdataviewer-core.version>
 		<bigdataviewer-vistools.version>1.0.0-beta-36</bigdataviewer-vistools.version>
 -->
+		<bigdataviewer-core.version>10.6.3-SNAPSHOT</bigdataviewer-core.version>
 		<spim_data.version>2.3.5</spim_data.version>
 
 		<!-- from n5-universe 

From bfac46c116a554f128ac3afd7459ccbeb6339947 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 29 Oct 2024 16:31:28 -0400
Subject: [PATCH 03/24] added ImageLoader for Allen OME ZARR

---
 .../imgloaders/AllenOMEZarrLoader.java        | 221 ++++++++++++++++++
 .../imgloaders/XmlIoAllenOMEZarrLoader.java   | 153 ++++++++++++
 2 files changed, 374 insertions(+)
 create mode 100644 src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
 create mode 100644 src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
new file mode 100644
index 00000000..b2df0b76
--- /dev/null
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -0,0 +1,221 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.concurrent.ForkJoinPool;
+
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.N5Reader;
+import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata.OmeNgffDataset;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
+
+import bdv.ViewerImgLoader;
+import bdv.img.cache.VolatileCachedCellImg;
+import bdv.img.n5.N5ImageLoader;
+import bdv.img.n5.N5Properties;
+import ij.ImageJ;
+import mpicbg.spim.data.SpimDataException;
+import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.sequence.ViewId;
+import net.imglib2.RandomAccessibleInterval;
+import net.imglib2.img.display.imagej.ImageJFunctions;
+import net.imglib2.type.NativeType;
+import net.imglib2.view.Views;
+import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
+import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
+import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer;
+import util.URITools;
+
+public class AllenOMEZarrLoader extends N5ImageLoader
+{
+	final HashMap< ViewId, String > viewIdToPath;
+	final HashMap< Integer, double[][] > setupIdToMultiRes;
+	final HashMap< Integer, DataType > setupIdToDataType;
+
+	public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
+	{
+		this( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription, viewIdToPath );
+	}
+
+	public AllenOMEZarrLoader( final N5Reader n5, final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
+	{
+		super( n5, n5URI, sequenceDescription );
+
+		this.viewIdToPath = viewIdToPath;
+
+		final HashSet< Integer > viewSetupIds = new HashSet<>();
+		viewIdToPath.keySet().forEach( viewId -> viewSetupIds.add( viewId.getViewSetupId() ) );
+
+		this.setupIdToMultiRes = new HashMap<>();
+		this.setupIdToDataType = new HashMap<>();
+
+		System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
+
+		// assemble metadata in advance in parallel (we should store this to the XML)
+		final ForkJoinPool myPool = new ForkJoinPool( URITools.cloudThreads );
+
+		myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId ->
+		{
+			final int timePointId = getRandomTimepointId( setupId );
+
+			// multiresolution pyramid
+
+			//org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata
+			// for this to work you need to register an adapter in the N5Factory class
+			// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
+			final OmeNgffMultiScaleMetadata[] multiscales = n5.getAttribute( n5properties.getPath( setupId, timePointId ), "multiscales", OmeNgffMultiScaleMetadata[].class );
+
+			if ( multiscales == null || multiscales.length == 0 )
+				throw new RuntimeException( "Could not parse OME-ZARR multiscales object. stopping." );
+
+			if ( multiscales.length != 1 )
+				System.out.println( "This dataset has " + multiscales.length + " objects, we expected 1. Picking the first one." );
+
+			//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": found " + multiscales[ 0 ].datasets.length + " multi-resolution levels." );
+
+			double[][] mipMapResolutions = new double[ multiscales[ 0 ].datasets.length ][ 3 ];
+			double[] firstScale = null;
+
+			for ( int i = 0; i < multiscales[ 0 ].datasets.length; ++i )
+			{
+				final OmeNgffDataset ds = multiscales[ 0 ].datasets[ i ];
+
+				for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
+				{
+					if ( ScaleCoordinateTransformation.class.isInstance( c ) )
+					{
+						final ScaleCoordinateTransformation s = (ScaleCoordinateTransformation)c;
+
+						if ( firstScale == null )
+							firstScale = s.getScale().clone();
+
+						for ( int d = 0; d < mipMapResolutions[ i ].length; ++d )
+						{
+							mipMapResolutions[ i ][ d ] = s.getScale()[ d ] / firstScale[ d ];
+							mipMapResolutions[ i ][ d ] = Math.round(mipMapResolutions[ i ][ d ]*10000)/10000d; // round to the 5th digit
+						}
+						//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions(), level " + i + ": " + Arrays.toString( s.getScale() ) + " >> " + Arrays.toString( mipMapResolutions[ i ] ) );
+					}
+				}
+			}
+
+			setupIdToMultiRes.put( setupId, mipMapResolutions );
+
+			// data type
+			final DatasetAttributes attrib =
+					n5.getDatasetAttributes( n5properties.getPath( setupId, timePointId, 0 ) );
+
+			//System.out.println( "AllenOMEZarrLoader.getDataType() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": " + attrib.getDataType() );
+
+			setupIdToDataType.put( setupId, attrib.getDataType() );
+		})).join();
+
+		myPool.shutdown();
+
+		viewSetupIds.stream().sorted().forEach( setupId ->
+		{
+			System.out.println( "ViewSetupId: " + setupId + ": " + setupIdToDataType.get( setupId ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setupId ) ) );
+		});
+
+		// more threads for cloud-based fetching
+		System.out.println( "Setting num fetcher threads to " + URITools.cloudThreads + " for cloud access." );
+		setNumFetcherThreads( URITools.cloudThreads );
+	}
+
+	public int getRandomTimepointId( final int setupId )
+	{
+		return getRandomTimepointId( viewIdToPath, setupId );
+	}
+
+	public static int getRandomTimepointId( final HashMap< ViewId, String > viewIdToPath, final int setupId )
+	{
+		// find any timepoint of this setupId
+		ViewId v = null;
+
+		for ( final ViewId viewId : viewIdToPath.keySet() )
+			if ( viewId.getViewSetupId() == setupId )
+			{
+				v = viewId;
+				break;
+			}
+
+		if ( v == null )
+			throw new RuntimeException( "Could not find a ViewSetup that contains SetupId: " + setupId );
+
+		return v.getTimePointId();
+	}
+
+	public N5Properties createN5PropertiesInstance()
+	{
+		return new N5Properties()
+		{
+			@Override
+			public String getPath( final int setupId )
+			{
+				return ".";
+			}
+	
+			@Override
+			public String getPath( final int setupId, final int timepointId )
+			{
+				return viewIdToPath.get( new ViewId(timepointId, setupId) );
+			}
+	
+			@Override
+			public String getPath( final int setupId, final int timepointId, final int level )
+			{
+				return String.format( getPath( setupId, timepointId )+ "/%d", level );
+			}
+
+			@Override
+			public DataType getDataType( final N5Reader n5, final int setupId )
+			{
+				return setupIdToDataType.get( setupId );
+			}
+
+			@Override
+			public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
+			{
+				return setupIdToMultiRes.get( setupId );
+			}
+
+			@Override
+			public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
+					final VolatileCachedCellImg<T, ?> img,
+					final int setupId,
+					final int timepointId)
+			{
+				return Views.hyperSlice( Views.hyperSlice( img, 4, 0 ), 3, 0 );
+			}
+		};
+	}
+
+	public static void main( String[] args ) throws SpimDataException
+	{
+		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
+		//URI xml = URITools.toURI( "/Users/preibischs/SparkTest/IP/dataset.xml" );
+
+		XmlIoSpimData2 io = new XmlIoSpimData2();
+		SpimData2 data = io.load( xml );
+
+		SetupImgLoader sil = (SetupImgLoader)data.getSequenceDescription().getImgLoader().getSetupImgLoader( 0 );
+
+		final int tp = data.getSequenceDescription().getTimePoints().getTimePointsOrdered().get( 0 ).getId();
+
+		//RandomAccessibleInterval img = sil.getImage( tp, sil.getMipmapResolutions().length - 1 );
+		RandomAccessibleInterval vol = sil.getVolatileImage( tp, sil.getMipmapResolutions().length - 1 );
+
+		new ImageJ();
+		//ImageJFunctions.show( img );
+		ImageJFunctions.show( vol );
+
+		final ViewSetupExplorer< SpimData2 > explorer = new ViewSetupExplorer<>( data, xml, io );
+		explorer.getFrame().toFront();
+	}
+}
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
new file mode 100644
index 00000000..df4d88b4
--- /dev/null
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
@@ -0,0 +1,153 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+/*
+ * #%L
+ * BigDataViewer core classes with minimal dependencies.
+ * %%
+ * Copyright (C) 2012 - 2024 BigDataViewer developers.
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+
+import static mpicbg.spim.data.XmlKeys.IMGLOADER_FORMAT_ATTRIBUTE_NAME;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+
+import org.janelia.saalfeldlab.n5.N5Reader;
+import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
+import org.jdom2.Element;
+
+import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.generic.sequence.ImgLoaderIo;
+import mpicbg.spim.data.generic.sequence.XmlIoBasicImgLoader;
+import mpicbg.spim.data.sequence.ViewId;
+import util.URITools;
+
+@ImgLoaderIo( format = "bdv.multimg.zarr", type = AllenOMEZarrLoader.class )
+public class XmlIoAllenOMEZarrLoader implements XmlIoBasicImgLoader< AllenOMEZarrLoader >
+{
+	public static boolean PREFER_URI_FOR_LOCAL_FILES = true;
+
+	@Override
+	public Element toXml( final AllenOMEZarrLoader imgLoader, final File basePath )
+	{
+		return toXml( imgLoader, basePath == null ? null : basePath.toURI() );
+	}
+
+	@Override
+	public Element toXml( final AllenOMEZarrLoader imgLoader, final URI basePathURI )
+	{
+		final Element elem = new Element( "ImageLoader" );
+		elem.setAttribute( IMGLOADER_FORMAT_ATTRIBUTE_NAME, "bdv.multimg.zarr" );
+		elem.setAttribute( "version", "1.0" );
+
+		// TODO: implement
+		/*
+		final File n5File = getLocalFile( imgLoader.getN5URI() );
+		if ( !PREFER_URI_FOR_LOCAL_FILES && n5File != null )
+		{
+			final File basePath = basePathURI == null ? null : new File( basePathURI );
+			elem.addContent( XmlHelpers.pathElement( "n5", n5File, basePath ) );
+		}
+		else
+		{
+			elem.addContent( XmlHelpers.pathElementURI( "n5", imgLoader.getN5URI(), basePathURI ) );
+		}*/
+		return elem;
+	}
+
+	@Override
+	public AllenOMEZarrLoader fromXml( final Element elem, final File basePath, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription )
+	{
+		return fromXml( elem, basePath.toURI(), sequenceDescription );
+	}
+
+	@Override
+	public AllenOMEZarrLoader fromXml( final Element elem, final URI basePathURI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription )
+	{
+//		final String version = elem.getAttributeValue( "version" );
+//		final URI uri = XmlHelpers.loadPathURI( elem, "n5", basePathURI );
+
+		final HashMap<ViewId, String> zgroups = new HashMap<>();
+
+		final Element s3Bucket = elem.getChild( "s3bucket" );
+		final URI uri;
+
+		if (s3Bucket == null)
+		{
+			// TODO: we do not support that here for now
+			throw new RuntimeException( "Only S3 bucket storage is supported for now." );
+		}
+		else
+		{
+			// `File` class should not be used for uri manipulation as it replaces slashes
+			// with backslashes on Windows
+			final String bucket = s3Bucket.getText();
+			String folder = elem.getChildText( "zarr" );
+
+			if ( !folder.endsWith( "/" ) )
+				folder = folder + "/";
+
+			if ( !folder.startsWith( "/" ) )
+				folder = "/" + folder;
+
+			try
+			{
+				uri = new URI( "s3", bucket, folder, null );
+			}
+			catch (URISyntaxException e)
+			{
+				e.printStackTrace();
+				throw new RuntimeException( "Could not instantiate N5 reader for S3 bucket '" + bucket + "'." );
+			}
+
+			final Element zgroupsElem = elem.getChild( "zgroups" );
+			for (final Element c : zgroupsElem.getChildren( "zgroup" ))
+			{
+				final int timepointId = Integer.parseInt( c.getAttributeValue( "timepoint" ) );
+				final int setupId = Integer.parseInt( c.getAttributeValue( "setup" ) );
+				final String path = c.getChild( "path" ).getText();
+				zgroups.put( new ViewId( timepointId, setupId ), path );
+
+				//System.out.println( Group.pvid( new ViewId( timepointId, setupId ) )+ ": " + (folder+path) );
+			}
+			//System.exit( 0 );
+		}
+
+		try
+		{
+			System.out.println( "Opening N5 Zarr reader for '" + uri + "'" );
+
+			final N5Reader n5 = URITools.instantiateN5Reader( StorageFormat.ZARR, uri );
+			return new AllenOMEZarrLoader( n5, uri, sequenceDescription, zgroups );
+		}
+		catch ( Exception e )
+		{
+			e.printStackTrace();
+			return null;
+		}
+	}
+}

From 06a33041260523ff246275dda55079e5a653fa52 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Wed, 30 Oct 2024 13:28:57 -0400
Subject: [PATCH 04/24] change code to pre-fetch all dataset attributes

---
 .../imgloaders/AllenOMEZarrLoader.java        | 158 +++++++++---------
 .../imgloaders/XmlIoAllenOMEZarrLoader.java   |   3 +-
 2 files changed, 82 insertions(+), 79 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index b2df0b76..7d603856 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -4,7 +4,9 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map.Entry;
 import java.util.concurrent.ForkJoinPool;
+import java.util.stream.IntStream;
 
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
@@ -34,121 +36,113 @@
 
 public class AllenOMEZarrLoader extends N5ImageLoader
 {
+	public static boolean preFetchDatasetAttributes = true;
+
 	final HashMap< ViewId, String > viewIdToPath;
 	final HashMap< Integer, double[][] > setupIdToMultiRes;
 	final HashMap< Integer, DataType > setupIdToDataType;
+	final HashMap< ViewId, DatasetAttributes > viewIdToDatasetAttributes;
 
-	public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
-	{
-		this( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription, viewIdToPath );
-	}
+	final HashMap< String, DatasetAttributes > attributes = new HashMap<>();
 
-	public AllenOMEZarrLoader( final N5Reader n5, final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
+	public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
 	{
-		super( n5, n5URI, sequenceDescription );
+		super( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription );
 
 		this.viewIdToPath = viewIdToPath;
 
 		final HashSet< Integer > viewSetupIds = new HashSet<>();
 		viewIdToPath.keySet().forEach( viewId -> viewSetupIds.add( viewId.getViewSetupId() ) );
 
+		this.viewIdToDatasetAttributes = new HashMap<>();
 		this.setupIdToMultiRes = new HashMap<>();
 		this.setupIdToDataType = new HashMap<>();
 
-		System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
-
 		// assemble metadata in advance in parallel (we should store this to the XML)
 		final ForkJoinPool myPool = new ForkJoinPool( URITools.cloudThreads );
 
-		myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId ->
-		{
-			final int timePointId = getRandomTimepointId( setupId );
-
-			// multiresolution pyramid
-
-			//org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata
-			// for this to work you need to register an adapter in the N5Factory class
-			// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
-			final OmeNgffMultiScaleMetadata[] multiscales = n5.getAttribute( n5properties.getPath( setupId, timePointId ), "multiscales", OmeNgffMultiScaleMetadata[].class );
-
-			if ( multiscales == null || multiscales.length == 0 )
-				throw new RuntimeException( "Could not parse OME-ZARR multiscales object. stopping." );
-
-			if ( multiscales.length != 1 )
-				System.out.println( "This dataset has " + multiscales.length + " objects, we expected 1. Picking the first one." );
-
-			//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": found " + multiscales[ 0 ].datasets.length + " multi-resolution levels." );
-
-			double[][] mipMapResolutions = new double[ multiscales[ 0 ].datasets.length ][ 3 ];
-			double[] firstScale = null;
+		System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
 
-			for ( int i = 0; i < multiscales[ 0 ].datasets.length; ++i )
+		if ( preFetchDatasetAttributes )
+		{
+			// fetch multiresolution info for all viewsetups
+			myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId -> setupIdToMultiRes.put( setupId, getMipMapResolutions( setupId ) ) ) ).join();
+	
+			// fetch all DatasetAttributes
+			myPool.submit(() -> viewIdToPath.keySet().parallelStream().forEach( viewId ->
 			{
-				final OmeNgffDataset ds = multiscales[ 0 ].datasets[ i ];
-
-				for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
-				{
-					if ( ScaleCoordinateTransformation.class.isInstance( c ) )
-					{
-						final ScaleCoordinateTransformation s = (ScaleCoordinateTransformation)c;
-
-						if ( firstScale == null )
-							firstScale = s.getScale().clone();
-
-						for ( int d = 0; d < mipMapResolutions[ i ].length; ++d )
-						{
-							mipMapResolutions[ i ][ d ] = s.getScale()[ d ] / firstScale[ d ];
-							mipMapResolutions[ i ][ d ] = Math.round(mipMapResolutions[ i ][ d ]*10000)/10000d; // round to the 5th digit
-						}
-						//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions(), level " + i + ": " + Arrays.toString( s.getScale() ) + " >> " + Arrays.toString( mipMapResolutions[ i ] ) );
-					}
-				}
-			}
-
-			setupIdToMultiRes.put( setupId, mipMapResolutions );
+				myPool.submit(() -> IntStream.range( 0, setupIdToMultiRes.get( viewId.getViewSetupId() ).length ).parallel().forEach( level ->
+					this.viewIdToDatasetAttributes.put(
+							viewId,
+							n5.getDatasetAttributes( n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ) ) ) ) ).join();
+			})).join();
 
-			// data type
-			final DatasetAttributes attrib =
-					n5.getDatasetAttributes( n5properties.getPath( setupId, timePointId, 0 ) );
-
-			//System.out.println( "AllenOMEZarrLoader.getDataType() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": " + attrib.getDataType() );
-
-			setupIdToDataType.put( setupId, attrib.getDataType() );
-		})).join();
+			// fill up DataType from pre-fetched DatasetAttributes
+			for ( final Entry< ViewId, DatasetAttributes > entry : viewIdToDatasetAttributes.entrySet() )
+				setupIdToDataType.computeIfAbsent( entry.getKey().getViewSetupId(), setupId -> entry.getValue().getDataType() );
+		}
 
 		myPool.shutdown();
 
 		viewSetupIds.stream().sorted().forEach( setupId ->
-		{
-			System.out.println( "ViewSetupId: " + setupId + ": " + setupIdToDataType.get( setupId ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setupId ) ) );
-		});
+			System.out.println( "ViewSetupId: " + setupId + ": " + setupIdToDataType.get( setupId ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setupId ) ) ) );
 
 		// more threads for cloud-based fetching
 		System.out.println( "Setting num fetcher threads to " + URITools.cloudThreads + " for cloud access." );
 		setNumFetcherThreads( URITools.cloudThreads );
 	}
 
-	public int getRandomTimepointId( final int setupId )
+	public double[][] getMipMapResolutions( final int setupId )
 	{
-		return getRandomTimepointId( viewIdToPath, setupId );
-	}
+		final int timePointId = getFirstTimepointId( setupId );
 
-	public static int getRandomTimepointId( final HashMap< ViewId, String > viewIdToPath, final int setupId )
-	{
-		// find any timepoint of this setupId
-		ViewId v = null;
+		// multiresolution pyramid
+
+		//org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata
+		// for this to work you need to register an adapter in the N5Factory class
+		// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
+		final OmeNgffMultiScaleMetadata[] multiscales = n5.getAttribute( n5properties.getPath( setupId, timePointId ), "multiscales", OmeNgffMultiScaleMetadata[].class );
+
+		if ( multiscales == null || multiscales.length == 0 )
+			throw new RuntimeException( "Could not parse OME-ZARR multiscales object. stopping." );
 
-		for ( final ViewId viewId : viewIdToPath.keySet() )
-			if ( viewId.getViewSetupId() == setupId )
+		if ( multiscales.length != 1 )
+			System.out.println( "This dataset has " + multiscales.length + " objects, we expected 1. Picking the first one." );
+
+		//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": found " + multiscales[ 0 ].datasets.length + " multi-resolution levels." );
+
+		double[][] mipMapResolutions = new double[ multiscales[ 0 ].datasets.length ][ 3 ];
+		double[] firstScale = null;
+
+		for ( int i = 0; i < multiscales[ 0 ].datasets.length; ++i )
+		{
+			final OmeNgffDataset ds = multiscales[ 0 ].datasets[ i ];
+
+			for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
 			{
-				v = viewId;
-				break;
+				if ( ScaleCoordinateTransformation.class.isInstance( c ) )
+				{
+					final ScaleCoordinateTransformation s = (ScaleCoordinateTransformation)c;
+
+					if ( firstScale == null )
+						firstScale = s.getScale().clone();
+
+					for ( int d = 0; d < mipMapResolutions[ i ].length; ++d )
+					{
+						mipMapResolutions[ i ][ d ] = s.getScale()[ d ] / firstScale[ d ];
+						mipMapResolutions[ i ][ d ] = Math.round(mipMapResolutions[ i ][ d ]*10000)/10000d; // round to the 5th digit
+					}
+					//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions(), level " + i + ": " + Arrays.toString( s.getScale() ) + " >> " + Arrays.toString( mipMapResolutions[ i ] ) );
+				}
 			}
+		}
 
-		if ( v == null )
-			throw new RuntimeException( "Could not find a ViewSetup that contains SetupId: " + setupId );
+		return mipMapResolutions;
+	}
 
-		return v.getTimePointId();
+	public int getFirstTimepointId( final int setupId )
+	{
+		return seq.getTimePoints().getTimePointsOrdered().get( 0 ).getId();
 	}
 
 	public N5Properties createN5PropertiesInstance()
@@ -193,11 +187,21 @@ public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
 			{
 				return Views.hyperSlice( Views.hyperSlice( img, 4, 0 ), 3, 0 );
 			}
+
+			@Override
+			public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String pathName )
+			{
+				// attributes are cached by the N5 API
+				return n5.getDatasetAttributes( pathName );
+			}
 		};
 	}
 
 	public static void main( String[] args ) throws SpimDataException
 	{
+		//IntStream.range(0, 100 ).parallel().forEach( i -> System.out.println( i ));
+		//System.exit( 0 );
+
 		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
 		//URI xml = URITools.toURI( "/Users/preibischs/SparkTest/IP/dataset.xml" );
 
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
index df4d88b4..6a0afe25 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
@@ -141,8 +141,7 @@ public AllenOMEZarrLoader fromXml( final Element elem, final URI basePathURI, fi
 		{
 			System.out.println( "Opening N5 Zarr reader for '" + uri + "'" );
 
-			final N5Reader n5 = URITools.instantiateN5Reader( StorageFormat.ZARR, uri );
-			return new AllenOMEZarrLoader( n5, uri, sequenceDescription, zgroups );
+			return new AllenOMEZarrLoader( uri, sequenceDescription, zgroups );
 		}
 		catch ( Exception e )
 		{

From 429c96b98eff84eb74deca0973eee78bcbdda0e7 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Wed, 30 Oct 2024 13:45:10 -0400
Subject: [PATCH 05/24] fix prefetching, support not-prefetching option

---
 .../imgloaders/AllenOMEZarrLoader.java        | 68 +++++++++++++------
 1 file changed, 46 insertions(+), 22 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index 7d603856..977a50e3 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -8,6 +8,8 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.stream.IntStream;
 
+import javax.management.RuntimeErrorException;
+
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
 import org.janelia.saalfeldlab.n5.N5Reader;
@@ -24,6 +26,7 @@
 import ij.ImageJ;
 import mpicbg.spim.data.SpimDataException;
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.sequence.TimePoint;
 import mpicbg.spim.data.sequence.ViewId;
 import net.imglib2.RandomAccessibleInterval;
 import net.imglib2.img.display.imagej.ImageJFunctions;
@@ -39,11 +42,9 @@ public class AllenOMEZarrLoader extends N5ImageLoader
 	public static boolean preFetchDatasetAttributes = true;
 
 	final HashMap< ViewId, String > viewIdToPath;
-	final HashMap< Integer, double[][] > setupIdToMultiRes;
-	final HashMap< Integer, DataType > setupIdToDataType;
-	final HashMap< ViewId, DatasetAttributes > viewIdToDatasetAttributes;
-
-	final HashMap< String, DatasetAttributes > attributes = new HashMap<>();
+	final HashMap< Integer, double[][] > setupIdToMultiRes = new HashMap<>();
+	final HashMap< Integer, DataType > setupIdToDataType = new HashMap<>();
+	final HashMap< String, DatasetAttributes > pathToDatasetAttributes = new HashMap<>();
 
 	public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
 	{
@@ -54,32 +55,42 @@ public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?
 		final HashSet< Integer > viewSetupIds = new HashSet<>();
 		viewIdToPath.keySet().forEach( viewId -> viewSetupIds.add( viewId.getViewSetupId() ) );
 
-		this.viewIdToDatasetAttributes = new HashMap<>();
-		this.setupIdToMultiRes = new HashMap<>();
-		this.setupIdToDataType = new HashMap<>();
-
 		// assemble metadata in advance in parallel (we should store this to the XML)
 		final ForkJoinPool myPool = new ForkJoinPool( URITools.cloudThreads );
 
-		System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
+		if ( preFetchDatasetAttributes )
+			System.out.println( "Loading metadata and pre-fetching all DatasetAttributes from OME-ZARR containers (in parallel) ... " );
+		else
+			System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
+
+		// fetch multiresolution info for all viewsetups
+		myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId -> setupIdToMultiRes.put( setupId, getMipMapResolutions( setupId ) ) ) ).join();
 
 		if ( preFetchDatasetAttributes )
 		{
-			// fetch multiresolution info for all viewsetups
-			myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId -> setupIdToMultiRes.put( setupId, getMipMapResolutions( setupId ) ) ) ).join();
-	
 			// fetch all DatasetAttributes
 			myPool.submit(() -> viewIdToPath.keySet().parallelStream().forEach( viewId ->
 			{
 				myPool.submit(() -> IntStream.range( 0, setupIdToMultiRes.get( viewId.getViewSetupId() ).length ).parallel().forEach( level ->
-					this.viewIdToDatasetAttributes.put(
-							viewId,
+					this.pathToDatasetAttributes.put(
+							n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ),
 							n5.getDatasetAttributes( n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ) ) ) ) ).join();
 			})).join();
 
 			// fill up DataType from pre-fetched DatasetAttributes
-			for ( final Entry< ViewId, DatasetAttributes > entry : viewIdToDatasetAttributes.entrySet() )
-				setupIdToDataType.computeIfAbsent( entry.getKey().getViewSetupId(), setupId -> entry.getValue().getDataType() );
+			for ( final ViewId viewId : viewIdToPath.keySet() )
+				setupIdToDataType.computeIfAbsent( viewId.getViewSetupId(), setupId ->
+					pathToDatasetAttributes.get( n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), 0 ) ).getDataType() );
+		}
+		else
+		{
+			myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId ->
+			{
+				final int timePointId = getFirstAvailableTimepointId( setupId );
+				setupIdToDataType.put(
+						setupId,
+						n5.getDatasetAttributes( n5properties.getPath(setupId, timePointId, 0 ) ).getDataType() );
+			})).join();
 		}
 
 		myPool.shutdown();
@@ -94,7 +105,7 @@ public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?
 
 	public double[][] getMipMapResolutions( final int setupId )
 	{
-		final int timePointId = getFirstTimepointId( setupId );
+		final int timePointId = getFirstAvailableTimepointId( setupId );
 
 		// multiresolution pyramid
 
@@ -140,9 +151,15 @@ public double[][] getMipMapResolutions( final int setupId )
 		return mipMapResolutions;
 	}
 
-	public int getFirstTimepointId( final int setupId )
+	public int getFirstAvailableTimepointId( final int setupId )
 	{
-		return seq.getTimePoints().getTimePointsOrdered().get( 0 ).getId();
+		for ( final TimePoint tp : seq.getTimePoints().getTimePointsOrdered() )
+		{
+			if ( !seq.getMissingViews().getMissingViews().contains( new ViewId( tp.getId(), setupId ) ) )
+				return tp.getId();
+		}
+
+		throw new RuntimeException( "All timepoints for setupId " + setupId + " are declared missing. Stopping." );
 	}
 
 	public N5Properties createN5PropertiesInstance()
@@ -191,8 +208,15 @@ public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
 			@Override
 			public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String pathName )
 			{
-				// attributes are cached by the N5 API
-				return n5.getDatasetAttributes( pathName );
+				// attributes are cached by the N5 API, so this is technically not necessary ... maybe later if we store it in the XML
+				if ( preFetchDatasetAttributes )
+				{
+					return pathToDatasetAttributes.get( pathName );
+				}
+				else
+				{
+					return n5.getDatasetAttributes( pathName );
+				}
 			}
 		};
 	}

From 238eb3dd4b63977507b6a0ff3504b48e2797010c Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Wed, 30 Oct 2024 15:35:32 -0400
Subject: [PATCH 06/24] fixes

---
 .../imgloaders/AllenOMEZarrLoader.java        | 60 ++++++++++---------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index 977a50e3..a4b50609 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -3,13 +3,9 @@
 import java.net.URI;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map.Entry;
 import java.util.concurrent.ForkJoinPool;
 import java.util.stream.IntStream;
 
-import javax.management.RuntimeErrorException;
-
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
 import org.janelia.saalfeldlab.n5.N5Reader;
@@ -19,7 +15,6 @@
 import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
 import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
 
-import bdv.ViewerImgLoader;
 import bdv.img.cache.VolatileCachedCellImg;
 import bdv.img.n5.N5ImageLoader;
 import bdv.img.n5.N5Properties;
@@ -39,8 +34,6 @@
 
 public class AllenOMEZarrLoader extends N5ImageLoader
 {
-	public static boolean preFetchDatasetAttributes = true;
-
 	final HashMap< ViewId, String > viewIdToPath;
 	final HashMap< Integer, double[][] > setupIdToMultiRes = new HashMap<>();
 	final HashMap< Integer, DataType > setupIdToDataType = new HashMap<>();
@@ -52,11 +45,26 @@ public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?
 
 		this.viewIdToPath = viewIdToPath;
 
-		final HashSet< Integer > viewSetupIds = new HashSet<>();
-		viewIdToPath.keySet().forEach( viewId -> viewSetupIds.add( viewId.getViewSetupId() ) );
+		// this methods needs the assignment of viewIdToPath
+		preFetch();
+
+		seq.getViewSetupsOrdered().stream().forEach( setup ->
+			System.out.println( "ViewSetupId: " + setup.getId() + ": " + setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setup.getId() ) ) ) );
+
+		// more threads for cloud-based fetching
+		System.out.println( "Setting num fetcher threads to " + URITools.cloudThreads + " for cloud access." );
+		setNumFetcherThreads( URITools.cloudThreads );
+	}
+
+	@Override
+	public void preFetch()
+	{
+		// this is called from the superclass constructor once, but the we cannot assign the variable yet
+		if ( this.viewIdToPath == null )
+			return;
 
 		// assemble metadata in advance in parallel (we should store this to the XML)
-		final ForkJoinPool myPool = new ForkJoinPool( URITools.cloudThreads );
+		final ForkJoinPool myPool = new ForkJoinPool( cloudThreads );
 
 		if ( preFetchDatasetAttributes )
 			System.out.println( "Loading metadata and pre-fetching all DatasetAttributes from OME-ZARR containers (in parallel) ... " );
@@ -64,17 +72,23 @@ public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?
 			System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
 
 		// fetch multiresolution info for all viewsetups
-		myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId -> setupIdToMultiRes.put( setupId, getMipMapResolutions( setupId ) ) ) ).join();
+		myPool.submit(() -> seq.getViewSetupsOrdered().parallelStream().forEach( setup ->
+			setupIdToMultiRes.put( setup.getId(), getMipMapResolutions( setup.getId() ) ) ) ).join();
 
 		if ( preFetchDatasetAttributes )
 		{
 			// fetch all DatasetAttributes
 			myPool.submit(() -> viewIdToPath.keySet().parallelStream().forEach( viewId ->
 			{
-				myPool.submit(() -> IntStream.range( 0, setupIdToMultiRes.get( viewId.getViewSetupId() ).length ).parallel().forEach( level ->
-					this.pathToDatasetAttributes.put(
+				if ( !seq.getMissingViews().getMissingViews().contains( viewId ) )
+				{
+					final int numLevels = n5properties.getMipmapResolutions( n5, viewId.getViewSetupId() ).length;
+
+					myPool.submit(() -> IntStream.range( 0, numLevels ).parallel().forEach( level ->
+						pathToDatasetAttributes.put(
 							n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ),
 							n5.getDatasetAttributes( n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ) ) ) ) ).join();
+				}
 			})).join();
 
 			// fill up DataType from pre-fetched DatasetAttributes
@@ -84,23 +98,16 @@ public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?
 		}
 		else
 		{
-			myPool.submit(() -> viewSetupIds.parallelStream().forEach( setupId ->
+			myPool.submit(() -> seq.getViewSetupsOrdered().parallelStream().forEach( setup ->
 			{
-				final int timePointId = getFirstAvailableTimepointId( setupId );
+				final int timePointId = getFirstAvailableTimepointId( setup.getId() );
 				setupIdToDataType.put(
-						setupId,
-						n5.getDatasetAttributes( n5properties.getPath(setupId, timePointId, 0 ) ).getDataType() );
+						setup.getId(),
+						n5.getDatasetAttributes( n5properties.getPath( setup.getId(), timePointId, 0 ) ).getDataType() );
 			})).join();
 		}
 
 		myPool.shutdown();
-
-		viewSetupIds.stream().sorted().forEach( setupId ->
-			System.out.println( "ViewSetupId: " + setupId + ": " + setupIdToDataType.get( setupId ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setupId ) ) ) );
-
-		// more threads for cloud-based fetching
-		System.out.println( "Setting num fetcher threads to " + URITools.cloudThreads + " for cloud access." );
-		setNumFetcherThreads( URITools.cloudThreads );
 	}
 
 	public double[][] getMipMapResolutions( final int setupId )
@@ -223,10 +230,9 @@ public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String p
 
 	public static void main( String[] args ) throws SpimDataException
 	{
-		//IntStream.range(0, 100 ).parallel().forEach( i -> System.out.println( i ));
-		//System.exit( 0 );
-
 		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
+		//URI xml = URITools.toURI( "s3://janelia-bigstitcher-spark/Stitching/dataset.xml" );
+		//URI xml = URITools.toURI( "gs://janelia-spark-test/I2K-test/dataset.xml" );
 		//URI xml = URITools.toURI( "/Users/preibischs/SparkTest/IP/dataset.xml" );
 
 		XmlIoSpimData2 io = new XmlIoSpimData2();

From 0ed9eb6fea10a99dd8a86bc09bd4e57bbd02f75b Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Thu, 31 Oct 2024 10:04:10 -0400
Subject: [PATCH 07/24] finish XmlIoAllenOMEZarrLoader.java (support saving)

---
 .../imgloaders/AllenOMEZarrLoader.java        | 15 +++-
 .../imgloaders/XmlIoAllenOMEZarrLoader.java   | 69 ++++++++++++-------
 2 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index a4b50609..a0d8af1e 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -39,10 +39,19 @@ public class AllenOMEZarrLoader extends N5ImageLoader
 	final HashMap< Integer, DataType > setupIdToDataType = new HashMap<>();
 	final HashMap< String, DatasetAttributes > pathToDatasetAttributes = new HashMap<>();
 
-	public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription, final HashMap< ViewId, String > viewIdToPath )
+	final String bucket, folder;
+
+	public AllenOMEZarrLoader(
+			final URI n5URI,
+			final String bucket,
+			final String folder,
+			final AbstractSequenceDescription< ?, ?, ? > sequenceDescription,
+			final HashMap< ViewId, String > viewIdToPath )
 	{
 		super( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription );
 
+		this.bucket = bucket;
+		this.folder = folder;
 		this.viewIdToPath = viewIdToPath;
 
 		// this methods needs the assignment of viewIdToPath
@@ -56,6 +65,10 @@ public AllenOMEZarrLoader( final URI n5URI, final AbstractSequenceDescription< ?
 		setNumFetcherThreads( URITools.cloudThreads );
 	}
 
+	public HashMap< ViewId, String > getViewIdToPath() { return viewIdToPath; }
+	public String getBucket() { return bucket; }
+	public String getFolder() { return folder; }
+
 	@Override
 	public void preFetch()
 	{
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
index 6a0afe25..7c85e727 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
@@ -35,11 +35,11 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.HashMap;
+import java.util.Map.Entry;
 
-import org.janelia.saalfeldlab.n5.N5Reader;
-import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
 import org.jdom2.Element;
 
+import mpicbg.spim.data.XmlHelpers;
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
 import mpicbg.spim.data.generic.sequence.ImgLoaderIo;
 import mpicbg.spim.data.generic.sequence.XmlIoBasicImgLoader;
@@ -60,23 +60,42 @@ public Element toXml( final AllenOMEZarrLoader imgLoader, final File basePath )
 	@Override
 	public Element toXml( final AllenOMEZarrLoader imgLoader, final URI basePathURI )
 	{
-		final Element elem = new Element( "ImageLoader" );
-		elem.setAttribute( IMGLOADER_FORMAT_ATTRIBUTE_NAME, "bdv.multimg.zarr" );
-		elem.setAttribute( "version", "1.0" );
-
-		// TODO: implement
-		/*
-		final File n5File = getLocalFile( imgLoader.getN5URI() );
-		if ( !PREFER_URI_FOR_LOCAL_FILES && n5File != null )
+		final Element imgLoaderElement = new Element( "ImageLoader" );
+		imgLoaderElement.setAttribute( IMGLOADER_FORMAT_ATTRIBUTE_NAME, "bdv.multimg.zarr" );
+		imgLoaderElement.setAttribute( "version", "1.0" );
+
+		if ( URITools.isS3( imgLoader.getN5URI() ) || URITools.isGC( imgLoader.getN5URI() ) )
 		{
-			final File basePath = basePathURI == null ? null : new File( basePathURI );
-			elem.addContent( XmlHelpers.pathElement( "n5", n5File, basePath ) );
+			final Element bucketElement = new Element("s3bucket");
+			bucketElement.addContent( imgLoader.getBucket() );
+			imgLoaderElement.addContent(bucketElement);
+			
+			final Element zarrElement = new Element("zarr");
+			zarrElement.setAttribute("type", "absolute");
+			zarrElement.addContent( imgLoader.getFolder() );
+			imgLoaderElement.addContent( zarrElement );
 		}
 		else
 		{
-			elem.addContent( XmlHelpers.pathElementURI( "n5", imgLoader.getN5URI(), basePathURI ) );
-		}*/
-		return elem;
+			imgLoaderElement.addContent( XmlHelpers.pathElement( "zarr", new File( imgLoader.getN5URI() ), null));
+		}
+
+		final Element zgroupsElement = new Element( "zgroups" );
+
+		for ( final Entry<ViewId, String > entry : imgLoader.getViewIdToPath().entrySet() )
+		{
+			final Element zgroupElement = new Element("zgroup");
+			zgroupElement.setAttribute( "setup", String.valueOf( entry.getKey().getViewSetupId() ) );
+			zgroupElement.setAttribute( "timepoint", String.valueOf( entry.getKey().getTimePointId() ) );
+
+			final Element pathElement = new Element( "path" );
+			pathElement.addContent( entry.getValue() );
+			zgroupElement.addContent( pathElement );
+
+			zgroupsElement.addContent( zgroupElement );
+		}
+
+		return imgLoaderElement;
 	}
 
 	@Override
@@ -88,25 +107,26 @@ public AllenOMEZarrLoader fromXml( final Element elem, final File basePath, fina
 	@Override
 	public AllenOMEZarrLoader fromXml( final Element elem, final URI basePathURI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription )
 	{
-//		final String version = elem.getAttributeValue( "version" );
-//		final URI uri = XmlHelpers.loadPathURI( elem, "n5", basePathURI );
-
 		final HashMap<ViewId, String> zgroups = new HashMap<>();
 
 		final Element s3Bucket = elem.getChild( "s3bucket" );
 		final URI uri;
+		String bucket, folder;
 
 		if (s3Bucket == null)
 		{
-			// TODO: we do not support that here for now
-			throw new RuntimeException( "Only S3 bucket storage is supported for now." );
+			uri = XmlHelpers.loadPathURI( elem, "zarr", basePathURI );
+
+			bucket = null;
+			folder = null;
 		}
 		else
 		{
 			// `File` class should not be used for uri manipulation as it replaces slashes
 			// with backslashes on Windows
-			final String bucket = s3Bucket.getText();
-			String folder = elem.getChildText( "zarr" );
+			bucket = s3Bucket.getText();
+
+			folder = elem.getChildText( "zarr" );
 
 			if ( !folder.endsWith( "/" ) )
 				folder = folder + "/";
@@ -131,17 +151,14 @@ public AllenOMEZarrLoader fromXml( final Element elem, final URI basePathURI, fi
 				final int setupId = Integer.parseInt( c.getAttributeValue( "setup" ) );
 				final String path = c.getChild( "path" ).getText();
 				zgroups.put( new ViewId( timepointId, setupId ), path );
-
-				//System.out.println( Group.pvid( new ViewId( timepointId, setupId ) )+ ": " + (folder+path) );
 			}
-			//System.exit( 0 );
 		}
 
 		try
 		{
 			System.out.println( "Opening N5 Zarr reader for '" + uri + "'" );
 
-			return new AllenOMEZarrLoader( uri, sequenceDescription, zgroups );
+			return new AllenOMEZarrLoader( uri, bucket, folder, sequenceDescription, zgroups );
 		}
 		catch ( Exception e )
 		{

From d29e5a0705f5673333caab438d3d8bbe0fe7aee6 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Thu, 31 Oct 2024 10:29:44 -0400
Subject: [PATCH 08/24] fix toXML() call

---
 .../fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
index 7c85e727..35e5b985 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
@@ -95,6 +95,8 @@ public Element toXml( final AllenOMEZarrLoader imgLoader, final URI basePathURI
 			zgroupsElement.addContent( zgroupElement );
 		}
 
+		imgLoaderElement.addContent( zgroupsElement );
+
 		return imgLoaderElement;
 	}
 

From 51fea17a9a353cce6e7c96d4169dac6ac81fd17c Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Thu, 31 Oct 2024 10:29:52 -0400
Subject: [PATCH 09/24] pre-fetching is just happening at the point of open()
 call in N5ImageLoader

---
 .../spimdata/imgloaders/AllenOMEZarrLoader.java     | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index a0d8af1e..f2a0edf1 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -50,19 +50,14 @@ public AllenOMEZarrLoader(
 	{
 		super( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription );
 
+		setNumFetcherThreads( cloudThreads );
+
 		this.bucket = bucket;
 		this.folder = folder;
 		this.viewIdToPath = viewIdToPath;
 
-		// this methods needs the assignment of viewIdToPath
-		preFetch();
-
-		seq.getViewSetupsOrdered().stream().forEach( setup ->
-			System.out.println( "ViewSetupId: " + setup.getId() + ": " + setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setup.getId() ) ) ) );
-
-		// more threads for cloud-based fetching
-		System.out.println( "Setting num fetcher threads to " + URITools.cloudThreads + " for cloud access." );
-		setNumFetcherThreads( URITools.cloudThreads );
+		//seq.getViewSetupsOrdered().stream().forEach( setup ->
+		//	System.out.println( "ViewSetupId: " + setup.getId() + ": " + setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setup.getId() ) ) ) );
 	}
 
 	public HashMap< ViewId, String > getViewIdToPath() { return viewIdToPath; }

From e173a17aee27fa0459eff5841606a455b2ed2316 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Thu, 31 Oct 2024 10:33:00 -0400
Subject: [PATCH 10/24] fix prefetch() method, small re-arrangements

---
 .../imgloaders/AllenOMEZarrLoader.java         | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index f2a0edf1..a1f8e6ef 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -35,9 +35,9 @@
 public class AllenOMEZarrLoader extends N5ImageLoader
 {
 	final HashMap< ViewId, String > viewIdToPath;
-	final HashMap< Integer, double[][] > setupIdToMultiRes = new HashMap<>();
-	final HashMap< Integer, DataType > setupIdToDataType = new HashMap<>();
-	final HashMap< String, DatasetAttributes > pathToDatasetAttributes = new HashMap<>();
+	final HashMap< Integer, double[][] > setupIdToMultiRes;
+	final HashMap< Integer, DataType > setupIdToDataType;
+	final HashMap< String, DatasetAttributes > pathToDatasetAttributes;
 
 	final String bucket, folder;
 
@@ -56,8 +56,9 @@ public AllenOMEZarrLoader(
 		this.folder = folder;
 		this.viewIdToPath = viewIdToPath;
 
-		//seq.getViewSetupsOrdered().stream().forEach( setup ->
-		//	System.out.println( "ViewSetupId: " + setup.getId() + ": " + setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setup.getId() ) ) ) );
+		this.setupIdToMultiRes = new HashMap<>();
+		this.setupIdToDataType = new HashMap<>();
+		this.pathToDatasetAttributes = new HashMap<>();
 	}
 
 	public HashMap< ViewId, String > getViewIdToPath() { return viewIdToPath; }
@@ -67,10 +68,6 @@ public AllenOMEZarrLoader(
 	@Override
 	public void preFetch()
 	{
-		// this is called from the superclass constructor once, but the we cannot assign the variable yet
-		if ( this.viewIdToPath == null )
-			return;
-
 		// assemble metadata in advance in parallel (we should store this to the XML)
 		final ForkJoinPool myPool = new ForkJoinPool( cloudThreads );
 
@@ -116,6 +113,9 @@ public void preFetch()
 		}
 
 		myPool.shutdown();
+
+		seq.getViewSetupsOrdered().stream().forEach( setup ->
+			System.out.println( "ViewSetupId: " + setup.getId() + ": " + setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setup.getId() ) ) ) );
 	}
 
 	public double[][] getMipMapResolutions( final int setupId )

From 06bab4a02d61deca79c38476c4409d26bdbc7b4f Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Thu, 31 Oct 2024 13:55:01 -0400
Subject: [PATCH 11/24] finish loader including caching attributes

---
 .../imgloaders/AllenOMEZarrLoader.java        | 216 ++++--------------
 .../imgloaders/AllenOMEZarrProperties.java    | 188 +++++++++++++++
 2 files changed, 235 insertions(+), 169 deletions(-)
 create mode 100644 src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index a1f8e6ef..6d535846 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -6,27 +6,16 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.stream.IntStream;
 
-import org.janelia.saalfeldlab.n5.DataType;
-import org.janelia.saalfeldlab.n5.DatasetAttributes;
-import org.janelia.saalfeldlab.n5.N5Reader;
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
-import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata;
-import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata.OmeNgffDataset;
-import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
-import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
 
-import bdv.img.cache.VolatileCachedCellImg;
 import bdv.img.n5.N5ImageLoader;
 import bdv.img.n5.N5Properties;
 import ij.ImageJ;
 import mpicbg.spim.data.SpimDataException;
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
-import mpicbg.spim.data.sequence.TimePoint;
 import mpicbg.spim.data.sequence.ViewId;
 import net.imglib2.RandomAccessibleInterval;
 import net.imglib2.img.display.imagej.ImageJFunctions;
-import net.imglib2.type.NativeType;
-import net.imglib2.view.Views;
 import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer;
@@ -35,9 +24,6 @@
 public class AllenOMEZarrLoader extends N5ImageLoader
 {
 	final HashMap< ViewId, String > viewIdToPath;
-	final HashMap< Integer, double[][] > setupIdToMultiRes;
-	final HashMap< Integer, DataType > setupIdToDataType;
-	final HashMap< String, DatasetAttributes > pathToDatasetAttributes;
 
 	final String bucket, folder;
 
@@ -54,186 +40,78 @@ public AllenOMEZarrLoader(
 
 		this.bucket = bucket;
 		this.folder = folder;
-		this.viewIdToPath = viewIdToPath;
 
-		this.setupIdToMultiRes = new HashMap<>();
-		this.setupIdToDataType = new HashMap<>();
-		this.pathToDatasetAttributes = new HashMap<>();
+		this.viewIdToPath = viewIdToPath;
 	}
 
+	public AbstractSequenceDescription< ?, ?, ? > getSequenceDescription() { return seq; }
 	public HashMap< ViewId, String > getViewIdToPath() { return viewIdToPath; }
 	public String getBucket() { return bucket; }
 	public String getFolder() { return folder; }
 
+	@Override
+	public AllenOMEZarrProperties getN5properties()
+	{
+		return (AllenOMEZarrProperties)n5properties;
+	}
+
+	@Override
+	public N5Properties createN5PropertiesInstance()
+	{
+		return new AllenOMEZarrProperties( this );
+	}
+
 	@Override
 	public void preFetch()
 	{
+		final AllenOMEZarrProperties n5p = getN5properties();
+
 		// assemble metadata in advance in parallel (we should store this to the XML)
 		final ForkJoinPool myPool = new ForkJoinPool( cloudThreads );
 
-		if ( preFetchDatasetAttributes )
-			System.out.println( "Loading metadata and pre-fetching all DatasetAttributes from OME-ZARR containers (in parallel) ... " );
-		else
-			System.out.println( "Loading metadata from OME-ZARR containers (in parallel) ... " );
+		System.out.println( "Loading metadata and pre-fetching all DatasetAttributes from OME-ZARR containers (in parallel) ... " );
 
 		// fetch multiresolution info for all viewsetups
 		myPool.submit(() -> seq.getViewSetupsOrdered().parallelStream().forEach( setup ->
-			setupIdToMultiRes.put( setup.getId(), getMipMapResolutions( setup.getId() ) ) ) ).join();
-
-		if ( preFetchDatasetAttributes )
+			n5p.setupIdToMultiRes.put(
+				setup.getId(),
+				AllenOMEZarrProperties.getMipMapResolutions(
+					n5p,
+					n5,
+					setup.getId() ) ) ) ).join();
+
+		// fetch all DatasetAttributes
+		myPool.submit(() -> viewIdToPath.keySet().parallelStream().forEach( viewId ->
 		{
-			// fetch all DatasetAttributes
-			myPool.submit(() -> viewIdToPath.keySet().parallelStream().forEach( viewId ->
+			if ( !seq.getMissingViews().getMissingViews().contains( viewId ) )
 			{
-				if ( !seq.getMissingViews().getMissingViews().contains( viewId ) )
-				{
-					final int numLevels = n5properties.getMipmapResolutions( n5, viewId.getViewSetupId() ).length;
-
-					myPool.submit(() -> IntStream.range( 0, numLevels ).parallel().forEach( level ->
-						pathToDatasetAttributes.put(
-							n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ),
-							n5.getDatasetAttributes( n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), level ) ) ) ) ).join();
-				}
-			})).join();
-
-			// fill up DataType from pre-fetched DatasetAttributes
-			for ( final ViewId viewId : viewIdToPath.keySet() )
-				setupIdToDataType.computeIfAbsent( viewId.getViewSetupId(), setupId ->
-					pathToDatasetAttributes.get( n5properties.getPath( viewId.getViewSetupId(), viewId.getTimePointId(), 0 ) ).getDataType() );
-		}
-		else
-		{
-			myPool.submit(() -> seq.getViewSetupsOrdered().parallelStream().forEach( setup ->
-			{
-				final int timePointId = getFirstAvailableTimepointId( setup.getId() );
-				setupIdToDataType.put(
-						setup.getId(),
-						n5.getDatasetAttributes( n5properties.getPath( setup.getId(), timePointId, 0 ) ).getDataType() );
-			})).join();
-		}
-
-		myPool.shutdown();
-
-		seq.getViewSetupsOrdered().stream().forEach( setup ->
-			System.out.println( "ViewSetupId: " + setup.getId() + ": " + setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( setupIdToMultiRes.get( setup.getId() ) ) ) );
-	}
-
-	public double[][] getMipMapResolutions( final int setupId )
-	{
-		final int timePointId = getFirstAvailableTimepointId( setupId );
-
-		// multiresolution pyramid
-
-		//org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata
-		// for this to work you need to register an adapter in the N5Factory class
-		// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
-		final OmeNgffMultiScaleMetadata[] multiscales = n5.getAttribute( n5properties.getPath( setupId, timePointId ), "multiscales", OmeNgffMultiScaleMetadata[].class );
-
-		if ( multiscales == null || multiscales.length == 0 )
-			throw new RuntimeException( "Could not parse OME-ZARR multiscales object. stopping." );
-
-		if ( multiscales.length != 1 )
-			System.out.println( "This dataset has " + multiscales.length + " objects, we expected 1. Picking the first one." );
-
-		//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": found " + multiscales[ 0 ].datasets.length + " multi-resolution levels." );
-
-		double[][] mipMapResolutions = new double[ multiscales[ 0 ].datasets.length ][ 3 ];
-		double[] firstScale = null;
-
-		for ( int i = 0; i < multiscales[ 0 ].datasets.length; ++i )
-		{
-			final OmeNgffDataset ds = multiscales[ 0 ].datasets[ i ];
+				final int numLevels = n5properties.getMipmapResolutions( n5, viewId.getViewSetupId() ).length;
 
-			for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
-			{
-				if ( ScaleCoordinateTransformation.class.isInstance( c ) )
+				myPool.submit(() -> IntStream.range( 0, numLevels ).parallel().forEach( level ->
 				{
-					final ScaleCoordinateTransformation s = (ScaleCoordinateTransformation)c;
-
-					if ( firstScale == null )
-						firstScale = s.getScale().clone();
-
-					for ( int d = 0; d < mipMapResolutions[ i ].length; ++d )
-					{
-						mipMapResolutions[ i ][ d ] = s.getScale()[ d ] / firstScale[ d ];
-						mipMapResolutions[ i ][ d ] = Math.round(mipMapResolutions[ i ][ d ]*10000)/10000d; // round to the 5th digit
-					}
-					//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions(), level " + i + ": " + Arrays.toString( s.getScale() ) + " >> " + Arrays.toString( mipMapResolutions[ i ] ) );
-				}
-			}
-		}
-
-		return mipMapResolutions;
-	}
-
-	public int getFirstAvailableTimepointId( final int setupId )
-	{
-		for ( final TimePoint tp : seq.getTimePoints().getTimePointsOrdered() )
-		{
-			if ( !seq.getMissingViews().getMissingViews().contains( new ViewId( tp.getId(), setupId ) ) )
-				return tp.getId();
-		}
-
-		throw new RuntimeException( "All timepoints for setupId " + setupId + " are declared missing. Stopping." );
-	}
-
-	public N5Properties createN5PropertiesInstance()
-	{
-		return new N5Properties()
-		{
-			@Override
-			public String getPath( final int setupId )
-			{
-				return ".";
-			}
-	
-			@Override
-			public String getPath( final int setupId, final int timepointId )
-			{
-				return viewIdToPath.get( new ViewId(timepointId, setupId) );
-			}
-	
-			@Override
-			public String getPath( final int setupId, final int timepointId, final int level )
-			{
-				return String.format( getPath( setupId, timepointId )+ "/%d", level );
-			}
+					final String path = n5properties.getPath(
+						viewId.getViewSetupId(),
+						viewId.getTimePointId(),
+						level );
 
-			@Override
-			public DataType getDataType( final N5Reader n5, final int setupId )
-			{
-				return setupIdToDataType.get( setupId );
+					n5p.pathToDatasetAttributes.put( path, n5.getDatasetAttributes( path ) );
+				}) ).join();
 			}
+		})).join();
 
-			@Override
-			public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
-			{
-				return setupIdToMultiRes.get( setupId );
-			}
+		// fill up DataType from pre-fetched DatasetAttributes
+		for ( final ViewId viewId : viewIdToPath.keySet() )
+			n5p.setupIdToDataType.computeIfAbsent( viewId.getViewSetupId(), setupId ->
+				n5p.pathToDatasetAttributes.get(
+					n5properties.getPath(
+						viewId.getViewSetupId(),
+						viewId.getTimePointId(),
+						0 ) ).getDataType() );
 
-			@Override
-			public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
-					final VolatileCachedCellImg<T, ?> img,
-					final int setupId,
-					final int timepointId)
-			{
-				return Views.hyperSlice( Views.hyperSlice( img, 4, 0 ), 3, 0 );
-			}
+		myPool.shutdown();
 
-			@Override
-			public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String pathName )
-			{
-				// attributes are cached by the N5 API, so this is technically not necessary ... maybe later if we store it in the XML
-				if ( preFetchDatasetAttributes )
-				{
-					return pathToDatasetAttributes.get( pathName );
-				}
-				else
-				{
-					return n5.getDatasetAttributes( pathName );
-				}
-			}
-		};
+		seq.getViewSetupsOrdered().stream().forEach( setup ->
+			System.out.println( "ViewSetupId: " + setup.getId() + ": " + n5p.setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( n5p.setupIdToMultiRes.get( setup.getId() ) ) ) );
 	}
 
 	public static void main( String[] args ) throws SpimDataException
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
new file mode 100644
index 00000000..70687900
--- /dev/null
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
@@ -0,0 +1,188 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+import java.util.HashMap;
+
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.N5Reader;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata.OmeNgffDataset;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
+
+import bdv.img.cache.VolatileCachedCellImg;
+import bdv.img.n5.N5Properties;
+import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.sequence.TimePoint;
+import mpicbg.spim.data.sequence.ViewId;
+import net.imglib2.RandomAccessibleInterval;
+import net.imglib2.type.NativeType;
+import net.imglib2.view.Views;
+
+public class AllenOMEZarrProperties implements N5Properties
+{
+	final AllenOMEZarrLoader loader;
+
+	final HashMap< Integer, double[][] > setupIdToMultiRes;
+	final HashMap< Integer, DataType > setupIdToDataType;
+	final HashMap< String, DatasetAttributes > pathToDatasetAttributes;
+
+	public AllenOMEZarrProperties( final AllenOMEZarrLoader loader )
+	{
+		this.loader = loader;
+
+		this.setupIdToMultiRes = new HashMap<>();
+		this.setupIdToDataType = new HashMap<>();
+		this.pathToDatasetAttributes = new HashMap<>();
+	}
+
+	@Override
+	public String getPath( final int setupId )
+	{
+		// that does not really exist
+		return "";
+	}
+
+	@Override
+	public String getPath( final int setupId, final int timepointId )
+	{
+		return loader.viewIdToPath.get( new ViewId( timepointId, setupId ) );
+	}
+
+	@Override
+	public String getPath( final int setupId, final int timepointId, final int level )
+	{
+		return String.format( getPath( setupId, timepointId )+ "/%d", level );
+	}
+
+	@Override
+	public DataType getDataType( final N5Reader n5, final int setupId )
+	{
+		if ( !setupIdToDataType.containsKey( setupId ) )
+		{
+			synchronized ( this )
+			{
+				if ( !setupIdToDataType.containsKey( setupId ) )
+				{
+					setupIdToDataType.put( setupId, getDataType( this, n5, setupId ) );
+				}
+			}
+		}
+
+		return setupIdToDataType.get( setupId );
+	}
+
+	@Override
+	public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
+	{
+		if ( !setupIdToMultiRes.containsKey( setupId ) )
+		{
+			synchronized ( this )
+			{
+				if ( !setupIdToMultiRes.containsKey( setupId ) )
+				{
+					setupIdToMultiRes.put( setupId, getMipMapResolutions( this, n5, setupId ) );
+				}
+			}
+		}
+
+		return setupIdToMultiRes.get( setupId );
+	}
+
+	@Override
+	public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
+			final VolatileCachedCellImg<T, ?> img,
+			final int setupId,
+			final int timepointId)
+	{
+		return Views.hyperSlice( Views.hyperSlice( img, 4, 0 ), 3, 0 );
+	}
+
+	@Override
+	public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String pathName )
+	{
+		// attributes are cached by the N5 API, so this is technically not necessary ... maybe later if we store it in the XML
+		if ( !pathToDatasetAttributes.containsKey( pathName ) )
+		{
+			synchronized ( this )
+			{
+				if ( !pathToDatasetAttributes.containsKey( pathName ) )
+				{
+					pathToDatasetAttributes.put( pathName, n5.getDatasetAttributes( pathName ) );
+				}
+			}
+		}
+
+		return pathToDatasetAttributes.get( pathName );
+	}
+
+	//
+	// static methods
+	//
+
+	public static int getFirstAvailableTimepointId( final AbstractSequenceDescription< ?, ?, ? > seq, final int setupId )
+	{
+		for ( final TimePoint tp : seq.getTimePoints().getTimePointsOrdered() )
+		{
+			if ( !seq.getMissingViews().getMissingViews().contains( new ViewId( tp.getId(), setupId ) ) )
+				return tp.getId();
+		}
+
+		throw new RuntimeException( "All timepoints for setupId " + setupId + " are declared missing. Stopping." );
+	}
+
+	public static DataType getDataType( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
+	{
+		final int timePointId = getFirstAvailableTimepointId( n5properties.loader.getSequenceDescription(), setupId );
+		return n5.getDatasetAttributes( n5properties.getPath( setupId, timePointId, 0 ) ).getDataType();
+	}
+
+	public static double[][] getMipMapResolutions( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
+	{
+		final int timePointId = getFirstAvailableTimepointId( n5properties.loader.getSequenceDescription(), setupId );
+
+		// multiresolution pyramid
+
+		//org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata
+		// for this to work you need to register an adapter in the N5Factory class
+		// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
+		final OmeNgffMultiScaleMetadata[] multiscales = n5.getAttribute( n5properties.getPath( setupId, timePointId ), "multiscales", OmeNgffMultiScaleMetadata[].class );
+
+		if ( multiscales == null || multiscales.length == 0 )
+			throw new RuntimeException( "Could not parse OME-ZARR multiscales object. stopping." );
+
+		if ( multiscales.length != 1 )
+			System.out.println( "This dataset has " + multiscales.length + " objects, we expected 1. Picking the first one." );
+
+		//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": found " + multiscales[ 0 ].datasets.length + " multi-resolution levels." );
+
+		double[][] mipMapResolutions = new double[ multiscales[ 0 ].datasets.length ][ 3 ];
+		double[] firstScale = null;
+
+		for ( int i = 0; i < multiscales[ 0 ].datasets.length; ++i )
+		{
+			final OmeNgffDataset ds = multiscales[ 0 ].datasets[ i ];
+
+			for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
+			{
+				if ( ScaleCoordinateTransformation.class.isInstance( c ) )
+				{
+					final ScaleCoordinateTransformation s = (ScaleCoordinateTransformation)c;
+
+					if ( firstScale == null )
+						firstScale = s.getScale().clone();
+
+					for ( int d = 0; d < mipMapResolutions[ i ].length; ++d )
+					{
+						mipMapResolutions[ i ][ d ] = s.getScale()[ d ] / firstScale[ d ];
+						mipMapResolutions[ i ][ d ] = Math.round(mipMapResolutions[ i ][ d ]*10000)/10000d; // round to the 5th digit
+					}
+					//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions(), level " + i + ": " + Arrays.toString( s.getScale() ) + " >> " + Arrays.toString( mipMapResolutions[ i ] ) );
+				}
+			}
+		}
+
+		return mipMapResolutions;
+	}
+
+}

From 91179104bcb5eed15cb68631ffb75c1b9b011d71 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Thu, 31 Oct 2024 13:59:51 -0400
Subject: [PATCH 12/24] significantly simplify code to make use of generic
 N5ImageLoader code

---
 .../imgloaders/AllenOMEZarrLoader.java        | 62 -------------------
 .../imgloaders/AllenOMEZarrProperties.java    | 51 +--------------
 2 files changed, 3 insertions(+), 110 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index 6d535846..2542b0ad 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -1,10 +1,7 @@
 package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
 
 import java.net.URI;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.concurrent.ForkJoinPool;
-import java.util.stream.IntStream;
 
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
 
@@ -49,71 +46,12 @@ public AllenOMEZarrLoader(
 	public String getBucket() { return bucket; }
 	public String getFolder() { return folder; }
 
-	@Override
-	public AllenOMEZarrProperties getN5properties()
-	{
-		return (AllenOMEZarrProperties)n5properties;
-	}
-
 	@Override
 	public N5Properties createN5PropertiesInstance()
 	{
 		return new AllenOMEZarrProperties( this );
 	}
 
-	@Override
-	public void preFetch()
-	{
-		final AllenOMEZarrProperties n5p = getN5properties();
-
-		// assemble metadata in advance in parallel (we should store this to the XML)
-		final ForkJoinPool myPool = new ForkJoinPool( cloudThreads );
-
-		System.out.println( "Loading metadata and pre-fetching all DatasetAttributes from OME-ZARR containers (in parallel) ... " );
-
-		// fetch multiresolution info for all viewsetups
-		myPool.submit(() -> seq.getViewSetupsOrdered().parallelStream().forEach( setup ->
-			n5p.setupIdToMultiRes.put(
-				setup.getId(),
-				AllenOMEZarrProperties.getMipMapResolutions(
-					n5p,
-					n5,
-					setup.getId() ) ) ) ).join();
-
-		// fetch all DatasetAttributes
-		myPool.submit(() -> viewIdToPath.keySet().parallelStream().forEach( viewId ->
-		{
-			if ( !seq.getMissingViews().getMissingViews().contains( viewId ) )
-			{
-				final int numLevels = n5properties.getMipmapResolutions( n5, viewId.getViewSetupId() ).length;
-
-				myPool.submit(() -> IntStream.range( 0, numLevels ).parallel().forEach( level ->
-				{
-					final String path = n5properties.getPath(
-						viewId.getViewSetupId(),
-						viewId.getTimePointId(),
-						level );
-
-					n5p.pathToDatasetAttributes.put( path, n5.getDatasetAttributes( path ) );
-				}) ).join();
-			}
-		})).join();
-
-		// fill up DataType from pre-fetched DatasetAttributes
-		for ( final ViewId viewId : viewIdToPath.keySet() )
-			n5p.setupIdToDataType.computeIfAbsent( viewId.getViewSetupId(), setupId ->
-				n5p.pathToDatasetAttributes.get(
-					n5properties.getPath(
-						viewId.getViewSetupId(),
-						viewId.getTimePointId(),
-						0 ) ).getDataType() );
-
-		myPool.shutdown();
-
-		seq.getViewSetupsOrdered().stream().forEach( setup ->
-			System.out.println( "ViewSetupId: " + setup.getId() + ": " + n5p.setupIdToDataType.get( setup.getId() ) + ", " + Arrays.deepToString( n5p.setupIdToMultiRes.get( setup.getId() ) ) ) );
-	}
-
 	public static void main( String[] args ) throws SpimDataException
 	{
 		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
index 70687900..4ae6be06 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
@@ -1,7 +1,5 @@
 package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
 
-import java.util.HashMap;
-
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
 import org.janelia.saalfeldlab.n5.N5Reader;
@@ -23,17 +21,9 @@ public class AllenOMEZarrProperties implements N5Properties
 {
 	final AllenOMEZarrLoader loader;
 
-	final HashMap< Integer, double[][] > setupIdToMultiRes;
-	final HashMap< Integer, DataType > setupIdToDataType;
-	final HashMap< String, DatasetAttributes > pathToDatasetAttributes;
-
 	public AllenOMEZarrProperties( final AllenOMEZarrLoader loader )
 	{
 		this.loader = loader;
-
-		this.setupIdToMultiRes = new HashMap<>();
-		this.setupIdToDataType = new HashMap<>();
-		this.pathToDatasetAttributes = new HashMap<>();
 	}
 
 	@Override
@@ -58,35 +48,13 @@ public String getPath( final int setupId, final int timepointId, final int level
 	@Override
 	public DataType getDataType( final N5Reader n5, final int setupId )
 	{
-		if ( !setupIdToDataType.containsKey( setupId ) )
-		{
-			synchronized ( this )
-			{
-				if ( !setupIdToDataType.containsKey( setupId ) )
-				{
-					setupIdToDataType.put( setupId, getDataType( this, n5, setupId ) );
-				}
-			}
-		}
-
-		return setupIdToDataType.get( setupId );
+		return getDataType( this, n5, setupId );
 	}
 
 	@Override
 	public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
 	{
-		if ( !setupIdToMultiRes.containsKey( setupId ) )
-		{
-			synchronized ( this )
-			{
-				if ( !setupIdToMultiRes.containsKey( setupId ) )
-				{
-					setupIdToMultiRes.put( setupId, getMipMapResolutions( this, n5, setupId ) );
-				}
-			}
-		}
-
-		return setupIdToMultiRes.get( setupId );
+		return getMipMapResolutions( this, n5, setupId );
 	}
 
 	@Override
@@ -101,19 +69,7 @@ public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
 	@Override
 	public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String pathName )
 	{
-		// attributes are cached by the N5 API, so this is technically not necessary ... maybe later if we store it in the XML
-		if ( !pathToDatasetAttributes.containsKey( pathName ) )
-		{
-			synchronized ( this )
-			{
-				if ( !pathToDatasetAttributes.containsKey( pathName ) )
-				{
-					pathToDatasetAttributes.put( pathName, n5.getDatasetAttributes( pathName ) );
-				}
-			}
-		}
-
-		return pathToDatasetAttributes.get( pathName );
+		return n5.getDatasetAttributes( pathName );
 	}
 
 	//
@@ -184,5 +140,4 @@ public static double[][] getMipMapResolutions( final AllenOMEZarrProperties n5pr
 
 		return mipMapResolutions;
 	}
-
 }

From 66ceb36e2fdec08245b7c8f614c12683c3d6c011 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Sat, 2 Nov 2024 11:09:58 -0400
Subject: [PATCH 13/24] fix small bug (with no consequences)

---
 .../java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java  | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java
index 4ece0f95..324ead81 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java
@@ -207,11 +207,6 @@ public static boolean split( final SpimData2 data, final URI filePath )
 		else
 			assignIllum = false;
 
-		gd.addMessage( "" );
-
-		if ( data.getSequenceDescription().getAllIlluminationsOrdered().size() == 1 )
-			gd.addCheckbox( "Assign_old_tiles_as_illuminations (great for visualization)", defaultAssignIlluminations );
-
 		final String saveAs = gd.getNextString();
 		final int choice = defaultChoice = gd.getNextChoiceIndex();
 

From c2097c602a00abe6b24384e7f5e75cb5fd50a845 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Sat, 2 Nov 2024 11:16:49 -0400
Subject: [PATCH 14/24] generalize SplitViewerImgLoader

---
 .../splitting/SplitViewerImgLoader.java       |  12 +-
 .../splitting/SplitViewerSetupImgLoader.java  | 186 ++----------------
 2 files changed, 17 insertions(+), 181 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
index 8139aa4d..364e4360 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
@@ -35,8 +35,6 @@
 import mpicbg.spim.data.sequence.SequenceDescription;
 import net.imglib2.Interval;
 import net.imglib2.cache.queue.BlockingFetchQueues;
-import net.imglib2.type.numeric.integer.UnsignedShortType;
-import net.imglib2.type.volatiles.VolatileUnsignedShortType;
 
 public class SplitViewerImgLoader implements ViewerImgLoader, MultiResolutionImgLoader
 {
@@ -94,11 +92,11 @@ private final synchronized SplitViewerSetupImgLoader getSplitViewerSetupImgLoade
 		if ( sil == null )
 		{
 			final ViewerSetupImgLoader setupLoader = underlyingImgLoader.getSetupImgLoader( oldSetupId );
-			final Object imgType = setupLoader.getImageType();
-			final Object volTyoe = setupLoader.getVolatileImageType();
+			//final Object imgType = setupLoader.getImageType();
+			//final Object volTyoe = setupLoader.getVolatileImageType();
 
-			if ( !imgType.getClass().isInstance( new UnsignedShortType() ) || !volTyoe.getClass().isInstance( new VolatileUnsignedShortType() ) )
-				throw new RuntimeException( "The underlying ViewerSetupImgLoader is not typed for <UnsignedShortType, VolatileUnsignedShortType>, cannot split up for BDV." );
+			//if ( !imgType.getClass().isInstance( new UnsignedShortType() ) || !volTyoe.getClass().isInstance( new VolatileUnsignedShortType() ) )
+			//	throw new RuntimeException( "The underlying ViewerSetupImgLoader is not typed for <UnsignedShortType, VolatileUnsignedShortType>, cannot split up for BDV." );
 
 			sil = createNewSetupImgLoader( (ViewerSetupImgLoader)underlyingImgLoader.getSetupImgLoader( oldSetupId ), interval );
 			splitSetupImgLoaders.put( newSetupId, sil );
@@ -107,7 +105,7 @@ private final synchronized SplitViewerSetupImgLoader getSplitViewerSetupImgLoade
 	}
 
 	private final synchronized SplitViewerSetupImgLoader createNewSetupImgLoader(
-			final ViewerSetupImgLoader< UnsignedShortType, VolatileUnsignedShortType > setupImgLoader,
+			final ViewerSetupImgLoader setupImgLoader,
 			final Interval interval )
 	{
 		return new SplitViewerSetupImgLoader( setupImgLoader, interval );
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
index 138098eb..1f61f02a 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
@@ -22,39 +22,24 @@
  */
 package net.preibisch.mvrecon.fiji.spimdata.imgloaders.splitting;
 
-import java.util.ArrayList;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
 import bdv.ViewerSetupImgLoader;
 import mpicbg.spim.data.generic.sequence.ImgLoaderHint;
 import mpicbg.spim.data.sequence.MultiResolutionSetupImgLoader;
 import mpicbg.spim.data.sequence.VoxelDimensions;
-import net.imglib2.Cursor;
 import net.imglib2.Dimensions;
 import net.imglib2.FinalDimensions;
 import net.imglib2.FinalInterval;
 import net.imglib2.Interval;
-import net.imglib2.IterableInterval;
-import net.imglib2.RandomAccess;
 import net.imglib2.RandomAccessibleInterval;
-import net.imglib2.img.Img;
-import net.imglib2.img.ImgFactory;
-import net.imglib2.img.array.ArrayImgFactory;
-import net.imglib2.img.cell.CellImgFactory;
+import net.imglib2.Volatile;
 import net.imglib2.realtransform.AffineTransform3D;
-import net.imglib2.type.numeric.integer.UnsignedShortType;
-import net.imglib2.type.numeric.real.FloatType;
-import net.imglib2.type.volatiles.VolatileUnsignedShortType;
-import net.imglib2.util.Intervals;
+import net.imglib2.type.NativeType;
 import net.imglib2.view.Views;
 import net.preibisch.legacy.io.IOFunctions;
-import util.ImgLib2Tools;
 
-public class SplitViewerSetupImgLoader implements ViewerSetupImgLoader< UnsignedShortType, VolatileUnsignedShortType >, MultiResolutionSetupImgLoader< UnsignedShortType >
+public class SplitViewerSetupImgLoader< T extends NativeType< T >, V extends Volatile< T > & NativeType< V > > implements ViewerSetupImgLoader< T, V >, MultiResolutionSetupImgLoader< T >
 {
-	final ViewerSetupImgLoader< UnsignedShortType, VolatileUnsignedShortType > underlyingSetupImgLoader;
+	final ViewerSetupImgLoader< T, V > underlyingSetupImgLoader;
 	final Interval interval;
 	final Dimensions size;
 	final int n;
@@ -66,7 +51,7 @@ public class SplitViewerSetupImgLoader implements ViewerSetupImgLoader< Unsigned
 
 	private boolean[] isUpdated;
 
-	public SplitViewerSetupImgLoader( final ViewerSetupImgLoader< UnsignedShortType, VolatileUnsignedShortType > underlyingSetupImgLoader, final Interval interval )
+	public SplitViewerSetupImgLoader( final ViewerSetupImgLoader< T, V > underlyingSetupImgLoader, final Interval interval )
 	{
 		this.underlyingSetupImgLoader = underlyingSetupImgLoader;
 		this.interval = interval;
@@ -144,7 +129,7 @@ protected static final void setUpMultiRes(
 	}
 
 	@Override
-	public RandomAccessibleInterval< UnsignedShortType > getImage( final int timepointId, final ImgLoaderHint... hints )
+	public RandomAccessibleInterval< T > getImage( final int timepointId, final ImgLoaderHint... hints )
 	{
 		IOFunctions.println( "requesting full size: " );
 
@@ -152,20 +137,11 @@ public RandomAccessibleInterval< UnsignedShortType > getImage( final int timepoi
 	}
 
 	@Override
-	public UnsignedShortType getImageType()
+	public T getImageType()
 	{
 		return underlyingSetupImgLoader.getImageType();
 	}
 
-	@Override
-	public RandomAccessibleInterval< FloatType > getFloatImage( final int timepointId, final boolean normalize, final ImgLoaderHint... hints )
-	{
-		if ( normalize )
-			return ImgLib2Tools.normalizeVirtualRAI( getImage( timepointId, hints ) );
-		else
-			return ImgLib2Tools.convertVirtualRAI( getImage( timepointId, hints ) );
-	}
-
 	@Override
 	public Dimensions getImageSize( final int timepointId )
 	{
@@ -179,7 +155,7 @@ public VoxelDimensions getVoxelSize( final int timepointId )
 	}
 
 	@Override
-	public RandomAccessibleInterval< UnsignedShortType > getImage( final int timepointId, final int level, final ImgLoaderHint... hints )
+	public RandomAccessibleInterval< T > getImage( final int timepointId, final int level, final ImgLoaderHint... hints )
 	{
 		/*
 		System.out.println( "requesting: " + level );
@@ -210,7 +186,7 @@ public RandomAccessibleInterval< UnsignedShortType > getImage( final int timepoi
 		System.out.println( "size: " + Util.printInterval( img ) );
 		System.out.println( "interval: " + Util.printInterval( scaledIntervals[ level ] ) ); */
 
-		final RandomAccessibleInterval< UnsignedShortType > full = underlyingSetupImgLoader.getImage( timepointId, level, hints );
+		final RandomAccessibleInterval< T > full = underlyingSetupImgLoader.getImage( timepointId, level, hints );
 
 		updateScaledIntervals( this.scaledIntervals, level, n, full );
 
@@ -260,9 +236,9 @@ protected final void updateScaledIntervals( final Interval[] scaledIntervals, fi
 	}
 
 	@Override
-	public RandomAccessibleInterval< VolatileUnsignedShortType > getVolatileImage( final int timepointId, final int level, final ImgLoaderHint... hints )
+	public RandomAccessibleInterval< V > getVolatileImage( final int timepointId, final int level, final ImgLoaderHint... hints )
 	{
-		final RandomAccessibleInterval< VolatileUnsignedShortType > full = underlyingSetupImgLoader.getVolatileImage( timepointId, level, hints );
+		final RandomAccessibleInterval< V > full = underlyingSetupImgLoader.getVolatileImage( timepointId, level, hints );
 
 		updateScaledIntervals( this.scaledIntervals, level, n, full );
 
@@ -287,144 +263,6 @@ public int numMipmapLevels()
 		return underlyingSetupImgLoader.numMipmapLevels();
 	}
 
-	@Override
-	public RandomAccessibleInterval< FloatType > getFloatImage( final int timepointId, final int level, final boolean normalize, final ImgLoaderHint... hints )
-	{
-		final RandomAccessibleInterval< UnsignedShortType > ushortImg = getImage( timepointId, level, hints );
-
-		// copy unsigned short img to float img
-
-		// create float img
-		final FloatType f = new FloatType();
-		final ImgFactory< FloatType > imgFactory;
-		if ( Intervals.numElements( ushortImg ) <= Integer.MAX_VALUE )
-		{
-			imgFactory = new ArrayImgFactory<>( f );
-		}
-		else
-		{
-			final long[] dimsLong = new long[ ushortImg.numDimensions() ];
-			ushortImg.dimensions( dimsLong );
-			final int[] cellDimensions = new int[ dimsLong.length ];
-			for ( int i = 0; i < cellDimensions.length; ++i )
-				cellDimensions[ i ] = 64;
-
-			imgFactory = new CellImgFactory<>( f, cellDimensions );
-		}
-		final Img< FloatType > floatImg = imgFactory.create( ushortImg );
-
-		// set up executor service
-		final int numProcessors = Runtime.getRuntime().availableProcessors();
-		final ExecutorService taskExecutor = Executors.newFixedThreadPool( numProcessors );
-		final ArrayList< Callable< Void > > tasks = new ArrayList<>();
-
-		// set up all tasks
-		final int numPortions = numProcessors * 2;
-		final long threadChunkSize = floatImg.size() / numPortions;
-		final long threadChunkMod = floatImg.size() % numPortions;
-
-		for ( int portionID = 0; portionID < numPortions; ++portionID )
-		{
-			// move to the starting position of the current thread
-			final long startPosition = portionID * threadChunkSize;
-
-			// the last thread may has to run longer if the number of pixels cannot be divided by the number of threads
-			final long loopSize = ( portionID == numPortions - 1 ) ? threadChunkSize + threadChunkMod : threadChunkSize;
-
-			if ( Views.iterable( ushortImg ).iterationOrder().equals( floatImg.iterationOrder() ) )
-			{
-				tasks.add( new Callable< Void >()
-				{
-					@Override
-					public Void call() throws Exception
-					{
-						final Cursor< UnsignedShortType > in = Views.iterable( ushortImg ).cursor();
-						final Cursor< FloatType > out = floatImg.cursor();
-
-						in.jumpFwd( startPosition );
-						out.jumpFwd( startPosition );
-
-						for ( long j = 0; j < loopSize; ++j )
-							out.next().set( in.next().getRealFloat() );
-
-						return null;
-					}
-				} );
-			}
-			else
-			{
-				tasks.add( new Callable< Void >()
-				{
-					@Override
-					public Void call() throws Exception
-					{
-						final Cursor< UnsignedShortType > in = Views.iterable( ushortImg ).localizingCursor();
-						final RandomAccess< FloatType > out = floatImg.randomAccess();
-
-						in.jumpFwd( startPosition );
-
-						for ( long j = 0; j < loopSize; ++j )
-						{
-							final UnsignedShortType vin = in.next();
-							out.setPosition( in );
-							out.get().set( vin.getRealFloat() );
-						}
-
-						return null;
-					}
-				} );
-			}
-		}
-
-		try
-		{
-			// invokeAll() returns when all tasks are complete
-			taskExecutor.invokeAll( tasks );
-			taskExecutor.shutdown();
-		}
-		catch ( final InterruptedException e )
-		{
-			return null;
-		}
-
-		if ( normalize )
-			// normalize the image to 0...1
-			normalize( floatImg );
-
-		return floatImg;
-	}
-
-	/*
-	 * normalize img to 0...1
-	 */
-	protected static void normalize( final IterableInterval< FloatType > img )
-	{
-		float currentMax = img.firstElement().get();
-		float currentMin = currentMax;
-		for ( final FloatType t : img )
-		{
-			final float f = t.get();
-			if ( f > currentMax )
-				currentMax = f;
-			else if ( f < currentMin )
-				currentMin = f;
-		}
-
-		final float scale = ( float ) ( 1.0 / ( currentMax - currentMin ) );
-		for ( final FloatType t : img )
-			t.set( ( t.get() - currentMin ) * scale );
-
-		/*
-		 * Once we do not need Img's for DoG anymore ....
-
-		final RandomAccessibleInterval< UnsignedShortType > image = getImage( timepointId, level, hints );
-		final RandomAccessibleInterval< FloatType > floatImg = Converters.convert( image, new RealFloatConverter< UnsignedShortType >(), new FloatType() );
-		if (normalize)
-			AbstractImgLoader.normalize( floatImg );
-		return floatImg;
-		*/
-	}
-
 	@Override
 	public Dimensions getImageSize( final int timepointId, final int level )
 	{
@@ -432,7 +270,7 @@ public Dimensions getImageSize( final int timepointId, final int level )
 	}
 
 	@Override
-	public VolatileUnsignedShortType getVolatileImageType()
+	public V getVolatileImageType()
 	{
 		return underlyingSetupImgLoader.getVolatileImageType();
 	}

From 49f59360f4cf47bad8faaddbc84efe8bf3ce2b49 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Sun, 3 Nov 2024 08:10:54 -0500
Subject: [PATCH 15/24] remove outdated methods, comment code

---
 .../SplitMultiResolutionSetupImgLoader.java   | 48 ++-----------------
 1 file changed, 3 insertions(+), 45 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java
index a74cad5a..2eccc7b8 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java
@@ -34,7 +34,6 @@
 import net.imglib2.type.numeric.real.FloatType;
 import net.imglib2.view.Views;
 import net.preibisch.legacy.io.IOFunctions;
-import util.ImgLib2Tools;
 
 public class SplitMultiResolutionSetupImgLoader< T > implements MultiResolutionSetupImgLoader< T >
 {
@@ -88,22 +87,6 @@ public T getImageType()
 		return underlyingSetupImgLoader.getImageType();
 	}
 
-	@Override
-	public RandomAccessibleInterval< FloatType > getFloatImage( final int timepointId, final boolean normalize, final ImgLoaderHint... hints )
-	{
-		final RandomAccessibleInterval< FloatType > img = Views.zeroMin( Views.interval( underlyingSetupImgLoader.getFloatImage( timepointId, false, hints ), interval ) );
-
-		// TODO: this is stupid, remove capablitity to get FloatType images!
-		if ( normalize )
-		{
-			return ImgLib2Tools.normalizeVirtualRAI( img );
-		}
-		else
-		{
-			return img;
-		}
-	}
-
 	@Override
 	public Dimensions getImageSize( final int timepointId )
 	{
@@ -119,36 +102,11 @@ public VoxelDimensions getVoxelSize( final int timepointId )
 	@Override
 	public RandomAccessibleInterval< T > getImage( final int timepointId, final int level, final ImgLoaderHint... hints )
 	{
-		/*IOFunctions.println( "requesting: " + level );
+		final RandomAccessibleInterval< T > full = underlyingSetupImgLoader.getImage( timepointId, level, hints );
 
-		for ( int l = 0; l < mipmapResolutions.length; ++l )
-		{
-			System.out.println( "level " + l + ": " + mipmapTransforms[ l ] );
-			System.out.println( "level " + l + ": " + Util.printInterval( scaledIntervals[ l ] ) );
-			System.out.print( "level " + l + ": " );
-			for ( int d = 0; d < mipmapResolutions[ l ].length; ++d )
-				System.out.print( mipmapResolutions[ l ][ d ] + "x" );
-			System.out.println();
-		}
-		/*
-		if ( level == 0 )
-		{
-			return getImage( timepointId, hints );
-		}
-		else*/
-		{
-			//final RandomAccessibleInterval img = Views.zeroMin( Views.interval(underlyingSetupImgLoader.getImage( timepointId, level, hints ), scaledIntervals[ level ] ) );
-			//DisplayImage.getImagePlusInstance( img, false, "level=" + level, 0.0, 255.0 ).show();;
+		updateScaledIntervals( this.scaledIntervals, level, n, full );
 
-			//IOFunctions.println( "size: " + Util.printInterval( img ) );
-			//IOFunctions.println( "interval: " + Util.printInterval( scaledIntervals[ level ] ) );
-
-			final RandomAccessibleInterval< T > full = underlyingSetupImgLoader.getImage( timepointId, level, hints );
-
-			updateScaledIntervals( this.scaledIntervals, level, n, full );
-
-			return Views.zeroMin( Views.interval( full, scaledIntervals[ level ] ) );
-		}
+		return Views.zeroMin( Views.interval( full, scaledIntervals[ level ] ) );
 	}
 
 	/**

From 53860af89ffdec76140f57ee661c82548d06259f Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 5 Nov 2024 11:08:00 -0500
Subject: [PATCH 16/24] remove unnecessary debug msgs

---
 .../splitting/SplitMultiResolutionSetupImgLoader.java          | 3 ---
 .../imgloaders/splitting/SplitViewerSetupImgLoader.java        | 2 --
 2 files changed, 5 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java
index 2eccc7b8..02d13888 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitMultiResolutionSetupImgLoader.java
@@ -33,7 +33,6 @@
 import net.imglib2.realtransform.AffineTransform3D;
 import net.imglib2.type.numeric.real.FloatType;
 import net.imglib2.view.Views;
-import net.preibisch.legacy.io.IOFunctions;
 
 public class SplitMultiResolutionSetupImgLoader< T > implements MultiResolutionSetupImgLoader< T >
 {
@@ -76,8 +75,6 @@ public SplitMultiResolutionSetupImgLoader( final MultiResolutionSetupImgLoader<
 	@Override
 	public RandomAccessibleInterval< T > getImage( final int timepointId, final ImgLoaderHint... hints )
 	{
-		IOFunctions.println( "requesting full size: " );
-
 		return Views.zeroMin( Views.interval( underlyingSetupImgLoader.getImage( timepointId, hints ), interval ) );
 	}
 
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
index 1f61f02a..962604b7 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
@@ -131,8 +131,6 @@ protected static final void setUpMultiRes(
 	@Override
 	public RandomAccessibleInterval< T > getImage( final int timepointId, final ImgLoaderHint... hints )
 	{
-		IOFunctions.println( "requesting full size: " );
-
 		return Views.zeroMin( Views.interval( underlyingSetupImgLoader.getImage( timepointId, hints ), interval ) );
 	}
 

From f48b2a93e5bcc1226dc506cdacf248ec312f7d57 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 5 Nov 2024 11:19:04 -0500
Subject: [PATCH 17/24] support more fetcher threads for
 SplitViewerImgLoader.java fix generics

---
 .../splitting/SplitViewerImgLoader.java       | 34 ++++++++++---------
 .../splitting/SplitViewerSetupImgLoader.java  |  1 -
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
index 364e4360..fe07cfe1 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
@@ -35,6 +35,7 @@
 import mpicbg.spim.data.sequence.SequenceDescription;
 import net.imglib2.Interval;
 import net.imglib2.cache.queue.BlockingFetchQueues;
+import net.imglib2.util.Cast;
 
 public class SplitViewerImgLoader implements ViewerImgLoader, MultiResolutionImgLoader
 {
@@ -58,13 +59,15 @@ public class SplitViewerImgLoader implements ViewerImgLoader, MultiResolutionImg
 	/**
 	 * Remembers instances of SplitSetupImgLoader
 	 */
-	private final HashMap< Integer, SplitViewerSetupImgLoader > splitSetupImgLoaders;
+	private final HashMap< Integer, SplitViewerSetupImgLoader<?,?> > splitSetupImgLoaders;
 
 	/**
 	 * Its own cell cache
 	 */
 	protected VolatileGlobalCellCache cache;
 
+	private int requestedNumFetcherThreads = -1;
+
 	public SplitViewerImgLoader(
 			final ViewerImgLoader underlyingImgLoader,
 			final HashMap< Integer, Integer > new2oldSetupId,
@@ -81,34 +84,33 @@ public SplitViewerImgLoader(
 	private boolean isOpen = false;
 
 	@Override
-	public SplitViewerSetupImgLoader getSetupImgLoader( final int setupId )
+	public SplitViewerSetupImgLoader<?,?> getSetupImgLoader( final int setupId )
 	{
 		return getSplitViewerSetupImgLoader( underlyingImgLoader, new2oldSetupId.get( setupId ), setupId, newSetupId2Interval.get( setupId ) );
 	}
 
-	private final synchronized SplitViewerSetupImgLoader getSplitViewerSetupImgLoader( final ViewerImgLoader underlyingImgLoader, final int oldSetupId, final int newSetupId, final Interval interval )
+	private final synchronized SplitViewerSetupImgLoader<?,?> getSplitViewerSetupImgLoader( final ViewerImgLoader underlyingImgLoader, final int oldSetupId, final int newSetupId, final Interval interval )
 	{
-		SplitViewerSetupImgLoader sil = splitSetupImgLoaders.get( newSetupId );
+		SplitViewerSetupImgLoader<?,?> sil = splitSetupImgLoaders.get( newSetupId );
 		if ( sil == null )
 		{
-			final ViewerSetupImgLoader setupLoader = underlyingImgLoader.getSetupImgLoader( oldSetupId );
-			//final Object imgType = setupLoader.getImageType();
-			//final Object volTyoe = setupLoader.getVolatileImageType();
-
-			//if ( !imgType.getClass().isInstance( new UnsignedShortType() ) || !volTyoe.getClass().isInstance( new VolatileUnsignedShortType() ) )
-			//	throw new RuntimeException( "The underlying ViewerSetupImgLoader is not typed for <UnsignedShortType, VolatileUnsignedShortType>, cannot split up for BDV." );
-
-			sil = createNewSetupImgLoader( (ViewerSetupImgLoader)underlyingImgLoader.getSetupImgLoader( oldSetupId ), interval );
+			sil = createNewSetupImgLoader( (ViewerSetupImgLoader<?,?>)underlyingImgLoader.getSetupImgLoader( oldSetupId ), interval );
 			splitSetupImgLoaders.put( newSetupId, sil );
 		}
 		return sil;
 	}
 
-	private final synchronized SplitViewerSetupImgLoader createNewSetupImgLoader(
-			final ViewerSetupImgLoader setupImgLoader,
+	private final synchronized SplitViewerSetupImgLoader<?,?> createNewSetupImgLoader(
+			final ViewerSetupImgLoader<?,?> setupImgLoader,
 			final Interval interval )
 	{
-		return new SplitViewerSetupImgLoader( setupImgLoader, interval );
+		return new SplitViewerSetupImgLoader<>( Cast.unchecked( setupImgLoader ), interval );
+	}
+
+	@Override
+	public synchronized void setNumFetcherThreads( final int n )
+	{
+		requestedNumFetcherThreads = n;
 	}
 
 	private void open()
@@ -132,7 +134,7 @@ private void open()
 						maxNumLevels = resolutions.length;
 				}
 
-				final int numFetcherThreads = 1;
+				final int numFetcherThreads = ( requestedNumFetcherThreads > 0 ) ? requestedNumFetcherThreads : Runtime.getRuntime().availableProcessors();
 				final BlockingFetchQueues< Callable< ? > > queue = new BlockingFetchQueues<>( maxNumLevels, numFetcherThreads );
 				cache = new VolatileGlobalCellCache( queue );
 			}
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
index 962604b7..d6964eb9 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerSetupImgLoader.java
@@ -35,7 +35,6 @@
 import net.imglib2.realtransform.AffineTransform3D;
 import net.imglib2.type.NativeType;
 import net.imglib2.view.Views;
-import net.preibisch.legacy.io.IOFunctions;
 
 public class SplitViewerSetupImgLoader< T extends NativeType< T >, V extends Volatile< T > & NativeType< V > > implements ViewerSetupImgLoader< T, V >, MultiResolutionSetupImgLoader< T >
 {

From da3725b1d0e671d7401e94c6e08016335f86e858 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 5 Nov 2024 11:23:04 -0500
Subject: [PATCH 18/24] propagate num fetcher threads downstream

---
 .../fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
index fe07cfe1..f38c47c4 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
@@ -111,6 +111,7 @@ private final synchronized SplitViewerSetupImgLoader<?,?> createNewSetupImgLoade
 	public synchronized void setNumFetcherThreads( final int n )
 	{
 		requestedNumFetcherThreads = n;
+		underlyingImgLoader.setNumFetcherThreads( n );
 	}
 
 	private void open()

From 74052cfc16cb245a24805644585dc5882a6c5639 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Tue, 5 Nov 2024 11:23:36 -0500
Subject: [PATCH 19/24] set fetcher threads

---
 .../mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index 2542b0ad..0c367953 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -5,6 +5,7 @@
 
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
 
+import bdv.ViewerImgLoader;
 import bdv.img.n5.N5ImageLoader;
 import bdv.img.n5.N5Properties;
 import ij.ImageJ;
@@ -54,7 +55,7 @@ public N5Properties createN5PropertiesInstance()
 
 	public static void main( String[] args ) throws SpimDataException
 	{
-		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
+		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.split.xml" );
 		//URI xml = URITools.toURI( "s3://janelia-bigstitcher-spark/Stitching/dataset.xml" );
 		//URI xml = URITools.toURI( "gs://janelia-spark-test/I2K-test/dataset.xml" );
 		//URI xml = URITools.toURI( "/Users/preibischs/SparkTest/IP/dataset.xml" );
@@ -62,6 +63,8 @@ public static void main( String[] args ) throws SpimDataException
 		XmlIoSpimData2 io = new XmlIoSpimData2();
 		SpimData2 data = io.load( xml );
 
+		ViewerImgLoader imgL = (ViewerImgLoader)data.getSequenceDescription().getImgLoader();
+		imgL.setNumFetcherThreads( N5ImageLoader.cloudThreads );
 		SetupImgLoader sil = (SetupImgLoader)data.getSequenceDescription().getImgLoader().getSetupImgLoader( 0 );
 
 		final int tp = data.getSequenceDescription().getTimePoints().getTimePointsOrdered().get( 0 ).getId();

From 617ff2067d9b734ce03af3bf8714a79e9aae75c9 Mon Sep 17 00:00:00 2001
From: tpietzsch <tobias.pietzsch@gmail.com>
Date: Wed, 6 Nov 2024 10:55:37 +0100
Subject: [PATCH 20/24] Revise AllenOMEZarrLoader for bigdataviewer-core-10.6.3

---
 pom.xml                                       |  2 +-
 .../imgloaders/AllenOMEZarrLoader.java        | 52 +++++++++++----
 .../imgloaders/AllenOMEZarrProperties.java    | 63 ++++++++-----------
 .../imgloaders/XmlIoAllenOMEZarrLoader.java   | 11 ++--
 4 files changed, 73 insertions(+), 55 deletions(-)

diff --git a/pom.xml b/pom.xml
index 65535a02..83189816 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,7 +114,7 @@ like Selective Plane Illumination Microscopy (SPIM) Data.</description>
 		<n5-imglib2.version>7.0.2</n5-imglib2.version>
 		<bigdataviewer-vistools.version>1.0.0-beta-36</bigdataviewer-vistools.version>
 -->
-		<bigdataviewer-core.version>10.6.3-SNAPSHOT</bigdataviewer-core.version>
+		<bigdataviewer-core.version>10.6.3</bigdataviewer-core.version>
 		<spim_data.version>2.3.5</spim_data.version>
 
 		<!-- from n5-universe 
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index 0c367953..365cf020 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -1,7 +1,7 @@
 package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
 
 import java.net.URI;
-import java.util.HashMap;
+import java.util.Map;
 
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
 
@@ -13,7 +13,9 @@
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
 import mpicbg.spim.data.sequence.ViewId;
 import net.imglib2.RandomAccessibleInterval;
+import net.imglib2.cache.volatiles.CacheHints;
 import net.imglib2.img.display.imagej.ImageJFunctions;
+import net.imglib2.type.NativeType;
 import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer;
@@ -21,18 +23,23 @@
 
 public class AllenOMEZarrLoader extends N5ImageLoader
 {
-	final HashMap< ViewId, String > viewIdToPath;
+	private final AbstractSequenceDescription< ?, ?, ? > sequenceDescription;
 
-	final String bucket, folder;
+	private final Map< ViewId, String > viewIdToPath;
+
+	private final String bucket, folder;
+
+	private static final int cloudThreads = 256;
 
 	public AllenOMEZarrLoader(
 			final URI n5URI,
 			final String bucket,
 			final String folder,
 			final AbstractSequenceDescription< ?, ?, ? > sequenceDescription,
-			final HashMap< ViewId, String > viewIdToPath )
+			final Map< ViewId, String > viewIdToPath )
 	{
 		super( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription );
+		this.sequenceDescription = sequenceDescription;
 
 		setNumFetcherThreads( cloudThreads );
 
@@ -42,15 +49,37 @@ public AllenOMEZarrLoader(
 		this.viewIdToPath = viewIdToPath;
 	}
 
-	public AbstractSequenceDescription< ?, ?, ? > getSequenceDescription() { return seq; }
-	public HashMap< ViewId, String > getViewIdToPath() { return viewIdToPath; }
-	public String getBucket() { return bucket; }
-	public String getFolder() { return folder; }
+	public Map< ViewId, String > getViewIdToPath()
+	{
+		return viewIdToPath;
+	}
+
+	public String getBucket()
+	{
+		return bucket;
+	}
+
+	public String getFolder()
+	{
+		return folder;
+	}
+
+	@Override
+	protected N5Properties createN5PropertiesInstance()
+	{
+		return new AllenOMEZarrProperties( sequenceDescription, viewIdToPath );
+	}
 
 	@Override
-	public N5Properties createN5PropertiesInstance()
+	protected < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage(
+			final String datasetPath,
+			final int setupId, final int timepointId, final int level,
+			final CacheHints cacheHints,
+			final T type )
 	{
-		return new AllenOMEZarrProperties( this );
+		return super.prepareCachedImage( datasetPath, setupId, 0, level, cacheHints, type ).view()
+				.slice( 4, 0 )
+				.slice( 3, 0 );
 	}
 
 	public static void main( String[] args ) throws SpimDataException
@@ -64,7 +93,8 @@ public static void main( String[] args ) throws SpimDataException
 		SpimData2 data = io.load( xml );
 
 		ViewerImgLoader imgL = (ViewerImgLoader)data.getSequenceDescription().getImgLoader();
-		imgL.setNumFetcherThreads( N5ImageLoader.cloudThreads );
+		( ( N5ImageLoader ) imgL ).prefetch( cloudThreads );
+		imgL.setNumFetcherThreads( cloudThreads );
 		SetupImgLoader sil = (SetupImgLoader)data.getSequenceDescription().getImgLoader().getSetupImgLoader( 0 );
 
 		final int tp = data.getSequenceDescription().getTimePoints().getTimePointsOrdered().get( 0 ).getId();
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
index 4ae6be06..e878f1e5 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java
@@ -1,46 +1,41 @@
 package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
 
+import java.util.Arrays;
+import java.util.Map;
+
 import org.janelia.saalfeldlab.n5.DataType;
-import org.janelia.saalfeldlab.n5.DatasetAttributes;
 import org.janelia.saalfeldlab.n5.N5Reader;
 import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata;
 import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata.OmeNgffDataset;
 import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
 import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
 
-import bdv.img.cache.VolatileCachedCellImg;
 import bdv.img.n5.N5Properties;
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
 import mpicbg.spim.data.sequence.TimePoint;
 import mpicbg.spim.data.sequence.ViewId;
-import net.imglib2.RandomAccessibleInterval;
-import net.imglib2.type.NativeType;
-import net.imglib2.view.Views;
 
 public class AllenOMEZarrProperties implements N5Properties
 {
-	final AllenOMEZarrLoader loader;
+	private final AbstractSequenceDescription< ?, ?, ? > sequenceDescription;
 
-	public AllenOMEZarrProperties( final AllenOMEZarrLoader loader )
-	{
-		this.loader = loader;
-	}
+	private final Map< ViewId, String > viewIdToPath;
 
-	@Override
-	public String getPath( final int setupId )
+	public AllenOMEZarrProperties(
+			final AbstractSequenceDescription< ?, ?, ? > sequenceDescription,
+			final Map< ViewId, String > viewIdToPath )
 	{
-		// that does not really exist
-		return "";
+		this.sequenceDescription = sequenceDescription;
+		this.viewIdToPath = viewIdToPath;
 	}
 
-	@Override
-	public String getPath( final int setupId, final int timepointId )
+	private String getPath( final int setupId, final int timepointId )
 	{
-		return loader.viewIdToPath.get( new ViewId( timepointId, setupId ) );
+		return viewIdToPath.get( new ViewId( timepointId, setupId ) );
 	}
 
 	@Override
-	public String getPath( final int setupId, final int timepointId, final int level )
+	public String getDatasetPath( final int setupId, final int timepointId, final int level )
 	{
 		return String.format( getPath( setupId, timepointId )+ "/%d", level );
 	}
@@ -58,25 +53,19 @@ public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
 	}
 
 	@Override
-	public <T extends NativeType<T>> RandomAccessibleInterval<T> extractImg(
-			final VolatileCachedCellImg<T, ?> img,
-			final int setupId,
-			final int timepointId)
-	{
-		return Views.hyperSlice( Views.hyperSlice( img, 4, 0 ), 3, 0 );
-	}
-
-	@Override
-	public DatasetAttributes getDatasetAttributes( final N5Reader n5, final String pathName )
+	public long[] getDimensions( final N5Reader n5, final int setupId, final int timepointId, final int level )
 	{
-		return n5.getDatasetAttributes( pathName );
+		final String path = getDatasetPath( setupId, timepointId, level );
+		final long[] dimensions = n5.getDatasetAttributes( path ).getDimensions();
+		// dataset dimensions is 5D, remove the channel and time dimensions
+		return Arrays.copyOf( dimensions, 3 );
 	}
 
 	//
 	// static methods
 	//
 
-	public static int getFirstAvailableTimepointId( final AbstractSequenceDescription< ?, ?, ? > seq, final int setupId )
+	private static int getFirstAvailableTimepointId( final AbstractSequenceDescription< ?, ?, ? > seq, final int setupId )
 	{
 		for ( final TimePoint tp : seq.getTimePoints().getTimePointsOrdered() )
 		{
@@ -87,15 +76,15 @@ public static int getFirstAvailableTimepointId( final AbstractSequenceDescriptio
 		throw new RuntimeException( "All timepoints for setupId " + setupId + " are declared missing. Stopping." );
 	}
 
-	public static DataType getDataType( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
+	private static DataType getDataType( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
 	{
-		final int timePointId = getFirstAvailableTimepointId( n5properties.loader.getSequenceDescription(), setupId );
-		return n5.getDatasetAttributes( n5properties.getPath( setupId, timePointId, 0 ) ).getDataType();
+		final int timePointId = getFirstAvailableTimepointId( n5properties.sequenceDescription, setupId );
+		return n5.getDatasetAttributes( n5properties.getDatasetPath( setupId, timePointId, 0 ) ).getDataType();
 	}
 
-	public static double[][] getMipMapResolutions( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
+	private static double[][] getMipMapResolutions( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
 	{
-		final int timePointId = getFirstAvailableTimepointId( n5properties.loader.getSequenceDescription(), setupId );
+		final int timePointId = getFirstAvailableTimepointId( n5properties.sequenceDescription, setupId );
 
 		// multiresolution pyramid
 
@@ -121,9 +110,9 @@ public static double[][] getMipMapResolutions( final AllenOMEZarrProperties n5pr
 
 			for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
 			{
-				if ( ScaleCoordinateTransformation.class.isInstance( c ) )
+				if ( c instanceof ScaleCoordinateTransformation )
 				{
-					final ScaleCoordinateTransformation s = (ScaleCoordinateTransformation)c;
+					final ScaleCoordinateTransformation s = ( ScaleCoordinateTransformation ) c;
 
 					if ( firstScale == null )
 						firstScale = s.getScale().clone();
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
index 35e5b985..cb06b14b 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/XmlIoAllenOMEZarrLoader.java
@@ -35,6 +35,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.HashMap;
+import java.util.Map;
 import java.util.Map.Entry;
 
 import org.jdom2.Element;
@@ -49,8 +50,6 @@
 @ImgLoaderIo( format = "bdv.multimg.zarr", type = AllenOMEZarrLoader.class )
 public class XmlIoAllenOMEZarrLoader implements XmlIoBasicImgLoader< AllenOMEZarrLoader >
 {
-	public static boolean PREFER_URI_FOR_LOCAL_FILES = true;
-
 	@Override
 	public Element toXml( final AllenOMEZarrLoader imgLoader, final File basePath )
 	{
@@ -69,7 +68,7 @@ public Element toXml( final AllenOMEZarrLoader imgLoader, final URI basePathURI
 			final Element bucketElement = new Element("s3bucket");
 			bucketElement.addContent( imgLoader.getBucket() );
 			imgLoaderElement.addContent(bucketElement);
-			
+
 			final Element zarrElement = new Element("zarr");
 			zarrElement.setAttribute("type", "absolute");
 			zarrElement.addContent( imgLoader.getFolder() );
@@ -109,7 +108,7 @@ public AllenOMEZarrLoader fromXml( final Element elem, final File basePath, fina
 	@Override
 	public AllenOMEZarrLoader fromXml( final Element elem, final URI basePathURI, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription )
 	{
-		final HashMap<ViewId, String> zgroups = new HashMap<>();
+		final Map<ViewId, String> zgroups = new HashMap<>();
 
 		final Element s3Bucket = elem.getChild( "s3bucket" );
 		final URI uri;
@@ -140,14 +139,14 @@ public AllenOMEZarrLoader fromXml( final Element elem, final URI basePathURI, fi
 			{
 				uri = new URI( "s3", bucket, folder, null );
 			}
-			catch (URISyntaxException e)
+			catch ( URISyntaxException e )
 			{
 				e.printStackTrace();
 				throw new RuntimeException( "Could not instantiate N5 reader for S3 bucket '" + bucket + "'." );
 			}
 
 			final Element zgroupsElem = elem.getChild( "zgroups" );
-			for (final Element c : zgroupsElem.getChildren( "zgroup" ))
+			for ( final Element c : zgroupsElem.getChildren( "zgroup" ) )
 			{
 				final int timepointId = Integer.parseInt( c.getAttributeValue( "timepoint" ) );
 				final int setupId = Integer.parseInt( c.getAttributeValue( "setup" ) );

From 2cf7c3c50c5eac9c5e1bcaafca8eb49733d8af9f Mon Sep 17 00:00:00 2001
From: tpietzsch <tobias.pietzsch@gmail.com>
Date: Wed, 6 Nov 2024 11:00:03 +0100
Subject: [PATCH 21/24] REMOVE LATER: interactive AllenOMEZarrLoader test (show
 in BDV)

---
 .../spimdata/imgloaders/AllenOMEZarrTest.java | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrTest.java

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrTest.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrTest.java
new file mode 100644
index 00000000..8d6c6175
--- /dev/null
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrTest.java
@@ -0,0 +1,33 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+import java.io.File;
+import java.util.concurrent.Future;
+
+import bdv.BigDataViewer;
+import bdv.export.ProgressWriterConsole;
+import bdv.img.n5.N5ImageLoader;
+import bdv.spimdata.SpimDataMinimal;
+import bdv.spimdata.XmlIoSpimDataMinimal;
+import bdv.tools.InitializeViewerState;
+import bdv.ui.UIUtils;
+import bdv.viewer.ViewerOptions;
+import mpicbg.spim.data.SpimDataException;
+
+public class AllenOMEZarrTest
+{
+	public static void main( String[] args ) throws SpimDataException
+	{
+		final String fn = "/Users/pietzsch/Desktop/data/Allen/704522.xml";
+
+		System.setProperty( "apple.laf.useScreenMenuBar", "true" );
+		UIUtils.installFlatLafInfos();
+
+		final SpimDataMinimal spimData = new XmlIoSpimDataMinimal().load( fn );
+		final N5ImageLoader n5ImageLoader = ( N5ImageLoader ) spimData.getSequenceDescription().getImgLoader();
+		final Future< Void > f = n5ImageLoader.prefetch( 256 );
+		n5ImageLoader.setNumFetcherThreads( 64 );
+
+		final BigDataViewer bdv = BigDataViewer.open( spimData, new File( fn ).getName(), new ProgressWriterConsole(), ViewerOptions.options() );
+		InitializeViewerState.initBrightness( 0, 1, bdv.getViewerFrame() );
+	}
+}

From 8b6aea50c6cb59e07ade0421511c8299cb178208 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Wed, 6 Nov 2024 14:28:26 -0500
Subject: [PATCH 22/24] fixed code to work with SplitImgLoaders

---
 .../imgloaders/AllenOMEZarrLoader.java        | 18 ++++++---
 .../splitting/SplitViewerImgLoader.java       | 10 +++++
 src/main/java/util/URITools.java              | 38 +++++++++++++++++--
 3 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index 365cf020..c12b4ce6 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -6,11 +6,13 @@
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
 
 import bdv.ViewerImgLoader;
+import bdv.ViewerSetupImgLoader;
 import bdv.img.n5.N5ImageLoader;
 import bdv.img.n5.N5Properties;
 import ij.ImageJ;
 import mpicbg.spim.data.SpimDataException;
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.generic.sequence.BasicMultiResolutionSetupImgLoader;
 import mpicbg.spim.data.sequence.ViewId;
 import net.imglib2.RandomAccessibleInterval;
 import net.imglib2.cache.volatiles.CacheHints;
@@ -19,6 +21,7 @@
 import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer;
+import net.preibisch.mvrecon.fiji.spimdata.imgloaders.splitting.SplitViewerImgLoader;
 import util.URITools;
 
 public class AllenOMEZarrLoader extends N5ImageLoader
@@ -85,6 +88,7 @@ protected < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCac
 	public static void main( String[] args ) throws SpimDataException
 	{
 		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.split.xml" );
+		//URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
 		//URI xml = URITools.toURI( "s3://janelia-bigstitcher-spark/Stitching/dataset.xml" );
 		//URI xml = URITools.toURI( "gs://janelia-spark-test/I2K-test/dataset.xml" );
 		//URI xml = URITools.toURI( "/Users/preibischs/SparkTest/IP/dataset.xml" );
@@ -92,15 +96,19 @@ public static void main( String[] args ) throws SpimDataException
 		XmlIoSpimData2 io = new XmlIoSpimData2();
 		SpimData2 data = io.load( xml );
 
-		ViewerImgLoader imgL = (ViewerImgLoader)data.getSequenceDescription().getImgLoader();
-		( ( N5ImageLoader ) imgL ).prefetch( cloudThreads );
-		imgL.setNumFetcherThreads( cloudThreads );
-		SetupImgLoader sil = (SetupImgLoader)data.getSequenceDescription().getImgLoader().getSetupImgLoader( 0 );
+		System.out.println( "Setting cloud fetcher threads to: " + URITools.cloudThreads );
+		URITools.setNumFetcherThreads( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+		System.out.println( "Prefetching with threads: " + URITools.cloudThreads );
+		URITools.prefetch( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+		//imgL.setNumFetcherThreads( cloudThreads );
+		ViewerSetupImgLoader sil = (ViewerSetupImgLoader)data.getSequenceDescription().getImgLoader().getSetupImgLoader( 0 );
 
 		final int tp = data.getSequenceDescription().getTimePoints().getTimePointsOrdered().get( 0 ).getId();
 
 		//RandomAccessibleInterval img = sil.getImage( tp, sil.getMipmapResolutions().length - 1 );
-		RandomAccessibleInterval vol = sil.getVolatileImage( tp, sil.getMipmapResolutions().length - 1 );
+		RandomAccessibleInterval vol = sil.getVolatileImage( tp, ((BasicMultiResolutionSetupImgLoader)sil).getMipmapResolutions().length - 1 );
 
 		new ImageJ();
 		//ImageJFunctions.show( img );
diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
index f38c47c4..7deeca59 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/splitting/SplitViewerImgLoader.java
@@ -25,11 +25,13 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
 
 import bdv.ViewerImgLoader;
 import bdv.ViewerSetupImgLoader;
 import bdv.cache.CacheControl;
 import bdv.img.cache.VolatileGlobalCellCache;
+import bdv.img.n5.N5ImageLoader;
 import mpicbg.spim.data.generic.sequence.BasicViewSetup;
 import mpicbg.spim.data.sequence.MultiResolutionImgLoader;
 import mpicbg.spim.data.sequence.SequenceDescription;
@@ -107,6 +109,14 @@ private final synchronized SplitViewerSetupImgLoader<?,?> createNewSetupImgLoade
 		return new SplitViewerSetupImgLoader<>( Cast.unchecked( setupImgLoader ), interval );
 	}
 
+	public Future< Void > prefetch( final int parallelism )
+	{
+		if ( N5ImageLoader.class.isInstance( underlyingImgLoader ) )
+			return ((N5ImageLoader)underlyingImgLoader).prefetch( parallelism );
+		else
+			return null;
+	}
+
 	@Override
 	public synchronized void setNumFetcherThreads( final int n )
 	{
diff --git a/src/main/java/util/URITools.java b/src/main/java/util/URITools.java
index 4cab21b7..f0caa7ee 100644
--- a/src/main/java/util/URITools.java
+++ b/src/main/java/util/URITools.java
@@ -63,12 +63,15 @@
 import com.google.gson.GsonBuilder;
 
 import bdv.ViewerImgLoader;
+import bdv.img.n5.N5ImageLoader;
 import mpicbg.spim.data.SpimDataException;
 import mpicbg.spim.data.SpimDataIOException;
 import mpicbg.spim.data.generic.AbstractSpimData;
+import mpicbg.spim.data.sequence.ImgLoader;
 import net.preibisch.legacy.io.IOFunctions;
 import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
+import net.preibisch.mvrecon.fiji.spimdata.imgloaders.splitting.SplitViewerImgLoader;
 
 public class URITools
 {
@@ -437,10 +440,6 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
 
 			final SpimData2 data = io.fromXml( docRoot, xmlURI );
 
-			// more threads for cloud-based fetching
-			System.out.println( "Setting num fetcher threads to " + cloudThreads + " for cloud access." );
-			( (ViewerImgLoader) data.getSequenceDescription().getImgLoader() ).setNumFetcherThreads( cloudThreads );
-
 			return data;
 		}
 		else
@@ -449,6 +448,37 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
 		}
 	}
 
+	public static boolean setNumFetcherThreads( final ImgLoader loader, final int threads )
+	{
+		if ( ViewerImgLoader.class.isInstance( loader ) )
+		{
+			( (ViewerImgLoader) loader ).setNumFetcherThreads( threads );
+			return true;
+		}
+		else
+		{
+			return false;
+		}
+	}
+
+	public static boolean prefetch( final ImgLoader loader, final int threads )
+	{
+		if ( N5ImageLoader.class.isInstance( loader ) )
+		{
+			( ( N5ImageLoader ) loader ).prefetch( threads );
+			return true;
+		}
+		else if ( SplitViewerImgLoader.class.isInstance( loader ) )
+		{
+			if ( ( ( SplitViewerImgLoader ) loader ).prefetch( threads ) != null )
+				return true;
+			else
+				return false;
+		}
+		else
+			return false;
+	}
+
 	/**
 	 * This is an abstraction that is only required because the Google cloud and AWS KeyValueAccesses behave differently
 	 *

From 494370509c41ae9251a7829526087127b454b9dd Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Wed, 6 Nov 2024 14:35:48 -0500
Subject: [PATCH 23/24] centralized cloud-prefetching & setting cloud threads
 to GenericLoadParseQueryXML.java (i.e. when using a GUI to open an XML)

---
 .../queryXML/GenericLoadParseQueryXML.java    | 22 +++++++++++++++++--
 src/main/java/util/URITools.java              |  6 ++---
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/queryXML/GenericLoadParseQueryXML.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/queryXML/GenericLoadParseQueryXML.java
index 6898c56f..b97923e1 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/queryXML/GenericLoadParseQueryXML.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/queryXML/GenericLoadParseQueryXML.java
@@ -128,6 +128,8 @@ public abstract class GenericLoadParseQueryXML<
 	protected ArrayList< ActionListener > listener = null;
 	protected GenericDialog gd = null;
 
+	protected boolean isPrefetched = false;
+
 	public Button defineNewDataset = null;
 
 	/*
@@ -149,12 +151,28 @@ public GenericLoadParseQueryXML( final X io )
 	/**
 	 * @return the SpimData object parsed from the xml
 	 */
-	public AS getData() { return data; }
+	public AS getData()
+	{
+		final URI uri = getXMLURI();
+
+		if ( !isPrefetched && ( URITools.isS3( uri ) || URITools.isGC( uri ) ) )
+		{
+			IOFunctions.println( "Setting cloud fetcher threads for '" + uri + "' to: " + URITools.cloudThreads );
+			URITools.setNumFetcherThreads( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+	
+			IOFunctions.println( "Prefetching with " + URITools.cloudThreads + " threads: for '" + uri + "'");
+			URITools.prefetch( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+			isPrefetched = true;
+		}
+
+		return data;
+	}
 	
 	/**
 	 * @return The location of the xml file
 	 */
-	public URI getXMLURI() { return URITools.xmlFilenameToFullPath( getData(), xmlFileName ); }
+	public URI getXMLURI() { return URITools.xmlFilenameToFullPath( data, xmlFileName ); }
 
 	/**
 	 * @return ONLY the filename, without any path
diff --git a/src/main/java/util/URITools.java b/src/main/java/util/URITools.java
index f0caa7ee..a377f04b 100644
--- a/src/main/java/util/URITools.java
+++ b/src/main/java/util/URITools.java
@@ -67,7 +67,7 @@
 import mpicbg.spim.data.SpimDataException;
 import mpicbg.spim.data.SpimDataIOException;
 import mpicbg.spim.data.generic.AbstractSpimData;
-import mpicbg.spim.data.sequence.ImgLoader;
+import mpicbg.spim.data.generic.sequence.BasicImgLoader;
 import net.preibisch.legacy.io.IOFunctions;
 import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
@@ -448,7 +448,7 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
 		}
 	}
 
-	public static boolean setNumFetcherThreads( final ImgLoader loader, final int threads )
+	public static boolean setNumFetcherThreads( final BasicImgLoader loader, final int threads )
 	{
 		if ( ViewerImgLoader.class.isInstance( loader ) )
 		{
@@ -461,7 +461,7 @@ public static boolean setNumFetcherThreads( final ImgLoader loader, final int th
 		}
 	}
 
-	public static boolean prefetch( final ImgLoader loader, final int threads )
+	public static boolean prefetch( final BasicImgLoader loader, final int threads )
 	{
 		if ( N5ImageLoader.class.isInstance( loader ) )
 		{

From 97ec884e93a90c142c5a983066a0dfd24c641684 Mon Sep 17 00:00:00 2001
From: Stephan Preibisch <stephan.preibisch@gmx.de>
Date: Wed, 6 Nov 2024 14:36:52 -0500
Subject: [PATCH 24/24] no setting of fetcher threads by default

---
 .../fiji/spimdata/imgloaders/AllenOMEZarrLoader.java        | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
index c12b4ce6..037b06ce 100644
--- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
+++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java
@@ -5,7 +5,6 @@
 
 import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
 
-import bdv.ViewerImgLoader;
 import bdv.ViewerSetupImgLoader;
 import bdv.img.n5.N5ImageLoader;
 import bdv.img.n5.N5Properties;
@@ -21,7 +20,6 @@
 import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
 import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer;
-import net.preibisch.mvrecon.fiji.spimdata.imgloaders.splitting.SplitViewerImgLoader;
 import util.URITools;
 
 public class AllenOMEZarrLoader extends N5ImageLoader
@@ -32,8 +30,6 @@ public class AllenOMEZarrLoader extends N5ImageLoader
 
 	private final String bucket, folder;
 
-	private static final int cloudThreads = 256;
-
 	public AllenOMEZarrLoader(
 			final URI n5URI,
 			final String bucket,
@@ -44,8 +40,6 @@ public AllenOMEZarrLoader(
 		super( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription );
 		this.sequenceDescription = sequenceDescription;
 
-		setNumFetcherThreads( cloudThreads );
-
 		this.bucket = bucket;
 		this.folder = folder;