Merge branch 'allenomezarr'

pom.xml

@@ -112,9 +112,9 @@ like Selective Plane Illumination Microscopy (SPIM) Data.</description>
 		<imglib2-algorithm.version>0.16.0</imglib2-algorithm.version>
 		<imglib2-roi.version>0.15.1</imglib2-roi.version>
 		<n5-imglib2.version>7.0.2</n5-imglib2.version>
-		<bigdataviewer-core.version>10.6.1</bigdataviewer-core.version>
 		<bigdataviewer-vistools.version>1.0.0-beta-36</bigdataviewer-vistools.version>
 -->
+		<bigdataviewer-core.version>10.6.3</bigdataviewer-core.version>
 		<spim_data.version>2.3.5</spim_data.version>
 
 		<!-- from n5-universe 

src/main/java/net/preibisch/mvrecon/fiji/plugin/queryXML/GenericLoadParseQueryXML.java

@@ -128,6 +128,8 @@ public abstract class GenericLoadParseQueryXML<
 	protected ArrayList< ActionListener > listener = null;
 	protected GenericDialog gd = null;
 
+	protected boolean isPrefetched = false;
+
 	public Button defineNewDataset = null;
 
 	/*
@@ -149,12 +151,28 @@ public GenericLoadParseQueryXML( final X io )
 	/**
 	 * @return the SpimData object parsed from the xml
 	 */
-	public AS getData() { return data; }
+	public AS getData()
+	{
+		final URI uri = getXMLURI();
+
+		if ( !isPrefetched && ( URITools.isS3( uri ) || URITools.isGC( uri ) ) )
+		{
+			IOFunctions.println( "Setting cloud fetcher threads for '" + uri + "' to: " + URITools.cloudThreads );
+			URITools.setNumFetcherThreads( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+			IOFunctions.println( "Prefetching with " + URITools.cloudThreads + " threads: for '" + uri + "'");
+			URITools.prefetch( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+			isPrefetched = true;
+		}
+
+		return data;
+	}
 
 	/**
 	 * @return The location of the xml file
 	 */
-	public URI getXMLURI() { return URITools.xmlFilenameToFullPath( getData(), xmlFileName ); }
+	public URI getXMLURI() { return URITools.xmlFilenameToFullPath( data, xmlFileName ); }
 
 	/**
 	 * @return ONLY the filename, without any path

src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrLoader.java

@@ -0,0 +1,114 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+import java.net.URI;
+import java.util.Map;
+
+import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
+
+import bdv.ViewerSetupImgLoader;
+import bdv.img.n5.N5ImageLoader;
+import bdv.img.n5.N5Properties;
+import ij.ImageJ;
+import mpicbg.spim.data.SpimDataException;
+import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.generic.sequence.BasicMultiResolutionSetupImgLoader;
+import mpicbg.spim.data.sequence.ViewId;
+import net.imglib2.RandomAccessibleInterval;
+import net.imglib2.cache.volatiles.CacheHints;
+import net.imglib2.img.display.imagej.ImageJFunctions;
+import net.imglib2.type.NativeType;
+import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
+import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
+import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer;
+import util.URITools;
+
+public class AllenOMEZarrLoader extends N5ImageLoader
+{
+	private final AbstractSequenceDescription< ?, ?, ? > sequenceDescription;
+
+	private final Map< ViewId, String > viewIdToPath;
+
+	private final String bucket, folder;
+
+	public AllenOMEZarrLoader(
+			final URI n5URI,
+			final String bucket,
+			final String folder,
+			final AbstractSequenceDescription< ?, ?, ? > sequenceDescription,
+			final Map< ViewId, String > viewIdToPath )
+	{
+		super( URITools.instantiateN5Reader( StorageFormat.ZARR, n5URI ), n5URI, sequenceDescription );
+		this.sequenceDescription = sequenceDescription;
+
+		this.bucket = bucket;
+		this.folder = folder;
+
+		this.viewIdToPath = viewIdToPath;
+	}
+
+	public Map< ViewId, String > getViewIdToPath()
+	{
+		return viewIdToPath;
+	}
+
+	public String getBucket()
+	{
+		return bucket;
+	}
+
+	public String getFolder()
+	{
+		return folder;
+	}
+
+	@Override
+	protected N5Properties createN5PropertiesInstance()
+	{
+		return new AllenOMEZarrProperties( sequenceDescription, viewIdToPath );
+	}
+
+	@Override
+	protected < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage(
+			final String datasetPath,
+			final int setupId, final int timepointId, final int level,
+			final CacheHints cacheHints,
+			final T type )
+	{
+		return super.prepareCachedImage( datasetPath, setupId, 0, level, cacheHints, type ).view()
+				.slice( 4, 0 )
+				.slice( 3, 0 );
+	}
+
+	public static void main( String[] args ) throws SpimDataException
+	{
+		URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.split.xml" );
+		//URI xml = URITools.toURI( "/Users/preibischs/Documents/Janelia/Projects/BigStitcher/Allen/bigstitcher_708373/708373.xml" );
+		//URI xml = URITools.toURI( "s3://janelia-bigstitcher-spark/Stitching/dataset.xml" );
+		//URI xml = URITools.toURI( "gs://janelia-spark-test/I2K-test/dataset.xml" );
+		//URI xml = URITools.toURI( "/Users/preibischs/SparkTest/IP/dataset.xml" );
+
+		XmlIoSpimData2 io = new XmlIoSpimData2();
+		SpimData2 data = io.load( xml );
+
+		System.out.println( "Setting cloud fetcher threads to: " + URITools.cloudThreads );
+		URITools.setNumFetcherThreads( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+		System.out.println( "Prefetching with threads: " + URITools.cloudThreads );
+		URITools.prefetch( data.getSequenceDescription().getImgLoader(), URITools.cloudThreads );
+
+		//imgL.setNumFetcherThreads( cloudThreads );
+		ViewerSetupImgLoader sil = (ViewerSetupImgLoader)data.getSequenceDescription().getImgLoader().getSetupImgLoader( 0 );
+
+		final int tp = data.getSequenceDescription().getTimePoints().getTimePointsOrdered().get( 0 ).getId();
+
+		//RandomAccessibleInterval img = sil.getImage( tp, sil.getMipmapResolutions().length - 1 );
+		RandomAccessibleInterval vol = sil.getVolatileImage( tp, ((BasicMultiResolutionSetupImgLoader)sil).getMipmapResolutions().length - 1 );
+
+		new ImageJ();
+		//ImageJFunctions.show( img );
+		ImageJFunctions.show( vol );
+
+		final ViewSetupExplorer< SpimData2 > explorer = new ViewSetupExplorer<>( data, xml, io );
+		explorer.getFrame().toFront();
+	}
+}

src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrProperties.java

@@ -0,0 +1,132 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+import java.util.Arrays;
+import java.util.Map;
+
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.N5Reader;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMultiScaleMetadata.OmeNgffDataset;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
+import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
+
+import bdv.img.n5.N5Properties;
+import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import mpicbg.spim.data.sequence.TimePoint;
+import mpicbg.spim.data.sequence.ViewId;
+
+public class AllenOMEZarrProperties implements N5Properties
+{
+	private final AbstractSequenceDescription< ?, ?, ? > sequenceDescription;
+
+	private final Map< ViewId, String > viewIdToPath;
+
+	public AllenOMEZarrProperties(
+			final AbstractSequenceDescription< ?, ?, ? > sequenceDescription,
+			final Map< ViewId, String > viewIdToPath )
+	{
+		this.sequenceDescription = sequenceDescription;
+		this.viewIdToPath = viewIdToPath;
+	}
+
+	private String getPath( final int setupId, final int timepointId )
+	{
+		return viewIdToPath.get( new ViewId( timepointId, setupId ) );
+	}
+
+	@Override
+	public String getDatasetPath( final int setupId, final int timepointId, final int level )
+	{
+		return String.format( getPath( setupId, timepointId )+ "/%d", level );
+	}
+
+	@Override
+	public DataType getDataType( final N5Reader n5, final int setupId )
+	{
+		return getDataType( this, n5, setupId );
+	}
+
+	@Override
+	public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
+	{
+		return getMipMapResolutions( this, n5, setupId );
+	}
+
+	@Override
+	public long[] getDimensions( final N5Reader n5, final int setupId, final int timepointId, final int level )
+	{
+		final String path = getDatasetPath( setupId, timepointId, level );
+		final long[] dimensions = n5.getDatasetAttributes( path ).getDimensions();
+		// dataset dimensions is 5D, remove the channel and time dimensions
+		return Arrays.copyOf( dimensions, 3 );
+	}
+
+	//
+	// static methods
+	//
+
+	private static int getFirstAvailableTimepointId( final AbstractSequenceDescription< ?, ?, ? > seq, final int setupId )
+	{
+		for ( final TimePoint tp : seq.getTimePoints().getTimePointsOrdered() )
+		{
+			if ( !seq.getMissingViews().getMissingViews().contains( new ViewId( tp.getId(), setupId ) ) )
+				return tp.getId();
+		}
+
+		throw new RuntimeException( "All timepoints for setupId " + setupId + " are declared missing. Stopping." );
+	}
+
+	private static DataType getDataType( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
+	{
+		final int timePointId = getFirstAvailableTimepointId( n5properties.sequenceDescription, setupId );
+		return n5.getDatasetAttributes( n5properties.getDatasetPath( setupId, timePointId, 0 ) ).getDataType();
+	}
+
+	private static double[][] getMipMapResolutions( final AllenOMEZarrProperties n5properties, final N5Reader n5, final int setupId )
+	{
+		final int timePointId = getFirstAvailableTimepointId( n5properties.sequenceDescription, setupId );
+
+		// multiresolution pyramid
+
+		//org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata
+		// for this to work you need to register an adapter in the N5Factory class
+		// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
+		final OmeNgffMultiScaleMetadata[] multiscales = n5.getAttribute( n5properties.getPath( setupId, timePointId ), "multiscales", OmeNgffMultiScaleMetadata[].class );
+
+		if ( multiscales == null || multiscales.length == 0 )
+			throw new RuntimeException( "Could not parse OME-ZARR multiscales object. stopping." );
+
+		if ( multiscales.length != 1 )
+			System.out.println( "This dataset has " + multiscales.length + " objects, we expected 1. Picking the first one." );
+
+		//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions() for " + setupId + " using " + n5properties.getPath( setupId, timePointId ) + ": found " + multiscales[ 0 ].datasets.length + " multi-resolution levels." );
+
+		double[][] mipMapResolutions = new double[ multiscales[ 0 ].datasets.length ][ 3 ];
+		double[] firstScale = null;
+
+		for ( int i = 0; i < multiscales[ 0 ].datasets.length; ++i )
+		{
+			final OmeNgffDataset ds = multiscales[ 0 ].datasets[ i ];
+
+			for ( final CoordinateTransformation< ? > c : ds.coordinateTransformations )
+			{
+				if ( c instanceof ScaleCoordinateTransformation )
+				{
+					final ScaleCoordinateTransformation s = ( ScaleCoordinateTransformation ) c;
+
+					if ( firstScale == null )
+						firstScale = s.getScale().clone();
+
+					for ( int d = 0; d < mipMapResolutions[ i ].length; ++d )
+					{
+						mipMapResolutions[ i ][ d ] = s.getScale()[ d ] / firstScale[ d ];
+						mipMapResolutions[ i ][ d ] = Math.round(mipMapResolutions[ i ][ d ]*10000)/10000d; // round to the 5th digit
+					}
+					//System.out.println( "AllenOMEZarrLoader.getMipmapResolutions(), level " + i + ": " + Arrays.toString( s.getScale() ) + " >> " + Arrays.toString( mipMapResolutions[ i ] ) );
+				}
+			}
+		}
+
+		return mipMapResolutions;
+	}
+}

src/main/java/net/preibisch/mvrecon/fiji/spimdata/imgloaders/AllenOMEZarrTest.java

@@ -0,0 +1,33 @@
+package net.preibisch.mvrecon.fiji.spimdata.imgloaders;
+
+import java.io.File;
+import java.util.concurrent.Future;
+
+import bdv.BigDataViewer;
+import bdv.export.ProgressWriterConsole;
+import bdv.img.n5.N5ImageLoader;
+import bdv.spimdata.SpimDataMinimal;
+import bdv.spimdata.XmlIoSpimDataMinimal;
+import bdv.tools.InitializeViewerState;
+import bdv.ui.UIUtils;
+import bdv.viewer.ViewerOptions;
+import mpicbg.spim.data.SpimDataException;
+
+public class AllenOMEZarrTest
+{
+	public static void main( String[] args ) throws SpimDataException
+	{
+		final String fn = "/Users/pietzsch/Desktop/data/Allen/704522.xml";
+
+		System.setProperty( "apple.laf.useScreenMenuBar", "true" );
+		UIUtils.installFlatLafInfos();
+
+		final SpimDataMinimal spimData = new XmlIoSpimDataMinimal().load( fn );
+		final N5ImageLoader n5ImageLoader = ( N5ImageLoader ) spimData.getSequenceDescription().getImgLoader();
+		final Future< Void > f = n5ImageLoader.prefetch( 256 );
+		n5ImageLoader.setNumFetcherThreads( 64 );
+
+		final BigDataViewer bdv = BigDataViewer.open( spimData, new File( fn ).getName(), new ProgressWriterConsole(), ViewerOptions.options() );
+		InitializeViewerState.initBrightness( 0, 1, bdv.getViewerFrame() );
+	}
+}