diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java index 602196ce4..394c46cda 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/Split_Views.java @@ -23,13 +23,17 @@ package net.preibisch.mvrecon.fiji.plugin; import java.awt.Color; +import java.util.Arrays; import java.util.HashMap; import fiji.util.gui.GenericDialogPlus; import ij.ImageJ; import ij.plugin.PlugIn; +import mpicbg.spim.data.SpimDataException; import mpicbg.spim.data.generic.AbstractSpimData; +import mpicbg.spim.data.generic.sequence.BasicImgLoader; import mpicbg.spim.data.generic.sequence.BasicViewSetup; +import mpicbg.spim.data.sequence.MultiResolutionImgLoader; import net.imglib2.Dimensions; import net.imglib2.util.Pair; import net.imglib2.util.Util; @@ -45,13 +49,15 @@ public class Split_Views implements PlugIn { - public static int defaultImgX = 600; - public static int defaultImgY = 600; - public static int defaultImgZ = 200; + public static long defaultImgX = 500; + public static long defaultImgY = 500; + public static long defaultImgZ = 200; - public static int defaultOverlapX = 60; - public static int defaultOverlapY = 60; - public static int defaultOverlapZ = 20; + public static long defaultOverlapX = 60; + public static long defaultOverlapY = 60; + public static long defaultOverlapZ = 20; + + public static boolean defaultOptimize = true; public static String defaultPath = null; @@ -74,15 +80,13 @@ public void run(String arg) public static boolean split( final SpimData2 data, final String saveAs, - final int sx, - final int sy, - final int sz, - final int ox, - final int oy, - final int oz, + final long[] targetSize, + final long[] overlap, + final long[] minStepSize, + final boolean optimize, final boolean display ) { - final SpimData2 newSD = SplittingTools.splitImages( data, new long[] { ox, oy, oz }, new long[] { sx, sy, sz } ); + final SpimData2 newSD = SplittingTools.splitImages( data, overlap, targetSize, minStepSize, optimize ); if ( display ) { @@ -99,6 +103,8 @@ public static boolean split( public static boolean split( final SpimData2 data, final String fileName ) { + final long[] minStepSize = findMinStepSize( data ); + final Pair< HashMap< String, Integer >, long[] > imgSizes = collectImageSizes( data ); IOFunctions.println( "Current image sizes of dataset :"); @@ -108,16 +114,28 @@ public static boolean split( final SpimData2 data, final String fileName ) final GenericDialogPlus gd = new GenericDialogPlus( "Dataset splitting/subdividing" ); - gd.addSlider( "New_Image_Size_X", 100, 2000, defaultImgX ); - gd.addSlider( "New_Image_Size_Y", 100, 2000, defaultImgY ); - gd.addSlider( "New_Image_Size_Z", 100, 2000, defaultImgZ ); + defaultImgX = closestLargerLongDivisableBy( defaultImgX, minStepSize[ 0 ] ); + defaultImgY = closestLargerLongDivisableBy( defaultImgY, minStepSize[ 1 ] ); + defaultImgZ = closestLargerLongDivisableBy( defaultImgZ, minStepSize[ 2 ] ); + + defaultOverlapX = closestLargerLongDivisableBy( defaultOverlapX, minStepSize[ 0 ] ); + defaultOverlapY = closestLargerLongDivisableBy( defaultOverlapY, minStepSize[ 1 ] ); + defaultOverlapZ = closestLargerLongDivisableBy( defaultOverlapZ, minStepSize[ 2 ] ); + + gd.addSlider( "Target_Image_Size_X", 100, 2000, defaultImgX, minStepSize[ 0 ] ); + gd.addSlider( "Target_Image_Size_Y", 100, 2000, defaultImgY, minStepSize[ 1 ] ); + gd.addSlider( "Target_Image_Size_Z", 100, 2000, defaultImgZ, minStepSize[ 2 ] ); + gd.addCheckbox( "Optimize_image_sizes per view", defaultOptimize ); + + gd.addMessage( "Note: new sizes will be adjusted to be divisible by " + Arrays.toString( minStepSize ), GUIHelper.mediumstatusfont, Color.RED ); gd.addMessage( "" ); - gd.addSlider( "Overlap_X", 10, 200, defaultOverlapX ); - gd.addSlider( "Overlap_Y", 10, 200, defaultOverlapY ); - gd.addSlider( "Overlap_Z", 10, 200, defaultOverlapZ ); + gd.addSlider( "Overlap_X", 10, 200, defaultOverlapX, minStepSize[ 0 ] ); + gd.addSlider( "Overlap_Y", 10, 200, defaultOverlapY, minStepSize[ 1 ] ); + gd.addSlider( "Overlap_Z", 10, 200, defaultOverlapZ, minStepSize[ 2 ] ); + gd.addMessage( "Note: overlap will be adjusted to be divisible by " + Arrays.toString( minStepSize ), GUIHelper.mediumstatusfont, Color.RED ); gd.addMessage( "Minimal image sizes per dimension: " + Util.printCoordinates( imgSizes.getB() ), GUIHelper.mediumstatusfont, Color.DARK_GRAY ); gd.addMessage( "" ); @@ -140,18 +158,29 @@ public static boolean split( final SpimData2 data, final String fileName ) if ( gd.wasCanceled() ) return false; - final int sx = defaultImgX = (int)Math.round( gd.getNextNumber() ); - final int sy = defaultImgY = (int)Math.round( gd.getNextNumber() ); - final int sz = defaultImgZ = (int)Math.round( gd.getNextNumber() ); + final long sx = defaultImgX = closestLargerLongDivisableBy( Math.round( gd.getNextNumber() ), minStepSize[ 0 ] ); + final long sy = defaultImgY = closestLargerLongDivisableBy( Math.round( gd.getNextNumber() ), minStepSize[ 1 ] ); + final long sz = defaultImgZ = closestLargerLongDivisableBy( Math.round( gd.getNextNumber() ), minStepSize[ 2 ] ); + + final boolean optimize = defaultOptimize = gd.getNextBoolean(); - final int ox = defaultOverlapX = (int)Math.round( gd.getNextNumber() ); - final int oy = defaultOverlapY = (int)Math.round( gd.getNextNumber() ); - final int oz = defaultOverlapZ = (int)Math.round( gd.getNextNumber() ); + final long ox = defaultOverlapX = closestLargerLongDivisableBy( Math.round( gd.getNextNumber() ), minStepSize[ 0 ] ); + final long oy = defaultOverlapY = closestLargerLongDivisableBy( Math.round( gd.getNextNumber() ), minStepSize[ 1 ] ); + final long oz = defaultOverlapZ = closestLargerLongDivisableBy( Math.round( gd.getNextNumber() ), minStepSize[ 2 ] ); final String saveAs = defaultPath = gd.getNextString(); final int choice = defaultChoice = gd.getNextChoiceIndex(); - return split( data, saveAs, sx, sy, sz, ox, oy, oz, choice == 0 ); + System.out.println( sx + ", " + sy + ", " + sz + ", " + ox + ", " + oy + ", " + oz ); + + if ( ox > sx || oy > sy || oz > sz ) + { + IOFunctions.println( "overlap cannot be bigger than size" ); + + return false; + } + + return split( data, saveAs, new long[]{ sx, sy, sz }, new long[]{ ox, oy, oz }, minStepSize, optimize, choice == 0 ); } public static Pair< HashMap< String, Integer >, long[] > collectImageSizes( final AbstractSpimData< ? > data ) @@ -188,15 +217,97 @@ public static Pair< HashMap< String, Integer >, long[] > collectImageSizes( fina return new ValuePair, long[]>( sizes, minSize ); } - public static void main( String[] args ) + public static long greatestCommonDivisor( long a, long b ) { - new ImageJ(); + while (b > 0) + { + long temp = b; + b = a % b; + a = temp; + } + return a; + } - if ( !System.getProperty("os.name").toLowerCase().contains( "mac" ) ) - GenericLoadParseQueryXML.defaultXMLfilename = "/home/steffi/Desktop/HisYFP-SPIM/dataset.xml"; + public static long lowestCommonMultiplier( long a, long b ) + { + return a * (b / greatestCommonDivisor(a, b)); + } + + public static long closestSmallerLongDivisableBy( final long a, final long b ) + { + if ( a == b || a == 0 || a % b == 0 ) + return a; else - GenericLoadParseQueryXML.defaultXMLfilename = "/Users/spreibi/Documents/Microscopy/SPIM/HisYFP-SPIM/dataset.xml";//"/Users/spreibi/Documents/Microscopy/SPIM/HisYFP-SPIM//dataset.xml"; + return a - (a % b); + } + + public static long closestLargerLongDivisableBy( final long a, final long b ) + { + if ( a == b || a == 0 || a % b == 0 ) + return a; + else + return (a + b) - (a % b); + } + + public static long closestLongDivisableBy( final long a, final long b) + { + final long c1 = closestSmallerLongDivisableBy( a, b );//a - (a % b); + final long c2 = closestLargerLongDivisableBy( a, b ); //(a + b) - (a % b); + + if (a - c1 > c2 - a) + return c2; + else + return c1; + } + + public static long[] findMinStepSize( final AbstractSpimData< ? > data ) + { + final BasicImgLoader imgLoader = data.getSequenceDescription().getImgLoader(); + + final long[] minStepSize = new long[] { 1, 1, 1 }; + + if ( MultiResolutionImgLoader.class.isInstance( imgLoader ) ) + { + IOFunctions.println( "We have a multi-resolution image loader: " + imgLoader.getClass().getName() + ", finding resolution steps"); + + final MultiResolutionImgLoader mrImgLoader = ( MultiResolutionImgLoader ) imgLoader; + + for ( final BasicViewSetup vs : data.getSequenceDescription().getViewSetupsOrdered() ) + { + final double[][] mipmapResolutions = mrImgLoader.getSetupImgLoader( vs.getId() ).getMipmapResolutions(); + + IOFunctions.println( "ViewSetup: " + vs.getName() + " (id=" + vs.getId() + "): " + Arrays.deepToString( mipmapResolutions ) ); + + // lowest resolution defines the minimal steps size + final double[] lowestResolution = mipmapResolutions[ mipmapResolutions.length - 1 ]; + + for ( int d = 0; d < minStepSize.length; ++d ) + { + if ( lowestResolution[ d ] % 1 != 0.0 ) + throw new RuntimeException( "Downsampling has a fraction, cannot split dataset" ); + + minStepSize[ d ] = lowestCommonMultiplier( minStepSize[ d ], Math.round( lowestResolution[ d ] ) ); + } + } + } + else + { + IOFunctions.println( "Not a multi-resolution image loader, all data splits are possible." ); + } + + IOFunctions.println( "Final minimal step size: " + Arrays.toString( minStepSize ) ); + + return minStepSize; + } + + public static void main( String[] args ) throws SpimDataException + { + new ImageJ(); + + GenericLoadParseQueryXML.defaultXMLfilename = "/Users/preibischs/SparkTest/IP/dataset.xml"; new Split_Views().run( null ); + //SpimData2 data = new XmlIoSpimData2("").load( GenericLoadParseQueryXML.defaultXMLfilename ); + //findMinStepSize(data); } } diff --git a/src/main/java/net/preibisch/mvrecon/headless/splitting/TestSplitting.java b/src/main/java/net/preibisch/mvrecon/headless/splitting/TestSplitting.java index 7f9dd2f93..797d1a04d 100644 --- a/src/main/java/net/preibisch/mvrecon/headless/splitting/TestSplitting.java +++ b/src/main/java/net/preibisch/mvrecon/headless/splitting/TestSplitting.java @@ -24,6 +24,7 @@ import ij.ImageJ; import mpicbg.spim.data.SpimDataException; +import net.preibisch.mvrecon.fiji.plugin.Split_Views; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2; import net.preibisch.mvrecon.fiji.spimdata.explorer.ViewSetupExplorer; @@ -56,8 +57,19 @@ public static void main( String[] args ) throws SpimDataException // load drosophila spimData = new XmlIoSpimData2( "" ).load( file ); + final long[] minStepSize = Split_Views.findMinStepSize( spimData ); + //SpimData2 newSD = SplittingTools.splitImages( spimData, new long[] { 30, 30, 15 }, new long[] { 600, 600, 300 } ); - SpimData2 newSD = SplittingTools.splitImages( spimData, new long[] { 30, 30, 10 }, new long[] { 200, 200, 40 } ); + SpimData2 newSD = + SplittingTools.splitImages( + spimData, + new long[] { 30, 30, 10 }, + new long[] { + Split_Views.closestLongDivisableBy( 200, minStepSize[ 0 ] ), + Split_Views.closestLongDivisableBy( 200, minStepSize[ 1 ] ), + Split_Views.closestLongDivisableBy( 40, minStepSize[ 2 ] ) }, + minStepSize, + true ); // drosophila with 1000 views final ViewSetupExplorer< SpimData2 > explorer = new ViewSetupExplorer<>( newSD, fileOut, new XmlIoSpimData2( "" ) ); diff --git a/src/main/java/net/preibisch/mvrecon/process/splitting/SplittingTools.java b/src/main/java/net/preibisch/mvrecon/process/splitting/SplittingTools.java index daffb0997..f7244d00d 100644 --- a/src/main/java/net/preibisch/mvrecon/process/splitting/SplittingTools.java +++ b/src/main/java/net/preibisch/mvrecon/process/splitting/SplittingTools.java @@ -56,6 +56,8 @@ import net.imglib2.util.Pair; import net.imglib2.util.Util; import net.imglib2.util.ValuePair; +import net.preibisch.legacy.io.IOFunctions; +import net.preibisch.mvrecon.fiji.plugin.Split_Views; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.imgloaders.splitting.SplitImgLoader; import net.preibisch.mvrecon.fiji.spimdata.imgloaders.splitting.SplitMultiResolutionImgLoader; @@ -71,7 +73,7 @@ public class SplittingTools { - public static SpimData2 splitImages( final SpimData2 spimData, final long[] overlapPx, final long[] targetSize ) + public static SpimData2 splitImages( final SpimData2 spimData, final long[] overlapPx, final long[] targetSize, final long[] minStepSize, final boolean optimize ) { final TimePoints timepoints = spimData.getSequenceDescription().getTimePoints(); @@ -96,7 +98,7 @@ public static SpimData2 splitImages( final SpimData2 spimData, final long[] over // new tileId is locally computed based on the old tile ids // by multiplying it with maxspread and then +1 for each new tile // so each new one has to be the same across channel & illumination! - final int maxIntervalSpread = maxIntervalSpread( oldSetups, overlapPx, targetSize ); + final int maxIntervalSpread = maxIntervalSpread( oldSetups, overlapPx, targetSize, minStepSize, optimize ); for ( final ViewSetup oldSetup : oldSetups ) { @@ -110,12 +112,17 @@ public static SpimData2 splitImages( final SpimData2 spimData, final long[] over final VoxelDimensions voxDim = oldSetup.getVoxelSize(); final Interval input = new FinalInterval( oldSetup.getSize() ); - final ArrayList< Interval > intervals = distributeIntervalsFixedOverlap( input, overlapPx, targetSize ); + + IOFunctions.println( "ViewId " + oldSetup.getId() + " with interval " + Util.printInterval( input ) + " will be split as follows: " ); + + final ArrayList< Interval > intervals = distributeIntervalsFixedOverlap( input, overlapPx, targetSize, minStepSize, optimize ); for ( int i = 0; i < intervals.size(); ++i ) { final Interval interval = intervals.get( i ); + IOFunctions.println( "Interval " + (i+1) + ": " + Util.printInterval( interval ) ); + // from the new ID get the old ID and the corresponding interval new2oldSetupId.put( newId, oldID ); newSetupId2Interval.put( newId, interval ); @@ -261,20 +268,21 @@ else if ( MultiResolutionImgLoader.class.isInstance( underlyingImgLoader ) ) return spimDataNew; } - private static final int maxIntervalSpread( final List< ViewSetup > oldSetups, final long[] overlapPx, final long[] targetSize ) + private static final int maxIntervalSpread( final List< ViewSetup > oldSetups, final long[] overlapPx, final long[] targetSize, final long[] minStepSize, final boolean optimize ) { int max = 1; for ( final ViewSetup oldSetup : oldSetups ) { final Interval input = new FinalInterval( oldSetup.getSize() ); - final ArrayList< Interval > intervals = distributeIntervalsFixedOverlap( input, overlapPx, targetSize ); + final ArrayList< Interval > intervals = distributeIntervalsFixedOverlap( input, overlapPx, targetSize, minStepSize, optimize ); max = Math.max( max, intervals.size() ); } return max; } + private static final boolean contains( final double[] l, final Interval interval ) { for ( int d = 0; d < l.length; ++d ) @@ -284,12 +292,41 @@ private static final boolean contains( final double[] l, final Interval interval return true; } - public static ArrayList< Interval > distributeIntervalsFixedOverlap( final Interval input, final long[] overlapPx, final long[] targetSize ) + /** + * computes a set of overlapping intervals with desired target size and overlap. Importantly, minStepSize is computed from the multi-resolution pyramid and constrains + * that intervals need to be divisible by minStepSize (except the last one) AND that the offsets where images start are divisble by minStepSize. + * + * Otherwise one would need to recompute the multi-resolution pyramid. + * + * @param input + * @param overlapPx + * @param targetSize + * @param minStepSize + * @param optimize - optimize targetsize to make tiles as equal as possible + * @return + */ + public static ArrayList< Interval > distributeIntervalsFixedOverlap( final Interval input, final long[] overlapPx, final long[] targetSize, final long[] minStepSize, final boolean optimize ) { + for ( int d = 0; d < input.numDimensions(); ++d ) + { + if ( targetSize[ d ] % minStepSize[ d ] != 0 ) + { + IOFunctions.printErr( "targetSize " + targetSize[ d ] + " not divisible by minStepSize " + minStepSize[ d ] + " for dim=" + d + ". stopping." ); + return null; + } + + if ( overlapPx[ d ] % minStepSize[ d ] != 0 ) + { + IOFunctions.printErr( "overlapPx " + overlapPx[ d ] + " not divisible by minStepSize " + minStepSize[ d ] + " for dim=" + d + ". stopping." ); + return null; + } + } + final ArrayList< ArrayList< Pair< Long, Long > > > intervalBasis = new ArrayList<>(); for ( int d = 0; d < input.numDimensions(); ++d ) { + System.out.println( "dim="+ d); final ArrayList< Pair< Long, Long > > dimIntervals = new ArrayList<>(); final long length = input.dimension( d ); @@ -305,46 +342,70 @@ public static ArrayList< Interval > distributeIntervalsFixedOverlap( final Inter } else { - final double l = length; - final double s = targetSize[ d ]; - final double o = overlapPx[ d ]; - - final double numCenterBlocks = ( l - 2.0 * ( s-o ) - o ) / ( s - 2.0 * o + o ); - final long numCenterBlocksInt; + final long l = length; + final long s = targetSize[ d ]; + final long o = overlapPx[ d ]; - if ( numCenterBlocks <= 0.0 ) - numCenterBlocksInt = 0; - else - numCenterBlocksInt = Math.round( numCenterBlocks ); - - final double n = numCenterBlocksInt; + // now we iterate the targetsize until we are as close as possible to an equal distribution (ideally 0.0 fraction) - final double newSize = ( l + o + n * o ) / ( 2.0 + n ); - final long newSizeInt = Math.round( newSize ); + long lastImageSize = lastImageSize(l, s, o);// o + ( l - 2 * ( s-o ) - o ) % ( s - 2 * o + o ); - System.out.println( "numCenterBlocks: " + numCenterBlocks ); - System.out.println( "numCenterBlocksInt: " + numCenterBlocksInt ); - System.out.println( "numBlocks: " + (numCenterBlocksInt + 2) ); - System.out.println( "newSize: " + newSize ); - System.out.println( "newSizeInt: " + newSizeInt ); + System.out.println( "length: " + l ); + System.out.println( "overlap: " + o ); + System.out.println( "targetSize: " + s ); + System.out.println( "lastImageSize: " + lastImageSize ); - System.out.println(); - //System.out.println( "block 0: " + input.min( d ) + " " + (input.min( d ) + Math.round( newSize ) - 1) ); + final long finalSize; - for ( int i = 0; i <= numCenterBlocksInt; ++i ) + if ( optimize && lastImageSize != s ) { - final long from = Math.round( input.min( d ) + i * newSize - i * o ); - final long to = from + newSizeInt - 1; + long lastSize = s; + long delta, currentLastImageSize; + + if ( lastImageSize <= s / 2 ) + { + // increase image size until lastImageSize goes towards zero, then large + + do + { + lastSize += minStepSize[ d ]; + currentLastImageSize = lastImageSize(l, lastSize, o); + delta = lastImageSize - currentLastImageSize; + + lastImageSize = currentLastImageSize; + //System.out.println( lastSize + ": " + lastImageSize + ", delta=" + delta ); + } + while ( delta > 0 ); + + finalSize = lastSize; + } + else + { + // decrease image size until lastImageSize is maximal - System.out.println( "block " + (numCenterBlocksInt) + ": " + from + " " + to ); - dimIntervals.add( new ValuePair< Long, Long >( from, to ) ); + do + { + lastSize -= minStepSize[ d ]; + currentLastImageSize = lastImageSize(l, lastSize, o); + delta = lastImageSize - currentLastImageSize; + + lastImageSize = currentLastImageSize; + //System.out.println( lastSize + ": " + lastImageSize + ", delta=" + delta ); + } + while ( delta < 0 ); + + finalSize = lastSize + minStepSize[ d ]; + } + } + else + { + finalSize = s; } - final long from = ( input.max( d ) - Math.round( newSize ) + 1 ); - final long to = input.max( d ); - - System.out.println( "block " + (numCenterBlocksInt + 1) + ": " + from + " " + to ); - dimIntervals.add( new ValuePair< Long, Long >( from, to ) ); + System.out.println( "finalSize: " + finalSize ); + System.out.println( "finalLastImageSize: " + lastImageSize(l, finalSize, o) ); + + dimIntervals.addAll( splitDim( input, d, finalSize, overlapPx[ d ] ) ); } intervalBasis.add( dimIntervals ); @@ -381,13 +442,58 @@ public static ArrayList< Interval > distributeIntervalsFixedOverlap( final Inter return intervalList; } + public static long lastImageSize( final long l, final long s, final long o) + { + return o + ( l - 2 * ( s-o ) - o ) % ( s - 2 * o + o ); + } + + public static double numCenterBlocks( final double l, final double s, final double o ) + { + return ( l - 2.0 * ( s-o ) - o ) / ( s - 2.0 * o + o ); + } + + public static ArrayList< Pair< Long, Long > > splitDim( + final Interval input, + final int d, + final long s, + final long o ) + { + System.out.println( "min=" + input.min( d ) + ", max=" + input.max( d ) ); + + final ArrayList< Pair< Long, Long > > dimIntervals = new ArrayList<>(); + + long from = input.min( d ); + long to; + + do + { + to = Math.min( input.max( d ), from + s - 1 ); + dimIntervals.add( new ValuePair<>( from, to ) ); + + System.out.println( "block " + (dimIntervals.size() - 1) + ": " + from + " " + to + " (size=" + (to-from+1) + ")" ); + + //SimpleMultiThreading.threadWait( 100 ); + from = to - o + 1; + } + while ( to < input.max( d ) ); + + return dimIntervals; + } + public static void main( String[] args ) { - Interval input = new FinalInterval( new long[]{ 0 }, new long[] { 1915 - 1 } ); - long[] overlapPx = new long[] { 10 }; + Interval input = new FinalInterval( new long[]{ 0 }, new long[] { 2048 - 1 } ); + + long[] overlapPx = new long[] { 17 }; long[] targetSize = new long[] { 500 }; + long[] minStepSize = new long[] { 32 }; + + targetSize[ 0 ] = Split_Views.closestLongDivisableBy( targetSize[ 0 ], minStepSize[ 0 ] ); + overlapPx[ 0 ] = Split_Views.closestLargerLongDivisableBy( overlapPx[ 0 ], minStepSize[ 0 ] ); + + boolean optimize = true; - ArrayList< Interval > intervals = distributeIntervalsFixedOverlap( input, overlapPx, targetSize ); + ArrayList< Interval > intervals = distributeIntervalsFixedOverlap( input, overlapPx, targetSize, minStepSize, optimize ); System.out.println();