diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java index eb741355..b538a63b 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java @@ -46,6 +46,7 @@ import mpicbg.spim.data.sequence.ViewId; import mpicbg.spim.data.sequence.ViewSetup; import net.imglib2.util.Util; +import net.imglib2.util.ValuePair; import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.plugin.queryXML.LoadParseQueryXML; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; @@ -127,25 +128,31 @@ public static SpimData2 resaveN5( final int[][] downsamplings = N5ApiTools.mipMapInfoToDownsamplings( n5Params.proposedMipmaps ); - final ArrayList grid = - N5ApiTools.assembleS0Jobs( vidsToResave, dimensions, blockSize, computeBlockSize ); + final List grid = + vidsToResave.stream().map( viewId -> + N5ApiTools.assembleJobs( + viewId, + dimensions.get( viewId.getViewSetupId() ), + blockSize, + computeBlockSize ) ).flatMap(List::stream).collect( Collectors.toList() ); final Map dataTypes = N5ApiTools.assembleDataTypes( data, dimensions.keySet() ); - // create all datasets and write BDV metadata for all ViewIds (including downsampling) - final HashMap< ViewId, MultiResolutionLevelInfo[] > viewIdToMrInfo = new HashMap<>(); - + // create all datasets and write BDV metadata for all ViewIds (including downsampling) in parallel long time = System.currentTimeMillis(); - for ( final ViewId viewId : vidsToResave ) - viewIdToMrInfo.put( viewId , N5ApiTools.setupBdvDatasetsN5( - n5Writer, viewId, - dataTypes.get( viewId.getViewSetupId() ), - dimensions.get( viewId.getViewSetupId() ), - compression, - blockSize, - downsamplings) ); + final Map< ViewId, MultiResolutionLevelInfo[] > viewIdToMrInfo = + vidsToResave.parallelStream().map( + viewId -> new ValuePair<>( + viewId, + N5ApiTools.setupBdvDatasetsN5( + n5Writer, viewId, + dataTypes.get( viewId.getViewSetupId() ), + dimensions.get( viewId.getViewSetupId() ), + compression, + blockSize, + downsamplings ) ) ).collect(Collectors.toMap( e -> e.getA(), e -> e.getB() )); IOFunctions.println( "Created BDV-metadata, took: " + (System.currentTimeMillis() - time ) + " ms." ); IOFunctions.println( "Dimensions of raw images: " ); @@ -185,8 +192,14 @@ public static SpimData2 resaveN5( for ( int level = 1; level < downsamplings.length; ++level ) { final int s = level; - final ArrayList allBlocks = - N5ApiTools.assembleDownsamplingJobs( vidsToResave, viewIdToMrInfo, level ); + //final ArrayList allBlocks = + // N5ApiTools.assembleJobs( vidsToResave, viewIdToMrInfo, level ); + + final List allBlocks = + vidsToResave.stream().map( viewId -> + N5ApiTools.assembleJobs( + viewId, + viewIdToMrInfo.get(viewId)[s] )).flatMap(List::stream).collect( Collectors.toList() ); IOFunctions.println( "Downsampling level s" + s + "... " ); IOFunctions.println( "Number of compute blocks: " + allBlocks.size() ); diff --git a/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java b/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java index 3476473a..da9c8d7e 100644 --- a/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java +++ b/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java @@ -263,15 +263,13 @@ else if ( FloatType.class.isInstance( type ) ) this.downsampling ); } - final List grid = - Grid.create( - bb.dimensionsAsLongArray(), - new int[] { - blocksize()[0] * computeBlocksizeFactor()[ 0 ], - blocksize()[1] * computeBlocksizeFactor()[ 1 ], - blocksize()[2] * computeBlocksizeFactor()[ 2 ] - }, - blocksize() ); + final List grid = N5ApiTools.assembleJobs( + mrInfo[ 0 ], + new int[] { + blocksize()[0] * computeBlocksizeFactor()[ 0 ], + blocksize()[1] * computeBlocksizeFactor()[ 1 ], + blocksize()[2] * computeBlocksizeFactor()[ 2 ] + } ); IOFunctions.println( "num blocks = " + Grid.create( bb.dimensionsAsLongArray(), blocksize() ).size() + ", size = " + bsX + "x" + bsY + "x" + bsZ ); IOFunctions.println( "num compute blocks = " + grid.size() + ", size = " + bsX*bsFactorX + "x" + bsY*bsFactorY + "x" + bsZ*bsFactorZ ); @@ -325,40 +323,37 @@ else if ( FloatType.class.isInstance( type ) ) // // save multiresolution pyramid (s1 ... sN) // - if ( this.downsampling != null ) + for ( int level = 1; level < mrInfo.length; ++level ) { - for ( int level = 1; level < this.downsampling.length; ++level ) - { - final int s = level; - final ArrayList allBlocks = N5ApiTools.assembleDownsamplingJobs( mrInfo[ level ] ); + final int s = level; + final ArrayList allBlocks = N5ApiTools.assembleJobs( mrInfo[ level ] ); - IOFunctions.println( new Date( System.currentTimeMillis() ) + ": Downsampling: " + Util.printCoordinates( mrInfo[ level ].absoluteDownsampling ) + " with relative downsampling of " + Util.printCoordinates( mrInfo[ level ].relativeDownsampling )); - IOFunctions.println( new Date( System.currentTimeMillis() ) + ": s" + level + " num blocks=" + allBlocks.size() ); - IOFunctions.println( new Date( System.currentTimeMillis() ) + ": Loading '" + mrInfo[ level - 1 ].dataset + "', downsampled will be written as '" + mrInfo[ level ].dataset + "'." ); + IOFunctions.println( new Date( System.currentTimeMillis() ) + ": Downsampling: " + Util.printCoordinates( mrInfo[ level ].absoluteDownsampling ) + " with relative downsampling of " + Util.printCoordinates( mrInfo[ level ].relativeDownsampling )); + IOFunctions.println( new Date( System.currentTimeMillis() ) + ": s" + level + " num blocks=" + allBlocks.size() ); + IOFunctions.println( new Date( System.currentTimeMillis() ) + ": Loading '" + mrInfo[ level - 1 ].dataset + "', downsampled will be written as '" + mrInfo[ level ].dataset + "'." ); - time = System.currentTimeMillis(); - - try - { - myPool.submit( () -> allBlocks.parallelStream().forEach( - gridBlock -> N5ApiTools.writeDownsampledBlock( - driverVolumeWriter, - mrInfo[ s ], - mrInfo[ s - 1 ], - gridBlock ) ) ).get(); - - myPool.shutdown(); - myPool.awaitTermination( Long.MAX_VALUE, TimeUnit.HOURS); - } - catch (InterruptedException | ExecutionException e) - { - IOFunctions.println( "Failed to write HDF5/N5/ZARR dataset '" + mrInfo[ level ].dataset + "'. Error: " + e ); - e.printStackTrace(); - return false; - } + time = System.currentTimeMillis(); - IOFunctions.println( new Date( System.currentTimeMillis() ) + ": Saved level s " + level + ", took: " + (System.currentTimeMillis() - time ) + " ms." ); + try + { + myPool.submit( () -> allBlocks.parallelStream().forEach( + gridBlock -> N5ApiTools.writeDownsampledBlock( + driverVolumeWriter, + mrInfo[ s ], + mrInfo[ s - 1 ], + gridBlock ) ) ).get(); + + myPool.shutdown(); + myPool.awaitTermination( Long.MAX_VALUE, TimeUnit.HOURS); } + catch (InterruptedException | ExecutionException e) + { + IOFunctions.println( "Failed to write HDF5/N5/ZARR dataset '" + mrInfo[ level ].dataset + "'. Error: " + e ); + e.printStackTrace(); + return false; + } + + IOFunctions.println( new Date( System.currentTimeMillis() ) + ": Saved level s " + level + ", took: " + (System.currentTimeMillis() - time ) + " ms." ); } return true; diff --git a/src/main/java/net/preibisch/mvrecon/process/n5api/N5ApiTools.java b/src/main/java/net/preibisch/mvrecon/process/n5api/N5ApiTools.java index c2f74c43..5c9e1ff4 100644 --- a/src/main/java/net/preibisch/mvrecon/process/n5api/N5ApiTools.java +++ b/src/main/java/net/preibisch/mvrecon/process/n5api/N5ApiTools.java @@ -264,7 +264,7 @@ public static MultiResolutionLevelInfo[] setupBdvDatasetsHDF5( { final String subdivisionsDatasets = "s" + String.format("%02d", viewId.getViewSetupId()) + "/subdivisions"; final String resolutionsDatasets = "s" + String.format("%02d", viewId.getViewSetupId()) + "/resolutions"; - + if ( driverVolumeWriter.datasetExists( subdivisionsDatasets ) && driverVolumeWriter.datasetExists( resolutionsDatasets ) ) { // TODO: test that the values are consistent? @@ -455,7 +455,8 @@ else if ( dataType == DataType.FLOAT32 ) } } - public static ArrayList assembleDownsamplingJobs( + /* + public static ArrayList assembleJobs( final Collection< ? extends ViewId > viewIds, final Map< ViewId, MultiResolutionLevelInfo[] > viewIdToMrInfo, final int level ) @@ -463,30 +464,90 @@ public static ArrayList assembleDownsamplingJobs( // all blocks (a.k.a. grids) across all ViewId's final ArrayList allBlocks = new ArrayList<>(); - viewIds.forEach( viewId -> allBlocks.addAll( assembleDownsamplingJobs( viewId, viewIdToMrInfo.get( viewId )[ level ] ) ) ); + viewIds.forEach( viewId -> allBlocks.addAll( assembleJobs( viewId, viewIdToMrInfo.get( viewId )[ level ] ) ) ); return allBlocks; } - public static ArrayList assembleDownsamplingJobs( final MultiResolutionLevelInfo mrInfo ) + + public static ArrayList assembleJobs( + final Collection< ? extends ViewId > viewIds, + final Map< ViewId, MultiResolutionLevelInfo[] > viewIdToMrInfo, + final int level, + final int[] computeBlockSize ) { - return assembleDownsamplingJobs( null, mrInfo ); + // all blocks (a.k.a. grids) across all ViewId's + final ArrayList allBlocks = new ArrayList<>(); + + viewIds.forEach( viewId -> allBlocks.addAll( assembleJobs( viewId, viewIdToMrInfo.get( viewId )[ level ], computeBlockSize ) ) ); + + return allBlocks; + }*/ + + public static ArrayList assembleJobs( final MultiResolutionLevelInfo mrInfo ) + { + return assembleJobs( null, mrInfo ); } - public static ArrayList assembleDownsamplingJobs( + public static ArrayList assembleJobs( final MultiResolutionLevelInfo mrInfo, final int[] computeBlockSize ) + { + return assembleJobs( null, mrInfo, computeBlockSize ); + } + + public static ArrayList assembleJobs( final ViewId viewId, final MultiResolutionLevelInfo mrInfo ) { - // all blocks (a.k.a. grids) across all ViewId's + return assembleJobs( viewId, mrInfo.dimensions, mrInfo.blockSize, mrInfo.blockSize ); + } + + public static ArrayList assembleJobs( + final ViewId viewId, + final MultiResolutionLevelInfo mrInfo, + final int[] computeBlockSize ) + { + return assembleJobs( viewId, mrInfo.dimensions, mrInfo.blockSize, computeBlockSize ); + } + + public static ArrayList assembleJobs( + final long[] dimensions, + final int[] blockSize ) + { + return assembleJobs(null, dimensions, blockSize, blockSize ); + } + + public static ArrayList assembleJobs( + final long[] dimensions, + final int[] blockSize, + final int[] computeBlockSize ) + { + return assembleJobs(null, dimensions, blockSize, computeBlockSize ); + } + + public static ArrayList assembleJobs( + final ViewId viewId, //can be null + final long[] dimensions, + final int[] blockSize ) + { + return assembleJobs(viewId, dimensions, blockSize, blockSize ); + } + + public static ArrayList assembleJobs( + final ViewId viewId, //can be null + final long[] dimensions, + final int[] blockSize, + final int[] computeBlockSize ) + { + // all blocks (a.k.a. grids) final ArrayList allBlocks = new ArrayList<>(); final List grid = Grid.create( - mrInfo.dimensions, - mrInfo.blockSize, - mrInfo.blockSize); + dimensions, + computeBlockSize, + blockSize); if ( viewId != null ) { - // add timepointId and ViewSetupId to the gridblock + // add timepointId and ViewSetupId & dimensions to the gridblock for ( final long[][] gridBlock : grid ) allBlocks.add( new long[][]{ gridBlock[ 0 ].clone(), @@ -573,35 +634,6 @@ public static int[][] mipMapInfoToDownsamplings( final Map< Integer, ExportMipma return downsamplings; } - public static ArrayList assembleS0Jobs( - final Collection< ? extends ViewId > viewIds, - final HashMap< Integer, long[] > viewSetupIdToDimensions, - final int[] blockSize, - final int[] computeBlockSize ) - { - // all blocks (a.k.a. grids) across all ViewId's - final ArrayList allBlocks = new ArrayList<>(); - - for ( final ViewId viewId : viewIds ) - { - final List grid = Grid.create( - viewSetupIdToDimensions.get( viewId.getViewSetupId() ), - computeBlockSize, - blockSize); - - // add timepointId and ViewSetupId & dimensions to the gridblock - for ( final long[][] gridBlock : grid ) - allBlocks.add( new long[][]{ - gridBlock[ 0 ].clone(), - gridBlock[ 1 ].clone(), - gridBlock[ 2 ].clone(), - new long[] { viewId.getTimePointId(), viewId.getViewSetupId() } - }); - } - - return allBlocks; - } - public static HashMap assembleDimensions( final SpimData data, final Collection< ? extends ViewId > viewIds )