From da5d069789adbd616b8ded65bf0622105c436333 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Sat, 7 Sep 2024 19:11:35 -0400 Subject: [PATCH] add compute block size as it is very useful and more explanations --- .../plugin/resave/ParametersResaveN5.java | 42 ++++++++++++++----- .../mvrecon/fiji/plugin/resave/Resave_N5.java | 1 - 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java index e0453da1..b9d17e8b 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java @@ -22,6 +22,7 @@ */ package net.preibisch.mvrecon.fiji.plugin.resave; +import java.awt.Color; import java.awt.Font; import java.net.URI; import java.net.URISyntaxException; @@ -42,7 +43,9 @@ import bdv.export.ProposeMipmaps; import fiji.util.gui.GenericDialogPlus; import mpicbg.spim.data.sequence.ViewSetup; +import net.imglib2.Dimensions; import net.preibisch.legacy.io.IOFunctions; +import net.preibisch.mvrecon.fiji.plugin.util.GUIHelper; public class ParametersResaveN5 { @@ -50,21 +53,26 @@ public class ParametersResaveN5 public static int defaultBlockSize = 64; public static int defaultBlockSizeXY = 128; public static int defaultCompression = 1; + + public static int defaultBlockSizeFactorX = 16; + public static int defaultBlockSizeFactorY = 16; + public static int defaultBlockSizeFactorZ = 1; + public static int defaultNumThreads = Math.max( 1, Runtime.getRuntime().availableProcessors() - 1 ); public URI xmlURI, n5URI; + public int[] blockSizeFactor; public int[][] resolutions, subdivisions; public Map< Integer, ExportMipmapInfo > proposedMipmaps; public Compression compression; public int numCellCreatorThreads = 1; - //public boolean saveXML = true; // mostly important for cluster-based re-saving - //public boolean saveData = true; // mostly important for cluster-based re-saving - - //public boolean setFinishedAttributeInN5 = true; // required if double-checking that all ViewId were written - //final public static String finishedAttrib = "saved_completely"; // required if double-checking that all ViewId were written + public static URI createN5URIfromXMLURI( final URI xmlURI ) + { + return URI.create( xmlURI.toString().subSequence( 0, xmlURI.toString().length() - 4 ) + ".n5" ); + } public static ParametersResaveN5 getParamtersIJ( final Collection< ViewSetup > setupsToProcess ) @@ -82,11 +90,6 @@ public static ParametersResaveN5 getParamtersIJ( return getParamtersIJ( xmlURI, n5URI, setupsToProcess, askForPaths ); } - public static URI createN5URIfromXMLURI( final URI xmlURI ) - { - return URI.create( xmlURI.toString().subSequence( 0, xmlURI.toString().length() - 4 ) + ".n5" ); - } - public static ParametersResaveN5 getParamtersIJ( final URI xmlURI, final URI n5URI, @@ -108,6 +111,14 @@ public static ParametersResaveN5 getParamtersIJ( row[ 1 ] = ParametersResaveN5.defaultBlockSizeXY; } + final long[] maxDimensions = setupsToProcess.iterator().next().getSize().dimensionsAsLongArray(); + + setupsToProcess.forEach( viewSetup -> + { + for ( int d = 0; d < maxDimensions.length; ++d ) + maxDimensions[ d ] = Math.max( maxDimensions[ d ], viewSetup.getSize().dimension( d ) ); + } ); + final GenericDialogPlus gdp = new GenericDialogPlus( "Options" ); gdp.addMessage( "N5 saving options", new Font( Font.SANS_SERIF, Font.BOLD, 13 ) ); @@ -115,6 +126,12 @@ public static ParametersResaveN5 getParamtersIJ( gdp.addChoice( "Compression", compressions, compressions[ defaultCompression ] ); gdp.addStringField( "Downsampling_factors", ProposeMipmaps.getArrayString( autoMipmapSettings.getExportResolutions() ), 40 ); gdp.addStringField( "Block_size (all the same)", ProposeMipmaps.getArrayString( autoMipmapSettings.getSubdivisions() ), 40 ); + gdp.addSlider( "Compute_block_size_factor_X", 1, 128, defaultBlockSizeFactorX ); + gdp.addSlider( "Compute_block_size_factor_Y", 1, 128, defaultBlockSizeFactorY ); + gdp.addSlider( "Compute_block_size_factor_Z", 1, 128, defaultBlockSizeFactorZ ); + gdp.addMessage( "Defines how many blocks are written at once; e.g. 4,4,1 & blockSize 128,128,32 means each thread writes 512,512,32 chunks.\n" + + "For optimal performance, the size in XY is as large as each XY plane, as usually entire planes must be read.", GUIHelper.smallStatusFont ); + gdp.addMessage( "Max dimensions of the images that will be re-saved: " + Arrays.toString( maxDimensions ), GUIHelper.smallStatusFont, Color.red ); gdp.addNumericField( "Number_of_threads (CPUs:" + Runtime.getRuntime().availableProcessors() + ")", defaultNumThreads, 0 ); if ( askForPaths ) @@ -134,6 +151,11 @@ public static ParametersResaveN5 getParamtersIJ( final String subsampling = gdp.getNextString(); final String chunkSizes = gdp.getNextString(); + final int blockSizeFactorX = defaultBlockSizeFactorX = Math.max( 1, (int)Math.round( gdp.getNextNumber() ) ); + final int blockSizeFactorY = defaultBlockSizeFactorY = Math.max( 1, (int)Math.round( gdp.getNextNumber() ) ); + final int blockSizeFactorZ = defaultBlockSizeFactorZ = Math.max( 1, (int)Math.round( gdp.getNextNumber() ) ); + + n5params.blockSizeFactor = new int[] { blockSizeFactorX, blockSizeFactorY, blockSizeFactorZ }; n5params.numCellCreatorThreads = defaultNumThreads = Math.max( 1, (int)Math.round( gdp.getNextNumber() ) ); if ( askForPaths ) diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java index f1ca3fbd..3e5b8394 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java @@ -183,7 +183,6 @@ else if ( URITools.isS3( n5Params.n5URI ) || URITools.isGC( n5Params.n5URI ) )*/ // // Save remaining downsampling levels (s1 ... sN) // - for ( int level = 1; level < downsamplings.length; ++level ) { final int s = level;