Skip to content

Commit

Permalink
consolidate Fusion and Resave methods for BDV metadata writing
Browse files Browse the repository at this point in the history
  • Loading branch information
@StephanPreibisch
StephanPreibisch committed Sep 10, 2024
1 parent f4396c6 commit b691668
Show file tree
Hide file tree
Showing 4 changed files with 237 additions and 157 deletions.
165 changes: 71 additions & 94 deletions src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
import net.preibisch.mvrecon.fiji.plugin.queryXML.LoadParseQueryXML;
import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
import net.preibisch.mvrecon.process.export.ExportN5API.StorageType;
import net.preibisch.mvrecon.process.export.ExportTools;
import net.preibisch.mvrecon.process.resave.N5ResaveTools;
import net.preibisch.mvrecon.process.resave.SpimData2Tools;
import util.Grid;
Expand Down Expand Up @@ -109,126 +111,101 @@ public static SpimData2 resaveN5(
});
}

/*
// re-save data to file
if ( URITools.isFile( n5Params.n5URI ) )
{
try
{
WriteSequenceToN5.writeN5File(
sdReduced.getSequenceDescription(),
n5Params.proposedMipmaps,
n5Params.compression, //new GzipCompression()
new File( URITools.removeFilePrefix( n5Params.n5URI ) ),
new bdv.export.ExportScalePyramid.DefaultLoopbackHeuristic(),
null,
n5Params.numCellCreatorThreads, // Runtime.getRuntime().availableProcessors()
progressWriter );
}
catch ( IOException e )
{
e.printStackTrace();
}
}
else if ( URITools.isS3( n5Params.n5URI ) || URITools.isGC( n5Params.n5URI ) )*/
{
// save to cloud or file
final N5Writer n5Writer = URITools.instantiateGuessedN5Writer( n5Params.n5URI );
// save to cloud or file
final N5Writer n5Writer = URITools.instantiateGuessedN5Writer( n5Params.n5URI );

final int[] blockSize = n5Params.subdivisions[ 0 ];
final int[] computeBlockSize = new int[ blockSize.length ];
final Compression compression = n5Params.compression;

final int[] blockSize = n5Params.subdivisions[ 0 ];
final int[] computeBlockSize = new int[ blockSize.length ];
final Compression compression = n5Params.compression;
for ( int d = 0; d < blockSize.length; ++d )
computeBlockSize[ d ] = blockSize[ d ] * n5Params.blockSizeFactor[ d ];

for ( int d = 0; d < blockSize.length; ++d )
computeBlockSize[ d ] = blockSize[ d ] * n5Params.blockSizeFactor[ d ];
final HashMap<Integer, long[]> dimensions =
N5ResaveTools.assembleDimensions( data, vidsToResave );

//final ArrayList<ViewSetup> viewSetups =
// N5ResaveTools.assembleViewSetups( data, vidsToResave );
final int[][] downsamplings =
N5ResaveTools.mipMapInfoToDownsamplings( n5Params.proposedMipmaps );

final HashMap<Integer, long[]> viewSetupIdToDimensions =
N5ResaveTools.assembleDimensions( data, vidsToResave );
final ArrayList<long[][]> grid =
N5ResaveTools.assembleS0Jobs( vidsToResave, dimensions, blockSize, computeBlockSize );

IOFunctions.println( "Dimensions of raw images: " );
viewSetupIdToDimensions.forEach( (id,dim ) -> IOFunctions.println( "ViewSetup " + id + ": " + Arrays.toString( dim )) );
final Map<Integer, DataType> dataTypes =
N5ResaveTools.assembleDataTypes( data, dimensions.keySet() );

final int[][] downsamplings =
N5ResaveTools.mipMapInfoToDownsamplings( n5Params.proposedMipmaps );
// write BDV metadata for all ViewIds (including downsampling)
vidsToResave.forEach( viewId -> ExportTools.writeBDVDatasetMetadataN5(
n5Writer, viewId, dataTypes.get( viewId.getViewSetupId() ), dimensions.get( viewId.getViewSetupId() ), compression, blockSize, downsamplings));

IOFunctions.println( "Downsamplings: " + Arrays.deepToString( downsamplings ) );
IOFunctions.println( "Dimensions of raw images: " );
dimensions.forEach( ( id,dim ) -> IOFunctions.println( "ViewSetup " + id + ": " + Arrays.toString( dim )) );

final ArrayList<long[][]> grid =
N5ResaveTools.assembleAllS0Jobs( vidsToResave, viewSetupIdToDimensions, blockSize, computeBlockSize );
IOFunctions.println( "Downsamplings: " + Arrays.deepToString( downsamplings ) );

final Map<Integer, DataType> dataTypes =
N5ResaveTools.createGroups( n5Writer, data, viewSetupIdToDimensions, blockSize, downsamplings, compression );
//
// Save full resolution dataset (s0)
//
final ForkJoinPool myPool = new ForkJoinPool( n5Params.numCellCreatorThreads );

N5ResaveTools.createS0Datasets( n5Writer, vidsToResave, dataTypes, viewSetupIdToDimensions, blockSize, compression );
long time = System.currentTimeMillis();

try
{
myPool.submit(() -> grid.parallelStream().forEach(
gridBlock -> N5ResaveTools.resaveS0Block(
data,
n5Writer,
dataTypes.get( (int)gridBlock[ 3 ][ 1 ] ),
N5ResaveTools.datasetMappingFunctionBdv( 0, StorageType.N5 ),
gridBlock ) ) ).get();
}
catch (InterruptedException | ExecutionException e)
{
IOFunctions.println( "Failed to write s0 for N5 '" + n5Params.n5URI + "'. Error: " + e );
e.printStackTrace();
return null;
}

//
// Save full resolution dataset (s0)
//
final ForkJoinPool myPool = new ForkJoinPool( n5Params.numCellCreatorThreads );
IOFunctions.println( "Saved level s0, took: " + (System.currentTimeMillis() - time ) + " ms." );

long time = System.currentTimeMillis();
//
// Save remaining downsampling levels (s1 ... sN)
//
for ( int level = 1; level < downsamplings.length; ++level )
{
final int s = level;
final int[] ds = N5ResaveTools.computeRelativeDownsampling( downsamplings, s );
IOFunctions.println( "Downsampling: " + Util.printCoordinates( downsamplings[ s ] ) + " with relative downsampling of " + Util.printCoordinates( ds ));

final ArrayList<long[][]> allBlocks =
N5ResaveTools.prepareDownsampling( vidsToResave, n5Writer, level, ds, downsamplings[ s ], blockSize, compression );

time = System.currentTimeMillis();

try
{
myPool.submit(() -> grid.parallelStream().forEach(
gridBlock -> N5ResaveTools.resaveS0Block(
data,
myPool.submit(() -> allBlocks.parallelStream().forEach(
gridBlock -> N5ResaveTools.writeDownsampledBlock(
n5Writer,
dataTypes.get( (int)gridBlock[ 3 ][ 1 ] ),
N5ResaveTools.datasetMappingFunctionBdvN5( 0 ),
N5ResaveTools.datasetMappingFunctionBdv( s, StorageType.N5 ),
N5ResaveTools.datasetMappingFunctionBdv( s - 1, StorageType.N5 ),
ds,
gridBlock ) ) ).get();
}
catch (InterruptedException | ExecutionException e)
{
IOFunctions.println( "Failed to write s0 for N5 '" + n5Params.n5URI + "'. Error: " + e );
IOFunctions.println( "Failed to write downsample step s" + s +" for N5 '" + n5Params.n5URI + "'. Error: " + e );
e.printStackTrace();
return null;
}

IOFunctions.println( "Saved level s0, took: " + (System.currentTimeMillis() - time ) + " ms." );

//
// Save remaining downsampling levels (s1 ... sN)
//
for ( int level = 1; level < downsamplings.length; ++level )
{
final int s = level;
final int[] ds = N5ResaveTools.computeRelativeDownsampling( downsamplings, s );
IOFunctions.println( "Downsampling: " + Util.printCoordinates( downsamplings[ s ] ) + " with relative downsampling of " + Util.printCoordinates( ds ));

final ArrayList<long[][]> allBlocks =
N5ResaveTools.prepareDownsampling( vidsToResave, n5Writer, level, ds, downsamplings[ s ], blockSize, compression );

time = System.currentTimeMillis();

try
{
myPool.submit(() -> allBlocks.parallelStream().forEach(
gridBlock -> N5ResaveTools.writeDownsampledBlock(
n5Writer,
N5ResaveTools.datasetMappingFunctionBdvN5( s ),
N5ResaveTools.datasetMappingFunctionBdvN5( s - 1 ),
ds,
gridBlock ) ) ).get();
}
catch (InterruptedException | ExecutionException e)
{
IOFunctions.println( "Failed to write downsample step s" + s +" for N5 '" + n5Params.n5URI + "'. Error: " + e );
e.printStackTrace();
return null;
}

IOFunctions.println( "Resaved N5 s" + s + " level, took: " + (System.currentTimeMillis() - time ) + " ms." );
}
IOFunctions.println( "Resaved N5 s" + s + " level, took: " + (System.currentTimeMillis() - time ) + " ms." );
}

myPool.shutdown();
try { myPool.awaitTermination( Long.MAX_VALUE, TimeUnit.HOURS ); } catch (InterruptedException e) { e.printStackTrace(); }
myPool.shutdown();
try { myPool.awaitTermination( Long.MAX_VALUE, TimeUnit.HOURS ); } catch (InterruptedException e) { e.printStackTrace(); }

n5Writer.close();
}
n5Writer.close();

sdReduced.getSequenceDescription().setImgLoader( new N5ImageLoader( n5Params.n5URI, sdReduced.getSequenceDescription() ) );
sdReduced.setBasePathURI( URITools.getParent( n5Params.xmlURI ) );
Expand Down
8 changes: 1 addition & 7 deletions src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,27 +25,22 @@
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;

import org.janelia.saalfeldlab.n5.Compression;
import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.GzipCompression;
import org.janelia.saalfeldlab.n5.N5FSWriter;
import org.janelia.saalfeldlab.n5.N5Writer;
import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer;
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
import org.janelia.saalfeldlab.n5.universe.N5Factory;
import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat;
import org.janelia.saalfeldlab.n5.zarr.N5ZarrWriter;

import bdv.export.ProposeMipmaps;
import fiji.util.gui.GenericDialogPlus;
Expand All @@ -70,7 +65,6 @@
import net.preibisch.mvrecon.fiji.plugin.resave.PluginHelper;
import net.preibisch.mvrecon.fiji.plugin.util.GUIHelper;
import net.preibisch.mvrecon.process.deconvolution.DeconViews;
import net.preibisch.mvrecon.process.downsampling.lazy.LazyHalfPixelDownsample2x;
import net.preibisch.mvrecon.process.export.ExportTools.InstantiateViewSetupBigStitcher;
import net.preibisch.mvrecon.process.interestpointregistration.pairwise.constellation.grouping.Group;
import net.preibisch.mvrecon.process.resave.N5ResaveTools;
Expand Down Expand Up @@ -241,7 +235,7 @@ else if ( FloatType.class.isInstance( type ) )

IOFunctions.println( "Assigning ViewId " + Group.pvid( viewId ) );

dataset = ExportTools.createBDVPath( viewId, this.storageType );
dataset = ExportTools.createBDVPath( viewId, 0, this.storageType );
}

//
Expand Down
Loading

0 comments on commit b691668

Please sign in to comment.