diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java index b9e5c9e8..ebe4f461 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java @@ -29,19 +29,18 @@ import java.util.Collection; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.universe.N5Factory; import bdv.export.ExportMipmapInfo; import bdv.export.ProgressWriter; import bdv.export.n5.WriteSequenceToN5; import bdv.img.n5.N5ImageLoader; -import ij.ImageJ; import ij.plugin.PlugIn; import mpicbg.spim.data.sequence.TimePoint; import mpicbg.spim.data.sequence.ViewId; @@ -53,6 +52,7 @@ import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2; import net.preibisch.mvrecon.process.resave.N5ResaveTools; import net.preibisch.mvrecon.process.resave.SpimData2Tools; +import util.Grid; import util.URITools; public class Resave_N5 implements PlugIn @@ -189,7 +189,7 @@ else if ( URITools.isS3( n5Params.n5URI ) || URITools.isGC( n5Params.n5URI ) ) try { - myPool.submit(() -> allBlocks.parallelStream().forEach( gridBlock -> N5ResaveTools.writeDownsampledBlock( data, n5Writer, s, ds, gridBlock ) ) ).get(); + myPool.submit(() -> allBlocks.parallelStream().forEach( gridBlock -> N5ResaveTools.writeDownsampledBlock( n5Writer, s, ds, gridBlock ) ) ).get(); } catch (InterruptedException | ExecutionException e) { @@ -224,7 +224,15 @@ else if ( URITools.isS3( n5Params.n5URI ) || URITools.isGC( n5Params.n5URI ) ) public static void main(String[] args) { - new ImageJ(); - new Resave_N5().run( null ); + List grid = Grid.create( new long[] { 500, 500 }, new int[] { 400, 400 }, new int[] { 200, 200 } ); + + grid.forEach( b -> { + System.out.println( Arrays.toString( b[0])); + System.out.println( Arrays.toString( b[1])); + System.out.println( Arrays.toString( b[2])); + System.out.println(); + }); + //new ImageJ(); + //new Resave_N5().run( null ); } } diff --git a/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java b/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java index 11eddd0f..987f99c0 100644 --- a/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java +++ b/src/main/java/net/preibisch/mvrecon/process/export/ExportN5API.java @@ -25,11 +25,13 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -70,6 +72,7 @@ import net.preibisch.mvrecon.process.downsampling.lazy.LazyHalfPixelDownsample2x; import net.preibisch.mvrecon.process.export.ExportTools.InstantiateViewSetupBigStitcher; import net.preibisch.mvrecon.process.interestpointregistration.pairwise.constellation.grouping.Group; +import net.preibisch.mvrecon.process.resave.N5ResaveTools; import util.Grid; import util.URITools; @@ -363,8 +366,31 @@ else if ( FloatType.class.isInstance( type ) ) // // save multiresolution pyramid (s1 ... sN) // + /* + for ( int level = 1; level < this.downsampling.length; ++level ) + { + final int s = level; + final int[] ds = N5ResaveTools.computeRelativeDownsampling( this.downsampling, s ); + IOFunctions.println( "Downsampling: " + Util.printCoordinates( this.downsampling[ s ] ) + " with relative downsampling of " + Util.printCoordinates( ds )); + + final ArrayList allBlocks = N5ResaveTools.prepareDownsampling( vidsToResave, n5Writer, level, blockSize, ds, this.downsampling[ s ], compression ); + + time = System.currentTimeMillis(); - // TODO: use code from N5ResaveTools + try + { + myPool.submit(() -> allBlocks.parallelStream().forEach( gridBlock -> N5ResaveTools.writeDownsampledBlock( data, n5Writer, s, ds, gridBlock ) ) ).get(); + } + catch (InterruptedException | ExecutionException e) + { + IOFunctions.println( "Failed to write HDF5/N5/ZARR. Error: " + e ); + e.printStackTrace(); + return false; + } + + IOFunctions.println( "Resaved N5 s" + s + " level, took: " + (System.currentTimeMillis() - time ) + " ms." ); + } + */ if ( this.downsampling != null ) { long[] previousDim = bb.dimensionsAsLongArray(); @@ -372,10 +398,8 @@ else if ( FloatType.class.isInstance( type ) ) for ( int level = 1; level < this.downsampling.length; ++level ) { - final int[] ds = new int[ this.downsampling[ 0 ].length ]; - - for ( int d = 0; d < ds.length; ++d ) - ds[ d ] = this.downsampling[ level ][ d ] / this.downsampling[ level - 1 ][ d ]; + final int s = level; + final int[] ds = N5ResaveTools.computeRelativeDownsampling( this.downsampling, level ); IOFunctions.println( "Downsampling: " + Util.printCoordinates( this.downsampling[ level ] ) + " with relative downsampling of " + Util.printCoordinates( ds )); @@ -383,8 +407,8 @@ else if ( FloatType.class.isInstance( type ) ) for ( int d = 0; d < dim.length; ++d ) dim[ d ] = previousDim[ d ] / ds[ d ]; - final String datasetDownsampling = - bdv ? ExportTools.createDownsampledBDVPath(dataset, level, storageType) : dataset.substring(0, dataset.length() - 3) + "/s" + level; + final String datasetDownsampling = bdv ? + ExportTools.createDownsampledBDVPath(dataset, level, storageType) : dataset.substring(0, dataset.length() - 3) + "/s" + level; try { @@ -401,14 +425,7 @@ else if ( FloatType.class.isInstance( type ) ) return false; } - final List gridDS = Grid.create( - dim, - new int[] { - blocksize()[0], - blocksize()[1], - blocksize()[2] - }, - blocksize()); + final List gridDS = Grid.create( dim, blocksize()); IOFunctions.println( new Date( System.currentTimeMillis() ) + ": s" + level + " num blocks=" + gridDS.size() ); @@ -419,6 +436,9 @@ else if ( FloatType.class.isInstance( type ) ) time = System.currentTimeMillis(); + e.submit( () -> gridDS.parallelStream().forEach( gridBlock -> N5ResaveTools.writeDownsampledBlock( driverVolumeWriter, s, ds, gridBlock ) ) ); + + /* e.submit(() -> gridDS.parallelStream().forEach( gridBlock -> @@ -473,33 +493,6 @@ else if ( dataType == DataType.FLOAT32 ) final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(downsampled, gridBlock[0], gridBlock[1]); N5Utils.saveNonEmptyBlock(sourceGridBlock, driverVolumeWriter, datasetDownsampling, gridBlock[2], new FloatType()); } - // this can be removed because of: https://github.com/bigdataviewer/bigdataviewer-core/pull/157 - /* - else if ( dataType == DataType.INT16 ) - { - // Tobias: unfortunately I store as short and treat it as unsigned short in Java. - // The reason is, that when I wrote this, the jhdf5 library did not support unsigned short. It's terrible and should be fixed. - // https://github.com/bigdataviewer/bigdataviewer-core/issues/154 - // https://imagesc.zulipchat.com/#narrow/stream/327326-BigDataViewer/topic/XML.2FHDF5.20specification - RandomAccessibleInterval downsampled = - Converters.convertRAI( - (RandomAccessibleInterval)(Object)N5Utils.open(driverVolumeWriter, datasetPrev), - (i,o)->o.set( i.getShort() ), - new UnsignedShortType()); - - for ( int d = 0; d < downsampled.numDimensions(); ++d ) - if ( ds[ d ] > 1 ) - downsampled = LazyHalfPixelDownsample2x.init( - downsampled, - new FinalInterval( downsampled ), - new UnsignedShortType(), - blocksize(), - d); - - final RandomAccessibleInterval sourceGridBlock = - Converters.convertRAI( Views.offsetInterval(downsampled, gridBlock[0], gridBlock[1]), (i,o)->o.set( i.getShort() ), new ShortType() ); - N5Utils.saveNonEmptyBlock(sourceGridBlock, driverVolumeWriter, datasetDownsampling, gridBlock[2], new ShortType()); - }*/ else { IOFunctions.println( "Unsupported pixel type: " + dataType ); @@ -512,7 +505,7 @@ else if ( dataType == DataType.INT16 ) exc.printStackTrace(); } } ) - ); + );*/ try { diff --git a/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java b/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java index 11041ff4..09a6f216 100644 --- a/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java +++ b/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java @@ -36,7 +36,6 @@ public class N5ResaveTools { public static void writeDownsampledBlock( - final SpimData2 data, final N5Writer n5, final int level, final int[] relativeDownsampling, @@ -51,7 +50,7 @@ public static void writeDownsampledBlock( if ( dataType == DataType.UINT16 ) { - RandomAccessibleInterval downsampled = N5Utils.open(n5, datasetPrev);; + RandomAccessibleInterval downsampled = N5Utils.open(n5, datasetPrev); for ( int d = 0; d < downsampled.numDimensions(); ++d ) if ( relativeDownsampling[ d ] > 1 ) @@ -129,12 +128,20 @@ public static ArrayList prepareDownsampling( final String dataset = "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s" + level; - n5.createDataset( - dataset, - dim, // dimensions - blockSize, - dataType, - compression ); + try + { + n5.createDataset( + dataset, + dim, // dimensions + blockSize, + dataType, + compression ); + } + catch ( Exception e ) + { + IOFunctions.println( "Couldn't create downsampling level " + level + ", dataset '" + dataset + "': " + e ); + return null; + } final List grid = Grid.create( dim,