diff --git a/src/main/java/net/preibisch/mvrecon/fiji/datasetmanager/FileListDatasetDefinition.java b/src/main/java/net/preibisch/mvrecon/fiji/datasetmanager/FileListDatasetDefinition.java index dc91cc12..b309aaa0 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/datasetmanager/FileListDatasetDefinition.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/datasetmanager/FileListDatasetDefinition.java @@ -1108,7 +1108,7 @@ else if (choice.equals( Z_VARIABLE_CHOICE )) gdSave.addDirectoryField( "metadata_save_path (XML)", prefixPath.getAbsolutePath(), 65 ); gdSave.addDirectoryField( "image_data_save_path", prefixPath.getAbsolutePath(), 65 ); - gdSave.addMessage( "Note: image data save path will be ignored if not re-saved as N5/HDF5.", GUIHelper.smallStatusFont ); + gdSave.addMessage( "Note: image data save path will be ignored if not re-saved as N5/HDF5.\nOnly provide the path, the actual .zarr, .n5 or .h5 path/file will be appended!", GUIHelper.smallStatusFont ); // check if all stack sizes are the same (in each file) boolean zSizeEqualInEveryFile = LegacyFileMapImgLoaderLOCI.isZSizeEqualInEveryFile( data, (FileMapGettable)data.getSequenceDescription().getImgLoader() ); @@ -1155,7 +1155,10 @@ else if (choice.equals( Z_VARIABLE_CHOICE )) chosenPathData = gdSave.getNextString(); // will be stored in the img loader (if identical to chosenPathXML then relative, otherwise absolute) } - final URI chosenPathXMLURI, chosenPathDataURI; + final boolean resaveAsHDF5 = (loadChoice == 0); + final boolean resaveAsN5 = (loadChoice == 1); + + URI chosenPathXMLURI, chosenPathDataURI; try { @@ -1181,6 +1184,7 @@ else if (choice.equals( Z_VARIABLE_CHOICE )) } IOFunctions.println( "XML & metadata path: " + chosenPathXMLURI ); + IOFunctions.println( "XML: " + URITools.appendName( chosenPathXMLURI, xmlFileName ) ); IOFunctions.println( "Image data path: " + chosenPathDataURI ); data.setBasePathURI( chosenPathXMLURI ); @@ -1244,9 +1248,6 @@ public int compare(Group< ViewDescription > o1, Group< ViewDescription > o2) if (applyAxis) Apply_Transformation.applyAxisGrouped( data ); - boolean resaveAsHDF5 = (loadChoice == 0); - boolean resaveAsN5 = (loadChoice == 1); - if (resaveAsHDF5) { if ( !URITools.isFile( chosenPathDataURI ) ) @@ -1303,26 +1304,31 @@ else if (resaveAsN5) final SequenceDescription sd = data.getSequenceDescription(); + final URI xmlURI = URI.create( URITools.appendName(chosenPathXMLURI, xmlFileName ) ); + final URI n5DatasetURI = URI.create( URITools.appendName(chosenPathDataURI, xmlFileName.subSequence( 0, xmlFileName.length() - 4 ) + ".n5" ) ); + + IOFunctions.println( "N5 path: " + n5DatasetURI ); + final ParametersResaveN5 n5params = ParametersResaveN5.getParamtersIJ( - chosenPathXMLURI, - chosenPathDataURI, + xmlURI, + n5DatasetURI, viewIds.stream().map( vid -> sd.getViewSetups().get( vid.getViewSetupId() ) ).collect( Collectors.toSet() ), false ); // do not ask for paths again if ( n5params == null ) return null; - Resave_N5.resaveN5( data, viewIds, n5params ); + data = Resave_N5.resaveN5( data, viewIds, n5params, false ); // Re-assemble a new SpimData object containing the subset of viewsetups and timepoints selected - final SpimData2 newSpimData = Resave_TIFF.assemblePartialSpimData2( data, viewIds, n5params.n5File.getParentFile(), new ArrayList<>() ); + //final SpimData2 newSpimData = Resave_TIFF.assemblePartialSpimData2( data, viewIds, chosenPathXMLURI , new ArrayList<>() ); // replace imgLoader - newSpimData.getSequenceDescription().setImgLoader( new N5ImageLoader( n5params.n5File, newSpimData.getSequenceDescription() ) ); - newSpimData.setBasePath( n5params.n5File.getParentFile() ); + //newSpimData.getSequenceDescription().setImgLoader( new N5ImageLoader( n5DatasetURI, newSpimData.getSequenceDescription() ) ); + //newSpimData.setBasePathURI( chosenPathXMLURI ); // replace the spimdata object - data = newSpimData; + //data = newSpimData; IOFunctions.println( "(" + new Date( System.currentTimeMillis() ) + "): N5 resave finished." ); } @@ -1333,7 +1339,6 @@ else if (resaveAsN5) } return data; - } public static File getLongestPathPrefix(Collection paths) diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java index f4f33710..b353ce83 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/ParametersResaveN5.java @@ -66,16 +66,27 @@ public class ParametersResaveN5 //public boolean setFinishedAttributeInN5 = true; // required if double-checking that all ViewId were written //final public static String finishedAttrib = "saved_completely"; // required if double-checking that all ViewId were written + public static ParametersResaveN5 getParamtersIJ( + final Collection< ViewSetup > setupsToProcess ) + { + return getParamtersIJ( null, null, setupsToProcess, false ); + } + public static ParametersResaveN5 getParamtersIJ( final URI xmlURI, final Collection< ViewSetup > setupsToProcess, final boolean askForPaths ) { - final URI n5URI = URI.create( xmlURI.toString().subSequence( 0, xmlURI.toString().length() - 4 ) + ".n5" ); + final URI n5URI = createN5URIfromXMLURI( xmlURI ); return getParamtersIJ( xmlURI, n5URI, setupsToProcess, askForPaths ); } + public static URI createN5URIfromXMLURI( final URI xmlURI ) + { + return URI.create( xmlURI.toString().subSequence( 0, xmlURI.toString().length() - 4 ) + ".n5" ); + } + public static ParametersResaveN5 getParamtersIJ( final URI xmlURI, final URI n5URI, diff --git a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java index 334a4028..f0e277dd 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/plugin/resave/Resave_N5.java @@ -29,12 +29,11 @@ import java.util.Collection; import java.util.Date; import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.Compression; -import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.universe.N5Factory; @@ -44,18 +43,15 @@ import bdv.img.n5.N5ImageLoader; import ij.ImageJ; import ij.plugin.PlugIn; -import mpicbg.spim.data.SpimData; -import mpicbg.spim.data.SpimDataException; -import mpicbg.spim.data.generic.AbstractSpimData; import mpicbg.spim.data.sequence.TimePoint; import mpicbg.spim.data.sequence.ViewId; import mpicbg.spim.data.sequence.ViewSetup; -import net.imglib2.type.numeric.integer.UnsignedByteType; -import net.imglib2.type.numeric.integer.UnsignedShortType; -import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.util.Util; +import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.plugin.queryXML.LoadParseQueryXML; import net.preibisch.mvrecon.fiji.spimdata.SpimData2; import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2; +import net.preibisch.mvrecon.process.resave.N5ResaveTools; import util.URITools; public class Resave_N5 implements PlugIn @@ -78,14 +74,15 @@ public void run(String arg) for ( final ViewSetup vs : xml.getViewSetupsToProcess() ) vidsToProcess.add( new ViewId( tp.getId(), vs.getId() ) ); - resaveN5( xml.getData(), vidsToProcess, n5params ); + resaveN5( xml.getData(), vidsToProcess, n5params, true ); } - public static void resaveN5( + public static SpimData2 resaveN5( final SpimData2 data, final Collection vidsToResave, - final ParametersResaveN5 n5Params ) + final ParametersResaveN5 n5Params, + final boolean saveXML ) { final SpimData2 sdReduced = Resave_HDF5.reduceSpimData2( data, vidsToResave.stream().collect( Collectors.toList() ) ); @@ -133,58 +130,95 @@ public static void resaveN5( } else if ( URITools.isS3( n5Params.n5URI ) || URITools.isGC( n5Params.n5URI ) ) { - final N5Writer = new N5Factory().openWriter( URITools.appendName( baseDir, baseN5 ) ); // cloud support, avoid dependency hell if it is a local file // TODO: save to cloud - } + final N5Writer n5Writer = new N5Factory().openWriter( n5Params.n5URI.toString() ); // cloud support, avoid dependency hell if it is a local file - sdReduced.getSequenceDescription().setImgLoader( new N5ImageLoader( n5Params.n5URI, sdReduced.getSequenceDescription() ) ); - sdReduced.setBasePathURI( n5Params.xmlURI ); + final int[] blockSize = null; + final int[] computeBlockSize = null; + final Compression compression = null; - progressWriter.out().println( new Date( System.currentTimeMillis() ) + ": Saving " + n5Params.xmlURI ); + //final ArrayList viewSetups = + // N5ResaveTools.assembleViewSetups( data, vidsToResave ); - new XmlIoSpimData2().save( sdReduced, n5Params.xmlURI ); + final HashMap viewSetupIdToDimensions = + N5ResaveTools.assembleDimensions( data, vidsToResave ); - progressWriter.setProgress( 1.0 ); - progressWriter.out().println( new Date( System.currentTimeMillis() ) + ": Finished saving " + n5Params.n5URI + " and " + n5Params.xmlURI ); - } + final int[][] downsamplings = + N5ResaveTools.mipMapInfoToDownsamplings( n5Params.proposedMipmaps ); - public static void createDatasets( - final N5Writer n5, - final AbstractSpimData data, - final int[] blockSize, - final int[][] downsamplingFactors, - final Compression compression, - final Map viewSetupIdToDimensions ) - { - for ( final Entry viewSetup : viewSetupIdToDimensions.entrySet() ) + final ArrayList grid = + N5ResaveTools.assembleAllS0Jobs( vidsToResave, viewSetupIdToDimensions, blockSize, computeBlockSize ); + + N5ResaveTools.createGroups( n5Writer, data, viewSetupIdToDimensions, blockSize, downsamplings, compression ); + N5ResaveTools.createS0Datasets( n5Writer, vidsToResave, viewSetupIdToDimensions, blockSize, compression ); + + // + // Save full resolution dataset (s0) + // + final ForkJoinPool myPool = new ForkJoinPool( n5Params.numCellCreatorThreads ); + + long time = System.currentTimeMillis(); + + try + { + myPool.submit(() -> grid.parallelStream().forEach( gridBlock -> N5ResaveTools.writeS0Block( data, n5Writer, gridBlock ) ) ).get(); + } + catch (InterruptedException | ExecutionException e) + { + IOFunctions.println( "Failed to write s0 for N5 '" + n5Params.n5URI + "'. Error: " + e ); + e.printStackTrace(); + return null; + } + + IOFunctions.println( "Saved level s0, took: " + (System.currentTimeMillis() - time ) + " ms." ); + + // + // Save remaining downsampling levels (s1 ... sN) + // + + for ( int level = 1; level < downsamplings.length; ++level ) + { + final int s = level; + final int[] ds = N5ResaveTools.computeRelativeDownsampling( downsamplings, s ); + IOFunctions.println( "Downsampling: " + Util.printCoordinates( downsamplings[ s ] ) + " with relative downsampling of " + Util.printCoordinates( ds )); + + final ArrayList allBlocks = N5ResaveTools.prepareDownsampling( vidsToResave, n5Writer, level, blockSize, ds, downsamplings[ s ], compression ); + + time = System.currentTimeMillis(); + + try + { + myPool.submit(() -> allBlocks.parallelStream().forEach( gridBlock -> N5ResaveTools.writeDownsampledBlock( data, n5Writer, s, ds, gridBlock ) ) ).get(); + } + catch (InterruptedException | ExecutionException e) + { + IOFunctions.println( "Failed to write downsample step s" + s +" for N5 '" + n5Params.n5URI + "'. Error: " + e ); + e.printStackTrace(); + return null; + } + + IOFunctions.println( "Resaved N5 s" + s + " level, took: " + (System.currentTimeMillis() - time ) + " ms." ); + } + + myPool.shutdown(); + //myPool.awaitTermination( Long.MAX_VALUE, TimeUnit.HOURS ); + + n5Writer.close(); + } + + sdReduced.getSequenceDescription().setImgLoader( new N5ImageLoader( n5Params.n5URI, sdReduced.getSequenceDescription() ) ); + sdReduced.setBasePathURI( URITools.getParent( n5Params.xmlURI ) ); + + if ( saveXML ) { - final Object type = data.getSequenceDescription().getImgLoader().getSetupImgLoader( viewSetup.getKey() ).getImageType(); - final DataType dataType; - - if ( UnsignedShortType.class.isInstance( type ) ) - dataType = DataType.UINT16; - else if ( UnsignedByteType.class.isInstance( type ) ) - dataType = DataType.UINT8; - else if ( FloatType.class.isInstance( type ) ) - dataType = DataType.FLOAT32; - else - throw new RuntimeException("Unsupported pixel type: " + type.getClass().getCanonicalName() ); - - // TODO: ViewSetupId needs to contain: {"downsamplingFactors":[[1,1,1],[2,2,1]],"dataType":"uint16"} - final String n5Dataset = "setup" + viewSetup.getKey(); - - System.out.println( "Creating group: " + "'setup" + viewSetup.getKey() + "'" ); - - n5.createGroup( n5Dataset ); - - System.out.println( "setting attributes for '" + "setup" + viewSetup.getKey() + "'"); - - n5.setAttribute( n5Dataset, "downsamplingFactors", downsamplingFactors ); - n5.setAttribute( n5Dataset, "dataType", dataType ); - n5.setAttribute( n5Dataset, "blockSize", blockSize ); - n5.setAttribute( n5Dataset, "dimensions", viewSetup.getValue() ); - n5.setAttribute( n5Dataset, "compression", compression ); + progressWriter.out().println( new Date( System.currentTimeMillis() ) + ": Saving " + n5Params.xmlURI ); + new XmlIoSpimData2().save( sdReduced, n5Params.xmlURI ); } + + progressWriter.setProgress( 1.0 ); + progressWriter.out().println( new Date( System.currentTimeMillis() ) + ": Finished saving " + n5Params.n5URI ); + + return sdReduced; } public static void main(String[] args) diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/SpimData2.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/SpimData2.java index f669252f..cca1517c 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/SpimData2.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/SpimData2.java @@ -592,9 +592,12 @@ public static void main( String[] args ) //u = URI.create( "s3://myBucket/" ) ; URI u2 = URI.create( u.toString() + ( u.toString().endsWith( "/" ) ? "" : "/") + "test.xml" ); + System.out.println( u ); System.out.println( u2 ); + System.out.println( URITools.getParent( u2 ) ); + System.out.println( URITools.getParent( URI.create("/nrs/test.xml") ) ); System.out.println( new File( u2 ) ); System.out.println( new File( URI.create("file:/nrs/test.xml") ) ); // System.out.println( new File( URI.create("/nrs/test.xml") ) ); // FAILS diff --git a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/explorer/popup/ResavePopup.java b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/explorer/popup/ResavePopup.java index 313c1f68..47b42667 100644 --- a/src/main/java/net/preibisch/mvrecon/fiji/spimdata/explorer/popup/ResavePopup.java +++ b/src/main/java/net/preibisch/mvrecon/fiji/spimdata/explorer/popup/ResavePopup.java @@ -25,6 +25,7 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.io.File; +import java.net.URI; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -43,7 +44,7 @@ import net.preibisch.legacy.io.IOFunctions; import net.preibisch.mvrecon.fiji.ImgLib2Temp.Pair; import net.preibisch.mvrecon.fiji.plugin.resave.Generic_Resave_HDF5; -import net.preibisch.mvrecon.fiji.plugin.resave.N5Parameters; +import net.preibisch.mvrecon.fiji.plugin.resave.ParametersResaveN5; import net.preibisch.mvrecon.fiji.plugin.resave.ProgressWriterIJ; import net.preibisch.mvrecon.fiji.plugin.resave.Resave_HDF5; import net.preibisch.mvrecon.fiji.plugin.resave.Resave_N5; @@ -288,23 +289,26 @@ else if (index == 4) panel.saveXML(); - final N5Parameters n5params = N5Parameters.getParamtersIJ( + final URI n5DatasetURI = ParametersResaveN5.createN5URIfromXMLURI( panel.xml() ); + + final ParametersResaveN5 n5params = ParametersResaveN5.getParamtersIJ( panel.xml(), + n5DatasetURI, viewIds.stream().map( vid -> data.getSequenceDescription().getViewSetups().get( vid.getViewSetupId() ) ).collect( Collectors.toSet() ), - true ); + false ); if ( n5params == null ) return; - Resave_N5.resaveN5( data, viewIds, n5params ); + final SpimData2 newSpimData = Resave_N5.resaveN5( data, viewIds, n5params, false ); // Re-assemble a new SpimData object containing the subset of viewsetups and timepoints selected - final List< String > filesToCopy = new ArrayList< String >(); - final SpimData2 newSpimData = Resave_TIFF.assemblePartialSpimData2( data, viewIds, n5params.n5File.getParentFile(), filesToCopy ); + //final List< String > filesToCopy = new ArrayList< String >(); + //final SpimData2 newSpimData = Resave_TIFF.assemblePartialSpimData2( data, viewIds, data.getBasePathURI(), filesToCopy ); // replace imgLoader - newSpimData.getSequenceDescription().setImgLoader( new N5ImageLoader( n5params.n5File, newSpimData.getSequenceDescription() ) ); - newSpimData.setBasePath( n5params.n5File.getParentFile() ); + newSpimData.getSequenceDescription().setImgLoader( new N5ImageLoader( n5DatasetURI, newSpimData.getSequenceDescription() ) ); + newSpimData.setBasePathURI( data.getBasePathURI() ); // replace the spimdata object panel.setSpimData( newSpimData ); diff --git a/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java b/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java new file mode 100644 index 00000000..11041ff4 --- /dev/null +++ b/src/main/java/net/preibisch/mvrecon/process/resave/N5ResaveTools.java @@ -0,0 +1,358 @@ +package net.preibisch.mvrecon.process.resave; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.imglib2.N5Utils; + +import bdv.export.ExportMipmapInfo; +import mpicbg.spim.data.SpimData; +import mpicbg.spim.data.generic.AbstractSpimData; +import mpicbg.spim.data.sequence.SetupImgLoader; +import mpicbg.spim.data.sequence.ViewDescription; +import mpicbg.spim.data.sequence.ViewId; +import mpicbg.spim.data.sequence.ViewSetup; +import net.imglib2.FinalInterval; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedShortType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.util.Util; +import net.imglib2.view.Views; +import net.preibisch.legacy.io.IOFunctions; +import net.preibisch.mvrecon.fiji.spimdata.SpimData2; +import net.preibisch.mvrecon.process.downsampling.lazy.LazyHalfPixelDownsample2x; +import net.preibisch.mvrecon.process.interestpointregistration.pairwise.constellation.grouping.Group; +import util.Grid; + +public class N5ResaveTools +{ + public static void writeDownsampledBlock( + final SpimData2 data, + final N5Writer n5, + final int level, + final int[] relativeDownsampling, + final long[][] gridBlock ) + { + final ViewId viewId = new ViewId( (int)gridBlock[ 3 ][ 0 ], (int)gridBlock[ 3 ][ 1 ]); + + final DataType dataType = n5.getAttribute( "setup" + viewId.getViewSetupId(), DatasetAttributes.DATA_TYPE_KEY, DataType.class ); + final int[] blockSize = n5.getAttribute( "setup" + viewId.getViewSetupId(), DatasetAttributes.BLOCK_SIZE_KEY, int[].class ); + final String datasetPrev = "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s" + (level-1); + final String dataset = "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s" + (level); + + if ( dataType == DataType.UINT16 ) + { + RandomAccessibleInterval downsampled = N5Utils.open(n5, datasetPrev);; + + for ( int d = 0; d < downsampled.numDimensions(); ++d ) + if ( relativeDownsampling[ d ] > 1 ) + downsampled = LazyHalfPixelDownsample2x.init( + downsampled, + new FinalInterval( downsampled ), + new UnsignedShortType(), + blockSize, + d); + + final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(downsampled, gridBlock[0], gridBlock[1]); + N5Utils.saveNonEmptyBlock(sourceGridBlock, n5, dataset, gridBlock[2], new UnsignedShortType()); + } + else if ( dataType == DataType.UINT8 ) + { + RandomAccessibleInterval downsampled = N5Utils.open(n5, datasetPrev); + + for ( int d = 0; d < downsampled.numDimensions(); ++d ) + if ( relativeDownsampling[ d ] > 1 ) + downsampled = LazyHalfPixelDownsample2x.init( + downsampled, + new FinalInterval( downsampled ), + new UnsignedByteType(), + blockSize, + d); + + final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(downsampled, gridBlock[0], gridBlock[1]); + N5Utils.saveNonEmptyBlock(sourceGridBlock, n5, dataset, gridBlock[2], new UnsignedByteType()); + } + else if ( dataType == DataType.FLOAT32 ) + { + RandomAccessibleInterval downsampled = N5Utils.open(n5, datasetPrev);; + + for ( int d = 0; d < downsampled.numDimensions(); ++d ) + if ( relativeDownsampling[ d ] > 1 ) + downsampled = LazyHalfPixelDownsample2x.init( + downsampled, + new FinalInterval( downsampled ), + new FloatType(), + blockSize, + d); + + final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(downsampled, gridBlock[0], gridBlock[1]); + N5Utils.saveNonEmptyBlock(sourceGridBlock, n5, dataset, gridBlock[2], new FloatType()); + } + else + { + n5.close(); + throw new RuntimeException("Unsupported pixel type: " + dataType ); + } + } + + public static ArrayList prepareDownsampling( + final Collection< ? extends ViewId > viewIds, + final N5Writer n5, + final int level, + final int[] relativeDownsampling, + final int[] absoluteDownsampling, + final int[] blockSize, + final Compression compression ) + { + // all blocks (a.k.a. grids) across all ViewId's + final ArrayList allBlocks = new ArrayList<>(); + + // adjust dimensions + for ( final ViewId viewId : viewIds ) + { + final long[] previousDim = n5.getAttribute( "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s" + (level-1), "dimensions", long[].class ); + final long[] dim = new long[ previousDim.length ]; + for ( int d = 0; d < dim.length; ++d ) + dim[ d ] = previousDim[ d ] / relativeDownsampling[ d ]; + final DataType dataType = n5.getAttribute( "setup" + viewId.getViewSetupId(), "dataType", DataType.class ); + + System.out.println( Group.pvid( viewId ) + ": s" + (level-1) + " dim=" + Util.printCoordinates( previousDim ) + ", s" + level + " dim=" + Util.printCoordinates( dim ) + ", datatype=" + dataType ); + + final String dataset = "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s" + level; + + n5.createDataset( + dataset, + dim, // dimensions + blockSize, + dataType, + compression ); + + final List grid = Grid.create( + dim, + new int[] { + blockSize[0], + blockSize[1], + blockSize[2] + }, + blockSize); + + // add timepointId and ViewSetupId to the gridblock + for ( final long[][] gridBlock : grid ) + allBlocks.add( new long[][]{ + gridBlock[ 0 ].clone(), + gridBlock[ 1 ].clone(), + gridBlock[ 2 ].clone(), + new long[] { viewId.getTimePointId(), viewId.getViewSetupId() } + }); + + // set additional N5 attributes for sN dataset + n5.setAttribute(dataset, "downsamplingFactors", absoluteDownsampling[ level ] ); + } + + return allBlocks; + } + + public static int[] computeRelativeDownsampling( + final int[][] downsamplings, + final int level ) + { + final int[] ds = new int[ downsamplings[ 0 ].length ]; + + for ( int d = 0; d < ds.length; ++d ) + ds[ d ] = downsamplings[ level ][ d ] / downsamplings[ level - 1 ][ d ]; + + return ds; + } + + public static void createS0Datasets( + final N5Writer n5, + final Collection< ? extends ViewId > viewIds, + final Map viewSetupIdToDimensions, + final int[] blockSize, + final Compression compression ) + { + for ( final ViewId viewId : viewIds ) + { + IOFunctions.println( "Creating dataset for " + Group.pvid( viewId ) ); + + final String dataset = "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s0"; + final DataType dataType = n5.getAttribute( "setup" + viewId.getViewSetupId(), "dataType", DataType.class ); + + n5.createDataset( + dataset, + viewSetupIdToDimensions.get( viewId.getViewSetupId() ), // dimensions + blockSize, + dataType, + compression ); + + System.out.println( "Setting attributes for " + Group.pvid( viewId ) ); + + // set N5 attributes for timepoint + // e.g. {"resolution":[1.0,1.0,3.0],"saved_completely":true,"multiScale":true} + String ds ="setup" + viewId.getViewSetupId() + "/" + "timepoint" + viewId.getTimePointId(); + n5.setAttribute(ds, "resolution", new double[] {1,1,1} ); + n5.setAttribute(ds, "saved_completely", true ); + n5.setAttribute(ds, "multiScale", true ); + + // set additional N5 attributes for s0 dataset + ds = ds + "/s0"; + n5.setAttribute(ds, "downsamplingFactors", new int[] {1,1,1} ); + } + } + + public static void writeS0Block( + final SpimData2 data, + final N5Writer n5, + final long[][] gridBlock ) + { + final ViewId viewId = new ViewId( (int)gridBlock[ 3 ][ 0 ], (int)gridBlock[ 3 ][ 1 ]); + + final SetupImgLoader< ? > imgLoader = data.getSequenceDescription().getImgLoader().getSetupImgLoader( viewId.getViewSetupId() ); + + @SuppressWarnings("rawtypes") + final RandomAccessibleInterval img = imgLoader.getImage( viewId.getTimePointId() ); + + final DataType dataType = n5.getAttribute( "setup" + viewId.getViewSetupId(), "dataType", DataType.class ); + final String dataset = "setup" + viewId.getViewSetupId() + "/timepoint" + viewId.getTimePointId() + "/s0"; + + if ( dataType == DataType.UINT16 ) + { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(img, gridBlock[0], gridBlock[1]); + N5Utils.saveNonEmptyBlock(sourceGridBlock, n5, dataset, gridBlock[2], new UnsignedShortType()); + } + else if ( dataType == DataType.UINT8 ) + { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(img, gridBlock[0], gridBlock[1]); + N5Utils.saveNonEmptyBlock(sourceGridBlock, n5, dataset, gridBlock[2], new UnsignedByteType()); + } + else if ( dataType == DataType.FLOAT32 ) + { + @SuppressWarnings("unchecked") + final RandomAccessibleInterval sourceGridBlock = Views.offsetInterval(img, gridBlock[0], gridBlock[1]); + N5Utils.saveNonEmptyBlock(sourceGridBlock, n5, dataset, gridBlock[2], new FloatType()); + } + else + { + n5.close(); + throw new RuntimeException("Unsupported pixel type: " + dataType ); + } + + System.out.println( "ViewId " + Group.pvid( viewId ) + ", written block: offset=" + Util.printCoordinates( gridBlock[0] ) + ", dimension=" + Util.printCoordinates( gridBlock[1] ) ); + } + + public static void createGroups( + final N5Writer n5, + final AbstractSpimData data, + final Map viewSetupIdToDimensions, + final int[] blockSize, + final int[][] downsamplingFactors, + final Compression compression ) + { + for ( final Entry< Integer, long[] > viewSetup : viewSetupIdToDimensions.entrySet() ) + { + final Object type = data.getSequenceDescription().getImgLoader().getSetupImgLoader( viewSetup.getKey() ).getImageType(); + final DataType dataType; + + if ( UnsignedShortType.class.isInstance( type ) ) + dataType = DataType.UINT16; + else if ( UnsignedByteType.class.isInstance( type ) ) + dataType = DataType.UINT8; + else if ( FloatType.class.isInstance( type ) ) + dataType = DataType.FLOAT32; + else + throw new RuntimeException("Unsupported pixel type: " + type.getClass().getCanonicalName() ); + + // ViewSetupId needs to contain: {"downsamplingFactors":[[1,1,1],[2,2,1]],"dataType":"uint16"} + final String n5Dataset = "setup" + viewSetup.getKey(); + + System.out.println( "Creating group: " + "'setup" + viewSetup.getKey() + "'" ); + + n5.createGroup( n5Dataset ); + + System.out.println( "setting attributes for '" + "setup" + viewSetup.getKey() + "'"); + + n5.setAttribute( n5Dataset, "downsamplingFactors", downsamplingFactors ); + n5.setAttribute( n5Dataset, "dataType", dataType ); + n5.setAttribute( n5Dataset, "blockSize", blockSize ); + n5.setAttribute( n5Dataset, "dimensions", viewSetup.getValue() ); + n5.setAttribute( n5Dataset, "compression", compression ); + } + } + + public static int[][] mipMapInfoToDownsamplings( final Map< Integer, ExportMipmapInfo > mipmaps ) + { + int[][] downsamplings = mipmaps.values().iterator().next().getExportResolutions(); + + // just find the biggest one (most steps) and use it for all + for ( final ExportMipmapInfo info : mipmaps.values() ) + if (info.getExportResolutions().length > downsamplings.length) + downsamplings = info.getExportResolutions(); + + return downsamplings; + } + + public static ArrayList assembleAllS0Jobs( + final Collection< ? extends ViewId > viewIds, + final HashMap< Integer, long[] > viewSetupIdToDimensions, + final int[] blockSize, + final int[] computeBlockSize ) + { + // all blocks (a.k.a. grids) across all ViewId's + final ArrayList allBlocks = new ArrayList<>(); + + for ( final ViewId viewId : viewIds ) + { + final List grid = Grid.create( + viewSetupIdToDimensions.get( viewId.getViewSetupId() ), + computeBlockSize, + blockSize); + + // add timepointId and ViewSetupId & dimensions to the gridblock + for ( final long[][] gridBlock : grid ) + allBlocks.add( new long[][]{ + gridBlock[ 0 ].clone(), + gridBlock[ 1 ].clone(), + gridBlock[ 2 ].clone(), + new long[] { viewId.getTimePointId(), viewId.getViewSetupId() }, + viewSetupIdToDimensions.get( viewId.getViewSetupId() ) // TODO: do we need the dimensions? + }); + } + + return allBlocks; + } + + public static HashMap assembleDimensions( + final SpimData data, + final Collection< ? extends ViewId > viewIds ) + { + final HashMap< Integer, long[] > viewSetupIdToDimensions = new HashMap<>(); + final Map map = data.getSequenceDescription().getViewDescriptions(); + + viewIds.forEach( viewId -> viewSetupIdToDimensions.put( viewId.getViewSetupId(), map.get( viewId ).getViewSetup().getSize().dimensionsAsLongArray() ) ); + + return viewSetupIdToDimensions; + } + + public static ArrayList< ViewSetup > assembleViewSetups( + final SpimData data, + final Collection< ? extends ViewId > viewIds ) + { + final ArrayList< ViewSetup > list = new ArrayList<>(); + final Map map = data.getSequenceDescription().getViewDescriptions(); + + viewIds.forEach( viewId -> list.add( map.get( viewId ).getViewSetup() ) ); + + return list; + } +} diff --git a/src/main/java/util/URITools.java b/src/main/java/util/URITools.java index 6308da54..b97fe2ff 100644 --- a/src/main/java/util/URITools.java +++ b/src/main/java/util/URITools.java @@ -12,6 +12,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.net.URI; +import java.nio.file.Paths; import java.util.regex.Pattern; import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils; @@ -305,6 +306,16 @@ public static String removeFilePrefix( URI uri ) return uri.toString(); } + public static URI getParent( final URI uri ) + { + return uri.getPath().endsWith("/") ? uri.resolve("..") : uri.resolve("."); + } + + public static String getFileName( final URI uri ) + { + return Paths.get( uri.getPath() ).getFileName().toString(); + } + public static String appendName( final URI uri, final String name ) { return uri.toString() + ( uri.toString().endsWith( "/" ) ? "" : "/") + name;