diff --git a/pom.xml b/pom.xml index c2e860f4..d9d5555d 100644 --- a/pom.xml +++ b/pom.xml @@ -143,11 +143,11 @@ bsd_2 BigDataViewer developers. - 7.1.1 + 7.1.2 4.0.3 - 1.0.0-beta-18 - 0.15.3 - 2.3.2 + 1.0.0-beta-19 + 0.16.0 + 2.3.5 sign,deploy-to-scijava diff --git a/src/main/java/bdv/export/CopyBlock.java b/src/main/java/bdv/export/CopyBlock.java deleted file mode 100644 index 8c3fc35b..00000000 --- a/src/main/java/bdv/export/CopyBlock.java +++ /dev/null @@ -1,147 +0,0 @@ -/*- - * #%L - * BigDataViewer core classes with minimal dependencies. - * %% - * Copyright (C) 2012 - 2024 BigDataViewer developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package bdv.export; - -import java.util.Arrays; -import net.imglib2.RandomAccess; -import net.imglib2.loops.ClassCopyProvider; -import net.imglib2.type.numeric.RealType; - -public interface CopyBlock< T extends RealType< T > > -{ - void copyBlock( final RandomAccess< T > in, final RandomAccess< T > out, final int[] dimensions ); - - static < T extends RealType< T > > CopyBlock< T > create( - final int numDimensions, - final Class< ? > pixelTypeClass, - final Class< ? > inAccessClass ) - { - return CopyBlockInstances.create( numDimensions, pixelTypeClass, inAccessClass ); - } -} - -class CopyBlockInstances -{ - @SuppressWarnings( "rawtypes" ) - private static ClassCopyProvider< CopyBlock > provider; - - @SuppressWarnings( "unchecked" ) - public static < T extends RealType< T > > CopyBlock< T > create( - final int numDimensions, - final Class< ? > pixelTypeClass, - final Class< ? > inAccessClass ) - { - if ( provider == null ) - { - synchronized ( CopyBlockInstances.class ) - { - if ( provider == null ) - provider = new ClassCopyProvider<>( Imp.class, CopyBlock.class, int.class ); - } - } - - Object key = Arrays.asList( numDimensions, pixelTypeClass, inAccessClass ); - return provider.newInstanceForKey( key, numDimensions ); - } - - public static class Imp< T extends RealType< T > > implements CopyBlock< T > - { - private final int n; - - public Imp( final int n ) - { - if ( n < 1 || n > 3 ) - throw new IllegalArgumentException(); - - this.n = n; - } - - @Override - public void copyBlock( - final RandomAccess< T > in, - final RandomAccess< T > out, - final int[] dimensions ) - { - if ( n == 3 ) - copyBlock3D( out, dimensions[ 0 ], dimensions[ 1 ], dimensions[ 2 ], in ); - else if ( n == 2 ) - copyBlock2D( out, dimensions[ 0 ], dimensions[ 1 ], in ); - else - copyBlock1D( out, dimensions[ 0 ], in ); - } - - private void copyBlock3D( - final RandomAccess< T > out, - final int sx, // size of output image - final int sy, - final int sz, - final RandomAccess< T > in ) - { - for ( int z = 0; z < sz; ++z ) - { - copyBlock2D( out, sx, sy, in ); - out.fwd( 2 ); - in.fwd( 2 ); - } - out.move( -sz, 2 ); - in.move( -sz, 2 ); - } - - private void copyBlock2D( - final RandomAccess< T > out, - final int sx, // size of output image - final int sy, - final RandomAccess< T > in ) - { - for ( int y = 0; y < sy; ++y ) - { - copyBlock1D( out, sx, in ); - out.fwd( 1 ); - in.fwd( 1 ); - } - out.move( -sy, 1 ); - in.move( -sy, 1 ); - } - - private void copyBlock1D( - final RandomAccess< T > out, - final int sx, // size of output image - final RandomAccess< T > in ) - { - for ( int x = 0; x < sx; ++x ) - { - out.get().set( in.get() ); - out.fwd( 0 ); - in.fwd( 0 ); - } - out.move( -sx, 0 ); - in.move( -sx, 0 ); - } - } -} diff --git a/src/main/java/bdv/export/Downsample.java b/src/main/java/bdv/export/Downsample.java deleted file mode 100644 index 6470e5c5..00000000 --- a/src/main/java/bdv/export/Downsample.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * #%L - * BigDataViewer core classes with minimal dependencies. - * %% - * Copyright (C) 2012 - 2024 BigDataViewer developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package bdv.export; - -import net.imglib2.Cursor; -import net.imglib2.FinalInterval; -import net.imglib2.Interval; -import net.imglib2.RandomAccess; -import net.imglib2.RandomAccessible; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.algorithm.neighborhood.Neighborhood; -import net.imglib2.algorithm.neighborhood.RectangleNeighborhoodFactory; -import net.imglib2.algorithm.neighborhood.RectangleNeighborhoodUnsafe; -import net.imglib2.algorithm.neighborhood.RectangleShape; -import net.imglib2.type.numeric.RealType; -import net.imglib2.view.Views; - -public class Downsample -{ - /** - * TODO: Revise. This is probably not very efficient - */ - public static < T extends RealType< T > > void downsample( final RandomAccessible< T > input, final RandomAccessibleInterval< T > output, final int[] factor ) - { - assert input.numDimensions() == output.numDimensions(); - assert input.numDimensions() == factor.length; - - final int n = input.numDimensions(); - final RectangleNeighborhoodFactory< T > f = RectangleNeighborhoodUnsafe.< T >factory(); - final long[] dim = new long[ n ]; - for ( int d = 0; d < n; ++d ) - dim[ d ] = factor[ d ]; - final Interval spanInterval = new FinalInterval( dim ); - - final long[] minRequiredInput = new long[ n ]; - final long[] maxRequiredInput = new long[ n ]; - output.min( minRequiredInput ); - output.max( maxRequiredInput ); - for ( int d = 0; d < n; ++d ) - { - minRequiredInput[ d ] *= factor[ d ]; - maxRequiredInput[ d ] *= factor[ d ]; - maxRequiredInput[ d ] += factor[ d ] - 1; - } - final RandomAccessibleInterval< T > requiredInput = Views.interval( input, new FinalInterval( minRequiredInput, maxRequiredInput ) ); - - final RectangleShape.NeighborhoodsAccessible< T > neighborhoods = new RectangleShape.NeighborhoodsAccessible<>( requiredInput, spanInterval, f ); - final RandomAccess< Neighborhood< T > > block = neighborhoods.randomAccess(); - - long size = 1; - for ( int d = 0; d < n; ++d ) - size *= factor[ d ]; - final double scale = 1.0 / size; - - final Cursor< T > out = Views.iterable( output ).localizingCursor(); - while( out.hasNext() ) - { - final T o = out.next(); - for ( int d = 0; d < n; ++d ) - block.setPosition( out.getLongPosition( d ) * factor[ d ], d ); - double sum = 0; - for ( final T i : block.get() ) - sum += i.getRealDouble(); - o.setReal( sum * scale ); - } - } -} diff --git a/src/main/java/bdv/export/DownsampleBlock.java b/src/main/java/bdv/export/DownsampleBlock.java deleted file mode 100644 index 14dabe9f..00000000 --- a/src/main/java/bdv/export/DownsampleBlock.java +++ /dev/null @@ -1,250 +0,0 @@ -/*- - * #%L - * BigDataViewer core classes with minimal dependencies. - * %% - * Copyright (C) 2012 - 2024 BigDataViewer developers. - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package bdv.export; - -import java.util.Arrays; -import net.imglib2.Cursor; -import net.imglib2.RandomAccess; -import net.imglib2.img.array.ArrayImgs; -import net.imglib2.loops.ClassCopyProvider; -import net.imglib2.type.numeric.RealType; -import net.imglib2.type.numeric.real.DoubleType; -import net.imglib2.util.Intervals; - -public interface DownsampleBlock< T extends RealType< T > > -{ - void downsampleBlock( final RandomAccess< T > in, final Cursor< T > out, final int[] dimensions ); - - static < T extends RealType< T > > DownsampleBlock< T > create( - final int[] blockDimensions, - final int[] downsamplingFactors, - final Class< ? > pixelTypeClass, - final Class< ? > inAccessClass ) - { - return DownsampleBlockInstances.create( blockDimensions, downsamplingFactors, pixelTypeClass, inAccessClass ); - } -} - -class DownsampleBlockInstances -{ - @SuppressWarnings( "rawtypes" ) - private static ClassCopyProvider< DownsampleBlock > provider; - - @SuppressWarnings( "unchecked" ) - public static < T extends RealType< T > > DownsampleBlock< T > create( - final int[] blockDimensions, - final int[] downsamplingFactors, - final Class< ? > pixelTypeClass, - final Class< ? > inAccessClass ) - { - if ( provider == null ) - { - synchronized ( DownsampleBlockInstances.class ) - { - if ( provider == null ) - provider = new ClassCopyProvider<>( Imp.class, DownsampleBlock.class, int[].class, int[].class ); - } - } - - final int numDimensions = blockDimensions.length; - - Object key = Arrays.asList( numDimensions, pixelTypeClass, inAccessClass ); - return provider.newInstanceForKey( key, blockDimensions, downsamplingFactors ); - } - - public static class Imp< T extends RealType< T > > implements DownsampleBlock< T > - { - private final int n; - - private final int[] downsamplingFactors; - - private final double scale; - - private final double[] accumulator; - - private final RandomAccess< DoubleType > acc; - - public Imp( - final int[] blockDimensions, - final int[] downsamplingFactors ) - { - n = blockDimensions.length; - if ( n < 1 || n > 3 ) - throw new IllegalArgumentException(); - - this.downsamplingFactors = downsamplingFactors; - scale = 1.0 / Intervals.numElements( downsamplingFactors ); - - accumulator = new double[ ( int ) Intervals.numElements( blockDimensions ) ]; - - final long[] dims = new long[ n ]; - Arrays.setAll( dims, d -> blockDimensions[ d ] ); - acc = ArrayImgs.doubles( accumulator, dims ).randomAccess(); - } - - @Override - public void downsampleBlock( - final RandomAccess< T > in, - final Cursor< T > out, // must be flat iteration order - final int[] dimensions ) - { - clearAccumulator(); - - if ( n == 3 ) - { - downsampleBlock3D( acc, dimensions[ 0 ], dimensions[ 1 ], dimensions[ 2 ], in ); - writeOutput3D( out, dimensions[ 0 ], dimensions[ 1 ], dimensions[ 2 ], acc ); - } - else if ( n == 2 ) - { - downsampleBlock2D( acc, dimensions[ 0 ], dimensions[ 1 ], in ); - writeOutput2D( out, dimensions[ 0 ], dimensions[ 1 ], acc ); - } - else - { - downsampleBlock1D( acc, dimensions[ 0 ], in ); - writeOutput1D( out, dimensions[ 0 ], acc ); - } - } - - private void clearAccumulator() - { - Arrays.fill( accumulator, 0, accumulator.length, 0 ); - } - - private void downsampleBlock3D( - final RandomAccess< DoubleType > acc, - final int asx, // size of output (resp accumulator) image - final int asy, - final int asz, - final RandomAccess< T > in ) - { - final int bsz = downsamplingFactors[ 2 ]; - final int sz = asz * bsz; - for ( int z = 0, bz = 0; z < sz; ++z ) - { - downsampleBlock2D( acc, asx, asy, in ); - in.fwd( 2 ); - if ( ++bz == bsz ) - { - bz = 0; - acc.fwd( 2 ); - } - } - in.move( -sz, 2 ); - acc.move( -asz, 2 ); - } - - private void downsampleBlock2D( - final RandomAccess< DoubleType > acc, - final int asx, // size of output (resp accumulator) image - final int asy, - final RandomAccess< T > in ) - { - final int bsy = downsamplingFactors[ 1 ]; - final int sy = asy * bsy; - for ( int y = 0, by = 0; y < sy; ++y ) - { - downsampleBlock1D( acc, asx, in ); - in.fwd( 1 ); - if ( ++by == bsy ) - { - by = 0; - acc.fwd( 1 ); - } - } - in.move( -sy, 1 ); - acc.move( -asy, 1 ); - } - - private void downsampleBlock1D( - final RandomAccess< DoubleType > acc, - final int asx, // size of output (resp accumulator) image - final RandomAccess< T > in ) - { - final int bsx = downsamplingFactors[ 0 ]; - final int sx = asx * bsx; - for ( int x = 0, bx = 0; x < sx; ++x ) - { - acc.get().set( acc.get().get() + in.get().getRealDouble() ); - in.fwd( 0 ); - if ( ++bx == bsx ) - { - bx = 0; - acc.fwd( 0 ); - } - } - in.move( -sx, 0 ); - acc.move( -asx, 0 ); - } - - private void writeOutput3D( - final Cursor< T > out, // must be flat iteration order - final int asx, // size of output (resp accumulator) image - final int asy, - final int asz, - final RandomAccess< DoubleType > acc ) - { - for ( int z = 0; z < asz; ++z ) - { - writeOutput2D( out, asx, asy, acc ); - acc.fwd( 2 ); - } - acc.move( -asz, 2 ); - } - - private void writeOutput2D( - final Cursor< T > out, // must be flat iteration order - final int asx, // size of output (resp accumulator) image - final int asy, - final RandomAccess< DoubleType > acc ) - { - for ( int y = 0; y < asy; ++y ) - { - writeOutput1D( out, asx, acc ); - acc.fwd( 1 ); - } - acc.move( -asy, 1 ); - } - - private void writeOutput1D( - final Cursor< T > out, // must be flat iteration order - final int asx, // size of output (resp accumulator) image - final RandomAccess< DoubleType > acc ) - { - final double scale = this.scale; - for ( int x = 0; x < asx; ++x ) - { - out.next().setReal( acc.get().get() * scale ); - acc.fwd( 0 ); - } - acc.move( -asx, 0 ); - } - } -} diff --git a/src/main/java/bdv/export/ExportScalePyramid.java b/src/main/java/bdv/export/ExportScalePyramid.java index ad1d7e71..c1a1901b 100644 --- a/src/main/java/bdv/export/ExportScalePyramid.java +++ b/src/main/java/bdv/export/ExportScalePyramid.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -37,19 +37,18 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; -import net.imglib2.FinalInterval; -import net.imglib2.RandomAccess; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; + +import bdv.img.n5.DataTypeProperties; import net.imglib2.RandomAccessibleInterval; -import net.imglib2.cache.img.SingleCellArrayImg; -import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory; -import net.imglib2.img.basictypeaccess.array.ArrayDataAccess; +import net.imglib2.algorithm.blocks.BlockSupplier; +import net.imglib2.algorithm.blocks.downsample.Downsample; import net.imglib2.img.cell.CellGrid; import net.imglib2.type.NativeType; -import net.imglib2.type.NativeTypeFactory; import net.imglib2.type.numeric.RealType; -import net.imglib2.util.Cast; import net.imglib2.util.Intervals; -import net.imglib2.view.Views; +import net.imglib2.view.fluent.RandomAccessibleIntervalView.Extension; /** * Write an image to a chunked mipmap representation. @@ -112,39 +111,6 @@ public interface AfterEachPlane void afterEachPlane( final boolean usedLoopBack ); } - /** - * A block to be written. See {@link DatasetIO#writeBlock(Object, Block) - * DatasetIO.writeBlock()}. - */ - public static class Block< T extends NativeType< T > > - { - final SingleCellArrayImg< T, ? > data; - final int[] size; - final long[] position; - - Block( final SingleCellArrayImg< T, ? > data, final int[] size, final long[] position ) - { - this.data = data; - this.size = size.clone(); - this.position = position.clone(); - } - - public SingleCellArrayImg< T, ? > getData() - { - return data; - } - - public int[] getSize() - { - return size; - } - - public long[] getGridPosition() - { - return position; - } - } - /** * Writing and reading back data for each resolution level. * @@ -170,7 +136,7 @@ D createDataset( */ void writeBlock( final D dataset, - final Block< T > dataBlock ) throws IOException; + final DataBlock< ? > dataBlock ) throws IOException; /** * Blocks until all pending data was written to {@code dataset}. @@ -234,7 +200,7 @@ public static < T extends RealType< T > & NativeType< T >, D > void writeScalePy final AfterEachPlane afterEachPlane, ProgressWriter progressWriter ) throws IOException { - final BlockCreator< T > blockCreator = BlockCreator.forType( type ); + final DataType dataType = DataTypeProperties.n5DataType( type ); if ( progressWriter == null ) progressWriter = new ProgressWriterNull(); @@ -246,7 +212,6 @@ public static < T extends RealType< T > & NativeType< T >, D > void writeScalePy // write image data for all views to the HDF5 file final int n = 3; // TODO checkNumDimensions( img.numDimensions() ); - final long[] dimensions = new long[ n ]; final int[][] resolutions = mipmapInfo.getExportResolutions(); final int[][] subdivisions = mipmapInfo.getSubdivisions(); @@ -315,28 +280,16 @@ public static < T extends RealType< T > & NativeType< T >, D > void writeScalePy factor = resolutions[ level ]; } - sourceImg.dimensions( dimensions ); - - final long size = Intervals.numElements( factor ); - final boolean fullResolution = size == 1; - if ( !fullResolution ) - { - for ( int d = 0; d < n; ++d ) - dimensions[ d ] = Math.max( dimensions[ d ] / factor[ d ], 1 ); - } - - final long[] minRequiredInput = new long[ n ]; - final long[] maxRequiredInput = new long[ n ]; - sourceImg.min( minRequiredInput ); - for ( int d = 0; d < n; ++d ) - maxRequiredInput[ d ] = minRequiredInput[ d ] + dimensions[ d ] * factor[ d ] - 1; - // TODO: pass OutOfBoundsFactory - final RandomAccessibleInterval< T > extendedImg = Views.interval( Views.extendBorder( sourceImg ), new FinalInterval( minRequiredInput, maxRequiredInput ) ); + final long[] dimensions = Downsample.getDownsampledDimensions( sourceImg.dimensionsAsLongArray(), factor ); + final boolean fullResolution = (Intervals.numElements( factor ) == 1); final int[] cellDimensions = subdivisions[ level ]; final D dataset = io.createDataset( level, dimensions, cellDimensions ); + final BlockSupplier< T > imgBlocks = BlockSupplier.of( sourceImg.view().extend( Extension.border() ) ); + final BlockSupplier< T > blocks = ( fullResolution ? imgBlocks : imgBlocks.andThen( Downsample.downsample( factor ) ) ).threadSafe(); + final ProgressWriter subProgressWriter = new SubTaskProgressWriter( progressWriter, ( double ) numCompletedTasks / numTasks, ( double ) ( numCompletedTasks + 1 ) / numTasks ); @@ -354,39 +307,14 @@ public static < T extends RealType< T > & NativeType< T >, D > void writeScalePy { tasks.add( () -> { final long[] currentCellMin = new long[ n ]; - final int[] currentCellDim = new int[ n ]; - final long[] currentCellPos = new long[ n ]; - final long[] blockMin = new long[ n ]; - final RandomAccess< T > in = extendedImg.randomAccess(); - - final Class< ? extends RealType > kl1 = type.getClass(); - final Class< ? extends RandomAccess > kl2 = in.getClass(); - final CopyBlock< T > copyBlock = fullResolution ? CopyBlock.create( n, kl1, kl2 ) : null; - final DownsampleBlock< T > downsampleBlock = fullResolution ? null : DownsampleBlock.create( cellDimensions, factor, kl1, kl2 ); - for ( int i = nextCellInPlane.getAndIncrement(); i < numBlocksPerPlane; i = nextCellInPlane.getAndIncrement() ) { final long index = planeBaseIndex + i; - - grid.getCellDimensions( index, currentCellMin, currentCellDim ); - grid.getCellGridPositionFlat( index, currentCellPos ); - final Block< T > block = blockCreator.create( currentCellDim, currentCellMin, currentCellPos ); - - if ( fullResolution ) - { - final RandomAccess< T > out = block.getData().randomAccess(); - in.setPosition( currentCellMin ); - out.setPosition( currentCellMin ); - copyBlock.copyBlock( in, out, currentCellDim ); - } - else - { - for ( int d = 0; d < n; ++d ) - blockMin[ d ] = currentCellMin[ d ] * factor[ d ]; - in.setPosition( blockMin ); - downsampleBlock.downsampleBlock( in, block.getData().cursor(), currentCellDim ); - } - + final int[] blockSize = grid.getCellDimensions( index, currentCellMin ).dimensions(); + final long[] gridPosition = new long[ n ]; + grid.getCellGridPositionFlat( index, gridPosition ); + final DataBlock< ? > block = dataType.createDataBlock( blockSize, gridPosition ); + blocks.copy( currentCellMin, block.getData(), blockSize ); io.writeBlock( dataset, block ); } return null; @@ -421,21 +349,4 @@ private static long numElements( final long[] size, final int mind, final int ma numElements *= size[ d ]; return numElements; } - - private interface BlockCreator< T extends NativeType< T > > - { - Block< T > create( final int[] blockSize, final long[] blockMin, final long[] gridPosition ); - - static < T extends NativeType< T > & RealType< T >, A extends ArrayDataAccess< A > > BlockCreator< T > forType( final T type ) - { - final A accessFactory = Cast.unchecked( ArrayDataAccessFactory.get( type ) ); - final NativeTypeFactory< T, A > nativeTypeFactory = Cast.unchecked( type.getNativeTypeFactory() ); - return ( blockSize, blockMin, gridPosition ) -> { - final A data = accessFactory.createArray( ( int ) Intervals.numElements( blockSize ) ); - final SingleCellArrayImg< T, A > img = new SingleCellArrayImg<>( blockSize, blockMin, data, null ); - img.setLinkedType( nativeTypeFactory.createLinkedType( img ) ); - return new Block<>( img, blockSize, gridPosition ); - }; - } - } } diff --git a/src/main/java/bdv/export/WriteSequenceToHdf5.java b/src/main/java/bdv/export/WriteSequenceToHdf5.java index e75a195b..01b30f4b 100644 --- a/src/main/java/bdv/export/WriteSequenceToHdf5.java +++ b/src/main/java/bdv/export/WriteSequenceToHdf5.java @@ -37,23 +37,15 @@ import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.Function; -import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; -import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.IntArrayDataBlock; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import bdv.export.ExportScalePyramid.AfterEachPlane; -import bdv.export.ExportScalePyramid.Block; import bdv.export.ExportScalePyramid.DatasetIO; import bdv.export.ExportScalePyramid.LoopbackHeuristic; import bdv.img.hdf5.Hdf5ImageLoader; @@ -501,8 +493,6 @@ static class HDF5DatasetIO< T extends RealType< T > & NativeType< T > > implemen private final int setupIdPartition; private final int timepointIdPartition; private final DataType dataType; - private final T type; - private final Function< Block< T >, DataBlock< ? > > getDataBlock; private final LoopBackImageLoader loopback; public HDF5DatasetIO( @@ -518,44 +508,7 @@ public HDF5DatasetIO( this.timepointIdPartition= timepointIdPartition; this.setupIdPartition = setupIdPartition; this.dataType = DataTypeProperties.n5DataType( type ); - this.type = type; this.loopback = loopback; - - switch ( dataType ) - { - case UINT8: - getDataBlock = b -> new ByteArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case UINT16: - getDataBlock = b -> new ShortArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case UINT32: - getDataBlock = b -> new IntArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case UINT64: - getDataBlock = b -> new LongArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT8: - getDataBlock = b -> new ByteArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT16: - getDataBlock = b -> new ShortArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT32: - getDataBlock = b -> new IntArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT64: - getDataBlock = b -> new LongArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case FLOAT32: - getDataBlock = b -> new FloatArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case FLOAT64: - getDataBlock = b -> new DoubleArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - default: - throw new IllegalArgumentException(); - } } @Override @@ -567,9 +520,9 @@ public H5Dataset createDataset( final int level, final long[] dimensions, final } @Override - public void writeBlock( final H5Dataset dataset, final ExportScalePyramid.Block< T > dataBlock ) + public void writeBlock( final H5Dataset dataset, final DataBlock< ? > dataBlock ) { - writerQueue.writeBlock( dataset.pathName, dataset.attributes, getDataBlock.apply( dataBlock ) ); + writerQueue.writeBlock( dataset.pathName, dataset.attributes, dataBlock ); } @Override @@ -602,7 +555,6 @@ static < T extends RealType< T > & NativeType< T > > void writeViewToHdf5Partiti final BasicSetupImgLoader< T > setupImgLoader = Cast.unchecked( imgLoader.getSetupImgLoader( setupId ) ); final RandomAccessibleInterval< T > img = setupImgLoader.getImage( timepointId ); final T type = setupImgLoader.getImageType(); - System.out.println( "typed ... WriteSequenceToHdf5.writeViewToHdf5PartitionFile" ); writeViewToHdf5PartitionFile( img, type, timepointId, setupId, mipmapInfo, writeMipmapInfo, deflate, writerQueue, executorService, numThreads, loopbackHeuristic, afterEachPlane, progressWriter); } diff --git a/src/main/java/bdv/export/n5/WriteSequenceToN5.java b/src/main/java/bdv/export/n5/WriteSequenceToN5.java index 86cecb5d..fb838439 100644 --- a/src/main/java/bdv/export/n5/WriteSequenceToN5.java +++ b/src/main/java/bdv/export/n5/WriteSequenceToN5.java @@ -41,22 +41,15 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.Function; import java.util.stream.Collectors; -import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; -import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; -import org.janelia.saalfeldlab.n5.IntArrayDataBlock; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import bdv.export.ExportMipmapInfo; import bdv.export.ExportScalePyramid; @@ -281,7 +274,6 @@ static class N5DatasetIO< T extends RealType< T > & NativeType< T > > implements private final int timepointId; private final DataType dataType; private final T type; - private final Function< ExportScalePyramid.Block< T >, DataBlock< ? > > getDataBlock; public N5DatasetIO( final N5Writer n5, final Compression compression, final int setupId, final int timepointId, final T type ) { @@ -291,42 +283,6 @@ public N5DatasetIO( final N5Writer n5, final Compression compression, final int this.timepointId = timepointId; this.dataType = n5DataType( type ); this.type = type; - - switch ( dataType ) - { - case UINT8: - getDataBlock = b -> new ByteArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case UINT16: - getDataBlock = b -> new ShortArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case UINT32: - getDataBlock = b -> new IntArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case UINT64: - getDataBlock = b -> new LongArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT8: - getDataBlock = b -> new ByteArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT16: - getDataBlock = b -> new ShortArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT32: - getDataBlock = b -> new IntArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case INT64: - getDataBlock = b -> new LongArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case FLOAT32: - getDataBlock = b -> new FloatArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - case FLOAT64: - getDataBlock = b -> new DoubleArrayDataBlock( b.getSize(), b.getGridPosition(), Cast.unchecked( b.getData().getStorageArray() ) ); - break; - default: - throw new IllegalArgumentException(); - } } @Override @@ -347,11 +303,11 @@ public N5Dataset createDataset( final int level, final long[] dimensions, final } @Override - public void writeBlock( final N5Dataset dataset, final ExportScalePyramid.Block< T > dataBlock ) throws IOException + public void writeBlock( final N5Dataset dataset, final DataBlock< ? > dataBlock ) throws IOException { try { - n5.writeBlock( dataset.pathName, dataset.attributes, getDataBlock.apply( dataBlock ) ); + n5.writeBlock( dataset.pathName, dataset.attributes, dataBlock ); } catch ( final N5Exception e ) { diff --git a/src/main/java/bdv/img/n5/N5ImageLoader.java b/src/main/java/bdv/img/n5/N5ImageLoader.java index 243b01fd..e4ce3873 100644 --- a/src/main/java/bdv/img/n5/N5ImageLoader.java +++ b/src/main/java/bdv/img/n5/N5ImageLoader.java @@ -68,6 +68,7 @@ import net.imglib2.FinalInterval; import net.imglib2.RandomAccessibleInterval; import net.imglib2.Volatile; +import net.imglib2.blocks.SubArrayCopy; import net.imglib2.cache.volatiles.CacheHints; import net.imglib2.cache.volatiles.LoadingStrategy; import net.imglib2.img.basictypeaccess.DataAccess; @@ -360,22 +361,27 @@ private static class N5CacheArrayLoader< T, A extends DataAccess > implements Si private final DatasetAttributes attributes; private final IntFunction< T > createPrimitiveArray; private final Function< T, A > createVolatileArrayAccess; + private final SubArrayCopy.Typed< T, T > subArrayCopy; N5CacheArrayLoader( final N5Reader n5, final String pathName, final DatasetAttributes attributes, final DataTypeProperties< ?, ?, T, A > dataTypeProperties ) { - this( n5, pathName, attributes, dataTypeProperties.createPrimitiveArray(), dataTypeProperties.createVolatileArrayAccess() ); + this( n5, pathName, attributes, dataTypeProperties.createPrimitiveArray(), dataTypeProperties.createVolatileArrayAccess(), + SubArrayCopy.forPrimitiveType( dataTypeProperties.type().getNativeTypeFactory().getPrimitiveType() ) ); } N5CacheArrayLoader( final N5Reader n5, final String pathName, final DatasetAttributes attributes, final IntFunction< T > createPrimitiveArray, - final Function< T, A > createVolatileArrayAccess ) + final Function< T, A > createVolatileArrayAccess, + final SubArrayCopy.Typed< T, T > subArrayCopy ) { + this.n5 = n5; this.pathName = pathName; this.attributes = attributes; this.createPrimitiveArray = createPrimitiveArray; this.createVolatileArrayAccess = createVolatileArrayAccess; + this.subArrayCopy = subArrayCopy; } @Override @@ -404,7 +410,7 @@ public A loadArray( final long[] gridPosition, final int[] cellDimensions ) thro final int[] pos = new int[ srcDims.length ]; final int[] size = new int[ srcDims.length ]; Arrays.setAll( size, d -> Math.min( srcDims[ d ], cellDimensions[ d ] ) ); - ndArrayCopy( src, srcDims, pos, data, cellDimensions, pos, size ); + subArrayCopy.copy( src, srcDims, pos, data, cellDimensions, pos, size ); } return createVolatileArrayAccess.apply( data ); } @@ -424,74 +430,4 @@ public A loadArray( final long[] gridPosition, final int[] cellDimensions ) thro } return new N5CacheArrayLoader<>( n5, pathName, attributes, DataTypeProperties.of( attributes.getDataType() ) ); } - - // TODO: replace ndArrayCopy(...) below with new SubArrayCopy from imglib2 core ??? - - /** - * Like `System.arrayCopy()` but for flattened nD arrays. - * - * @param src - * the (flattened) source array. - * @param srcSize - * dimensions of the source array. - * @param srcPos - * starting position in the source array. - * @param dest - * the (flattened destination array. - * @param destSize - * dimensions of the source array. - * @param destPos - * starting position in the destination data. - * @param size - * the number of array elements to be copied. - */ - // TODO: This will be moved to a new imglib2-blk artifact later. Re-use it from there when that happens. - private static < T > void ndArrayCopy( - final T src, final int[] srcSize, final int[] srcPos, - final T dest, final int[] destSize, final int[] destPos, - final int[] size) - { - final int n = srcSize.length; - int srcStride = 1; - int destStride = 1; - int srcOffset = 0; - int destOffset = 0; - for ( int d = 0; d < n; ++d ) - { - srcOffset += srcStride * srcPos[ d ]; - srcStride *= srcSize[ d ]; - destOffset += destStride * destPos[ d ]; - destStride *= destSize[ d ]; - } - ndArrayCopy( n - 1, src, srcSize, srcOffset, dest, destSize, destOffset, size ); - } - - private static void ndArrayCopy( - final int d, - final T src, final int[] srcSize, final int srcPos, - final T dest, final int[] destSize, final int destPos, - final int[] size) - { - if ( d == 0 ) - System.arraycopy( src, srcPos, dest, destPos, size[ d ] ); - else - { - int srcStride = 1; - int destStride = 1; - for ( int dd = 0; dd < d; ++dd ) - { - srcStride *= srcSize[ dd ]; - destStride *= destSize[ dd ]; - } - - final int w = size[ d ]; - for ( int x = 0; x < w; ++x ) - { - ndArrayCopy( d - 1, - src, srcSize, srcPos + x * srcStride, - dest, destSize, destPos + x * destStride, - size ); - } - } - } }