+ * Can be overridden by subclasses (rather than only having the option to + * provide an instance at construction time). + * + * @return a new N5Reader instance + */ + protected N5Reader instantiateN5Reader() + { + return new N5FSReader( getN5File().getAbsolutePath() ); + } + + /** + * Create a new N5Properties (provides metadata and resolves dataset paths). + *
+ * Can be overridden by subclasses to adapt to different metadata schemes + * and storage layouts. + * + * @return a new N5Properties instance. + */ + protected N5Properties createN5PropertiesInstance() + { + return new DefaultN5Properties(); + } + + /** + * Touch all metadata in advance in parallel so the N5-API caches them. + *
+ * {@code prefetch()} completes immediately. The prefetching is done + * asynchronously in a new {@code ForkJoinPool} with the indicated {@code + * parallelism} level. The returned {@code Future} can be used to wait for + * completion of the async prefetching. + * + * @param parallelism + * the parallelism level + * + * @return represents completion of the prefetching + */ + public Future< Void > prefetch( final int parallelism ) + { + openReader(); + + final CompletableFuture< Void > future = new CompletableFuture<>(); + + new Thread( () -> { + final ForkJoinPool myPool = new ForkJoinPool( parallelism ); + try + { + // prefetch all datatypes and MipmapResolutions + final Collection< ? extends BasicViewSetup > vss = seq.getViewSetupsOrdered(); + final Map< Integer, Integer > setupIdToNumLevels = new ConcurrentHashMap<>(); + myPool.submit( () -> + vss.parallelStream() + .mapToInt( BasicViewSetup::getId ) + .forEach( setup -> { + n5properties.getDataType( n5, setup ); + setupIdToNumLevels.put( setup, n5properties.getMipmapResolutions( n5, setup ).length ); + } ) + ).join(); +// System.out.println( "Pre-fetched " + vss.size() + " data types and mipmap resolutions." ); + + // prefetch all DatasetAttributes for all views + myPool.submit( () -> { + final Collection< TimePoint > tps = seq.getTimePoints().getTimePointsOrdered(); + Stream< ViewId > viewIds = vss.parallelStream().flatMap( + setup -> tps.parallelStream().map( + tp -> new ViewId( tp.getId(), setup.getId() ) ) ); + + // filter missing ViewIds + if ( seq.getMissingViews() != null ) + { + final Set< ViewId > missing = seq.getMissingViews().getMissingViews(); + viewIds = viewIds.filter( v -> !missing.contains( v ) ); + } + + viewIds.forEach( viewId -> { + final int setup = viewId.getViewSetupId(); + final int tp = viewId.getTimePointId(); + final int numLevels = setupIdToNumLevels.get( setup ); + IntStream.range( 0, numLevels ).parallel().forEach( level -> n5.getDatasetAttributes( n5properties.getDatasetPath( setup, tp, level ) ) ); + } ); + } ).join(); +// System.out.println( "Pre-fetched dataset attributes." ); + + future.complete(null); + } + catch ( Exception e ) + { + // no drama if this fails ... could be that some data is actually missing, one can still look at what's there + future.completeExceptionally(e); + } + myPool.shutdown(); + } ).start(); + + return future; + } + private volatile boolean isOpen = false; private SharedQueue createdSharedQueue; private VolatileGlobalCellCache cache; private N5Reader n5; - + private N5Properties n5properties; private int requestedNumFetcherThreads = -1; private SharedQueue requestedSharedQueue; @@ -141,6 +248,19 @@ public void setCreatedSharedQueue( final SharedQueue createdSharedQueue ) requestedSharedQueue = createdSharedQueue; } + private synchronized void openReader() + { + if ( n5 == null ) + { + n5 = instantiateN5Reader(); + } + + if ( n5properties == null ) + { + n5properties = createN5PropertiesInstance(); + } + } + private void open() { if ( !isOpen ) @@ -152,10 +272,7 @@ private void open() try { - if ( n5 == null ) - { - n5 = new N5FSReader( getN5File().getAbsolutePath() ); - } + openReader(); int maxNumLevels = 0; final List< ? extends BasicViewSetup > setups = seq.getViewSetupsOrdered(); @@ -185,6 +302,7 @@ private void open() } } + // TODO: the N5 is not re-opened /** * Clear the cache. Images that were obtained from * this loader before {@link #close()} will stop working. Requesting images @@ -219,17 +337,15 @@ public void close() private < T extends NativeType< T >, V extends Volatile< T > & NativeType< V > > SetupImgLoader< T, V > createSetupImgLoader( final int setupId ) throws IOException { - final String pathName = getPathName( setupId ); - final DataType dataType; try { - dataType = n5.getAttribute( pathName, DATA_TYPE_KEY, DataType.class ); + final DataType dataType = n5properties.getDataType( n5, setupId ); + return new SetupImgLoader<>( setupId, Cast.unchecked( DataTypeProperties.of( dataType ) ) ); } catch ( final N5Exception e ) { throw new IOException( e ); } - return new SetupImgLoader<>( setupId, Cast.unchecked( DataTypeProperties.of( dataType ) ) ); } @Override @@ -251,17 +367,16 @@ public class SetupImgLoader< T extends NativeType< T >, V extends Volatile< T > public SetupImgLoader( final int setupId, final DataTypeProperties< T, V, ?, ? > props ) throws IOException { - this(setupId, props.type(), props.volatileType() ); + this( setupId, props.type(), props.volatileType() ); } public SetupImgLoader( final int setupId, final T type, final V volatileType ) throws IOException { super( type, volatileType ); this.setupId = setupId; - final String pathName = getPathName( setupId ); try { - mipmapResolutions = n5.getAttribute( pathName, DOWNSAMPLING_FACTORS_KEY, double[][].class ); + mipmapResolutions = n5properties.getMipmapResolutions( n5, setupId ); } catch ( final N5Exception e ) { @@ -275,13 +390,24 @@ public SetupImgLoader( final int setupId, final T type, final V volatileType ) t @Override public RandomAccessibleInterval< V > getVolatileImage( final int timepointId, final int level, final ImgLoaderHint... hints ) { - return prepareCachedImage( timepointId, level, LoadingStrategy.BUDGETED, volatileType ); + return prepareCachedImage( timepointId, level, BUDGETED, volatileType ); } @Override public RandomAccessibleInterval< T > getImage( final int timepointId, final int level, final ImgLoaderHint... hints ) { - return prepareCachedImage( timepointId, level, LoadingStrategy.BLOCKING, type ); + return prepareCachedImage( timepointId, level, BLOCKING, type ); + } + + /** + * Create a {@link CellImg} backed by the cache. + */ + private < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage( final int timepointId, final int level, final LoadingStrategy loadingStrategy, final T type ) + { + final int priority = numMipmapLevels() - 1 - level; + final CacheHints cacheHints = new CacheHints( loadingStrategy, priority, false ); + final String datasetPath = n5properties.getDatasetPath( setupId, timepointId, level ); + return N5ImageLoader.this.prepareCachedImage( datasetPath, setupId, timepointId, level, cacheHints, type ); } @Override @@ -289,11 +415,9 @@ public Dimensions getImageSize( final int timepointId, final int level ) { try { - final String pathName = getPathName( setupId, timepointId, level ); - final DatasetAttributes attributes = n5.getDatasetAttributes( pathName ); - return new FinalDimensions( attributes.getDimensions() ); + return new FinalDimensions( n5properties.getDimensions( n5, setupId, timepointId, level ) ); } - catch( final RuntimeException e ) + catch ( final RuntimeException e ) { return null; } @@ -322,35 +446,37 @@ public VoxelDimensions getVoxelSize( final int timepointId ) { return null; } + } - /** - * Create a {@link CellImg} backed by the cache. - */ - private < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage( final int timepointId, final int level, final LoadingStrategy loadingStrategy, final T type ) + /** + * Create a {@link CellImg} backed by the cache. + */ + protected < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage( + final String datasetPath, + final int setupId, + final int timepointId, + final int level, + final CacheHints cacheHints, + final T type ) + { + try { - try - { - final String pathName = getPathName( setupId, timepointId, level ); - final DatasetAttributes attributes = n5.getDatasetAttributes( pathName ); - final long[] dimensions = attributes.getDimensions(); - final int[] cellDimensions = attributes.getBlockSize(); - final CellGrid grid = new CellGrid( dimensions, cellDimensions ); - - final int priority = numMipmapLevels() - 1 - level; - final CacheHints cacheHints = new CacheHints( loadingStrategy, priority, false ); - - final SimpleCacheArrayLoader< ? > loader = createCacheArrayLoader( n5, pathName ); - return cache.createImg( grid, timepointId, setupId, level, cacheHints, loader, type ); - } - catch ( final IOException | N5Exception e ) - { - System.err.println( String.format( - "image data for timepoint %d setup %d level %d could not be found.", - timepointId, setupId, level ) ); - return Views.interval( - new ConstantRandomAccessible<>( type.createVariable(), 3 ), - new FinalInterval( 1, 1, 1 ) ); - } + final DatasetAttributes attributes = n5.getDatasetAttributes( datasetPath ); + final long[] dimensions = attributes.getDimensions(); + final int[] cellDimensions = attributes.getBlockSize(); + final CellGrid grid = new CellGrid( dimensions, cellDimensions ); + final DataTypeProperties< ?, ?, ?, ? > dataTypeProperties = DataTypeProperties.of( attributes.getDataType() ); + final SimpleCacheArrayLoader< ? > loader = new N5CacheArrayLoader<>( n5, datasetPath, attributes, dataTypeProperties ); + return cache.createImg( grid, timepointId, setupId, level, cacheHints, loader, type ); + } + catch ( final N5Exception e ) + { + System.err.println( String.format( + "image data for timepoint %d setup %d level %d could not be found.", + timepointId, setupId, level ) ); + return Views.interval( + new ConstantRandomAccessible<>( type.createVariable(), 3 ), + new FinalInterval( 1, 1, 1 ) ); } } diff --git a/src/main/java/bdv/img/n5/N5Properties.java b/src/main/java/bdv/img/n5/N5Properties.java new file mode 100644 index 00000000..e6480918 --- /dev/null +++ b/src/main/java/bdv/img/n5/N5Properties.java @@ -0,0 +1,22 @@ +package bdv.img.n5; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.N5Reader; + +/** + * Provides metadata and resolves dataset path names for (`setupId`, + * `timePointId` and multiresolution `level`) triples. + *
+ * This allows to adapt {@link N5ImageLoader} subclasses for different storage + * schemes. + */ +public interface N5Properties +{ + DataType getDataType( N5Reader n5, int setupId ); + + double[][] getMipmapResolutions( N5Reader n5, int setupId ); + + long[] getDimensions( N5Reader n5, int setupId, int timepointId, int level ); + + String getDatasetPath( int setupId, int timepointId, int level ); +}