diff --git a/pom.xml b/pom.xml index 9baf555..c46cb01 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5-universe - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT N5-Universe Utilities spanning all of the N5 repositories @@ -111,8 +111,19 @@ sign,deploy-to-scijava + 3.2.0 + 2.2.0 + 7.0.0 + 4.1.0 + + 1.3.0 + 4.1.0 + 1.0.0-preview.20191208 1.4.1 + + 0.2.5 + 2.2.2 @@ -153,13 +164,71 @@ compile - - - - - - - + + org.janelia.saalfeldlab + n5 + tests + test + + + org.janelia.saalfeldlab + n5-zarr + ${n5-zarr.version} + tests + test + + + org.janelia.saalfeldlab + n5-aws-s3 + ${n5-aws-s3.version} + tests + test + + + org.janelia.saalfeldlab + n5-google-cloud + ${n5-google-cloud.version} + tests + test + + + com.google.cloud + google-cloud-nio + test + + + org.janelia.saalfeldlab + n5-hdf5 + ${n5-hdf5.version} + tests + test + + + com.googlecode.json-simple + json-simple + 1.1.1 + test + + + commons-io + commons-io + ${commons-io.version} + test + + + + + io.findify + s3mock_2.12 + ${s3mock_2.12.version} + test + + + javax.xml.bind + jaxb-api + ${jaxb-api.version} + test + @@ -199,5 +268,33 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + + **Backend** + + + + + + + + run-backend-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 000edd4..96d79a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -1,17 +1,17 @@ /** * Copyright (c) 2017-2021, Saalfeld lab, HHMI Janelia * All rights reserved. - * + *

* Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

* Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * + * list of conditions and the following disclaimer. + *

* Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + *

* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -26,59 +26,44 @@ */ package org.janelia.saalfeldlab.n5.universe; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.Serializable; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.janelia.saalfeldlab.googlecloud.GoogleCloudResourceManagerClient; -import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageClient; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Storage; +import com.google.gson.GsonBuilder; +import net.imglib2.util.Pair; +import net.imglib2.util.ValuePair; +import org.apache.commons.lang3.function.TriFunction; import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI; +import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.N5FSReader; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5KeyValueReader; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.hdf5.HDF5Utils; import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.AmazonS3Utils; import org.janelia.saalfeldlab.n5.zarr.N5ZarrReader; import org.janelia.saalfeldlab.n5.zarr.N5ZarrWriter; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.AmazonS3URI; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.google.cloud.resourcemanager.Project; -import com.google.cloud.resourcemanager.ResourceManager; -import com.google.cloud.storage.Storage; -import com.google.gson.GsonBuilder; +import javax.annotation.Nullable; +import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystems; +import java.util.function.BiFunction; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Factory for various N5 readers and writers. Implementation specific @@ -92,29 +77,43 @@ */ public class N5Factory implements Serializable { - private static final long serialVersionUID = -6823715427289454617L; - - private static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)"); + private static final N5Factory FACTORY = new N5Factory(); - private static byte[] HDF5_SIG = {(byte)137, 72, 68, 70, 13, 10, 26, 10}; + private static final long serialVersionUID = -6823715427289454617L; + private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); + private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); private int[] hdf5DefaultBlockSize = {64, 64, 64, 1, 1}; private boolean hdf5OverrideBlockSize = false; - private GsonBuilder gsonBuilder = new GsonBuilder(); private boolean cacheAttributes = true; - - private String zarrDimensionSeparator = "."; + private String zarrDimensionSeparator = "/"; private boolean zarrMapN5DatasetAttributes = true; private boolean zarrMergeAttributes = true; - private String googleCloudProjectId = null; - private String s3Region = null; private AWSCredentials s3Credentials = null; private boolean s3Anonymous = true; - private boolean s3RetryWithCredentials = false; private String s3Endpoint; + private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { + + final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); + return new GoogleCloudStorageKeyValueAccess(factory.createGoogleCloudStorage(), googleCloudUri, true); + } + + private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, final N5Factory factory) { + + final String uriString = uri.toString(); + final AmazonS3 s3 = factory.createS3(uriString); + + return new AmazonS3KeyValueAccess(s3, uri, true); + } + + private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { + + return new FileSystemKeyValueAccess(FileSystems.getDefault()); + } + public N5Factory hdf5DefaultBlockSize(final int... blockSize) { hdf5DefaultBlockSize = blockSize; @@ -181,9 +180,9 @@ public N5Factory s3UseCredentials(final AWSCredentials credentials) { return this; } + @Deprecated public N5Factory s3RetryWithCredentials() { - s3RetryWithCredentials = true; return this; } @@ -199,495 +198,496 @@ public N5Factory s3Region(final String s3Region) { return this; } - private static boolean isHDF5Writer(final String path) { - - if (path.matches("(?i).*\\.(h5|hdf|hdf5)")) - return true; - else - return false; - } - - private static boolean isHDF5Reader(final String path) throws N5IOException { - - if (Files.isRegularFile(Paths.get(path))) { - /* optimistic */ - if (isHDF5Writer(path)) - return true; - else { - try (final FileInputStream in = new FileInputStream(new File(path))) { - final byte[] sig = new byte[8]; - in.read(sig); - return Arrays.equals(sig, HDF5_SIG); - } catch (final IOException e) { - throw new N5Exception.N5IOException(e); - } - } - } - return false; - } - - private AmazonS3 createS3(final String uri) { - - try { - return createS3(new AmazonS3URI(uri)); - } catch (final IllegalArgumentException e) { - // if AmazonS3URI does not like the form of the uri - try { - final URI buri = new URI(uri); - final URI endpointUrl = new URI(buri.getScheme(), buri.getHost(), null, null); - return createS3(getS3Credentials(), new EndpointConfiguration(endpointUrl.toString(), null), null, getS3Bucket(uri)); - } catch (final URISyntaxException e1) {} - } - throw new N5Exception("Could not create s3 client from uri: " + uri); - } - - private AmazonS3 createS3( - final AWSCredentialsProvider credentialsProvider, - final EndpointConfiguration endpointConfiguration, - final Regions region, - final String bucketName ) { - - final boolean isAmazon = endpointConfiguration == null || AWS_ENDPOINT_PATTERN.matcher(endpointConfiguration.getServiceEndpoint()).find(); - final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); - - if (!isAmazon) - builder.withPathStyleAccessEnabled(true); - - if (credentialsProvider != null) - builder.withCredentials(credentialsProvider); - - if (endpointConfiguration != null) - builder.withEndpointConfiguration(endpointConfiguration); - else if (region != null) - builder.withRegion(region); - else - builder.withRegion("us-east-1"); - - AmazonS3 s3 = builder.build(); - // if we used anonymous credentials and the factory requests a retry with credentials: - if( s3RetryWithCredentials && areAnonymous(credentialsProvider)) { - - // I initially tried checking whether the bucket exists, but - // that, apparently, returns even when the client does not have access - if (!canListBucket(s3, bucketName)) { - // bucket not detected with anonymous credentials, try detecting credentials - // and return it even if it can't detect the bucket, since there's nothing else to do - s3 = createS3(new DefaultAWSCredentialsProviderChain(), endpointConfiguration, region, null ); - } - } - return s3; - } - - private boolean canListBucket( final AmazonS3 s3, final String bucket) { - - final ListObjectsV2Request request = new ListObjectsV2Request(); - request.setBucketName(bucket); - request.setMaxKeys(1); + AmazonS3 createS3(final String uri) { try { - // list objects will throw an AmazonS3Exception (Access Denied) if this client does not have access - s3.listObjectsV2(request); - return true; - } catch (final AmazonS3Exception e) { - return false; - } - } - - private AWSStaticCredentialsProvider getS3Credentials() { - - AWSCredentials credentials = null; - final AWSStaticCredentialsProvider credentialsProvider; - if (s3Credentials != null) { - credentials = s3Credentials; - credentialsProvider = new AWSStaticCredentialsProvider(credentials); - } else { - // if not anonymous, try finding credentials - if (!s3Anonymous) { - try { - credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); - } catch (final Exception e) { - System.out.println("Could not load AWS credentials, falling back to anonymous."); - } - credentialsProvider = new AWSStaticCredentialsProvider( - credentials == null ? new AnonymousAWSCredentials() : credentials); - } else - credentialsProvider = new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()); + return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region); + } catch (final Throwable e) { + throw new N5Exception("Could not create s3 client from uri: " + uri, e); } - - return credentialsProvider; - } - - private boolean areAnonymous(final AWSCredentialsProvider credsProvider) { - - final AWSCredentials creds = credsProvider.getCredentials(); - // AnonymousAWSCredentials do not have an equals method - if (creds.getClass().equals(AnonymousAWSCredentials.class)) - return true; - - return creds.getAWSAccessKeyId() == null && creds.getAWSSecretKey() == null; - } - - private Regions getS3Region(final AmazonS3URI uri) { - - final Regions region = Optional.ofNullable(uri.getRegion()).map(Regions::fromName) // get the region from the uri - .orElse(Optional.ofNullable(s3Region).map(Regions::fromName) // next use whatever is passed in - .orElse(null)); // fallback to null (amazon picks a default) - return region; } - private AmazonS3 createS3(final AmazonS3URI uri) { + Storage createGoogleCloudStorage() { - AwsClientBuilder.EndpointConfiguration endpointConfiguration = null; - if (!"s3".equalsIgnoreCase(uri.getURI().getScheme())) { - - if (s3Endpoint != null) - endpointConfiguration = new EndpointConfiguration(s3Endpoint, null); - else { - final Matcher matcher = AWS_ENDPOINT_PATTERN.matcher(uri.getURI().getHost()); - if (matcher.find()) - endpointConfiguration = new EndpointConfiguration(matcher.group(2), uri.getRegion()); - else - endpointConfiguration = new EndpointConfiguration(uri.getURI().getHost(), uri.getRegion()); - } - } - - return createS3(getS3Credentials(), endpointConfiguration, getS3Region(uri), uri.getBucket()); + return GoogleCloudUtils.createGoogleCloudStorage(googleCloudProjectId); } - private String getS3Bucket(final String uri) { - - try { - return new AmazonS3URI(uri).getBucket(); - } catch (final IllegalArgumentException e) {} - try { - // parse bucket manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(0, path.indexOf('/')); - } catch (final URISyntaxException e) { + /** + * Test the provided {@link URI} to and return the appropriate {@link KeyValueAccess}. + * If no appropriate {@link KeyValueAccess} is found, may be null + * + * @param uri to create a {@link KeyValueAccess} from. + * @return the {@link KeyValueAccess} and container path, or null if none are valid + */ + @Nullable + KeyValueAccess getKeyValueAccess(final URI uri) { + + /*NOTE: The order of these tests is very important, as the predicates for each + * backend take into account reasonable defaults when possible. + * Here we test from most to least restrictive. + * See the Javadoc for more details. */ + for (KeyValueAccessBackend backend : KeyValueAccessBackend.values()) { + final KeyValueAccess kva = backend.apply(uri, this); + if (kva != null) + return kva; } return null; } - private String getS3Key(final String uri) { - - try { - // if key is null, return the empty string - final String key = new AmazonS3URI(uri).getKey(); - return key == null ? "" : key; - } catch (final IllegalArgumentException e) {} - try { - // parse key manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(path.indexOf('/') + 1); - } catch (final URISyntaxException e) {} - return null; - } - /** - * Open an {@link N5Reader} for N5 filesystem. + * Open an {@link N5Reader} over an N5 Container. + *

+ * NOTE: The name seems to imply that this will open any N5Reader, over a + * {@link FileSystemKeyValueAccess} however that is misleading. Instead + * this will open any N5Container that is a valid {@link StorageFormat#N5}. + * This is partially why it is deprecated, as well as the redundant + * implementation with {@link N5Factory#openReader(StorageFormat, URI)}. * - * @param path - * path to the n5 root folder - * @return the N5FsReader + * @param path path to the n5 root folder + * @return the N5Reader + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5FSReader openFSReader(final String path) { + @Deprecated + public N5Reader openFSReader(final String path) { - return new N5FSReader(path, gsonBuilder); + return openN5ContainerWithStorageFormat(StorageFormat.N5, path, this::openReader); } /** * Open an {@link N5Reader} for Zarr. - * + *

* For more options of the Zarr backend study the {@link N5ZarrReader} * constructors.} * - * @param path - * path to the zarr directory - * @return the N5ZarrReader + * @param path path to the zarr directory + * @return the N5Reader + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5ZarrReader openZarrReader(final String path) { + @Deprecated + public N5Reader openZarrReader(final String path) { - return new N5ZarrReader(path, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); + return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openReader); } /** * Open an {@link N5Reader} for HDF5. Close the reader when you do not need * it any more. - * + *

* For more options of the HDF5 backend study the {@link N5HDF5Reader} * constructors. * - * @param path - * path to the hdf5 file - * @return the N5HDF5Reader + * @param path path to the hdf5 file + * @return the N5Reader + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5HDF5Reader openHDF5Reader(final String path) { + @Deprecated + public N5Reader openHDF5Reader(final String path) { - return new N5HDF5Reader(path, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + return openN5ContainerWithStorageFormat( + StorageFormat.HDF5, + path, + (format, uri) -> openReader(format, null, uri.getPath()) + ); } /** * Open an {@link N5Reader} for Google Cloud. * - * @param uri - * uri to the google cloud object - * @return the N5GoogleCloudStorageReader - * @throws URISyntaxException - * if uri is malformed + * @param uri uri to the google cloud object + * @return the N5Reader + * @throws URISyntaxException if uri is malformed */ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxException { - final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(N5URI.encodeAsUri(uri)); - final GoogleCloudStorageClient storageClient = new GoogleCloudStorageClient(); - final Storage storage = storageClient.create(); - final GoogleCloudStorageKeyValueAccess googleCloudBackend = new GoogleCloudStorageKeyValueAccess(storage, - googleCloudUri.getBucket(), false); - - return openReader(uri, googleCloudUri.getKey(), googleCloudBackend); + return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, this::openReader); } /** * Open an {@link N5Reader} for AWS S3. * - * @param uri - * uri to the amazon s3 object + * @param uri uri to the amazon s3 object * @return the N5Reader - * @throws URISyntaxException - * if uri is malformed + * @throws URISyntaxException if uri is malformed */ public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException { - final AmazonS3 s3 = createS3(N5URI.encodeAsUri(uri).toString()); - // when, if ever do we want to create a bucket? - final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, getS3Bucket(uri), false); - return openReader(uri, getS3Key(uri), s3kv); + return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, this::openReader); } /** - * Open an {@link N5Writer} for N5 filesystem. + * Open an {@link N5Reader} over a FileSytem. * - * @param path - * path to the n5 directory - * @return the N5FSWriter + * @param uri uri to the N5Reader + * @return the N5Reader + * @throws URISyntaxException if uri is malformed */ - public N5FSWriter openFSWriter(final String path) { + public N5Reader openFileSystemReader(final String uri) throws URISyntaxException { - return new N5FSWriter(path, gsonBuilder); + return openN5ContainerWithBackend(KeyValueAccessBackend.FILE, uri, this::openReader); + } + + public N5Reader openReader(final StorageFormat format, final String uri) { + + try { + return openN5Container(format, N5URI.encodeAsUri(uri), this::openReader); + } catch (URISyntaxException e) { + throw new N5Exception(e); + } + } + + public N5Reader openReader(final StorageFormat format, final URI uri) { + + return openN5Container(format, uri, this::openReader); } /** - * Open an {@link N5Writer} for Zarr. + * Open an {@link N5Reader} based on some educated guessing from the url. + * + * @param uri the location of the root location of the store + * @return the N5Reader + */ + public N5Reader openReader(final String uri) { + + return openN5Container(uri, this::openReader, this::openReader); + } + + private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, String containerPath) { + + if (storage == null) { + for (StorageFormat format : StorageFormat.values()) { + try { + return openReader(format, access, containerPath); + } catch (Throwable e) { + } + } + throw new N5Exception("Unable to open " + containerPath + " as N5Reader"); + + } else { + + switch (storage) { + case N5: + return new N5KeyValueReader(access, containerPath, gsonBuilder, cacheAttributes); + case ZARR: + return new ZarrKeyValueReader(access, containerPath, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); + case HDF5: + return new N5HDF5Reader(containerPath, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + } + return null; + } + } + + /** + * Open an {@link N5Writer} for N5 Container. + *

+ * NOTE: The name seems to imply that this will open any N5Writer, over a + * {@link FileSystemKeyValueAccess} however that is misleading. Instead + * this will open any N5Container that is a valid {@link StorageFormat#N5}. + * This is partially why it is deprecated, as well as the redundant + * implementation with {@link N5Factory#openWriter(StorageFormat, URI)}. * + * @param path path to the n5 directory + * @return the N5Writer + * @deprecated use {@link N5Factory#openWriter(StorageFormat, URI)} instead + */ + @Deprecated + public N5Writer openFSWriter(final String path) { + + return openN5ContainerWithStorageFormat(StorageFormat.N5, path, this::openWriter); + } + + /** + * Open an {@link N5Writer} for Zarr. + *

* For more options of the Zarr backend study the {@link N5ZarrWriter} * constructors. * - * @param path - * path to the zarr directory - * @return the N5ZarrWriter + * @param path path to the zarr directory + * @return the N5Writer + * @deprecated use {@link N5Factory#openWriter(StorageFormat, URI)}) instead */ - public N5ZarrWriter openZarrWriter(final String path) { + @Deprecated + public N5Writer openZarrWriter(final String path) { - return new N5ZarrWriter(path, gsonBuilder, zarrDimensionSeparator, zarrMapN5DatasetAttributes, true); + return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openWriter); } /** * Open an {@link N5Writer} for HDF5. Don't forget to close the writer after * writing to close the file and make it available to other processes. - * + *

* For more options of the HDF5 backend study the {@link N5HDF5Writer} * constructors. * - * @param path - * path to the hdf5 file + * @param path path to the hdf5 file * @return the N5HDF5Writer + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5HDF5Writer openHDF5Writer(final String path) { + @Deprecated + public N5Writer openHDF5Writer(final String path) { - return new N5HDF5Writer(path, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + return openN5ContainerWithStorageFormat( + StorageFormat.HDF5, + path, + (format, uri) -> openWriter(format, null, uri.getPath()) + ); } /** * Open an {@link N5Writer} for Google Cloud. * - * @param uri - * uri to the google cloud object + * @param uri uri to the google cloud object * @return the N5GoogleCloudStorageWriter - * @throws URISyntaxException - * if uri is malformed + * @throws URISyntaxException if uri is malformed */ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxException { - final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(N5URI.encodeAsUri(uri)); - final GoogleCloudStorageClient storageClient; - if (googleCloudProjectId == null) { - final ResourceManager resourceManager = new GoogleCloudResourceManagerClient().create(); - final Iterator projectsIterator = resourceManager.list().iterateAll().iterator(); - if (!projectsIterator.hasNext()) - return null; - storageClient = new GoogleCloudStorageClient(projectsIterator.next().getProjectId()); - } else - storageClient = new GoogleCloudStorageClient(googleCloudProjectId); - - final Storage storage = storageClient.create(); - final GoogleCloudStorageKeyValueAccess googleCloudBackend = new GoogleCloudStorageKeyValueAccess(storage, - googleCloudUri.getBucket(), false); - - return openWriter(uri, googleCloudUri.getKey(), googleCloudBackend); + return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, this::openWriter); } /** * Open an {@link N5Writer} for AWS S3. * - * @param uri - * uri to the s3 object + * @param uri uri to the s3 object * @return the N5Writer - * @throws URISyntaxException - * if the URI is malformed + * @throws URISyntaxException if the URI is malformed */ public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException { - final AmazonS3 s3 = createS3(N5URI.encodeAsUri(uri).toString()); - // when, if ever do we want to create a bucket? - final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, getS3Bucket(uri), false); - return openWriter(uri, getS3Key(uri), s3kv); + return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, this::openWriter); } - /** - * Open an {@link N5Reader} based on some educated guessing from the url. - *

- * If this fails for any reason, a {@link RuntimeException} will be thrown. - * - * @param uri - * the location of the root location of the store - * @return the N5Reader - */ - public N5Reader openReader(final String uri) { + public N5Writer openWriter(final StorageFormat format, final String uri) { try { - final URI encodedUri = N5URI.encodeAsUri(uri); - final String scheme = encodedUri.getScheme(); - if (scheme == null) - ; - else if (scheme.equals("file")) { - try { - return openFileBasedN5Reader(Paths.get(encodedUri).toFile().getCanonicalPath()); - } catch (final IOException e) { - throw new N5Exception.N5IOException(e); - } - } else if (scheme.equals("s3")) - return openAWSS3Reader(uri); - else if (scheme.equals("gs")) - return openGoogleCloudReader(uri); - else if (encodedUri.getHost() != null && scheme.equals("https") || scheme.equals("http")) { - if (encodedUri.getHost().matches(".*cloud\\.google\\.com") - || encodedUri.getHost().matches(".*storage\\.googleapis\\.com")) - return openGoogleCloudReader(uri); - else //if (encodedUri.getHost().matches(".*s3.*")) //< This is too fragile for what people in the wild are doing with their S3 instances, for now catch all - return openAWSS3Reader(uri); - } - } catch (final URISyntaxException ignored) {} - return openFileBasedN5Reader(uri); + return openN5Container(format, N5URI.encodeAsUri(uri), this::openWriter); + } catch (URISyntaxException e) { + throw new N5Exception(e); + } } - private N5Reader openFileBasedN5Reader(final String url) { + public N5Writer openWriter(final StorageFormat format, final URI uri) { - // TODO duplicates logic in openReader(final String uri, final String basePath, final KeyValueAccess kvAcess) - if (isHDF5Reader(url)) - return openHDF5Reader(url); - else if (lastExtension(url).startsWith(".n5")) - return openFSReader(url); - else - try { - return openZarrReader(url); - } catch (N5Exception e) { - return openFSReader(url); - } + return openN5Container(format, uri, this::openWriter); } /** - * Open an {@link N5Writer} based on some eapplyducated guessing from the uri. - *

- * If this fails for any reason, a {@link RuntimeException} will be thrown. + * Open an {@link N5Writer} based on some educated guessing from the uri. * - * @param uri - * the location of the root location of the store + * @param uri the location of the root location of the store * @return the N5Writer */ public N5Writer openWriter(final String uri) { - try { - final URI encodedUri = N5URI.encodeAsUri(uri); - final String scheme = encodedUri.getScheme(); - if (scheme == null) ; - else if (scheme.equals("file")) - return openFileBasedN5Writer(encodedUri.getPath()); - else if (scheme.equals("s3")) - return openAWSS3Writer(uri); - else if (scheme.equals("gs")) - return openGoogleCloudWriter(uri); - else if (encodedUri.getHost() != null && scheme.equals("https") || scheme.equals("http")) { - if (encodedUri.getHost().matches(".*s3.*")) - return openAWSS3Writer(uri); - else if (encodedUri.getHost().matches(".*cloud\\.google\\.com") - || encodedUri.getHost().matches(".*storage\\.googleapis\\.com")) - return openGoogleCloudWriter(uri); - } - } catch (final URISyntaxException e) {} - return openFileBasedN5Writer(uri); + return openN5Container(uri, this::openWriter, this::openWriter); } - private N5Writer openFileBasedN5Writer(final String url) { + private N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, final String containerPath) { - // TODO duplicates logic in openFileBasedN5Reader - if (isHDF5Reader(url)) - return openHDF5Writer(url); - else if (lastExtension(url).startsWith(".n5")) - return openFSWriter(url); - else - try { - return openZarrWriter(url); - } catch (N5Exception e) { - return openFSWriter(url); + if (storage == null) { + for (StorageFormat format : StorageFormat.values()) { + try { + return openWriter(format, access, containerPath); + } catch (Throwable ignored) { + } } + throw new N5Exception("Unable to open " + containerPath + " as N5Writer"); + + } else { + + switch (storage) { + case ZARR: + return new ZarrKeyValueWriter(access, containerPath, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes); + case N5: + return new N5KeyValueWriter(access, containerPath, gsonBuilder, cacheAttributes); + case HDF5: + return new N5HDF5Writer(containerPath, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + } + } + return null; + } + + private T openN5ContainerWithStorageFormat( + final StorageFormat format, + final String uri, + final BiFunction openWithFormat + ) { + + try { + final URI asUri = StorageFormat.parseUri(uri).getB(); + return openWithFormat.apply(format, asUri); + } catch (URISyntaxException e) { + throw new N5Exception("Cannot create N5 Container (" + format + ") at " + uri, e); + } + } + + private T openN5ContainerWithBackend( + final KeyValueAccessBackend backend, + final String containerUri, + final TriFunction openWithBackend + ) throws URISyntaxException { + + final Pair formatAndUri = StorageFormat.parseUri(containerUri); + final URI uri = formatAndUri.getB(); + final KeyValueAccess kva = backend.apply(uri, this); + return openWithBackend.apply(formatAndUri.getA(), kva, uri.toString()); } - private static String lastExtension(final String path) { + private T openN5Container( + final StorageFormat storageFormat, + final URI uri, + final TriFunction openWithKva) { - final int i = path.lastIndexOf('.'); - if (i >= 0) - return path.substring(path.lastIndexOf('.')); + final KeyValueAccess kva = getKeyValueAccess(uri); + if (kva == null) + throw new N5Exception("Cannot get KeyValueAccess at " + uri); + return openWithKva.apply(storageFormat, kva, uri.toString()); + } + + private T openN5Container( + final String containerUri, + final BiFunction openWithFormat, + final TriFunction openWithKva) { + + final Pair storageAndUri; + try { + storageAndUri = StorageFormat.parseUri(containerUri); + } catch (URISyntaxException e) { + throw new N5Exception("Unable to open " + containerUri + " as N5 Container", e); + } + final StorageFormat format = storageAndUri.getA(); + final URI uri = storageAndUri.getB(); + if (format != null) + return openWithFormat.apply(format, uri); else - return ""; + return openN5Container(null, uri, openWithKva); } - private N5Reader openReader(final String uri, final String basePath, final KeyValueAccess kvAcess) { + /** + * Enum to discover and provide {@link KeyValueAccess} for {@link N5Reader}s and {@link N5Writer}s. + * IMPORTANT: If ever new {@link KeyValueAccess} backends are adding, they MUST be re-ordered + * such that the earliest predicates are the most restrictive, and the later predicates + * are the least restrictive. This ensures that when iterating through the values of + * {@link KeyValueAccessBackend} you can test them in order, and stop at the first + * {@link KeyValueAccess} that is generated. + */ + enum KeyValueAccessBackend implements Predicate, BiFunction { + GOOGLE_CLOUD(uri -> { + final String scheme = uri.getScheme(); + final boolean hasScheme = scheme != null; + return hasScheme && GoogleCloudUtils.GS_SCHEME.asPredicate().test(scheme) + || hasScheme && HTTPS_SCHEME.asPredicate().test(scheme) + && uri.getHost() != null && GoogleCloudUtils.GS_HOST.asPredicate().test(uri.getHost()); + }, N5Factory::newGoogleCloudKeyValueAccess), + AWS(uri -> { + final String scheme = uri.getScheme(); + final boolean hasScheme = scheme != null; + return hasScheme && AmazonS3Utils.S3_SCHEME.asPredicate().test(scheme) + || uri.getHost() != null && hasScheme && HTTPS_SCHEME.asPredicate().test(scheme); + }, N5Factory::newAmazonS3KeyValueAccess), + FILE(uri -> { + final String scheme = uri.getScheme(); + final boolean hasScheme = scheme != null; + return !hasScheme || FILE_SCHEME.asPredicate().test(scheme); + }, N5Factory::newFileSystemKeyValueAccess); + + private final Predicate backendTest; + private final BiFunction backendGenerator; + + KeyValueAccessBackend(Predicate test, BiFunction generator) { + + backendTest = test; + backendGenerator = generator; + } + + @Override public KeyValueAccess apply(final URI uri, final N5Factory factory) { - // TODO duplicates logic in openFileBasedN5Reader - if (lastExtension(uri).startsWith(".n5")) { - return new N5KeyValueReader(kvAcess, basePath, gsonBuilder, cacheAttributes); - } else { - try { - return new ZarrKeyValueReader(kvAcess, basePath, gsonBuilder, - zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); - } catch (final N5Exception e) { - return new N5KeyValueReader(kvAcess, basePath, gsonBuilder, cacheAttributes); - } + if (test(uri)) + return backendGenerator.apply(uri, factory); + return null; + } + + @Override public boolean test(URI uri) { + + return backendTest.test(uri); } } - private N5Writer openWriter(final String uri, final String basePath, final KeyValueAccess kvAcess) { + public enum StorageFormat { + ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr(" + FileSystems.getDefault().getSeparator() + ")?$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), + N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5(" + FileSystems.getDefault().getSeparator() + ")?$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), + HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { + final boolean hasHdf5Extension = Pattern.compile("\\.h(df)?5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find(); + return hasHdf5Extension || HDF5Utils.isHDF5(uri.getPath()); + }); - // TODO duplicates logic in openReader(final String uri, final String basePath, final KeyValueAccess kvAcess) - if (lastExtension(uri).startsWith(".n5")) { - return new N5KeyValueWriter(kvAcess, basePath, gsonBuilder, cacheAttributes); - } else { - try { - return new ZarrKeyValueWriter(kvAcess, basePath, gsonBuilder, zarrMapN5DatasetAttributes, - zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes); - } catch (N5Exception e) { - return new N5KeyValueWriter(kvAcess, basePath, gsonBuilder, cacheAttributes); + static final Pattern STORAGE_SCHEME_PATTERN = Pattern.compile("^(\\s*(?(n5|h(df)?5|zarr)):(//)?)?(?.*)$", Pattern.CASE_INSENSITIVE); + private final static String STORAGE_SCHEME_GROUP = "storageScheme"; + private final static String URI_GROUP = "uri"; + + final Pattern schemePattern; + private final Predicate uriTest; + + StorageFormat(final Pattern schemePattern, final Predicate test) { + + this.schemePattern = schemePattern; + this.uriTest = test; + } + + public static StorageFormat guessStorageFromUri(URI uri) { + + for (StorageFormat format : StorageFormat.values()) { + if (format.uriTest.test(uri)) + return format; } + return null; + } + + public static Pair parseUri(String uri) throws URISyntaxException { + + final Pair storageFromScheme = getStorageFromNestedScheme(uri); + final URI asUri = N5URI.encodeAsUri(storageFromScheme.getB()); + if (storageFromScheme.getA() != null) + return new ValuePair<>(storageFromScheme.getA(), asUri); + else + return new ValuePair<>(guessStorageFromUri(asUri), asUri); + + } + + public static Pair getStorageFromNestedScheme(String uri) { + + final Matcher storageSchemeMatcher = StorageFormat.STORAGE_SCHEME_PATTERN.matcher(uri); + storageSchemeMatcher.matches(); + final String storageFormatScheme = storageSchemeMatcher.group(STORAGE_SCHEME_GROUP); + final String uriGroup = storageSchemeMatcher.group(URI_GROUP); + if (storageFormatScheme != null) { + for (StorageFormat format : StorageFormat.values()) { + if (format.schemePattern.asPredicate().test(storageFormatScheme)) + return new ValuePair<>(format, uriGroup); + } + } + return new ValuePair<>(null, uriGroup); } } + /** + * Creates an N5 writer for the specified container URI with default N5Factory configuration. + * + * @param containerUri location of the N5 container + * @return an N5Writer instance for the given containerURI + */ + public static N5Writer createWriter(String containerUri) { + + return FACTORY.openWriter(containerUri); + } + + /** + * Creates an N5Reader at containerURI with default N5Factory configuration. + * + * @param containerUri location of the N5 container + * @return an N5Reader instance for the given containerURI + */ + public static N5Reader createReader(String containerUri) { + + return FACTORY.openReader(containerUri); + } } \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java new file mode 100644 index 0000000..2d80d00 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java @@ -0,0 +1,9 @@ +package org.janelia.saalfeldlab.n5.universe; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({N5StorageTests.N5AmazonS3BackendTest.class, N5StorageTests.N5GoogleCloudBackendTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class, ZarrStorageTests.ZarrGoogleCloudBackendTest.class}) +public class BackendStorageFormatTests { +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java new file mode 100644 index 0000000..84dc7f5 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java @@ -0,0 +1,307 @@ +package org.janelia.saalfeldlab.n5.universe; + +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +public class N5FactoryTests { + + @Test + public void testStorageFormatGuesses() throws URISyntaxException { + + final URI noExt = new URI("file:///tmp/a"); + final URI h5Ext = new URI("file:///tmp/a.h5"); + final URI hdf5Ext = new URI("file:///tmp/a.hdf5"); + final URI n5Ext = new URI("file:///tmp/a.n5"); + final URI n5ExtSlash = new URI("file:///tmp/a.n5/"); + final URI zarrExt = new URI("file:///tmp/a.zarr"); + final URI zarrExtSlash = new URI("file:///tmp/a.zarr/"); + final URI unknownExt = new URI("file:///tmp/a.abc"); + + assertNull("no extension null", N5Factory.StorageFormat.guessStorageFromUri(noExt)); + + /** + * h5 tests fail now because these test whether the file exists. It + * should not do that, if, for example, we're making a writer. + */ + assertEquals("h5 extension == h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(h5Ext)); + assertNotEquals("h5 extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(h5Ext)); + assertNotEquals("h5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(h5Ext)); + + assertEquals("hdf5 extension == h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(hdf5Ext)); + assertNotEquals("hdf5 extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(hdf5Ext)); + assertNotEquals("hdf5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(hdf5Ext)); + + assertNotEquals("n5 extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + assertEquals("n5 extension == n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + assertNotEquals("n5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + + assertNotEquals("n5 extension slash != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(n5ExtSlash)); + assertEquals("n5 extension slash == n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(n5ExtSlash)); + assertNotEquals("n5 extension slash != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(n5ExtSlash)); + + assertNotEquals("zarr extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + assertNotEquals("zarr extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + assertEquals("zarr extension == zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + + assertNotEquals("zarr extension slash != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(zarrExtSlash)); + assertNotEquals("zarr extension slash != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(zarrExtSlash)); + assertEquals("zarr extension slash == zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(zarrExtSlash)); + + assertNull("unknown extension != h5", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNull("unknown extension != n5", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNull("unknown extension != zarr", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + } + + @Test + public void testWriterTypeByExtension() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] ext = new String[]{".h5", ".hdf5", ".n5", ".n5", ".zarr", ".zarr"}; + + // necessary because new File() removes trailing separator + final String separator = FileSystems.getDefault().getSeparator(); + final String[] trailing = new String[]{"", "", "", separator, "", separator}; + + final Class[] readerTypes = new Class[]{ + N5HDF5Writer.class, + N5HDF5Writer.class, + N5KeyValueWriter.class, + N5KeyValueWriter.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class + }; + + for (int i = 0; i < ext.length; i++) { + final File tmpWithExt = new File(tmp, "foo" + i + ext[i]); + final String extUri = new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString() + trailing[i]; + checkWriterTypeFromFactory( factory, extUri, readerTypes[i], " with extension"); + } + + } catch (IOException e) { + e.printStackTrace(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + + } + + @Test + public void testWriterTypeByPrefix() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] prefix = new String[]{"h5", "hdf5", "n5", "zarr"}; + final Class[] readerTypes = new Class[]{ + N5HDF5Writer.class, + N5HDF5Writer.class, + N5KeyValueWriter.class, + ZarrKeyValueWriter.class + }; + + for (int i = 0; i < prefix.length; i++) { + final File tmpNoExt = new File(tmp, "foo"+i); + + final String prefixUri = prefix[i] + ":" + new URI("file", null, tmpNoExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUri, readerTypes[i], " with prefix"); + + final String prefixUriSlashes = prefix[i] + "://" + new URI("file", null, tmpNoExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUriSlashes, readerTypes[i], " with prefix slashes"); + } + + // ensure that prefix is preferred to extensions + final String[] extensions = new String[]{".h5", ".hdf5", ".n5", ".zarr"}; + + for (int i = 0; i < prefix.length; i++) { + for (int j = 0; j < extensions.length; j++) { + + final File tmpWithExt = new File(tmp, "foo"+i+extensions[j]); + + final String prefixUri = prefix[i] + ":" + new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUri, readerTypes[i], " with prefix"); + + final String prefixUriSlashes = prefix[i] + "://" + new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUriSlashes, readerTypes[i], " with prefix slashes"); + } + } + + } catch (IOException e) { + e.printStackTrace(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + } + + @Test + public void testDefaultForAmbiguousWriters() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] paths = new String[]{ + "a_non_hdf5_file", + "an_hdf5_file", + "a_zarr_directory", + "an_n5_directory", + "an_empty_directory", + "a_non_existent_path" + }; + + final Path tmpPath = tmp.toPath(); + + final File tmpNonHdf5File = tmpPath.resolve(paths[0]).toFile(); + tmpNonHdf5File.createNewFile(); + tmpNonHdf5File.deleteOnExit(); + + factory.openWriter(StorageFormat.HDF5, tmpPath.resolve(paths[1]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.ZARR, tmpPath.resolve(paths[2]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.N5, tmpPath.resolve(paths[3]).toFile().getCanonicalPath()).close(); + + final File tmpEmptyDir = tmpPath.resolve(paths[4]).toFile(); + tmpEmptyDir.mkdirs(); + tmpEmptyDir.deleteOnExit(); + + + + final Class[] writerTypes = new Class[]{ + null, + N5HDF5Writer.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class + }; + + for (int i = 0; i < paths.length; i++) { + + final String prefixUri = new URI("file", null, tmpPath.resolve(paths[i]).normalize().toString(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUri, writerTypes[i], " with path " + paths[i]); + } + + } catch (IOException | URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + } + + + @Test + public void testDefaultForAmbiguousReaders() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] paths = new String[]{ + "a_non_hdf5_file", + "an_hdf5_file", + "a_zarr_directory", + "an_n5_directory", + "an_empty_directory", + "a_non_existent_path" + }; + + final Path tmpPath = tmp.toPath(); + + final File tmpNonHdf5File = tmpPath.resolve(paths[0]).toFile(); + tmpNonHdf5File.createNewFile(); + tmpNonHdf5File.deleteOnExit(); + + factory.openWriter(StorageFormat.HDF5, tmpPath.resolve(paths[1]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.ZARR, tmpPath.resolve(paths[2]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.N5, tmpPath.resolve(paths[3]).toFile().getCanonicalPath()).close(); + + final File tmpEmptyDir = tmpPath.resolve(paths[4]).toFile(); + tmpEmptyDir.mkdirs(); + tmpEmptyDir.deleteOnExit(); + + + + final Class[] readerTypes = new Class[]{ + null, + N5HDF5Reader.class, + ZarrKeyValueReader.class, + N5KeyValueReader.class, + N5KeyValueReader.class, + null + }; + + for (int i = 0; i < paths.length; i++) { + + final String prefixUri = new URI("file", null, tmpPath.resolve(paths[i]).normalize().toString(), null).toString(); + checkReaderTypeFromFactory( factory, prefixUri, readerTypes[i], " with path " + paths[i]); + } + + } catch (IOException | URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + } + + private void checkWriterTypeFromFactory(N5Factory factory, String uri, Class expected, String messageSuffix) { + + if (expected == null) { + assertThrows("Should throw exception for " + uri, N5Exception.class, () -> factory.openWriter(uri)); + return; + } + + final N5Writer n5 = factory.openWriter(uri); + assertNotNull( "null n5 for " + uri, n5); + assertEquals(expected.getName() + messageSuffix, expected, n5.getClass()); + n5.remove(); + } + + private void checkReaderTypeFromFactory(N5Factory factory, String uri, Class expected, String messageSuffix) { + + if (expected == null) { + assertThrows(N5Exception.class, () -> factory.openReader(uri)); + return; + } + + final N5Reader n5 = factory.openReader(uri); + assertNotNull( "null n5 for " + uri, n5); + assertEquals(expected.getName() + messageSuffix, expected, n5.getClass()); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java new file mode 100644 index 0000000..da0bb5a --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -0,0 +1,266 @@ +package org.janelia.saalfeldlab.n5.universe; + +import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.AbstractN5Test; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; +import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; +import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; +import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestWatcher; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; + +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; +import static org.junit.Assert.assertTrue; + +@RunWith(Suite.class) +@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5GoogleCloudMockTest.class}) +public class N5StorageTests { + + public static abstract class N5FactoryTest extends AbstractN5Test implements StorageSchemeWrappedN5Test { + + protected N5Factory factory; + + protected final ArrayList tempWriters = new ArrayList<>(); + + public N5FactoryTest() { + + this.factory = getFactory(); + } + + @Override abstract protected String tempN5Location(); + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory(); + } + return factory; + } + + @Override public N5Factory.StorageFormat getStorageFormat() { + + return N5Factory.StorageFormat.N5; + } + + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return createN5Writer(location); + } + + @Override protected N5Reader createN5Reader(String location) { + + return getReader(location); + } + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return createN5Writer(location); + } + + @Override protected N5Writer createN5Writer(String location) { + + return getWriter(location); + } + + @After + public void removeTempWriter() { + + synchronized (tempWriters) { + + for (N5Writer tempWriter : tempWriters) { + tempWriter.remove(); + } + tempWriters.clear(); + } + } + } + + public static class N5FileSystemTest extends N5FactoryTest { + + @Override public Class getBackendTargetClass() { + + return FileSystemKeyValueAccess.class; + } + + @Override protected String tempN5Location() { + + try { + return Files.createTempDirectory("n5-test").toUri().getPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static abstract class N5AmazonS3FactoryTest extends N5FactoryTest { + + public static AmazonS3 s3 = null; + + final static String testBucket = tempBucketName(); + + @Override public Class getBackendTargetClass() { + + return AmazonS3KeyValueAccess.class; + } + + @AfterClass + public static void removeTestBucket() { + + if (s3 != null && s3.doesBucketExistV2(testBucket)) + N5Factory.createWriter("s3://" + testBucket).remove(); + } + + @Override public N5Factory getFactory() { + + if (factory != null) + return factory; + factory = new N5Factory() { + + @Override AmazonS3 createS3(String uri) { + + if (N5AmazonS3FactoryTest.s3 == null) + N5AmazonS3FactoryTest.s3 = super.createS3(uri); + return s3; + } + }; + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("s3", testBucket, tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class N5AmazonS3MockTest extends N5AmazonS3FactoryTest { + public N5AmazonS3MockTest() { + + N5AmazonS3FactoryTest.s3 = MockS3Factory.getOrCreateS3(); + } + + @Override protected String tempN5Location() { + + try { + return new URI("http", "localhost:8001", "/" + testBucket + tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { + + @BeforeClass + public static void ensureBucketExists() { + + final N5Writer writer = N5Factory.createWriter("s3://" + testBucket + "/" + tempContainerPath()); + assertTrue(writer.exists("")); + writer.remove(); + } + + @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); + + public N5AmazonS3BackendTest() { + + N5AmazonS3FactoryTest.s3 = null; + } + } + + public static abstract class N5GoogleCloudFactoryTest extends N5FactoryTest { + + protected static String testBucket = N5GoogleCloudStorageTests.tempBucketName(); + protected static Storage storage = null; + + @Override public Class getBackendTargetClass() { + + return GoogleCloudStorageKeyValueAccess.class; + } + + @AfterClass + public static void removeTestBucket() { + + final Bucket bucket = storage.get(testBucket); + if (bucket != null && bucket.exists()) { + storage.delete(testBucket); + } + } + + @Override public N5Factory getFactory() { + + if (factory != null) + return factory; + factory = new N5Factory() { + + @Override Storage createGoogleCloudStorage() { + + return storage; + } + }; + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("gs", testBucket, tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class N5GoogleCloudMockTest extends N5GoogleCloudFactoryTest { + public N5GoogleCloudMockTest() { + + N5GoogleCloudFactoryTest.storage = MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + } + + public static class N5GoogleCloudBackendTest extends N5GoogleCloudFactoryTest { + + @BeforeClass + public static void ensureBucketExists() { + + final N5Writer writer = N5Factory.createWriter("gs://" + testBucket + "/" + tempContainerPath()); + assertTrue(writer.exists("")); + writer.remove(); + } + + public N5GoogleCloudBackendTest() { + + N5GoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); + } + + @Override public void testVersion() throws NumberFormatException, IOException, URISyntaxException { + + super.testVersion(); + } + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java new file mode 100644 index 0000000..0336647 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java @@ -0,0 +1,76 @@ +package org.janelia.saalfeldlab.n5.universe; + +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; + +import static org.junit.Assert.assertTrue; + +public interface StorageSchemeWrappedN5Test { + + N5Factory getFactory(); + + N5Factory.StorageFormat getStorageFormat(); + + Class getBackendTargetClass(); + + default N5Writer getWriter(String uri) { + + final String uriWithStorageScheme = prependStorageScheme(uri); + final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)getFactory().openWriter(uriWithStorageScheme); + switch (getStorageFormat()){ + case ZARR: + assertTrue(writer instanceof ZarrKeyValueWriter); + break; + case N5: + assertTrue(writer instanceof N5KeyValueWriter); + break; + case HDF5: + assertTrue(writer instanceof N5HDF5Writer); + break; + } + assertTrue(getBackendTargetClass().isAssignableFrom(writer.getKeyValueAccess().getClass())); + return writer; + } + + default N5Reader getReader(String uri) { + + final String uriWithStorageScheme = prependStorageScheme(uri); + final GsonKeyValueN5Reader reader = (GsonKeyValueN5Reader)getFactory().openReader(uriWithStorageScheme); + switch (getStorageFormat()){ + case ZARR: + assertTrue(reader instanceof ZarrKeyValueReader); + break; + case N5: + assertTrue(reader instanceof N5KeyValueReader); + break; + case HDF5: + assertTrue(reader instanceof N5HDF5Reader); + break; + } + assertTrue(getBackendTargetClass().isAssignableFrom(reader.getKeyValueAccess().getClass())); + return reader; + } + + default String prependStorageScheme(String uri) { + + final String schemePattern = getStorageFormat().schemePattern.pattern(); + final String doesntHaveStorageScheme = new StringBuilder("^(?:(?!(?i)") + .append(schemePattern) + .append(":))(?.*)") + .toString(); + + final String prependStorageScheme = new StringBuilder(getStorageFormat().toString().toLowerCase()) + .append(":${remaining}") + .toString(); + + return uri.replaceFirst(doesntHaveStorageScheme, prependStorageScheme); + } +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java new file mode 100644 index 0000000..a8a6e1b --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -0,0 +1,271 @@ +package org.janelia.saalfeldlab.n5.universe; + +import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; +import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; +import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; +import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; +import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.rules.TestWatcher; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; + +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; +import static org.junit.Assert.assertTrue; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3MockTest.class, ZarrStorageTests.ZarrGoogleCloudMockTest.class}) +public class ZarrStorageTests { + + public static abstract class ZarrFactoryTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { + + protected N5Factory factory; + + public ZarrFactoryTest() { + + this.factory = getFactory(); + } + + @Override abstract protected String tempN5Location(); + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory(); + } + return factory; + } + + @Override public N5Factory.StorageFormat getStorageFormat() { + + return N5Factory.StorageFormat.ZARR; + } + + @Override protected N5Writer createN5Writer() { + + return getWriter(tempN5Location()); + } + + @Override protected N5Writer createTempN5Writer(String location, GsonBuilder gsonBuilder, String dimensionSeparator, boolean mapN5DatasetAttributes) { + + factory.gsonBuilder(gsonBuilder); + factory.zarrDimensionSeparator(dimensionSeparator); + factory.zarrMapN5Attributes(mapN5DatasetAttributes); + final N5Writer writer = getWriter(location); + tempWriters.add(writer); + return writer; + } + + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getReader(location); + } + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getWriter(location); + } + + @Override protected N5Writer createN5Writer(String location) { + + return getWriter(location); + } + + @Override protected N5Reader createN5Reader(String location) { + + return getReader(location); + } + + @Ignore + @Override public void testReadZarrPython() { + + } + + @Ignore + @Override public void testReadZarrNestedPython() { + + } + } + + public static class ZarrFileSystemTest extends ZarrFactoryTest { + + @Override public Class getBackendTargetClass() { + + return FileSystemKeyValueAccess.class; + } + + @Override protected String tempN5Location() { + + try { + return Files.createTempDirectory("zarr-test").toUri().getPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static abstract class ZarrAmazonS3FactoryTest extends ZarrFactoryTest { + + public static AmazonS3 s3 = null; + + final static String testBucket = tempBucketName(); + + @Override public Class getBackendTargetClass() { + + return AmazonS3KeyValueAccess.class; + } + + @AfterClass + public static void removeTestBucket() { + + if (s3 != null && s3.doesBucketExistV2(testBucket)) + N5Factory.createWriter("s3://" + testBucket).remove(); + } + + @Override public N5Factory getFactory() { + + if (factory != null) + return factory; + factory = new N5Factory() { + + @Override AmazonS3 createS3(String uri) { + + if (ZarrAmazonS3FactoryTest.s3 == null) + ZarrAmazonS3FactoryTest.s3 = super.createS3(uri); + return s3; + } + }; + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("s3", testBucket, tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class ZarrAmazonS3MockTest extends ZarrAmazonS3FactoryTest { + public ZarrAmazonS3MockTest() { + + ZarrAmazonS3FactoryTest.s3 = MockS3Factory.getOrCreateS3(); + } + + @Override protected String tempN5Location() { + + try { + return new URI("http", "localhost:8001", "/" + testBucket + tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class ZarrAmazonS3BackendTest extends ZarrAmazonS3FactoryTest { + + @BeforeClass + public static void ensureBucketExists() { + + final N5Writer writer = N5Factory.createWriter("s3://" + testBucket + "/" + tempContainerPath()); + assertTrue(writer.exists("")); + writer.remove(); + } + + @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); + + public ZarrAmazonS3BackendTest() { + + ZarrAmazonS3FactoryTest.s3 = null; + } + } + + public static abstract class ZarrGoogleCloudFactoryTest extends ZarrFactoryTest { + + protected static String testBucket = N5GoogleCloudStorageTests.tempBucketName(); + protected static Storage storage = null; + + @Override public Class getBackendTargetClass() { + + return GoogleCloudStorageKeyValueAccess.class; + } + + @AfterClass + public static void removeTestBucket() { + + final Bucket bucket = storage.get(testBucket); + if (bucket != null && bucket.exists()) { + storage.delete(testBucket); + } + } + + @Override public N5Factory getFactory() { + + if (factory != null) + return factory; + factory = new N5Factory() { + + @Override Storage createGoogleCloudStorage() { + + return storage; + } + }; + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("gs", testBucket, tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class ZarrGoogleCloudMockTest extends ZarrGoogleCloudFactoryTest { + public ZarrGoogleCloudMockTest() { + + ZarrGoogleCloudFactoryTest.storage = MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + } + + public static class ZarrGoogleCloudBackendTest extends ZarrGoogleCloudFactoryTest { + + @BeforeClass + public static void ensureBucketExists() { + + final N5Writer writer = N5Factory.createWriter("gs://" + testBucket + "/" + tempContainerPath()); + assertTrue(writer.exists("")); + writer.remove(); + } + + public ZarrGoogleCloudBackendTest() { + + ZarrGoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); + } + } +}