From d4b6ffdf40c5019f9e736b5260341ea57b4a54eb Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 8 Jan 2024 16:32:34 -0500 Subject: [PATCH 01/31] Bump to next development cycle Signed-off-by: Stephan Saalfeld --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 68b4a7b..9baf555 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5-universe - 1.2.1-SNAPSHOT + 1.3.1-SNAPSHOT N5-Universe Utilities spanning all of the N5 repositories From 747143afb52053f7a3503f44bdf5e7297a7c0eee Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 31 Jan 2024 10:33:38 -0500 Subject: [PATCH 02/31] Bump to next development cycle Signed-off-by: John Bogovic --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9baf555..cc3f6df 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5-universe - 1.3.1-SNAPSHOT + 1.3.2-SNAPSHOT N5-Universe Utilities spanning all of the N5 repositories From 34caf479fe82b3c745e58a12652509ac2e9186ae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 6 Feb 2024 14:43:36 -0500 Subject: [PATCH 03/31] chore: bump n5-artifact versions * to match pom-scijava PR --- pom.xml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index cc3f6df..a354c6e 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,12 @@ sign,deploy-to-scijava + 3.1.1 + 2.1.0 + 7.0.0 + 4.0.2 + 1.2.0 + 1.0.0-preview.20191208 1.4.1 @@ -152,14 +158,6 @@ ${alphanumeric-comparator.version} compile - - - - - - - - From 97d0aa7fb107a9800320a4fa8d4bbc737e9dcdce Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Feb 2024 16:47:46 -0500 Subject: [PATCH 04/31] feat: separate storage format detection logic from KeyValueAccess logic --- .../saalfeldlab/n5/universe/N5Factory.java | 426 +++++++++++++++--- 1 file changed, 365 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 4d99be2..183550a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -32,17 +32,24 @@ import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; import java.util.Iterator; import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import net.imglib2.util.Pair; +import net.imglib2.util.ValuePair; import org.janelia.saalfeldlab.googlecloud.GoogleCloudResourceManagerClient; import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageClient; import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5FSReader; @@ -79,6 +86,8 @@ import com.google.cloud.storage.Storage; import com.google.gson.GsonBuilder; +import javax.annotation.Nullable; + /** * Factory for various N5 readers and writers. Implementation specific * parameters can be provided to the factory instance and will be used when such @@ -93,7 +102,7 @@ public class N5Factory implements Serializable { private static final long serialVersionUID = -6823715427289454617L; - private static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)"); + private static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)", Pattern.CASE_INSENSITIVE); private static byte[] HDF5_SIG = {(byte)137, 72, 68, 70, 13, 10, 26, 10}; private int[] hdf5DefaultBlockSize = {64, 64, 64, 1, 1}; @@ -113,6 +122,7 @@ public class N5Factory implements Serializable { private boolean s3Anonymous = true; private boolean s3RetryWithCredentials = false; private String s3Endpoint; + private boolean createS3Bucket = false; public N5Factory hdf5DefaultBlockSize(final int... blockSize) { @@ -200,7 +210,7 @@ public N5Factory s3Region(final String s3Region) { private static boolean isHDF5Writer(final String path) { - if (path.matches("(?i).*\\.(h5|hdf|hdf5)")) + if (path.matches("(?i).*\\.(h(df)?5)")) return true; else return false; @@ -212,20 +222,24 @@ private static boolean isHDF5Reader(final String path) throws N5IOException { /* optimistic */ if (isHDF5Writer(path)) return true; - else { - try (final FileInputStream in = new FileInputStream(new File(path))) { - final byte[] sig = new byte[8]; - in.read(sig); - return Arrays.equals(sig, HDF5_SIG); - } catch (final IOException e) { - throw new N5Exception.N5IOException(e); - } - } + else + return isHDF5(path); } return false; } - private AmazonS3 createS3(final String uri) { + private static boolean isHDF5(String path) { + + try (final FileInputStream in = new FileInputStream(new File(path))) { + final byte[] sig = new byte[8]; + in.read(sig); + return Arrays.equals(sig, HDF5_SIG); + } catch (final IOException e) { + throw new N5Exception.N5IOException(e); + } + } + + AmazonS3 createS3(final String uri) { try { return createS3(new AmazonS3URI(uri)); @@ -235,7 +249,8 @@ private AmazonS3 createS3(final String uri) { final URI buri = new URI(uri); final URI endpointUrl = new URI(buri.getScheme(), buri.getHost(), null, null); return createS3(getS3Credentials(), new EndpointConfiguration(endpointUrl.toString(), null), null, getS3Bucket(uri)); - } catch (final URISyntaxException e1) {} + } catch (final URISyntaxException e1) { + } } throw new N5Exception("Could not create s3 client from uri: " + uri); } @@ -244,7 +259,7 @@ private AmazonS3 createS3( final AWSCredentialsProvider credentialsProvider, final EndpointConfiguration endpointConfiguration, final Regions region, - final String bucketName ) { + final String bucketName) { final boolean isAmazon = endpointConfiguration == null || AWS_ENDPOINT_PATTERN.matcher(endpointConfiguration.getServiceEndpoint()).find(); final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); @@ -264,20 +279,20 @@ else if (region != null) AmazonS3 s3 = builder.build(); // if we used anonymous credentials and the factory requests a retry with credentials: - if( s3RetryWithCredentials && areAnonymous(credentialsProvider)) { + if (s3RetryWithCredentials && areAnonymous(credentialsProvider)) { // I initially tried checking whether the bucket exists, but // that, apparently, returns even when the client does not have access if (!canListBucket(s3, bucketName)) { // bucket not detected with anonymous credentials, try detecting credentials // and return it even if it can't detect the bucket, since there's nothing else to do - s3 = createS3(new DefaultAWSCredentialsProviderChain(), endpointConfiguration, region, null ); + s3 = createS3(new DefaultAWSCredentialsProviderChain(), endpointConfiguration, region, null); } } return s3; } - private boolean canListBucket( final AmazonS3 s3, final String bucket) { + private boolean canListBucket(final AmazonS3 s3, final String bucket) { final ListObjectsV2Request request = new ListObjectsV2Request(); request.setBucketName(bucket); @@ -330,7 +345,7 @@ private Regions getS3Region(final AmazonS3URI uri) { final Regions region = Optional.ofNullable(uri.getRegion()).map(Regions::fromName) // get the region from the uri .orElse(Optional.ofNullable(s3Region).map(Regions::fromName) // next use whatever is passed in - .orElse(null)); // fallback to null (amazon picks a default) + .orElse(null)); // fallback to null (amazon picks a default) return region; } @@ -353,11 +368,12 @@ private AmazonS3 createS3(final AmazonS3URI uri) { return createS3(getS3Credentials(), endpointConfiguration, getS3Region(uri), uri.getBucket()); } - private String getS3Bucket(final String uri) { + String getS3Bucket(final String uri) { try { return new AmazonS3URI(uri).getBucket(); - } catch (final IllegalArgumentException e) {} + } catch (final IllegalArgumentException e) { + } try { // parse bucket manually when AmazonS3URI can't final String path = new URI(uri).getPath().replaceFirst("^/", ""); @@ -373,20 +389,21 @@ private String getS3Key(final String uri) { // if key is null, return the empty string final String key = new AmazonS3URI(uri).getKey(); return key == null ? "" : key; - } catch (final IllegalArgumentException e) {} + } catch (final IllegalArgumentException e) { + } try { // parse key manually when AmazonS3URI can't final String path = new URI(uri).getPath().replaceFirst("^/", ""); return path.substring(path.indexOf('/') + 1); - } catch (final URISyntaxException e) {} + } catch (final URISyntaxException e) { + } return null; } /** * Open an {@link N5Reader} for N5 filesystem. * - * @param path - * path to the n5 root folder + * @param path path to the n5 root folder * @return the N5FsReader */ public N5FSReader openFSReader(final String path) { @@ -396,12 +413,11 @@ public N5FSReader openFSReader(final String path) { /** * Open an {@link N5Reader} for Zarr. - * + *

* For more options of the Zarr backend study the {@link N5ZarrReader} * constructors.} * - * @param path - * path to the zarr directory + * @param path path to the zarr directory * @return the N5ZarrReader */ public N5ZarrReader openZarrReader(final String path) { @@ -412,12 +428,11 @@ public N5ZarrReader openZarrReader(final String path) { /** * Open an {@link N5Reader} for HDF5. Close the reader when you do not need * it any more. - * + *

* For more options of the HDF5 backend study the {@link N5HDF5Reader} * constructors. * - * @param path - * path to the hdf5 file + * @param path path to the hdf5 file * @return the N5HDF5Reader */ public N5HDF5Reader openHDF5Reader(final String path) { @@ -428,11 +443,9 @@ public N5HDF5Reader openHDF5Reader(final String path) { /** * Open an {@link N5Reader} for Google Cloud. * - * @param uri - * uri to the google cloud object + * @param uri uri to the google cloud object * @return the N5GoogleCloudStorageReader - * @throws URISyntaxException - * if uri is malformed + * @throws URISyntaxException if uri is malformed */ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxException { @@ -453,11 +466,9 @@ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxExceptio /** * Open an {@link N5Reader} for AWS S3. * - * @param uri - * uri to the amazon s3 object + * @param uri uri to the amazon s3 object * @return the N5Reader - * @throws URISyntaxException - * if uri is malformed + * @throws URISyntaxException if uri is malformed */ public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException { @@ -476,8 +487,7 @@ public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException { /** * Open an {@link N5Writer} for N5 filesystem. * - * @param path - * path to the n5 directory + * @param path path to the n5 directory * @return the N5FSWriter */ public N5FSWriter openFSWriter(final String path) { @@ -487,12 +497,11 @@ public N5FSWriter openFSWriter(final String path) { /** * Open an {@link N5Writer} for Zarr. - * + *

* For more options of the Zarr backend study the {@link N5ZarrWriter} * constructors. * - * @param path - * path to the zarr directory + * @param path path to the zarr directory * @return the N5ZarrWriter */ public N5ZarrWriter openZarrWriter(final String path) { @@ -503,12 +512,11 @@ public N5ZarrWriter openZarrWriter(final String path) { /** * Open an {@link N5Writer} for HDF5. Don't forget to close the writer after * writing to close the file and make it available to other processes. - * + *

* For more options of the HDF5 backend study the {@link N5HDF5Writer} * constructors. * - * @param path - * path to the hdf5 file + * @param path path to the hdf5 file * @return the N5HDF5Writer */ public N5HDF5Writer openHDF5Writer(final String path) { @@ -519,11 +527,9 @@ public N5HDF5Writer openHDF5Writer(final String path) { /** * Open an {@link N5Writer} for Google Cloud. * - * @param uri - * uri to the google cloud object + * @param uri uri to the google cloud object * @return the N5GoogleCloudStorageWriter - * @throws URISyntaxException - * if uri is malformed + * @throws URISyntaxException if uri is malformed */ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxException { @@ -552,11 +558,9 @@ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxExceptio /** * Open an {@link N5Writer} for AWS S3. * - * @param uri - * uri to the s3 object + * @param uri uri to the s3 object * @return the N5Writer - * @throws URISyntaxException - * if the URI is malformed + * @throws URISyntaxException if the URI is malformed */ public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException { @@ -574,8 +578,7 @@ public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException { /** * Open an {@link N5Reader} based on some educated guessing from the url. * - * @param uri - * the location of the root location of the store + * @param uri the location of the root location of the store * @return the N5Reader */ public N5Reader openReader(final String uri) { @@ -602,7 +605,9 @@ else if (encodedUri.getHost() != null && scheme.equals("https") || scheme.equals else //if (encodedUri.getHost().matches(".*s3.*")) //< This is too fragile for what people in the wild are doing with their S3 instances, for now catch all return openAWSS3Reader(uri); } - } catch (final URISyntaxException ignored) {} + } catch (final URISyntaxException ignored) { + } + // return null; return openFileBasedN5Reader(uri); } @@ -620,8 +625,7 @@ else if (lastExtension(url).startsWith(".zarr")) /** * Open an {@link N5Writer} based on some educated guessing from the uri. * - * @param uri - * the location of the root location of the store + * @param uri the location of the root location of the store * @return the N5Writer */ public N5Writer openWriter(final String uri) { @@ -629,7 +633,8 @@ public N5Writer openWriter(final String uri) { try { final URI encodedUri = N5URI.encodeAsUri(uri); final String scheme = encodedUri.getScheme(); - if (scheme == null) ; + if (scheme == null) + ; else if (scheme.equals("file")) return openFileBasedN5Writer(encodedUri.getPath()); else if (scheme.equals("s3")) @@ -643,7 +648,8 @@ else if (encodedUri.getHost().matches(".*cloud\\.google\\.com") || encodedUri.getHost().matches(".*storage\\.googleapis\\.com")) return openGoogleCloudWriter(uri); } - } catch (final URISyntaxException e) {} + } catch (final URISyntaxException e) { + } return openFileBasedN5Writer(uri); } @@ -666,4 +672,302 @@ private static String lastExtension(final String path) { return ""; } + public N5Reader getReader(final String uri) { + + try { + final Pair storageAndUri = StorageFormat.parseUri(uri); + final StorageFormat format = storageAndUri.getA(); + final URI asUri = storageAndUri.getB(); + if (format != null) + return format.openReader(asUri, this); + + final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(asUri, this); + if (access == null) { + throw new N5Exception("Cannot get KeyValueAccess at " + asUri); + } + final String containerPath; + if (access instanceof AmazonS3KeyValueAccess) { + containerPath = getS3Key(asUri.toString()); + } else { + containerPath = asUri.getPath(); + } + + Exception exception = null; + for (StorageFormat storageFormat : StorageFormat.values()) { + if (storageFormat.compareTo(StorageFormat.HDF5) < 0) + break; + try { + return StorageFormat.getReader(storageFormat, access, containerPath, this); + } catch (Exception e) { + exception = e; + } + } + if (exception != null) + throw new N5Exception("Unable to open " + uri + " as N5 Container", exception); + } catch (final URISyntaxException ignored) { + } + return null; + } + + public N5Writer getWriter(final String uri) { + + try { + + final Pair storageAndUri = StorageFormat.parseUri(uri); + final StorageFormat format = storageAndUri.getA(); + final URI asUri = storageAndUri.getB(); + if (format != null) + return format.openWriter(asUri, this); + else { + try { + return openHDF5Writer(uri); + } catch (Exception ignored) { + } + } + final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(asUri, this); + if (access == null) { + throw new N5Exception("Cannot create KeyValueAcccess for URI " + uri); + } + final String containerPath; + if (access instanceof AmazonS3KeyValueAccess) { + containerPath = getS3Key(asUri.toString()); + } else { + containerPath = asUri.getPath(); + } + try { + final N5Writer zarrN5Writer = StorageFormat.getWriter(StorageFormat.ZARR, access, containerPath, this); + if (zarrN5Writer != null) + return zarrN5Writer; + } catch (Exception ignored) { + } + try { + final N5Writer n5Writer = StorageFormat.getWriter(StorageFormat.N5, access, containerPath, this); + if (n5Writer != null) + return n5Writer; + } catch (Exception ignored) { + } + } catch (final URISyntaxException ignored) { + } + return null; + } + + private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { + + final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); + final GoogleCloudStorageClient storageClient = new GoogleCloudStorageClient(); + final Storage storage = storageClient.create(); + return new GoogleCloudStorageKeyValueAccess(storage, googleCloudUri.getBucket(), false); + } + + private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, final N5Factory factory) { + + final String uriString = uri.toString(); + final AmazonS3 s3 = factory.createS3(uriString); + + return new AmazonS3KeyValueAccess(s3, factory.getS3Bucket(uriString), factory.createS3Bucket); + } + + private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { + + return new FileSystemKeyValueAccess(FileSystems.getDefault()); + } + + private final static Pattern GS_SCHEME = Pattern.compile("gs", Pattern.CASE_INSENSITIVE); + private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); + private final static Pattern GS_HOST = Pattern.compile("(cloud\\.google|storage\\.googleapis)\\.com", Pattern.CASE_INSENSITIVE); + private final static Pattern S3_SCHEME = Pattern.compile("s3", Pattern.CASE_INSENSITIVE); + private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); + + /** + * Enum to discover and provide {@link KeyValueAccess} for {@link N5Reader}s and {@link N5Writer}s. + * IMPORTANT: If ever new {@link KeyValueAccess} backends are adding, they MUST be re-ordered + * such that the earliest predicates are the most restrictive, and the later predicates + * are the least restrictive. This ensures that when iterating through the values of + * {@link KeyValueAccessBackend} you can test them in order, and stop at the first + * {@link KeyValueAccess} that is generated. + */ + enum KeyValueAccessBackend implements Predicate, BiFunction { + //TODO Caleb: Move all of the pattern matching, tests, and magic strings to static fields/methods in the respective KVA + GOOGLE_CLOUD(uri -> { + final String scheme = uri.getScheme(); + final boolean hasScheme = scheme != null; + return hasScheme && GS_SCHEME.asPredicate().test(scheme) + || hasScheme && HTTPS_SCHEME.asPredicate().test(scheme) + && uri.getHost() != null && GS_HOST.asPredicate().test(uri.getHost()); + }, N5Factory::newGoogleCloudKeyValueAccess), + AWS(uri -> { + final String scheme = uri.getScheme(); + final boolean hasScheme = scheme != null; + return hasScheme && S3_SCHEME.asPredicate().test(scheme) + || uri.getHost() != null && hasScheme && HTTPS_SCHEME.asPredicate().test(scheme); + }, N5Factory::newAmazonS3KeyValueAccess), + FILE(uri -> { + final String scheme = uri.getScheme(); + final boolean hasScheme = scheme != null; + return !hasScheme || hasScheme && FILE_SCHEME.asPredicate().test(scheme); + }, N5Factory::newFileSystemKeyValueAccess); + + private final Predicate backendTest; + private final BiFunction backendGenerator; + + KeyValueAccessBackend(Predicate test, BiFunction generator) { + + backendTest = test; + backendGenerator = generator; + } + + @Override public KeyValueAccess apply(final URI uri, final N5Factory factory) { + + if (test(uri)) + return backendGenerator.apply(uri, factory); + return null; + } + + /** + * Test the provided {@link URI} to and return the appropriate {@link KeyValueAccess}. + * If no appropriate {@link KeyValueAccess} is found, may be null + * + * @param uri to create a {@link KeyValueAccess} from. + * @return the {@link KeyValueAccess} or null if none are valid + */ + static KeyValueAccess getKeyValueAccess(final URI uri, final N5Factory factory) { + + /*NOTE: The order of these tests is very important, as the predicates for each + * backend take into account reasonable defaults when possible. + * Here we test from most to least restrictive. + * See the Javadoc for more details. */ + for (KeyValueAccessBackend backend : KeyValueAccessBackend.values()) { + final KeyValueAccess kva = backend.apply(uri, factory); + if (kva != null) + return kva; + } + return null; + } + + @Override public boolean test(URI uri) { + + return backendTest.test(uri); + } + } + + enum StorageFormat { + ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), + N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), + HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { + final boolean hasHdf5Extension = Pattern.compile("\\.h(df)5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath()); + return hasHdf5Extension || Paths.get(uri).toFile().exists() && isHDF5(uri.toString()); + }); + + static final Pattern STORAGE_SCHEME_PATTERN = Pattern.compile("^(\\s*(?(n5|h(df)?5|zarr)):(//)?)?(?.*)$", Pattern.CASE_INSENSITIVE); + private final static String STORAGE_SCHEME_GROUP = "storageScheme"; + private final static String URI_GROUP = "uri"; + + final Pattern schemePattern; + private final Predicate uriTest; + + StorageFormat(final Pattern schemePattern, final Predicate test) { + + this.schemePattern = schemePattern; + this.uriTest = test; + } + + private static StorageFormat guessStorageFromUri(URI uri) { + + for (StorageFormat format : StorageFormat.values()) { + if (format.uriTest.test(uri)) + return format; + } + return null; + } + + private static Pair parseUri(String uri) throws URISyntaxException { + + final Pair storageFromScheme = getStorageFromNestedScheme(uri); + final URI asUri = N5URI.encodeAsUri(storageFromScheme.getB()); + if (storageFromScheme.getA() != null) + return new ValuePair<>(storageFromScheme.getA(), asUri); + else + return new ValuePair<>(guessStorageFromUri(asUri), asUri); + + } + + private static Pair getStorageFromNestedScheme(String uri) { + + final Matcher storageSchemeMatcher = StorageFormat.STORAGE_SCHEME_PATTERN.matcher(uri); + storageSchemeMatcher.matches(); + final String storageFormatScheme = storageSchemeMatcher.group(STORAGE_SCHEME_GROUP); + final String uriGroup = storageSchemeMatcher.group(URI_GROUP); + if (storageFormatScheme != null) { + for (StorageFormat format : StorageFormat.values()) { + if (format.schemePattern.asPredicate().test(storageFormatScheme)) + return new ValuePair<>(format, uriGroup); + } + } + return new ValuePair<>(null, uriGroup); + } + + N5Reader openReader(final URI uri, final N5Factory factory) { + + return StorageFormat.getReader(this, uri, factory); + } + + N5Writer openWriter(final URI uri, final N5Factory factory) { + + return StorageFormat.getWriter(this, uri, factory); + } + + private static N5Reader getReader(StorageFormat storage, URI uri, N5Factory factory) { + + final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(uri, factory); + final String containerPath; + /* Any more special cases? google? */ + if (access instanceof AmazonS3KeyValueAccess) { + containerPath = factory.getS3Key(uri.toString()); + } else + containerPath = uri.getPath(); + return StorageFormat.getReader(storage, access, containerPath, factory); + } + + private static N5Writer getWriter(StorageFormat storage, URI uri, N5Factory factory) { + + factory.createS3Bucket = true; + final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(uri, factory); + final String containerPath; + /* Any more special cases? google? */ + if (access instanceof AmazonS3KeyValueAccess) { + containerPath = factory.getS3Key(uri.toString()); + } else + containerPath = uri.getPath(); + final N5Writer writer = StorageFormat.getWriter(storage, access, containerPath, factory); + factory.createS3Bucket = false; + return writer; + } + + private static N5Reader getReader(StorageFormat storage, @Nullable KeyValueAccess access, String containerPath, N5Factory factory) { + + switch (storage) { + case N5: + return new N5KeyValueReader(access, containerPath, factory.gsonBuilder, factory.cacheAttributes); + case ZARR: + return new ZarrKeyValueReader(access, containerPath, factory.gsonBuilder, factory.zarrMapN5DatasetAttributes, factory.zarrMergeAttributes, factory.cacheAttributes); + case HDF5: + return new N5HDF5Reader(containerPath, factory.hdf5OverrideBlockSize, factory.gsonBuilder, factory.hdf5DefaultBlockSize); + } + return null; + } + + private static N5Writer getWriter(StorageFormat storage, @Nullable KeyValueAccess access, String containerPath, N5Factory factory) { + + switch (storage) { + case N5: + return new N5KeyValueWriter(access, containerPath, factory.gsonBuilder, factory.cacheAttributes); + case ZARR: + return new ZarrKeyValueWriter(access, containerPath, factory.gsonBuilder, factory.zarrMapN5DatasetAttributes, factory.zarrMergeAttributes, factory.zarrDimensionSeparator, factory.cacheAttributes); + case HDF5: + return new N5HDF5Writer(containerPath, factory.hdf5OverrideBlockSize, factory.gsonBuilder, factory.hdf5DefaultBlockSize); + } + return null; + } + + } } \ No newline at end of file From b2510ce8418313c1739a07f69b830ffdfdf53fdb Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Feb 2024 16:48:24 -0500 Subject: [PATCH 05/31] feat(test): initial tests with different KVA backends per StorageFormat --- pom.xml | 68 ++++- .../n5/universe/N5FactoryTest.java | 103 ++++++++ .../universe/StorageSchemeWrappedN5Test.java | 40 +++ .../n5/universe/ZarrStorageTests.java | 237 ++++++++++++++++++ 4 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java diff --git a/pom.xml b/pom.xml index a354c6e..b10ed74 100644 --- a/pom.xml +++ b/pom.xml @@ -111,14 +111,17 @@ sign,deploy-to-scijava - 3.1.1 + 3.1.3 2.1.0 7.0.0 4.0.2 - 1.2.0 + 1.2.2-SNAPSHOT 1.0.0-preview.20191208 1.4.1 + + 0.2.5 + 2.2.2 @@ -158,6 +161,67 @@ ${alphanumeric-comparator.version} compile + + + org.janelia.saalfeldlab + n5 + tests + test + + + org.janelia.saalfeldlab + n5-zarr + ${n5-zarr.version} + tests + test + + + org.janelia.saalfeldlab + n5-aws-s3 + ${n5-aws-s3.version} + tests + test + + + org.janelia.saalfeldlab + n5-google-cloud + ${n5-google-cloud.version} + tests + test + + + org.janelia.saalfeldlab + n5-hdf5 + ${n5-hdf5.version} + tests + test + + + com.googlecode.json-simple + json-simple + 1.1.1 + test + + + commons-io + commons-io + ${commons-io.version} + test + + + + + io.findify + s3mock_2.12 + ${s3mock_2.12.version} + test + + + javax.xml.bind + jaxb-api + ${jaxb-api.version} + test + diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java new file mode 100644 index 0000000..d552249 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java @@ -0,0 +1,103 @@ +package org.janelia.saalfeldlab.n5.universe; + +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.AbstractN5Test; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.zarr.N5ZarrReader; +import org.janelia.saalfeldlab.n5.zarr.N5ZarrWriter; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; + +@RunWith(Parameterized.class) +public class N5FactoryTest extends AbstractN5Test { + + final N5Factory factory = new N5Factory(); + + @Parameterized.Parameter(0) + public String testName; + + @Parameterized.Parameter(1) + public String uri; + + @Parameterized.Parameters(name = "{0}") + public static Iterable parameters() { + + return Arrays.asList(new Object[][]{ + {"localPathOnly", "/Users/hulbertc/projects/paintera/test.n5"}, + {"n5StorageSchemeNoNesting", "n5:/Users/hulbertc/projects/paintera/test.n5"}, + {"n5StorageSchemeNoNestingDoubleSlash", "n5:///Users/hulbertc/projects/paintera/test.n5"}, + {"n5StorageSchemeAndFile", "n5:file:///Users/hulbertc/projects/paintera/test.n5"}, + {"n5StorageSchemeAndFileDoubleSlash", "n5://file:///Users/hulbertc/projects/paintera/test.n5"}, + {"n5StorageSchemeAndFileDoubleNoExtension", "n5:file:///Users/hulbertc/projects/paintera/test"}, + {"n5StorageSchemeAndFileDoubleMismatchExtension", "n5:file:///Users/hulbertc/projects/paintera/test.zarr"} + }); + } + + + @Override + protected String tempN5Location() { + + try { + return Files.createTempDirectory("n5-factory-test").toUri().toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + protected static String tmpPathName(final String prefix) { + + try { + final File tmpFile = Files.createTempDirectory(prefix).toFile(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected N5Writer createN5Writer() throws IOException { + + final String testDirPath = tmpPathName("n5-factory-test-"); + return factory.openWriter(testDirPath); + } + + @Override + protected N5Writer createN5Writer(final String location, final GsonBuilder gsonBuilder) throws IOException { + + return createN5Writer(location, gsonBuilder, ".", true); + } + + protected N5ZarrWriter createN5Writer(final String location, final String dimensionSeparator) throws IOException { + + return createN5Writer(location, new GsonBuilder(), dimensionSeparator, true); + } + + @Override + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException { + + return new N5ZarrReader(location, gson); + } + + protected N5ZarrWriter createN5Writer( + final String location, + final GsonBuilder gsonBuilder, + final String dimensionSeparator, + final boolean mapN5DatasetAttributes) throws IOException { + + return new N5ZarrWriter(location, gsonBuilder, dimensionSeparator, mapN5DatasetAttributes, false); + } + + @Test + public void storageSchemeTest() { + + final N5Reader reader = factory.getReader(uri); + } +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java new file mode 100644 index 0000000..c54ad03 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java @@ -0,0 +1,40 @@ +package org.janelia.saalfeldlab.n5.universe; + +import org.janelia.saalfeldlab.n5.AbstractN5Test; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.junit.Before; + +public interface StorageSchemeWrappedN5Test { + + abstract N5Factory getFactory(); + + N5Factory.StorageFormat getStorageFormat(); + + default N5Writer getWriter(String uri) { + + final String uriWithStorageScheme = prependStorageScheme(uri); + return getFactory().getWriter(uriWithStorageScheme); + } + + default N5Reader getReader(String uri) { + + final String uriWithStorageScheme = prependStorageScheme(uri); + return getFactory().getReader(uriWithStorageScheme); + } + + default String prependStorageScheme(String uri) { + + final String schemePattern = getStorageFormat().schemePattern.pattern(); + final String doesntHaveStorageScheme = new StringBuilder("^(?:(?!(?i)") + .append(schemePattern) + .append(":))(?.*)") + .toString(); + + final String prependStorageScheme = new StringBuilder(getStorageFormat().toString().toLowerCase()) + .append(":${remaining}") + .toString(); + + return uri.replaceFirst(doesntHaveStorageScheme, prependStorageScheme); + } +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java new file mode 100644 index 0000000..d481a71 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -0,0 +1,237 @@ +package org.janelia.saalfeldlab.n5.universe; + +import com.amazonaws.services.s3.AmazonS3; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3Test; +import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; +import org.janelia.saalfeldlab.n5.s3.mock.N5AmazonS3ContainerPathMockTest; +import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAwsS3MockTest.class}) +public class ZarrStorageTests { + + public static class ZarrFileSystemTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { + + private final N5Factory factory; + + public ZarrFileSystemTest() { + + this.factory = new N5Factory(); + } + + @Override public N5Factory getFactory() { + + return factory; + } + + @Override public N5Factory.StorageFormat getStorageFormat() { + + return N5Factory.StorageFormat.ZARR; + } + + @Override protected N5Writer createN5Writer() { + + return getWriter(tempN5Location()); + } + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gsonBuilder, String dimensionSeparator, boolean mapN5DatasetAttributes) { + + factory.gsonBuilder(gsonBuilder); + factory.zarrDimensionSeparator(dimensionSeparator); + factory.zarrMapN5Attributes(mapN5DatasetAttributes); + return getWriter(location); + } + + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getReader(location); + } + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getWriter(location); + } + + @Override protected N5Writer createN5Writer(String location) { + + return getWriter(location); + } + + @Override protected N5Reader createN5Reader(String location) { + + return getReader(location); + } + } + + public static class ZarrAwsS3MockTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { + private static String bucketName; + private final N5Factory factory; + + public ZarrAwsS3MockTest() { + + this.factory = new N5Factory() { + + @Override AmazonS3 createS3(String uri) { + + return MockS3Factory.getOrCreateS3(); + } + + @Override String getS3Bucket(String uri) { + + try { + final String path = new URI(uri).getPath().replaceFirst("^/", ""); + return path.substring(0, path.indexOf('/')); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + }; + } + + + @BeforeClass + public static void setup() { + bucketName = AbstractN5AmazonS3Test.tempBucketName(); + } + + @AfterClass + public static void cleanup() { + + MockS3Factory.getOrCreateS3().deleteBucket(bucketName); + } + + @Override + protected N5Writer createN5Writer(final String location, final String dimensionSeparator) { + + factory.zarrDimensionSeparator(dimensionSeparator); + return getWriter(location); + } + + @Override protected String tempN5Location() { + + try { + return new URI("http", "localhost:8001", "/" + bucketName + AbstractN5AmazonS3Test.tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override public N5Factory getFactory() { + + return factory; + } + + @Override public N5Factory.StorageFormat getStorageFormat() { + + return N5Factory.StorageFormat.ZARR; + } + + @Override protected N5Writer createN5Writer() { + + return getWriter(tempN5Location()); + } + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getWriter(location); + } + + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getReader(location); + } + + @Override protected N5Writer createN5Writer(String location) { + + return getWriter(location); + } + + @Override protected N5Reader createN5Reader(String location) { + + return getReader(location); + + } + + + /** + * Currently, {@code N5AmazonS3Reader#exists(String)} is implemented by listing objects under that group. + * This test case specifically tests its correctness. + * + * @throws IOException + */ + @Test + public void testExistsUsingListingObjects() { + + try (N5Writer n5 = createN5Writer()) { + n5.createGroup("/one/two/three"); + + Assert.assertTrue(n5.exists("")); + Assert.assertTrue(n5.exists("/")); + + Assert.assertTrue(n5.exists("one")); + Assert.assertTrue(n5.exists("one/")); + Assert.assertTrue(n5.exists("/one")); + Assert.assertTrue(n5.exists("/one/")); + + Assert.assertTrue(n5.exists("one/two")); + Assert.assertTrue(n5.exists("one/two/")); + Assert.assertTrue(n5.exists("/one/two")); + Assert.assertTrue(n5.exists("/one/two/")); + + Assert.assertTrue(n5.exists("one/two/three")); + Assert.assertTrue(n5.exists("one/two/three/")); + Assert.assertTrue(n5.exists("/one/two/three")); + Assert.assertTrue(n5.exists("/one/two/three/")); + + Assert.assertFalse(n5.exists("one/tw")); + Assert.assertFalse(n5.exists("one/tw/")); + Assert.assertFalse(n5.exists("/one/tw")); + Assert.assertFalse(n5.exists("/one/tw/")); + + Assert.assertArrayEquals(new String[]{"one"}, n5.list("/")); + Assert.assertArrayEquals(new String[]{"two"}, n5.list("/one")); + Assert.assertArrayEquals(new String[]{"three"}, n5.list("/one/two")); + + Assert.assertArrayEquals(new String[]{}, n5.list("/one/two/three")); + assertThrows(N5Exception.N5IOException.class, () -> n5.list("/one/tw")); + + Assert.assertTrue(n5.remove("/one/two/three")); + Assert.assertFalse(n5.exists("/one/two/three")); + Assert.assertTrue(n5.exists("/one/two")); + Assert.assertTrue(n5.exists("/one")); + + Assert.assertTrue(n5.remove("/one")); + Assert.assertFalse(n5.exists("/one/two")); + Assert.assertFalse(n5.exists("/one")); + } + } + } +} From 9a61e8ea271e1e56bb4829160f8abdcd831fd26a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 22 Feb 2024 14:42:17 -0500 Subject: [PATCH 06/31] fix: errors for HDF5 format guessing when uri has no scheme * isHdf5: check file existence & return false instead of throwing error --- .../saalfeldlab/n5/universe/N5Factory.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 183550a..4285e10 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -230,12 +230,16 @@ private static boolean isHDF5Reader(final String path) throws N5IOException { private static boolean isHDF5(String path) { - try (final FileInputStream in = new FileInputStream(new File(path))) { + final File f = new File(path); + if (!f.exists() || !f.isFile()) + return false; + + try (final FileInputStream in = new FileInputStream(f)) { final byte[] sig = new byte[8]; in.read(sig); return Arrays.equals(sig, HDF5_SIG); } catch (final IOException e) { - throw new N5Exception.N5IOException(e); + return false; } } @@ -694,8 +698,12 @@ public N5Reader getReader(final String uri) { Exception exception = null; for (StorageFormat storageFormat : StorageFormat.values()) { - if (storageFormat.compareTo(StorageFormat.HDF5) < 0) - break; + // all possible attempts at making an hdf5 reader will be done by now + // and HDF5 does not use a KeyValueAccess + // revisit this if more backends are added + if (storageFormat == StorageFormat.HDF5) + continue; + try { return StorageFormat.getReader(storageFormat, access, containerPath, this); } catch (Exception e) { @@ -855,7 +863,7 @@ enum StorageFormat { N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { final boolean hasHdf5Extension = Pattern.compile("\\.h(df)5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath()); - return hasHdf5Extension || Paths.get(uri).toFile().exists() && isHDF5(uri.toString()); + return hasHdf5Extension || isHDF5(uri.getPath()); }); static final Pattern STORAGE_SCHEME_PATTERN = Pattern.compile("^(\\s*(?(n5|h(df)?5|zarr)):(//)?)?(?.*)$", Pattern.CASE_INSENSITIVE); From bb263b7758a215b09f1a27f40ae6327dcb85efd2 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 Feb 2024 16:53:26 -0500 Subject: [PATCH 07/31] feat(test): add zarr format tests for FileSystem and S3 backends --- pom.xml | 2 + .../universe/StorageSchemeWrappedN5Test.java | 4 +- .../n5/universe/ZarrStorageTests.java | 329 +++++++++++++----- 3 files changed, 252 insertions(+), 83 deletions(-) diff --git a/pom.xml b/pom.xml index b10ed74..7e79568 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,9 @@ 2.1.0 7.0.0 4.0.2 + 1.2.2-SNAPSHOT + 4.0.3-SNAPSHOT 1.0.0-preview.20191208 1.4.1 diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java index c54ad03..0dedf15 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java @@ -1,13 +1,11 @@ package org.janelia.saalfeldlab.n5.universe; -import org.janelia.saalfeldlab.n5.AbstractN5Test; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; -import org.junit.Before; public interface StorageSchemeWrappedN5Test { - abstract N5Factory getFactory(); + N5Factory getFactory(); N5Factory.StorageFormat getStorageFormat(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index d481a71..555f5aa 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -4,34 +4,40 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonNull; +import com.google.gson.reflect.TypeToken; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3Test; -import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; -import org.janelia.saalfeldlab.n5.s3.mock.N5AmazonS3ContainerPathMockTest; +import org.janelia.saalfeldlab.n5.StringDataBlock; +import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; -import org.junit.After; -import org.junit.AfterClass; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; +import org.janelia.saalfeldlab.n5.zarr.ZarrStringDataBlock; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Suite; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.util.HashMap; +import java.util.Map; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; @RunWith(Suite.class) -@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAwsS3MockTest.class}) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAwsS3Tests.class}) public class ZarrStorageTests { public static class ZarrFileSystemTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { @@ -89,21 +95,23 @@ public ZarrFileSystemTest() { } } - public static class ZarrAwsS3MockTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { - private static String bucketName; + public static class ZarrAwsS3Tests extends N5AmazonS3Tests implements StorageSchemeWrappedN5Test { private final N5Factory factory; - public ZarrAwsS3MockTest() { + public ZarrAwsS3Tests() { this.factory = new N5Factory() { @Override AmazonS3 createS3(String uri) { - return MockS3Factory.getOrCreateS3(); + return getS3(); } @Override String getS3Bucket(String uri) { + if (useBackend) { + return super.getS3Bucket(uri); + } try { final String path = new URI(uri).getPath().replaceFirst("^/", ""); return path.substring(0, path.indexOf('/')); @@ -114,29 +122,10 @@ public ZarrAwsS3MockTest() { }; } - - @BeforeClass - public static void setup() { - bucketName = AbstractN5AmazonS3Test.tempBucketName(); - } - - @AfterClass - public static void cleanup() { - - MockS3Factory.getOrCreateS3().deleteBucket(bucketName); - } - - @Override - protected N5Writer createN5Writer(final String location, final String dimensionSeparator) { - - factory.zarrDimensionSeparator(dimensionSeparator); - return getWriter(location); - } - @Override protected String tempN5Location() { try { - return new URI("http", "localhost:8001", "/" + bucketName + AbstractN5AmazonS3Test.tempContainerPath(), null, null).toString(); + return new URI("http", "localhost:8001", "/" + tempBucketName(getS3()) + tempContainerPath(), null, null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } @@ -177,60 +166,240 @@ protected N5Writer createN5Writer(final String location, final String dimensionS @Override protected N5Reader createN5Reader(String location) { return getReader(location); + } + @Override + @Test + public void testCreateDataset() { + + final DatasetAttributes info; + try (N5Writer n5 = createN5Writer()) { + n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, getCompressions()[0]); + + assertTrue("Dataset does not exist", n5.exists(datasetName)); + + info = n5.getDatasetAttributes(datasetName); + assertArrayEquals(dimensions, info.getDimensions()); + assertArrayEquals(blockSize, info.getBlockSize()); + assertEquals(DataType.UINT64, info.getDataType()); + assertEquals(getCompressions()[0].getClass(), info.getCompression().getClass()); + } } + @Override + @Test + public void testVersion() throws NumberFormatException { + + try (final N5Writer writer = createN5Writer()) { + + final ZarrKeyValueWriter zarr = (ZarrKeyValueWriter)writer; + final N5Reader.Version n5Version = writer.getVersion(); + final N5Reader.Version expectedVersion = new N5Reader.Version(2, 0, 0); + assertEquals(n5Version, expectedVersion); + } + } - /** - * Currently, {@code N5AmazonS3Reader#exists(String)} is implemented by listing objects under that group. - * This test case specifically tests its correctness. - * - * @throws IOException - */ + @Override @Test - public void testExistsUsingListingObjects() { + @Ignore("Zarr does not currently support mode 1 data blocks.") + public void testMode1WriteReadByteBlock() { - try (N5Writer n5 = createN5Writer()) { - n5.createGroup("/one/two/three"); - - Assert.assertTrue(n5.exists("")); - Assert.assertTrue(n5.exists("/")); - - Assert.assertTrue(n5.exists("one")); - Assert.assertTrue(n5.exists("one/")); - Assert.assertTrue(n5.exists("/one")); - Assert.assertTrue(n5.exists("/one/")); - - Assert.assertTrue(n5.exists("one/two")); - Assert.assertTrue(n5.exists("one/two/")); - Assert.assertTrue(n5.exists("/one/two")); - Assert.assertTrue(n5.exists("/one/two/")); - - Assert.assertTrue(n5.exists("one/two/three")); - Assert.assertTrue(n5.exists("one/two/three/")); - Assert.assertTrue(n5.exists("/one/two/three")); - Assert.assertTrue(n5.exists("/one/two/three/")); - - Assert.assertFalse(n5.exists("one/tw")); - Assert.assertFalse(n5.exists("one/tw/")); - Assert.assertFalse(n5.exists("/one/tw")); - Assert.assertFalse(n5.exists("/one/tw/")); - - Assert.assertArrayEquals(new String[]{"one"}, n5.list("/")); - Assert.assertArrayEquals(new String[]{"two"}, n5.list("/one")); - Assert.assertArrayEquals(new String[]{"three"}, n5.list("/one/two")); - - Assert.assertArrayEquals(new String[]{}, n5.list("/one/two/three")); - assertThrows(N5Exception.N5IOException.class, () -> n5.list("/one/tw")); - - Assert.assertTrue(n5.remove("/one/two/three")); - Assert.assertFalse(n5.exists("/one/two/three")); - Assert.assertTrue(n5.exists("/one/two")); - Assert.assertTrue(n5.exists("/one")); - - Assert.assertTrue(n5.remove("/one")); - Assert.assertFalse(n5.exists("/one/two")); - Assert.assertFalse(n5.exists("/one")); + } + + @Override + @Test + @Ignore("Zarr does not currently support mode 2 data blocks and serialized objects.") + public void testWriteReadSerializableBlock() { + + } + + @Test + @Override + public void testAttributes() { + + try (final N5Writer n5 = createN5Writer()) { + n5.createGroup(groupName); + + n5.setAttribute(groupName, "key1", "value1"); + // length 2 because it includes "zarr_version" + Assert.assertEquals(2, n5.listAttributes(groupName).size()); + /* class interface */ + Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); + /* type interface */ + Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { + + }.getType())); + + final Map newAttributes = new HashMap<>(); + newAttributes.put("key2", "value2"); + newAttributes.put("key3", "value3"); + n5.setAttributes(groupName, newAttributes); + + Assert.assertEquals(4, n5.listAttributes(groupName).size()); + /* class interface */ + Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); + Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", String.class)); + Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); + /* type interface */ + Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { + + }.getType())); + Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", new TypeToken() { + + }.getType())); + Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { + + }.getType())); + + n5.setAttribute(groupName, "key1", 1); + n5.setAttribute(groupName, "key2", 2); + + Assert.assertEquals(4, n5.listAttributes(groupName).size()); + /* class interface */ + Assert.assertEquals(new Integer(1), n5.getAttribute(groupName, "key1", Integer.class)); + Assert.assertEquals(new Integer(2), n5.getAttribute(groupName, "key2", Integer.class)); + Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); + /* type interface */ + Assert + .assertEquals( + new Integer(1), + n5.getAttribute(groupName, "key1", new TypeToken() { + + }.getType())); + Assert + .assertEquals( + new Integer(2), + n5.getAttribute(groupName, "key2", new TypeToken() { + + }.getType())); + Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { + + }.getType())); + + n5.removeAttribute(groupName, "key1"); + n5.removeAttribute(groupName, "key2"); + n5.removeAttribute(groupName, "key3"); + Assert.assertEquals(1, n5.listAttributes(groupName).size()); + } + } + + @Test + @Override + @Ignore + public void testNullAttributes() throws IOException { + + // serializeNulls must be on for Zarr to be able to write datasets with raw compression + + /* serializeNulls*/ + try (N5Writer writer = createN5Writer(tempN5Location(), new GsonBuilder().serializeNulls())) { + + writer.createGroup(groupName); + writer.setAttribute(groupName, "nullValue", null); + assertNull(writer.getAttribute(groupName, "nullValue", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "nullValue", JsonElement.class)); + final HashMap nulls = new HashMap<>(); + nulls.put("anotherNullValue", null); + nulls.put("structured/nullValue", null); + nulls.put("implicitNulls[3]", null); + writer.setAttributes(groupName, nulls); + + assertNull(writer.getAttribute(groupName, "anotherNullValue", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); + + assertNull(writer.getAttribute(groupName, "structured/nullValue", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); + + assertNull(writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[3]", JsonElement.class)); + + assertNull(writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[1]", JsonElement.class)); + + /* Negative test; a value that truly doesn't exist will still return `null` but will also return `null` when querying as a `JsonElement` */ + assertNull(writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); + + assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); + assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); + + /* check existing value gets overwritten */ + writer.setAttribute(groupName, "existingValue", 1); + assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); + writer.setAttribute(groupName, "existingValue", null); + assertThrows(N5Exception.N5ClassCastException.class, () -> writer.getAttribute(groupName, "existingValue", Integer.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); + + writer.remove(); + } + } + + @Test + @Override + @Ignore + public void testRootLeaves() { + + // This tests serializing primitives, and arrays at the root of an n5's attributes, + // since .zattrs must be a json object, this would test invalide behavior for zarr, + // therefore this test is ignored. + } + + @Test + @Override + public void testReaderCreation() { + + final String canonicalPath = tempN5Location(); + try (N5Writer writer = createN5Writer(canonicalPath)) { + + final N5Reader n5r = createN5Reader(canonicalPath); + assertNotNull(n5r); + + // existing directory without attributes is okay; + // Remove and create to remove attributes store + writer.remove("/"); + writer.createGroup("/"); + final N5Reader na = createN5Reader(canonicalPath); + assertNotNull(na); + + // existing location with attributes, but no version + writer.remove("/"); + writer.createGroup("/"); + writer.setAttribute("/", "mystring", "ms"); + final N5Reader wa = createN5Reader(canonicalPath); + assertNotNull(wa); + + // non-existent directory should fail + writer.remove("/"); + assertThrows( + "Non-existant location throws error", + N5Exception.N5IOException.class, + () -> { + final N5Reader test = createN5Reader(canonicalPath); + test.list("/"); + }); + } + } + + @Test + @Override + public void testWriteReadStringBlock() { + + DataType dataType = DataType.STRING; + int[] blockSize = new int[]{3, 2, 1}; + String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-รพ"}; + Compression[] compressions = this.getCompressions(); + + for (Compression compression : compressions) { + System.out.println("Testing " + compression.getType() + " " + dataType); + + try (final N5Writer n5 = createN5Writer()) { + n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression); + DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); + StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); + n5.writeBlock("/test/group/dataset", attributes, dataBlock); + DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); + assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); + assertTrue(n5.remove("/test/group/dataset")); + } } } } From 2dcc9ff43329131acbce652b0f70fcedf2aa2ffb Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Feb 2024 15:42:00 -0500 Subject: [PATCH 08/31] refactor: extract S3 logic to n5-aws-s3 library --- pom.xml | 2 +- .../saalfeldlab/n5/universe/N5Factory.java | 214 +++--------------- 2 files changed, 30 insertions(+), 186 deletions(-) diff --git a/pom.xml b/pom.xml index 7e79568..2186f6b 100644 --- a/pom.xml +++ b/pom.xml @@ -111,7 +111,7 @@ sign,deploy-to-scijava - 3.1.3 + 3.1.4-SNAPSHOT 2.1.0 7.0.0 4.0.2 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 4285e10..b67170c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -1,17 +1,17 @@ /** * Copyright (c) 2017-2021, Saalfeld lab, HHMI Janelia * All rights reserved. - * + *

* Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

* Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * + * list of conditions and the following disclaimer. + *

* Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + *

* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -37,7 +37,6 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Iterator; -import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.regex.Matcher; @@ -63,24 +62,18 @@ import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.AmazonS3Utils; import org.janelia.saalfeldlab.n5.zarr.N5ZarrReader; import org.janelia.saalfeldlab.n5.zarr.N5ZarrWriter; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3URI; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.google.cloud.resourcemanager.Project; import com.google.cloud.resourcemanager.ResourceManager; import com.google.cloud.storage.Storage; @@ -102,8 +95,6 @@ public class N5Factory implements Serializable { private static final long serialVersionUID = -6823715427289454617L; - private static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)", Pattern.CASE_INSENSITIVE); - private static byte[] HDF5_SIG = {(byte)137, 72, 68, 70, 13, 10, 26, 10}; private int[] hdf5DefaultBlockSize = {64, 64, 64, 1, 1}; private boolean hdf5OverrideBlockSize = false; @@ -246,162 +237,10 @@ private static boolean isHDF5(String path) { AmazonS3 createS3(final String uri) { try { - return createS3(new AmazonS3URI(uri)); - } catch (final IllegalArgumentException e) { - // if AmazonS3URI does not like the form of the uri - try { - final URI buri = new URI(uri); - final URI endpointUrl = new URI(buri.getScheme(), buri.getHost(), null, null); - return createS3(getS3Credentials(), new EndpointConfiguration(endpointUrl.toString(), null), null, getS3Bucket(uri)); - } catch (final URISyntaxException e1) { - } - } - throw new N5Exception("Could not create s3 client from uri: " + uri); - } - - private AmazonS3 createS3( - final AWSCredentialsProvider credentialsProvider, - final EndpointConfiguration endpointConfiguration, - final Regions region, - final String bucketName) { - - final boolean isAmazon = endpointConfiguration == null || AWS_ENDPOINT_PATTERN.matcher(endpointConfiguration.getServiceEndpoint()).find(); - final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); - - if (!isAmazon) - builder.withPathStyleAccessEnabled(true); - - if (credentialsProvider != null) - builder.withCredentials(credentialsProvider); - - if (endpointConfiguration != null) - builder.withEndpointConfiguration(endpointConfiguration); - else if (region != null) - builder.withRegion(region); - else - builder.withRegion("us-east-1"); - - AmazonS3 s3 = builder.build(); - // if we used anonymous credentials and the factory requests a retry with credentials: - if (s3RetryWithCredentials && areAnonymous(credentialsProvider)) { - - // I initially tried checking whether the bucket exists, but - // that, apparently, returns even when the client does not have access - if (!canListBucket(s3, bucketName)) { - // bucket not detected with anonymous credentials, try detecting credentials - // and return it even if it can't detect the bucket, since there's nothing else to do - s3 = createS3(new DefaultAWSCredentialsProviderChain(), endpointConfiguration, region, null); - } - } - return s3; - } - - private boolean canListBucket(final AmazonS3 s3, final String bucket) { - - final ListObjectsV2Request request = new ListObjectsV2Request(); - request.setBucketName(bucket); - request.setMaxKeys(1); - - try { - // list objects will throw an AmazonS3Exception (Access Denied) if this client does not have access - s3.listObjectsV2(request); - return true; - } catch (final AmazonS3Exception e) { - return false; - } - } - - private AWSStaticCredentialsProvider getS3Credentials() { - - AWSCredentials credentials = null; - final AWSStaticCredentialsProvider credentialsProvider; - if (s3Credentials != null) { - credentials = s3Credentials; - credentialsProvider = new AWSStaticCredentialsProvider(credentials); - } else { - // if not anonymous, try finding credentials - if (!s3Anonymous) { - try { - credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); - } catch (final Exception e) { - System.out.println("Could not load AWS credentials, falling back to anonymous."); - } - credentialsProvider = new AWSStaticCredentialsProvider( - credentials == null ? new AnonymousAWSCredentials() : credentials); - } else - credentialsProvider = new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()); - } - - return credentialsProvider; - } - - private boolean areAnonymous(final AWSCredentialsProvider credsProvider) { - - final AWSCredentials creds = credsProvider.getCredentials(); - // AnonymousAWSCredentials do not have an equals method - if (creds.getClass().equals(AnonymousAWSCredentials.class)) - return true; - - return creds.getAWSAccessKeyId() == null && creds.getAWSSecretKey() == null; - } - - private Regions getS3Region(final AmazonS3URI uri) { - - final Regions region = Optional.ofNullable(uri.getRegion()).map(Regions::fromName) // get the region from the uri - .orElse(Optional.ofNullable(s3Region).map(Regions::fromName) // next use whatever is passed in - .orElse(null)); // fallback to null (amazon picks a default) - return region; - } - - private AmazonS3 createS3(final AmazonS3URI uri) { - - AwsClientBuilder.EndpointConfiguration endpointConfiguration = null; - if (!"s3".equalsIgnoreCase(uri.getURI().getScheme())) { - - if (s3Endpoint != null) - endpointConfiguration = new EndpointConfiguration(s3Endpoint, null); - else { - final Matcher matcher = AWS_ENDPOINT_PATTERN.matcher(uri.getURI().getHost()); - if (matcher.find()) - endpointConfiguration = new EndpointConfiguration(matcher.group(2), uri.getRegion()); - else - endpointConfiguration = new EndpointConfiguration(uri.getURI().getHost(), uri.getRegion()); - } - } - - return createS3(getS3Credentials(), endpointConfiguration, getS3Region(uri), uri.getBucket()); - } - - String getS3Bucket(final String uri) { - - try { - return new AmazonS3URI(uri).getBucket(); - } catch (final IllegalArgumentException e) { + return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region); + } catch (final Exception e) { + throw new N5Exception("Could not create s3 client from uri: " + uri, e); } - try { - // parse bucket manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(0, path.indexOf('/')); - } catch (final URISyntaxException e) { - } - return null; - } - - private String getS3Key(final String uri) { - - try { - // if key is null, return the empty string - final String key = new AmazonS3URI(uri).getKey(); - return key == null ? "" : key; - } catch (final IllegalArgumentException e) { - } - try { - // parse key manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(path.indexOf('/') + 1); - } catch (final URISyntaxException e) { - } - return null; } /** @@ -454,8 +293,7 @@ public N5HDF5Reader openHDF5Reader(final String path) { public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxException { final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(N5URI.encodeAsUri(uri)); - final GoogleCloudStorageClient storageClient = new GoogleCloudStorageClient(); - final Storage storage = storageClient.create(); + final Storage storage = createGoogleCloudStorage(); final GoogleCloudStorageKeyValueAccess googleCloudBackend = new GoogleCloudStorageKeyValueAccess(storage, googleCloudUri.getBucket(), false); @@ -467,6 +305,13 @@ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxExceptio } } + private Storage createGoogleCloudStorage() { + + final GoogleCloudStorageClient storageClient = new GoogleCloudStorageClient(); + final Storage storage = storageClient.create(); + return storage; + } + /** * Open an {@link N5Reader} for AWS S3. * @@ -479,12 +324,12 @@ public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException { final AmazonS3 s3 = createS3(N5URI.encodeAsUri(uri).toString()); // when, if ever do we want to creat a bucket? - final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, getS3Bucket(uri), false); + final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uri), false); if (lastExtension(uri).startsWith(".zarr")) { - return new ZarrKeyValueReader(s3kv, getS3Key(uri), gsonBuilder, zarrMapN5DatasetAttributes, + return new ZarrKeyValueReader(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); } else { - return new N5KeyValueReader(s3kv, getS3Key(uri), gsonBuilder, cacheAttributes); + return new N5KeyValueReader(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, cacheAttributes); } } @@ -570,12 +415,12 @@ public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException { final AmazonS3 s3 = createS3(N5URI.encodeAsUri(uri).toString()); // when, if ever do we want to creat a bucket? - final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, getS3Bucket(uri), false); + final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uri), false); if (lastExtension(uri).startsWith(".zarr")) { - return new ZarrKeyValueWriter(s3kv, getS3Key(uri), gsonBuilder, zarrMapN5DatasetAttributes, + return new ZarrKeyValueWriter(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes); } else { - return new N5KeyValueWriter(s3kv, getS3Key(uri), gsonBuilder, cacheAttributes); + return new N5KeyValueWriter(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, cacheAttributes); } } @@ -691,7 +536,7 @@ public N5Reader getReader(final String uri) { } final String containerPath; if (access instanceof AmazonS3KeyValueAccess) { - containerPath = getS3Key(asUri.toString()); + containerPath = AmazonS3Utils.getS3Key(asUri.toString()); } else { containerPath = asUri.getPath(); } @@ -738,7 +583,7 @@ public N5Writer getWriter(final String uri) { } final String containerPath; if (access instanceof AmazonS3KeyValueAccess) { - containerPath = getS3Key(asUri.toString()); + containerPath = AmazonS3Utils.getS3Key(asUri.toString()); } else { containerPath = asUri.getPath(); } @@ -772,7 +617,7 @@ private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, f final String uriString = uri.toString(); final AmazonS3 s3 = factory.createS3(uriString); - return new AmazonS3KeyValueAccess(s3, factory.getS3Bucket(uriString), factory.createS3Bucket); + return new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uriString), factory.createS3Bucket); } private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { @@ -783,7 +628,6 @@ private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI ur private final static Pattern GS_SCHEME = Pattern.compile("gs", Pattern.CASE_INSENSITIVE); private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); private final static Pattern GS_HOST = Pattern.compile("(cloud\\.google|storage\\.googleapis)\\.com", Pattern.CASE_INSENSITIVE); - private final static Pattern S3_SCHEME = Pattern.compile("s3", Pattern.CASE_INSENSITIVE); private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); /** @@ -806,7 +650,7 @@ enum KeyValueAccessBackend implements Predicate, BiFunction { final String scheme = uri.getScheme(); final boolean hasScheme = scheme != null; - return hasScheme && S3_SCHEME.asPredicate().test(scheme) + return hasScheme && AmazonS3Utils.S3_SCHEME.asPredicate().test(scheme) || uri.getHost() != null && hasScheme && HTTPS_SCHEME.asPredicate().test(scheme); }, N5Factory::newAmazonS3KeyValueAccess), FILE(uri -> { @@ -930,7 +774,7 @@ private static N5Reader getReader(StorageFormat storage, URI uri, N5Factory fact final String containerPath; /* Any more special cases? google? */ if (access instanceof AmazonS3KeyValueAccess) { - containerPath = factory.getS3Key(uri.toString()); + containerPath = AmazonS3Utils.getS3Key(uri.toString()); } else containerPath = uri.getPath(); return StorageFormat.getReader(storage, access, containerPath, factory); @@ -943,7 +787,7 @@ private static N5Writer getWriter(StorageFormat storage, URI uri, N5Factory fact final String containerPath; /* Any more special cases? google? */ if (access instanceof AmazonS3KeyValueAccess) { - containerPath = factory.getS3Key(uri.toString()); + containerPath = AmazonS3Utils.getS3Key(uri.toString()); } else containerPath = uri.getPath(); final N5Writer writer = StorageFormat.getWriter(storage, access, containerPath, factory); From c69cc7fda868177a6075fa3cf8a25ee52bc468bf Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Feb 2024 15:42:43 -0500 Subject: [PATCH 09/31] feat(test): improve tests, add N5StorageTest --- .../n5/universe/N5FactoryTest.java | 103 ------ .../n5/universe/N5StorageTests.java | 150 ++++++++ .../universe/StorageSchemeWrappedN5Test.java | 42 ++- .../n5/universe/ZarrStorageTests.java | 349 ++---------------- 4 files changed, 228 insertions(+), 416 deletions(-) delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java deleted file mode 100644 index d552249..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTest.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.janelia.saalfeldlab.n5.universe; - -import com.google.gson.GsonBuilder; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.AbstractN5Test; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.zarr.N5ZarrReader; -import org.janelia.saalfeldlab.n5.zarr.N5ZarrWriter; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Arrays; - -@RunWith(Parameterized.class) -public class N5FactoryTest extends AbstractN5Test { - - final N5Factory factory = new N5Factory(); - - @Parameterized.Parameter(0) - public String testName; - - @Parameterized.Parameter(1) - public String uri; - - @Parameterized.Parameters(name = "{0}") - public static Iterable parameters() { - - return Arrays.asList(new Object[][]{ - {"localPathOnly", "/Users/hulbertc/projects/paintera/test.n5"}, - {"n5StorageSchemeNoNesting", "n5:/Users/hulbertc/projects/paintera/test.n5"}, - {"n5StorageSchemeNoNestingDoubleSlash", "n5:///Users/hulbertc/projects/paintera/test.n5"}, - {"n5StorageSchemeAndFile", "n5:file:///Users/hulbertc/projects/paintera/test.n5"}, - {"n5StorageSchemeAndFileDoubleSlash", "n5://file:///Users/hulbertc/projects/paintera/test.n5"}, - {"n5StorageSchemeAndFileDoubleNoExtension", "n5:file:///Users/hulbertc/projects/paintera/test"}, - {"n5StorageSchemeAndFileDoubleMismatchExtension", "n5:file:///Users/hulbertc/projects/paintera/test.zarr"} - }); - } - - - @Override - protected String tempN5Location() { - - try { - return Files.createTempDirectory("n5-factory-test").toUri().toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - protected static String tmpPathName(final String prefix) { - - try { - final File tmpFile = Files.createTempDirectory(prefix).toFile(); - tmpFile.deleteOnExit(); - return tmpFile.getCanonicalPath(); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - @Override - protected N5Writer createN5Writer() throws IOException { - - final String testDirPath = tmpPathName("n5-factory-test-"); - return factory.openWriter(testDirPath); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gsonBuilder) throws IOException { - - return createN5Writer(location, gsonBuilder, ".", true); - } - - protected N5ZarrWriter createN5Writer(final String location, final String dimensionSeparator) throws IOException { - - return createN5Writer(location, new GsonBuilder(), dimensionSeparator, true); - } - - @Override - protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException { - - return new N5ZarrReader(location, gson); - } - - protected N5ZarrWriter createN5Writer( - final String location, - final GsonBuilder gsonBuilder, - final String dimensionSeparator, - final boolean mapN5DatasetAttributes) throws IOException { - - return new N5ZarrWriter(location, gsonBuilder, dimensionSeparator, mapN5DatasetAttributes, false); - } - - @Test - public void storageSchemeTest() { - - final N5Reader reader = factory.getReader(uri); - } -} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java new file mode 100644 index 0000000..0f5ac15 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -0,0 +1,150 @@ +package org.janelia.saalfeldlab.n5.universe; + +import com.amazonaws.services.s3.AmazonS3; +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.AbstractN5Test; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; + +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; + +@RunWith(Suite.class) +@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3Test.class}) +public class N5StorageTests { + + public static abstract class N5FactoryTest extends AbstractN5Test implements StorageSchemeWrappedN5Test { + + protected N5Factory factory; + + public N5FactoryTest() { + + this.factory = getFactory(); + } + + @Override abstract protected String tempN5Location(); + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory(); + } + return factory; + } + + @Override public N5Factory.StorageFormat getStorageFormat() { + + return N5Factory.StorageFormat.N5; + } + + @Override protected N5Writer createN5Writer() { + + return getWriter(tempN5Location()); + } + + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getReader(location); + } + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + + factory.gsonBuilder(gson); + return getWriter(location); + } + + @Override protected N5Writer createN5Writer(String location) { + + return getWriter(location); + } + + @Override protected N5Reader createN5Reader(String location) { + + return getReader(location); + } + } + + public static class N5FileSystemTest extends N5FactoryTest { + + @Override public Class getBackendTargetClass() { + + return FileSystemKeyValueAccess.class; + } + + @Override protected String tempN5Location() { + + try { + return Files.createTempDirectory("n5-test").toUri().getPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static class N5AmazonS3Test extends N5FactoryTest { + + @Override public Class getBackendTargetClass() { + + return AmazonS3KeyValueAccess.class; + } + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory() { + + @Override AmazonS3 createS3(String uri) { + + return MockS3Factory.getOrCreateS3(); + } + }; + } + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("http", "localhost:8001", "/" + tempBucketName(factory.createS3(null)) + tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + +// public static class N5GoogleCloudTest extends N5FactoryTest { +// +// @Override public Class getBackendTargetClass() { +// +// return GoogleCloudStorageKeyValueAccess.class; +// } +// +// @Override public N5Factory getFactory() { +// +// if (factory == null) { +// factory = new N5Factory() { +// +// +// }; +// } +// return factory; +// } +// +// @Override protected String tempN5Location() { +// +// return new URI("gs", tempBucketName(getGoogleCloudStorage()), tempContainerPath(), null).toString(); +// } +// } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java index 0dedf15..bfbe656 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java @@ -1,7 +1,17 @@ package org.janelia.saalfeldlab.n5.universe; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; + +import static org.junit.Assert.assertTrue; public interface StorageSchemeWrappedN5Test { @@ -9,16 +19,44 @@ public interface StorageSchemeWrappedN5Test { N5Factory.StorageFormat getStorageFormat(); + Class getBackendTargetClass(); + default N5Writer getWriter(String uri) { final String uriWithStorageScheme = prependStorageScheme(uri); - return getFactory().getWriter(uriWithStorageScheme); + final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)getFactory().getWriter(uriWithStorageScheme); + switch (getStorageFormat()){ + case ZARR: + assertTrue(writer instanceof ZarrKeyValueWriter); + break; + case N5: + assertTrue(writer instanceof N5KeyValueWriter); + break; + case HDF5: + assertTrue(writer instanceof N5HDF5Writer); + break; + } + assertTrue(getBackendTargetClass().isAssignableFrom(writer.getKeyValueAccess().getClass())); + return writer; } default N5Reader getReader(String uri) { final String uriWithStorageScheme = prependStorageScheme(uri); - return getFactory().getReader(uriWithStorageScheme); + final GsonKeyValueN5Reader reader = (GsonKeyValueN5Reader)getFactory().getReader(uriWithStorageScheme); + switch (getStorageFormat()){ + case ZARR: + assertTrue(reader instanceof ZarrKeyValueReader); + break; + case N5: + assertTrue(reader instanceof N5KeyValueReader); + break; + case HDF5: + assertTrue(reader instanceof N5HDF5Reader); + break; + } + assertTrue(getBackendTargetClass().isAssignableFrom(reader.getKeyValueAccess().getClass())); + return reader; } default String prependStorageScheme(String uri) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index 555f5aa..0a3fa56 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -2,55 +2,43 @@ import com.amazonaws.services.s3.AmazonS3; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.reflect.TypeToken; -import org.janelia.saalfeldlab.n5.Compression; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.StringDataBlock; -import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; -import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; -import org.janelia.saalfeldlab.n5.zarr.ZarrStringDataBlock; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Suite; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; +import java.nio.file.Files; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; +import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; @RunWith(Suite.class) -@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAwsS3Tests.class}) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3Test.class}) public class ZarrStorageTests { - public static class ZarrFileSystemTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { + public static abstract class ZarrFactoryTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { - private final N5Factory factory; + protected N5Factory factory; - public ZarrFileSystemTest() { + public ZarrFactoryTest() { - this.factory = new N5Factory(); + this.factory = getFactory(); } + @Override abstract protected String tempN5Location(); + @Override public N5Factory getFactory() { + if (factory == null) { + factory = new N5Factory(); + } return factory; } @@ -95,311 +83,50 @@ public ZarrFileSystemTest() { } } - public static class ZarrAwsS3Tests extends N5AmazonS3Tests implements StorageSchemeWrappedN5Test { - private final N5Factory factory; - - public ZarrAwsS3Tests() { - - this.factory = new N5Factory() { - - @Override AmazonS3 createS3(String uri) { - - return getS3(); - } + public static class ZarrFileSystemTest extends ZarrFactoryTest { - @Override String getS3Bucket(String uri) { + @Override public Class getBackendTargetClass() { - if (useBackend) { - return super.getS3Bucket(uri); - } - try { - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(0, path.indexOf('/')); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - }; + return FileSystemKeyValueAccess.class; } @Override protected String tempN5Location() { try { - return new URI("http", "localhost:8001", "/" + tempBucketName(getS3()) + tempContainerPath(), null, null).toString(); - } catch (URISyntaxException e) { + return Files.createTempDirectory("zarr-test").toUri().getPath(); + } catch (IOException e) { throw new RuntimeException(e); } } + } - @Override public N5Factory getFactory() { - - return factory; - } - - @Override public N5Factory.StorageFormat getStorageFormat() { - - return N5Factory.StorageFormat.ZARR; - } - - @Override protected N5Writer createN5Writer() { - - return getWriter(tempN5Location()); - } - - @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { - - factory.gsonBuilder(gson); - return getWriter(location); - } - - @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { - - factory.gsonBuilder(gson); - return getReader(location); - } - - @Override protected N5Writer createN5Writer(String location) { - - return getWriter(location); - } - - @Override protected N5Reader createN5Reader(String location) { - - return getReader(location); - } - - @Override - @Test - public void testCreateDataset() { - - final DatasetAttributes info; - try (N5Writer n5 = createN5Writer()) { - n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, getCompressions()[0]); - - assertTrue("Dataset does not exist", n5.exists(datasetName)); - - info = n5.getDatasetAttributes(datasetName); - assertArrayEquals(dimensions, info.getDimensions()); - assertArrayEquals(blockSize, info.getBlockSize()); - assertEquals(DataType.UINT64, info.getDataType()); - assertEquals(getCompressions()[0].getClass(), info.getCompression().getClass()); - } - } - - @Override - @Test - public void testVersion() throws NumberFormatException { - - try (final N5Writer writer = createN5Writer()) { - - final ZarrKeyValueWriter zarr = (ZarrKeyValueWriter)writer; - final N5Reader.Version n5Version = writer.getVersion(); - final N5Reader.Version expectedVersion = new N5Reader.Version(2, 0, 0); - assertEquals(n5Version, expectedVersion); - } - } - - @Override - @Test - @Ignore("Zarr does not currently support mode 1 data blocks.") - public void testMode1WriteReadByteBlock() { - - } + public static class ZarrAmazonS3Test extends ZarrFactoryTest { - @Override - @Test - @Ignore("Zarr does not currently support mode 2 data blocks and serialized objects.") - public void testWriteReadSerializableBlock() { + @Override public Class getBackendTargetClass() { + return AmazonS3KeyValueAccess.class; } - @Test - @Override - public void testAttributes() { - - try (final N5Writer n5 = createN5Writer()) { - n5.createGroup(groupName); - - n5.setAttribute(groupName, "key1", "value1"); - // length 2 because it includes "zarr_version" - Assert.assertEquals(2, n5.listAttributes(groupName).size()); - /* class interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); - /* type interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { - - }.getType())); - - final Map newAttributes = new HashMap<>(); - newAttributes.put("key2", "value2"); - newAttributes.put("key3", "value3"); - n5.setAttributes(groupName, newAttributes); - - Assert.assertEquals(4, n5.listAttributes(groupName).size()); - /* class interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); - Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", String.class)); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); - /* type interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { - - }.getType())); - Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", new TypeToken() { - - }.getType())); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { - - }.getType())); - - n5.setAttribute(groupName, "key1", 1); - n5.setAttribute(groupName, "key2", 2); - - Assert.assertEquals(4, n5.listAttributes(groupName).size()); - /* class interface */ - Assert.assertEquals(new Integer(1), n5.getAttribute(groupName, "key1", Integer.class)); - Assert.assertEquals(new Integer(2), n5.getAttribute(groupName, "key2", Integer.class)); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); - /* type interface */ - Assert - .assertEquals( - new Integer(1), - n5.getAttribute(groupName, "key1", new TypeToken() { - - }.getType())); - Assert - .assertEquals( - new Integer(2), - n5.getAttribute(groupName, "key2", new TypeToken() { - - }.getType())); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { - - }.getType())); - - n5.removeAttribute(groupName, "key1"); - n5.removeAttribute(groupName, "key2"); - n5.removeAttribute(groupName, "key3"); - Assert.assertEquals(1, n5.listAttributes(groupName).size()); - } - } - - @Test - @Override - @Ignore - public void testNullAttributes() throws IOException { - - // serializeNulls must be on for Zarr to be able to write datasets with raw compression - - /* serializeNulls*/ - try (N5Writer writer = createN5Writer(tempN5Location(), new GsonBuilder().serializeNulls())) { - - writer.createGroup(groupName); - writer.setAttribute(groupName, "nullValue", null); - assertNull(writer.getAttribute(groupName, "nullValue", Object.class)); - assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "nullValue", JsonElement.class)); - final HashMap nulls = new HashMap<>(); - nulls.put("anotherNullValue", null); - nulls.put("structured/nullValue", null); - nulls.put("implicitNulls[3]", null); - writer.setAttributes(groupName, nulls); - - assertNull(writer.getAttribute(groupName, "anotherNullValue", Object.class)); - assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); - - assertNull(writer.getAttribute(groupName, "structured/nullValue", Object.class)); - assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); - - assertNull(writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); - assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[3]", JsonElement.class)); - - assertNull(writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); - assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[1]", JsonElement.class)); - - /* Negative test; a value that truly doesn't exist will still return `null` but will also return `null` when querying as a `JsonElement` */ - assertNull(writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); - assertNull(writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); + @Override public N5Factory getFactory() { - assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); - assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); + if (factory == null) { + factory = new N5Factory() { - /* check existing value gets overwritten */ - writer.setAttribute(groupName, "existingValue", 1); - assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); - writer.setAttribute(groupName, "existingValue", null); - assertThrows(N5Exception.N5ClassCastException.class, () -> writer.getAttribute(groupName, "existingValue", Integer.class)); - assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); + @Override AmazonS3 createS3(String uri) { - writer.remove(); + return MockS3Factory.getOrCreateS3(); + } + }; } + return factory; } - @Test - @Override - @Ignore - public void testRootLeaves() { - - // This tests serializing primitives, and arrays at the root of an n5's attributes, - // since .zattrs must be a json object, this would test invalide behavior for zarr, - // therefore this test is ignored. - } - - @Test - @Override - public void testReaderCreation() { - - final String canonicalPath = tempN5Location(); - try (N5Writer writer = createN5Writer(canonicalPath)) { - - final N5Reader n5r = createN5Reader(canonicalPath); - assertNotNull(n5r); - - // existing directory without attributes is okay; - // Remove and create to remove attributes store - writer.remove("/"); - writer.createGroup("/"); - final N5Reader na = createN5Reader(canonicalPath); - assertNotNull(na); - - // existing location with attributes, but no version - writer.remove("/"); - writer.createGroup("/"); - writer.setAttribute("/", "mystring", "ms"); - final N5Reader wa = createN5Reader(canonicalPath); - assertNotNull(wa); - - // non-existent directory should fail - writer.remove("/"); - assertThrows( - "Non-existant location throws error", - N5Exception.N5IOException.class, - () -> { - final N5Reader test = createN5Reader(canonicalPath); - test.list("/"); - }); - } - } + @Override protected String tempN5Location() { - @Test - @Override - public void testWriteReadStringBlock() { - - DataType dataType = DataType.STRING; - int[] blockSize = new int[]{3, 2, 1}; - String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-รพ"}; - Compression[] compressions = this.getCompressions(); - - for (Compression compression : compressions) { - System.out.println("Testing " + compression.getType() + " " + dataType); - - try (final N5Writer n5 = createN5Writer()) { - n5.createDataset("/test/group/dataset", dimensions, blockSize, dataType, compression); - DatasetAttributes attributes = n5.getDatasetAttributes("/test/group/dataset"); - StringDataBlock dataBlock = new ZarrStringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); - n5.writeBlock("/test/group/dataset", attributes, dataBlock); - DataBlock loadedDataBlock = n5.readBlock("/test/group/dataset", attributes, 0L, 0L, 0L); - assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); - assertTrue(n5.remove("/test/group/dataset")); - } + try { + return new URI("http", "localhost:8001", "/" + tempBucketName(factory.createS3(null)) + tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); } } } From fb3282b2d0f3d62b4deba28d8b17c8213537dec4 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Feb 2024 16:32:18 -0500 Subject: [PATCH 10/31] refactor: extract GoogleCloudStorage logic to n5-google-cloud --- pom.xml | 6 +++ .../saalfeldlab/n5/universe/N5Factory.java | 39 +++++-------------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 2186f6b..d186d87 100644 --- a/pom.xml +++ b/pom.xml @@ -115,6 +115,7 @@ 2.1.0 7.0.0 4.0.2 + 4.0.1-SNAPSHOT 1.2.2-SNAPSHOT 4.0.3-SNAPSHOT @@ -191,6 +192,11 @@ tests test + + com.google.cloud + google-cloud-nio + test + org.janelia.saalfeldlab n5-hdf5 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index b67170c..cc7a313 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -36,7 +36,6 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; -import java.util.Iterator; import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.regex.Matcher; @@ -44,9 +43,8 @@ import net.imglib2.util.Pair; import net.imglib2.util.ValuePair; -import org.janelia.saalfeldlab.googlecloud.GoogleCloudResourceManagerClient; -import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageClient; import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI; +import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; @@ -69,13 +67,8 @@ import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3URI; -import com.google.cloud.resourcemanager.Project; -import com.google.cloud.resourcemanager.ResourceManager; import com.google.cloud.storage.Storage; import com.google.gson.GsonBuilder; @@ -113,7 +106,7 @@ public class N5Factory implements Serializable { private boolean s3Anonymous = true; private boolean s3RetryWithCredentials = false; private String s3Endpoint; - private boolean createS3Bucket = false; + private boolean createBucket = false; public N5Factory hdf5DefaultBlockSize(final int... blockSize) { @@ -305,11 +298,9 @@ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxExceptio } } - private Storage createGoogleCloudStorage() { + Storage createGoogleCloudStorage() { - final GoogleCloudStorageClient storageClient = new GoogleCloudStorageClient(); - final Storage storage = storageClient.create(); - return storage; + return GoogleCloudUtils.createGoogleCloudStorage(googleCloudProjectId); } /** @@ -383,17 +374,7 @@ public N5HDF5Writer openHDF5Writer(final String path) { public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxException { final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(N5URI.encodeAsUri(uri)); - final GoogleCloudStorageClient storageClient; - if (googleCloudProjectId == null) { - final ResourceManager resourceManager = new GoogleCloudResourceManagerClient().create(); - final Iterator projectsIterator = resourceManager.list().iterateAll().iterator(); - if (!projectsIterator.hasNext()) - return null; - storageClient = new GoogleCloudStorageClient(projectsIterator.next().getProjectId()); - } else - storageClient = new GoogleCloudStorageClient(googleCloudProjectId); - - final Storage storage = storageClient.create(); + final Storage storage = createGoogleCloudStorage(); final GoogleCloudStorageKeyValueAccess googleCloudBackend = new GoogleCloudStorageKeyValueAccess(storage, googleCloudUri.getBucket(), false); if (lastExtension(uri).startsWith(".zarr")) { @@ -607,9 +588,7 @@ public N5Writer getWriter(final String uri) { private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); - final GoogleCloudStorageClient storageClient = new GoogleCloudStorageClient(); - final Storage storage = storageClient.create(); - return new GoogleCloudStorageKeyValueAccess(storage, googleCloudUri.getBucket(), false); + return new GoogleCloudStorageKeyValueAccess(factory.createGoogleCloudStorage(), googleCloudUri.getBucket(), factory.createBucket); } private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, final N5Factory factory) { @@ -617,7 +596,7 @@ private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, f final String uriString = uri.toString(); final AmazonS3 s3 = factory.createS3(uriString); - return new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uriString), factory.createS3Bucket); + return new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uriString), factory.createBucket); } private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { @@ -782,7 +761,7 @@ private static N5Reader getReader(StorageFormat storage, URI uri, N5Factory fact private static N5Writer getWriter(StorageFormat storage, URI uri, N5Factory factory) { - factory.createS3Bucket = true; + factory.createBucket = true; final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(uri, factory); final String containerPath; /* Any more special cases? google? */ @@ -791,7 +770,7 @@ private static N5Writer getWriter(StorageFormat storage, URI uri, N5Factory fact } else containerPath = uri.getPath(); final N5Writer writer = StorageFormat.getWriter(storage, access, containerPath, factory); - factory.createS3Bucket = false; + factory.createBucket = false; return writer; } From d18da97b1b8a3b2b416c882e05b3c4e191fe49aa Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Feb 2024 16:34:16 -0500 Subject: [PATCH 11/31] feat(test): add GoogleCloudStorage backend tests --- .../saalfeldlab/n5/universe/N5Factory.java | 6 +- .../n5/universe/N5StorageTests.java | 56 +++++++++++-------- .../n5/universe/ZarrStorageTests.java | 35 ++++++++++++ 3 files changed, 70 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index cc7a313..979c47f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -604,9 +604,7 @@ private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI ur return new FileSystemKeyValueAccess(FileSystems.getDefault()); } - private final static Pattern GS_SCHEME = Pattern.compile("gs", Pattern.CASE_INSENSITIVE); private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); - private final static Pattern GS_HOST = Pattern.compile("(cloud\\.google|storage\\.googleapis)\\.com", Pattern.CASE_INSENSITIVE); private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); /** @@ -622,9 +620,9 @@ enum KeyValueAccessBackend implements Predicate, BiFunction { final String scheme = uri.getScheme(); final boolean hasScheme = scheme != null; - return hasScheme && GS_SCHEME.asPredicate().test(scheme) + return hasScheme && GoogleCloudUtils.GS_SCHEME.asPredicate().test(scheme) || hasScheme && HTTPS_SCHEME.asPredicate().test(scheme) - && uri.getHost() != null && GS_HOST.asPredicate().test(uri.getHost()); + && uri.getHost() != null && GoogleCloudUtils.GS_HOST.asPredicate().test(uri.getHost()); }, N5Factory::newGoogleCloudKeyValueAccess), AWS(uri -> { final String scheme = uri.getScheme(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index 0f5ac15..d34c7e2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -1,12 +1,15 @@ package org.janelia.saalfeldlab.n5.universe; import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Storage; import com.google.gson.GsonBuilder; import org.janelia.saalfeldlab.n5.AbstractN5Test; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTest; +import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.junit.runner.RunWith; @@ -124,27 +127,34 @@ public static class N5AmazonS3Test extends N5FactoryTest { } } -// public static class N5GoogleCloudTest extends N5FactoryTest { -// -// @Override public Class getBackendTargetClass() { -// -// return GoogleCloudStorageKeyValueAccess.class; -// } -// -// @Override public N5Factory getFactory() { -// -// if (factory == null) { -// factory = new N5Factory() { -// -// -// }; -// } -// return factory; -// } -// -// @Override protected String tempN5Location() { -// -// return new URI("gs", tempBucketName(getGoogleCloudStorage()), tempContainerPath(), null).toString(); -// } -// } + public static class N5GoogleCloudTest extends N5FactoryTest { + + @Override public Class getBackendTargetClass() { + + return GoogleCloudStorageKeyValueAccess.class; + } + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory() { + + @Override Storage createGoogleCloudStorage() { + + return MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + }; + } + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("gs", N5GoogleCloudStorageTest.tempBucketName(factory.createGoogleCloudStorage()), tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index 0a3fa56..8149e04 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -1,10 +1,14 @@ package org.janelia.saalfeldlab.n5.universe; import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Storage; import com.google.gson.GsonBuilder; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTest; +import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; @@ -130,4 +134,35 @@ public static class ZarrAmazonS3Test extends ZarrFactoryTest { } } } + + public static class ZarrGoogleCloudTest extends ZarrStorageTests.ZarrFactoryTest { + + @Override public Class getBackendTargetClass() { + + return GoogleCloudStorageKeyValueAccess.class; + } + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory() { + + @Override Storage createGoogleCloudStorage() { + + return MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + }; + } + return factory; + } + + @Override protected String tempN5Location() { + + try { + return new URI("gs", N5GoogleCloudStorageTest.tempBucketName(factory.createGoogleCloudStorage()), tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } } From 13ab7dd46654adc724f8e645ef4cb92781872661 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Feb 2024 16:39:05 -0500 Subject: [PATCH 12/31] feat(test): zarr ignore python tests in n5-universe --- .../janelia/saalfeldlab/n5/universe/ZarrStorageTests.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index 8149e04..b846e95 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -12,6 +12,8 @@ import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; +import org.junit.Ignore; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -85,6 +87,12 @@ public ZarrFactoryTest() { return getReader(location); } + + @Ignore + @Override public void testReadZarrPython() {} + + @Ignore + @Override public void testReadZarrNestedPython() {} } public static class ZarrFileSystemTest extends ZarrFactoryTest { From 34e1abe83bda4040e35f9a78994167f26432adef Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Feb 2024 16:40:01 -0500 Subject: [PATCH 13/31] feat: expose StorageFormat --- .../org/janelia/saalfeldlab/n5/universe/N5Factory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 979c47f..afa68e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -679,7 +679,7 @@ static KeyValueAccess getKeyValueAccess(final URI uri, final N5Factory factory) } } - enum StorageFormat { + public enum StorageFormat { ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { @@ -700,7 +700,7 @@ enum StorageFormat { this.uriTest = test; } - private static StorageFormat guessStorageFromUri(URI uri) { + public static StorageFormat guessStorageFromUri(URI uri) { for (StorageFormat format : StorageFormat.values()) { if (format.uriTest.test(uri)) @@ -709,7 +709,7 @@ private static StorageFormat guessStorageFromUri(URI uri) { return null; } - private static Pair parseUri(String uri) throws URISyntaxException { + public static Pair parseUri(String uri) throws URISyntaxException { final Pair storageFromScheme = getStorageFromNestedScheme(uri); final URI asUri = N5URI.encodeAsUri(storageFromScheme.getB()); @@ -720,7 +720,7 @@ private static Pair parseUri(String uri) throws URISyntaxExc } - private static Pair getStorageFromNestedScheme(String uri) { + public static Pair getStorageFromNestedScheme(String uri) { final Matcher storageSchemeMatcher = StorageFormat.STORAGE_SCHEME_PATTERN.matcher(uri); storageSchemeMatcher.matches(); From 7b2a9ecbd7e7456ec8771286e6a4d61edfb6640b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 28 Feb 2024 14:52:12 -0500 Subject: [PATCH 14/31] feat: support nested storage format schemes. Support explicitly opening by either storage format, key value access, or both refactor: major code refactor for deduplication and extracted backend specific logic to respective libraries. --- .../saalfeldlab/n5/universe/N5Factory.java | 658 ++++++++---------- 1 file changed, 278 insertions(+), 380 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index afa68e2..88df4a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -26,31 +26,19 @@ */ package org.janelia.saalfeldlab.n5.universe; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.Serializable; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.function.BiFunction; -import java.util.function.Predicate; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Storage; +import com.google.gson.GsonBuilder; import net.imglib2.util.Pair; import net.imglib2.util.ValuePair; +import org.apache.commons.lang3.function.TriFunction; import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI; import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.N5FSReader; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5KeyValueReader; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Reader; @@ -66,13 +54,20 @@ import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.services.s3.AmazonS3; -import com.google.cloud.storage.Storage; -import com.google.gson.GsonBuilder; - import javax.annotation.Nullable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystems; +import java.util.Arrays; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Factory for various N5 readers and writers. Implementation specific @@ -87,20 +82,17 @@ public class N5Factory implements Serializable { private static final long serialVersionUID = -6823715427289454617L; - + private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); + private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); private static byte[] HDF5_SIG = {(byte)137, 72, 68, 70, 13, 10, 26, 10}; private int[] hdf5DefaultBlockSize = {64, 64, 64, 1, 1}; private boolean hdf5OverrideBlockSize = false; - private GsonBuilder gsonBuilder = new GsonBuilder(); private boolean cacheAttributes = true; - private String zarrDimensionSeparator = "."; private boolean zarrMapN5DatasetAttributes = true; private boolean zarrMergeAttributes = true; - private String googleCloudProjectId = null; - private String s3Region = null; private AWSCredentials s3Credentials = null; private boolean s3Anonymous = true; @@ -108,6 +100,40 @@ public class N5Factory implements Serializable { private String s3Endpoint; private boolean createBucket = false; + private static boolean isHDF5(String path) { + + final File f = new File(path); + if (!f.exists() || !f.isFile()) + return false; + + try (final FileInputStream in = new FileInputStream(f)) { + final byte[] sig = new byte[8]; + in.read(sig); + return Arrays.equals(sig, HDF5_SIG); + } catch (final IOException e) { + return false; + } + } + + private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { + + final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); + return new GoogleCloudStorageKeyValueAccess(factory.createGoogleCloudStorage(), googleCloudUri.getBucket(), factory.createBucket); + } + + private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, final N5Factory factory) { + + final String uriString = uri.toString(); + final AmazonS3 s3 = factory.createS3(uriString); + + return new AmazonS3KeyValueAccess(s3, uri.toString(), factory.createBucket); + } + + private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { + + return new FileSystemKeyValueAccess(FileSystems.getDefault()); + } + public N5Factory hdf5DefaultBlockSize(final int... blockSize) { hdf5DefaultBlockSize = blockSize; @@ -192,59 +218,58 @@ public N5Factory s3Region(final String s3Region) { return this; } - private static boolean isHDF5Writer(final String path) { - - if (path.matches("(?i).*\\.(h(df)?5)")) - return true; - else - return false; - } - - private static boolean isHDF5Reader(final String path) throws N5IOException { + AmazonS3 createS3(final String uri) { - if (Files.isRegularFile(Paths.get(path))) { - /* optimistic */ - if (isHDF5Writer(path)) - return true; - else - return isHDF5(path); + try { + return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region); + } catch (final Exception e) { + throw new N5Exception("Could not create s3 client from uri: " + uri, e); } - return false; } - private static boolean isHDF5(String path) { - - final File f = new File(path); - if (!f.exists() || !f.isFile()) - return false; + Storage createGoogleCloudStorage() { - try (final FileInputStream in = new FileInputStream(f)) { - final byte[] sig = new byte[8]; - in.read(sig); - return Arrays.equals(sig, HDF5_SIG); - } catch (final IOException e) { - return false; - } + return GoogleCloudUtils.createGoogleCloudStorage(googleCloudProjectId); } - AmazonS3 createS3(final String uri) { - - try { - return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region); - } catch (final Exception e) { - throw new N5Exception("Could not create s3 client from uri: " + uri, e); + /** + * Test the provided {@link URI} to and return the appropriate {@link KeyValueAccess}. + * If no appropriate {@link KeyValueAccess} is found, may be null + * + * @param uri to create a {@link KeyValueAccess} from. + * @return the {@link KeyValueAccess} and container path, or null if none are valid + */ + Pair getKeyValueAccess(final URI uri) { + + /*NOTE: The order of these tests is very important, as the predicates for each + * backend take into account reasonable defaults when possible. + * Here we test from most to least restrictive. + * See the Javadoc for more details. */ + for (KeyValueAccessBackend backend : KeyValueAccessBackend.values()) { + final KeyValueAccess kva = backend.apply(uri, this); + if (kva != null) + return new ValuePair<>(kva, backend.parseContainerPath.apply(uri)); } + return null; } /** - * Open an {@link N5Reader} for N5 filesystem. + * Open an {@link N5Reader} over an N5 Container. + * + * NOTE: The name seems to imply that this will open any N5Reader, over a + * {@link FileSystemKeyValueAccess} however that is misleading. Instead + * this will open any N5Container that is a valid {@link StorageFormat#N5}. + * This is partially why it is deprecated, as well as the redundant + * implementation with {@link N5Factory#openReader(StorageFormat, URI)}. * * @param path path to the n5 root folder - * @return the N5FsReader + * @return the N5Reader + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5FSReader openFSReader(final String path) { + @Deprecated + public N5Reader openFSReader(final String path) { - return new N5FSReader(path, gsonBuilder); + return openN5ContainerWithStorageFormat(StorageFormat.N5, path, this::openReader); } /** @@ -254,11 +279,13 @@ public N5FSReader openFSReader(final String path) { * constructors.} * * @param path path to the zarr directory - * @return the N5ZarrReader + * @return the N5Reader + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5ZarrReader openZarrReader(final String path) { + @Deprecated + public N5Reader openZarrReader(final String path) { - return new N5ZarrReader(path, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); + return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openReader); } /** @@ -269,70 +296,122 @@ public N5ZarrReader openZarrReader(final String path) { * constructors. * * @param path path to the hdf5 file - * @return the N5HDF5Reader + * @return the N5Reader + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5HDF5Reader openHDF5Reader(final String path) { + @Deprecated + public N5Reader openHDF5Reader(final String path) { - return new N5HDF5Reader(path, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + return openN5ContainerWithStorageFormat( + StorageFormat.HDF5, + path, + (format, uri) -> openReader(format, null, uri.getPath()) + ); } /** * Open an {@link N5Reader} for Google Cloud. * * @param uri uri to the google cloud object - * @return the N5GoogleCloudStorageReader + * @return the N5Reader * @throws URISyntaxException if uri is malformed */ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxException { - final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(N5URI.encodeAsUri(uri)); - final Storage storage = createGoogleCloudStorage(); + return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, this::openReader); + } - final GoogleCloudStorageKeyValueAccess googleCloudBackend = new GoogleCloudStorageKeyValueAccess(storage, - googleCloudUri.getBucket(), false); - if (lastExtension(uri).startsWith(".zarr")) { - return new ZarrKeyValueReader(googleCloudBackend, googleCloudUri.getKey(), gsonBuilder, - zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); - } else { - return new N5KeyValueReader(googleCloudBackend, googleCloudUri.getKey(), gsonBuilder, cacheAttributes); + /** + * Open an {@link N5Reader} for AWS S3. + * + * @param uri uri to the amazon s3 object + * @return the N5Reader + * @throws URISyntaxException if uri is malformed + */ + public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException { + + return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, this::openReader); + } + + /** + * Open an {@link N5Reader} over a FileSytem. + * + * @param uri uri to the N5Reader + * @return the N5Reader + * @throws URISyntaxException if uri is malformed + */ + public N5Reader openFileSystemReader(final String uri) throws URISyntaxException { + + return openN5ContainerWithBackend(KeyValueAccessBackend.FILE, uri, this::openReader); + } + + public N5Reader openReader(final StorageFormat format, final String uri) { + + try { + return openN5Container(format, N5URI.encodeAsUri(uri), this::openReader); + } catch (URISyntaxException e) { + throw new N5Exception(e); } } - Storage createGoogleCloudStorage() { + public N5Reader openReader(final StorageFormat format, final URI uri) { - return GoogleCloudUtils.createGoogleCloudStorage(googleCloudProjectId); + return openN5Container(format, uri, this::openReader); } /** - * Open an {@link N5Reader} for AWS S3. + * Open an {@link N5Reader} based on some educated guessing from the url. * - * @param uri uri to the amazon s3 object + * @param uri the location of the root location of the store * @return the N5Reader - * @throws URISyntaxException if uri is malformed */ - public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException { + public N5Reader openReader(final String uri) { - final AmazonS3 s3 = createS3(N5URI.encodeAsUri(uri).toString()); + return openN5Container(uri, this::openReader, this::openReader); + } + + private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, String containerPath) { + + if (storage == null) { + for (StorageFormat format : StorageFormat.values()) { + try { + return openReader(format, access, containerPath); + } + catch (Exception e) {} + } + throw new N5Exception("Unable to open " + containerPath + " as N5Reader"); - // when, if ever do we want to creat a bucket? - final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uri), false); - if (lastExtension(uri).startsWith(".zarr")) { - return new ZarrKeyValueReader(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, zarrMapN5DatasetAttributes, - zarrMergeAttributes, cacheAttributes); } else { - return new N5KeyValueReader(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, cacheAttributes); + + switch (storage) { + case N5: + return new N5KeyValueReader(access, containerPath, gsonBuilder, cacheAttributes); + case ZARR: + return new ZarrKeyValueReader(access, containerPath, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes); + case HDF5: + return new N5HDF5Reader(containerPath, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + } + return null; } } /** - * Open an {@link N5Writer} for N5 filesystem. + * Open an {@link N5Writer} for N5 Container. + * + * NOTE: The name seems to imply that this will open any N5Writer, over a + * {@link FileSystemKeyValueAccess} however that is misleading. Instead + * this will open any N5Container that is a valid {@link StorageFormat#N5}. + * This is partially why it is deprecated, as well as the redundant + * implementation with {@link N5Factory#openWriter(StorageFormat, URI)}. * * @param path path to the n5 directory - * @return the N5FSWriter + * @return the N5Writer + * @deprecated use {@link N5Factory#openWriter(StorageFormat, URI)} instead */ - public N5FSWriter openFSWriter(final String path) { + @Deprecated + public N5Writer openFSWriter(final String path) { - return new N5FSWriter(path, gsonBuilder); + return openN5ContainerWithStorageFormat(StorageFormat.N5, path, this::openWriter); } /** @@ -342,11 +421,13 @@ public N5FSWriter openFSWriter(final String path) { * constructors. * * @param path path to the zarr directory - * @return the N5ZarrWriter + * @return the N5Writer + * @deprecated use {@link N5Factory#openWriter(StorageFormat, URI)}) instead */ - public N5ZarrWriter openZarrWriter(final String path) { + @Deprecated + public N5Writer openZarrWriter(final String path) { - return new N5ZarrWriter(path, gsonBuilder, zarrDimensionSeparator, zarrMapN5DatasetAttributes, true); + return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openWriter); } /** @@ -358,10 +439,16 @@ public N5ZarrWriter openZarrWriter(final String path) { * * @param path path to the hdf5 file * @return the N5HDF5Writer + * @deprecated use {@link N5Factory#openReader(StorageFormat, URI)} instead */ - public N5HDF5Writer openHDF5Writer(final String path) { + @Deprecated + public N5Writer openHDF5Writer(final String path) { - return new N5HDF5Writer(path, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); + return openN5ContainerWithStorageFormat( + StorageFormat.HDF5, + path, + (format, uri) -> openWriter(format, null, uri.getPath()) + ); } /** @@ -373,16 +460,7 @@ public N5HDF5Writer openHDF5Writer(final String path) { */ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxException { - final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(N5URI.encodeAsUri(uri)); - final Storage storage = createGoogleCloudStorage(); - final GoogleCloudStorageKeyValueAccess googleCloudBackend = new GoogleCloudStorageKeyValueAccess(storage, - googleCloudUri.getBucket(), false); - if (lastExtension(uri).startsWith(".zarr")) { - return new ZarrKeyValueWriter(googleCloudBackend, googleCloudUri.getKey(), gsonBuilder, - zarrMapN5DatasetAttributes, zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes); - } else { - return new N5KeyValueWriter(googleCloudBackend, googleCloudUri.getKey(), gsonBuilder, cacheAttributes); - } + return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, this::openWriter); } /** @@ -394,62 +472,15 @@ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxExceptio */ public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException { - final AmazonS3 s3 = createS3(N5URI.encodeAsUri(uri).toString()); - // when, if ever do we want to creat a bucket? - final AmazonS3KeyValueAccess s3kv = new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uri), false); - if (lastExtension(uri).startsWith(".zarr")) { - return new ZarrKeyValueWriter(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, zarrMapN5DatasetAttributes, - zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes); - } else { - return new N5KeyValueWriter(s3kv, AmazonS3Utils.getS3Key(uri), gsonBuilder, cacheAttributes); - } + return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, this::openWriter); } - /** - * Open an {@link N5Reader} based on some educated guessing from the url. - * - * @param uri the location of the root location of the store - * @return the N5Reader - */ - public N5Reader openReader(final String uri) { + public N5Writer openWriter(final StorageFormat format, final URI uri) { - try { - final URI encodedUri = N5URI.encodeAsUri(uri); - final String scheme = encodedUri.getScheme(); - if (scheme == null) - ; - else if (scheme.equals("file")) { - try { - return openFileBasedN5Reader(Paths.get(encodedUri).toFile().getCanonicalPath()); - } catch (final IOException e) { - throw new N5Exception.N5IOException(e); - } - } else if (scheme.equals("s3")) - return openAWSS3Reader(uri); - else if (scheme.equals("gs")) - return openGoogleCloudReader(uri); - else if (encodedUri.getHost() != null && scheme.equals("https") || scheme.equals("http")) { - if (encodedUri.getHost().matches(".*cloud\\.google\\.com") - || encodedUri.getHost().matches(".*storage\\.googleapis\\.com")) - return openGoogleCloudReader(uri); - else //if (encodedUri.getHost().matches(".*s3.*")) //< This is too fragile for what people in the wild are doing with their S3 instances, for now catch all - return openAWSS3Reader(uri); - } - } catch (final URISyntaxException ignored) { - } - // return null; - return openFileBasedN5Reader(uri); - } - - private N5Reader openFileBasedN5Reader(final String url) { - - if (isHDF5Reader(url)) - return openHDF5Reader(url); - else if (lastExtension(url).startsWith(".zarr")) - return openZarrReader(url); - - else - return openFSReader(url); + createBucket = true; + final N5Writer n5Writer = openN5Container(format, uri, this::openWriter); + createBucket = false; + return n5Writer; } /** @@ -460,152 +491,98 @@ else if (lastExtension(url).startsWith(".zarr")) */ public N5Writer openWriter(final String uri) { - try { - final URI encodedUri = N5URI.encodeAsUri(uri); - final String scheme = encodedUri.getScheme(); - if (scheme == null) - ; - else if (scheme.equals("file")) - return openFileBasedN5Writer(encodedUri.getPath()); - else if (scheme.equals("s3")) - return openAWSS3Writer(uri); - else if (scheme.equals("gs")) - return openGoogleCloudWriter(uri); - else if (encodedUri.getHost() != null && scheme.equals("https") || scheme.equals("http")) { - if (encodedUri.getHost().matches(".*s3.*")) - return openAWSS3Writer(uri); - else if (encodedUri.getHost().matches(".*cloud\\.google\\.com") - || encodedUri.getHost().matches(".*storage\\.googleapis\\.com")) - return openGoogleCloudWriter(uri); - } - } catch (final URISyntaxException e) { - } - return openFileBasedN5Writer(uri); - } - - private N5Writer openFileBasedN5Writer(final String url) { - - if (isHDF5Writer(url)) - return openHDF5Writer(url); - else if (lastExtension(url).startsWith(".zarr")) - return openZarrWriter(url); - else - return openFSWriter(url); - } - - private static String lastExtension(final String path) { - - final int i = path.lastIndexOf('.'); - if (i >= 0) - return path.substring(path.lastIndexOf('.')); - else - return ""; + return openN5Container(uri, this::openWriter, this::openWriter); } - public N5Reader getReader(final String uri) { + private N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, final String containerPath) { - try { - final Pair storageAndUri = StorageFormat.parseUri(uri); - final StorageFormat format = storageAndUri.getA(); - final URI asUri = storageAndUri.getB(); - if (format != null) - return format.openReader(asUri, this); - - final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(asUri, this); - if (access == null) { - throw new N5Exception("Cannot get KeyValueAccess at " + asUri); - } - final String containerPath; - if (access instanceof AmazonS3KeyValueAccess) { - containerPath = AmazonS3Utils.getS3Key(asUri.toString()); - } else { - containerPath = asUri.getPath(); + if (storage == null) { + for (StorageFormat format : StorageFormat.values()) { + try { + return openWriter(format, access, containerPath); + } + catch (Exception e) {} } + throw new N5Exception("Unable to open " + containerPath + " as N5Writer"); - Exception exception = null; - for (StorageFormat storageFormat : StorageFormat.values()) { - // all possible attempts at making an hdf5 reader will be done by now - // and HDF5 does not use a KeyValueAccess - // revisit this if more backends are added - if (storageFormat == StorageFormat.HDF5) - continue; + } else { - try { - return StorageFormat.getReader(storageFormat, access, containerPath, this); - } catch (Exception e) { - exception = e; - } + switch (storage) { + case ZARR: + return new ZarrKeyValueWriter(access, containerPath, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes); + case N5: + return new N5KeyValueWriter(access, containerPath, gsonBuilder, cacheAttributes); + case HDF5: + return new N5HDF5Writer(containerPath, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize); } - if (exception != null) - throw new N5Exception("Unable to open " + uri + " as N5 Container", exception); - } catch (final URISyntaxException ignored) { } return null; } - public N5Writer getWriter(final String uri) { + private T openN5ContainerWithStorageFormat( + final StorageFormat format, + final String uri, + final BiFunction openWithFormat + ) { try { - - final Pair storageAndUri = StorageFormat.parseUri(uri); - final StorageFormat format = storageAndUri.getA(); - final URI asUri = storageAndUri.getB(); - if (format != null) - return format.openWriter(asUri, this); - else { - try { - return openHDF5Writer(uri); - } catch (Exception ignored) { - } - } - final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(asUri, this); - if (access == null) { - throw new N5Exception("Cannot create KeyValueAcccess for URI " + uri); - } - final String containerPath; - if (access instanceof AmazonS3KeyValueAccess) { - containerPath = AmazonS3Utils.getS3Key(asUri.toString()); - } else { - containerPath = asUri.getPath(); - } - try { - final N5Writer zarrN5Writer = StorageFormat.getWriter(StorageFormat.ZARR, access, containerPath, this); - if (zarrN5Writer != null) - return zarrN5Writer; - } catch (Exception ignored) { - } - try { - final N5Writer n5Writer = StorageFormat.getWriter(StorageFormat.N5, access, containerPath, this); - if (n5Writer != null) - return n5Writer; - } catch (Exception ignored) { - } - } catch (final URISyntaxException ignored) { + final URI asUri = StorageFormat.parseUri(uri).getB(); + return openWithFormat.apply(format, asUri); + } catch (URISyntaxException e) { + throw new N5Exception("Cannot create N5 Container (" + format + ") at " + uri, e); } - return null; } - private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { + private T openN5ContainerWithBackend( + final KeyValueAccessBackend backend, + final String uri, + final TriFunction openWithBackend + ) throws URISyntaxException { - final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); - return new GoogleCloudStorageKeyValueAccess(factory.createGoogleCloudStorage(), googleCloudUri.getBucket(), factory.createBucket); + final Pair formatAndUri = StorageFormat.parseUri(uri); + final URI asUri = formatAndUri.getB(); + final KeyValueAccess kva = backend.apply(asUri, this); + final String containerPath = backend.parseContainerPath.apply(asUri); + return openWithBackend.apply(formatAndUri.getA(), kva, containerPath); } - private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, final N5Factory factory) { + private T openN5Container( + final StorageFormat storageFormat, + final URI uri, + final TriFunction openWithKva) { - final String uriString = uri.toString(); - final AmazonS3 s3 = factory.createS3(uriString); - - return new AmazonS3KeyValueAccess(s3, AmazonS3Utils.getS3Bucket(uriString), factory.createBucket); + final Pair accessAndContainerPath = getKeyValueAccess(uri); + if (accessAndContainerPath == null) + throw new N5Exception("Cannot get KeyValueAccess at " + uri); + final KeyValueAccess access = accessAndContainerPath.getA(); + final String containerPath = accessAndContainerPath.getB(); + return openWithKva.apply(storageFormat, access, containerPath); } - private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { + private T openN5Container( + final String uri, + final BiFunction openWithFormat, + final TriFunction openWithKva) { - return new FileSystemKeyValueAccess(FileSystems.getDefault()); - } + final Pair storageAndUri; + try { + storageAndUri = StorageFormat.parseUri(uri); + } catch (URISyntaxException e) { + throw new N5Exception("Unable to open " + uri + " as N5 Container", e); + } + final StorageFormat format = storageAndUri.getA(); + final URI asUri = storageAndUri.getB(); + if (format != null) + return openWithFormat.apply(format, asUri); - private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); - private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); + final Pair accessAndContainerPath = getKeyValueAccess(asUri); + if (accessAndContainerPath == null) + throw new N5Exception("Cannot get KeyValueAccess at " + asUri); + final KeyValueAccess access = accessAndContainerPath.getA(); + final String containerPath = accessAndContainerPath.getB(); + + return openWithKva.apply(null, access, containerPath); + } /** * Enum to discover and provide {@link KeyValueAccess} for {@link N5Reader}s and {@link N5Writer}s. @@ -616,7 +593,6 @@ private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI ur * {@link KeyValueAccess} that is generated. */ enum KeyValueAccessBackend implements Predicate, BiFunction { - //TODO Caleb: Move all of the pattern matching, tests, and magic strings to static fields/methods in the respective KVA GOOGLE_CLOUD(uri -> { final String scheme = uri.getScheme(); final boolean hasScheme = scheme != null; @@ -629,7 +605,7 @@ enum KeyValueAccessBackend implements Predicate, BiFunction { final String scheme = uri.getScheme(); final boolean hasScheme = scheme != null; @@ -638,11 +614,18 @@ enum KeyValueAccessBackend implements Predicate, BiFunction backendTest; private final BiFunction backendGenerator; + private final Function parseContainerPath; KeyValueAccessBackend(Predicate test, BiFunction generator) { + this(test, generator, URI::getPath); + } + + KeyValueAccessBackend(Predicate test, BiFunction generator, final Function getContainerPath) { + backendTest = test; backendGenerator = generator; + parseContainerPath = getContainerPath; } @Override public KeyValueAccess apply(final URI uri, final N5Factory factory) { @@ -652,27 +635,6 @@ enum KeyValueAccessBackend implements Predicate, BiFunction Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), - N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath())), + ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches()), + N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches()), HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { - final boolean hasHdf5Extension = Pattern.compile("\\.h(df)5$", Pattern.CASE_INSENSITIVE).asPredicate().test(uri.getPath()); + final boolean hasHdf5Extension = Pattern.compile("\\.h(df)5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches(); return hasHdf5Extension || isHDF5(uri.getPath()); }); @@ -734,69 +696,5 @@ public static Pair getStorageFromNestedScheme(String uri) } return new ValuePair<>(null, uriGroup); } - - N5Reader openReader(final URI uri, final N5Factory factory) { - - return StorageFormat.getReader(this, uri, factory); - } - - N5Writer openWriter(final URI uri, final N5Factory factory) { - - return StorageFormat.getWriter(this, uri, factory); - } - - private static N5Reader getReader(StorageFormat storage, URI uri, N5Factory factory) { - - final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(uri, factory); - final String containerPath; - /* Any more special cases? google? */ - if (access instanceof AmazonS3KeyValueAccess) { - containerPath = AmazonS3Utils.getS3Key(uri.toString()); - } else - containerPath = uri.getPath(); - return StorageFormat.getReader(storage, access, containerPath, factory); - } - - private static N5Writer getWriter(StorageFormat storage, URI uri, N5Factory factory) { - - factory.createBucket = true; - final KeyValueAccess access = KeyValueAccessBackend.getKeyValueAccess(uri, factory); - final String containerPath; - /* Any more special cases? google? */ - if (access instanceof AmazonS3KeyValueAccess) { - containerPath = AmazonS3Utils.getS3Key(uri.toString()); - } else - containerPath = uri.getPath(); - final N5Writer writer = StorageFormat.getWriter(storage, access, containerPath, factory); - factory.createBucket = false; - return writer; - } - - private static N5Reader getReader(StorageFormat storage, @Nullable KeyValueAccess access, String containerPath, N5Factory factory) { - - switch (storage) { - case N5: - return new N5KeyValueReader(access, containerPath, factory.gsonBuilder, factory.cacheAttributes); - case ZARR: - return new ZarrKeyValueReader(access, containerPath, factory.gsonBuilder, factory.zarrMapN5DatasetAttributes, factory.zarrMergeAttributes, factory.cacheAttributes); - case HDF5: - return new N5HDF5Reader(containerPath, factory.hdf5OverrideBlockSize, factory.gsonBuilder, factory.hdf5DefaultBlockSize); - } - return null; - } - - private static N5Writer getWriter(StorageFormat storage, @Nullable KeyValueAccess access, String containerPath, N5Factory factory) { - - switch (storage) { - case N5: - return new N5KeyValueWriter(access, containerPath, factory.gsonBuilder, factory.cacheAttributes); - case ZARR: - return new ZarrKeyValueWriter(access, containerPath, factory.gsonBuilder, factory.zarrMapN5DatasetAttributes, factory.zarrMergeAttributes, factory.zarrDimensionSeparator, factory.cacheAttributes); - case HDF5: - return new N5HDF5Writer(containerPath, factory.hdf5OverrideBlockSize, factory.gsonBuilder, factory.hdf5DefaultBlockSize); - } - return null; - } - } } \ No newline at end of file From a057b8280de1393903507bf7f8ac4e916886e026 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 28 Feb 2024 14:52:45 -0500 Subject: [PATCH 15/31] feat(test): more storage format and backend tests --- pom.xml | 28 +++++ .../n5/universe/N5StorageTests.java | 114 +++++++++++++++--- .../universe/StorageSchemeWrappedN5Test.java | 4 +- .../n5/universe/ZarrStorageTests.java | 48 +++++++- 4 files changed, 173 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index d186d87..609d6fb 100644 --- a/pom.xml +++ b/pom.xml @@ -269,5 +269,33 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + + **Backend*.java + + + + + + + + run-backend-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index d34c7e2..6b9ce0d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -1,6 +1,7 @@ package org.janelia.saalfeldlab.n5.universe; import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.gson.GsonBuilder; import org.janelia.saalfeldlab.n5.AbstractN5Test; @@ -12,6 +13,8 @@ import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; +import org.junit.After; +import org.junit.AfterClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -19,18 +22,21 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; +import java.util.ArrayList; import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; @RunWith(Suite.class) -@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3Test.class}) +@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5AmazonS3BackendTest.class}) public class N5StorageTests { public static abstract class N5FactoryTest extends AbstractN5Test implements StorageSchemeWrappedN5Test { protected N5Factory factory; + protected final ArrayList tempWriters = new ArrayList<>(); + public N5FactoryTest() { this.factory = getFactory(); @@ -51,21 +57,21 @@ public N5FactoryTest() { return N5Factory.StorageFormat.N5; } - @Override protected N5Writer createN5Writer() { + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { - return getWriter(tempN5Location()); + factory.gsonBuilder(gson); + return createN5Writer(location); } - @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + @Override protected N5Reader createN5Reader(String location) { - factory.gsonBuilder(gson); return getReader(location); } @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { factory.gsonBuilder(gson); - return getWriter(location); + return createN5Writer(location); } @Override protected N5Writer createN5Writer(String location) { @@ -73,9 +79,16 @@ public N5FactoryTest() { return getWriter(location); } - @Override protected N5Reader createN5Reader(String location) { + @After + public void removeTempWriter() { - return getReader(location); + synchronized (tempWriters) { + + for (N5Writer tempWriter : tempWriters) { + tempWriter.remove(); + } + tempWriters.clear(); + } } } @@ -96,13 +109,26 @@ public static class N5FileSystemTest extends N5FactoryTest { } } - public static class N5AmazonS3Test extends N5FactoryTest { + public static abstract class N5AmazonS3FactoryTest extends N5FactoryTest { + + public static AmazonS3 s3 = null; + + final static String testBucket = tempBucketName(); @Override public Class getBackendTargetClass() { return AmazonS3KeyValueAccess.class; } + @AfterClass + public static void removeTestBucket() { + if (s3.doesBucketExistV2(testBucket)) + s3.deleteBucket(testBucket); + } + + } + + public static class N5AmazonS3MockTest extends N5AmazonS3FactoryTest { @Override public N5Factory getFactory() { if (factory == null) { @@ -110,7 +136,9 @@ public static class N5AmazonS3Test extends N5FactoryTest { @Override AmazonS3 createS3(String uri) { - return MockS3Factory.getOrCreateS3(); + AmazonS3 s3 = MockS3Factory.getOrCreateS3(); + N5AmazonS3FactoryTest.s3 = s3; + return s3; } }; } @@ -120,14 +148,41 @@ public static class N5AmazonS3Test extends N5FactoryTest { @Override protected String tempN5Location() { try { - return new URI("http", "localhost:8001", "/" + tempBucketName(factory.createS3(null)) + tempContainerPath(), null, null).toString(); + + return new URI("s3", testBucket, tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { + + @Override public Class getBackendTargetClass() { + + return AmazonS3KeyValueAccess.class; + } + + @Override protected String tempN5Location() { + + try { + final String s3ContainerUri = new URI("s3", testBucket, tempContainerPath(), null).toString(); + if (s3 == null) { + s3 = factory.createS3(s3ContainerUri); + } + return s3ContainerUri; } catch (URISyntaxException e) { throw new RuntimeException(e); } } } - public static class N5GoogleCloudTest extends N5FactoryTest { + public static abstract class N5GoogleCloudFactoryTest extends N5FactoryTest { + + private static String testBucket = N5GoogleCloudStorageTest.tempBucketName(); + private static Storage storage = null; + + protected abstract Storage getOrCreateStorage(); @Override public Class getBackendTargetClass() { @@ -141,20 +196,49 @@ public static class N5GoogleCloudTest extends N5FactoryTest { @Override Storage createGoogleCloudStorage() { - return MockGoogleCloudStorageFactory.getOrCreateStorage(); + final Storage storage = getOrCreateStorage(); + N5GoogleCloudFactoryTest.storage = storage; + return storage; } }; } return factory; } - @Override protected String tempN5Location() { + @Override + protected String tempN5Location() { try { - return new URI("gs", N5GoogleCloudStorageTest.tempBucketName(factory.createGoogleCloudStorage()), tempContainerPath(), null).toString(); + return new URI("gs", testBucket, tempContainerPath(), null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } } + + + + @AfterClass + public static void removeTestBucket() { + + final Bucket bucket = storage.get(testBucket); + if (bucket != null && bucket.exists()) { + storage.delete(testBucket); + } + } + } + + public static class N5GoogleCloudMockTest extends N5GoogleCloudFactoryTest { + + @Override public Class getBackendTargetClass() { + + return GoogleCloudStorageKeyValueAccess.class; + } + + @Override protected Storage getOrCreateStorage() { + + return MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java index bfbe656..0336647 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/StorageSchemeWrappedN5Test.java @@ -24,7 +24,7 @@ public interface StorageSchemeWrappedN5Test { default N5Writer getWriter(String uri) { final String uriWithStorageScheme = prependStorageScheme(uri); - final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)getFactory().getWriter(uriWithStorageScheme); + final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)getFactory().openWriter(uriWithStorageScheme); switch (getStorageFormat()){ case ZARR: assertTrue(writer instanceof ZarrKeyValueWriter); @@ -43,7 +43,7 @@ default N5Writer getWriter(String uri) { default N5Reader getReader(String uri) { final String uriWithStorageScheme = prependStorageScheme(uri); - final GsonKeyValueN5Reader reader = (GsonKeyValueN5Reader)getFactory().getReader(uriWithStorageScheme); + final GsonKeyValueN5Reader reader = (GsonKeyValueN5Reader)getFactory().openReader(uriWithStorageScheme); switch (getStorageFormat()){ case ZARR: assertTrue(reader instanceof ZarrKeyValueReader); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index b846e95..ce14c05 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -10,10 +10,10 @@ import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTest; import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.backend.BackendS3Factory; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; import org.junit.Ignore; -import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -58,7 +58,7 @@ public ZarrFactoryTest() { return getWriter(tempN5Location()); } - @Override protected N5Writer createN5Writer(String location, GsonBuilder gsonBuilder, String dimensionSeparator, boolean mapN5DatasetAttributes) { + @Override protected N5Writer createTempN5Writer(String location, GsonBuilder gsonBuilder, String dimensionSeparator, boolean mapN5DatasetAttributes) { factory.gsonBuilder(gsonBuilder); factory.zarrDimensionSeparator(dimensionSeparator); @@ -133,10 +133,48 @@ public static class ZarrAmazonS3Test extends ZarrFactoryTest { return factory; } + final String testBucket = tempBucketName(); + @Override protected String tempN5Location() { try { - return new URI("http", "localhost:8001", "/" + tempBucketName(factory.createS3(null)) + tempContainerPath(), null, null).toString(); +// return new URI("s3", testBucket, tempContainerPath(), null).toString(); + return new URI("http", "localhost:8001", "/" + testBucket + tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + + + + public static class ZarrAmazonS3BackendTest extends N5StorageTests.N5FactoryTest { + + @Override public Class getBackendTargetClass() { + + return AmazonS3KeyValueAccess.class; + } + + @Override public N5Factory getFactory() { + + if (factory == null) { + factory = new N5Factory(); // { +// +// @Override AmazonS3 createS3(String uri) { +// +// return BackendS3Factory.getOrCreateS3(); +// } +// }; + } + return factory; + } + + final String testBucket = tempBucketName(); + + @Override protected String tempN5Location() { + + try { + return new URI("s3", testBucket, tempContainerPath(), null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } @@ -145,6 +183,8 @@ public static class ZarrAmazonS3Test extends ZarrFactoryTest { public static class ZarrGoogleCloudTest extends ZarrStorageTests.ZarrFactoryTest { + private String testBucket = N5GoogleCloudStorageTest.tempBucketName(); + @Override public Class getBackendTargetClass() { return GoogleCloudStorageKeyValueAccess.class; @@ -167,7 +207,7 @@ public static class ZarrGoogleCloudTest extends ZarrStorageTests.ZarrFactoryTest @Override protected String tempN5Location() { try { - return new URI("gs", N5GoogleCloudStorageTest.tempBucketName(factory.createGoogleCloudStorage()), tempContainerPath(), null).toString(); + return new URI("gs", testBucket, tempContainerPath(), null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } From b17bf7fc926a7187f0aa5f4c07e943e2396a3710 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 28 Feb 2024 15:03:26 -0500 Subject: [PATCH 16/31] fix(storageFormat): detection of hdf5 by file extension * add a test --- .../saalfeldlab/n5/universe/N5Factory.java | 2 +- .../n5/universe/N5FactoryTests.java | 153 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 88df4a7..642b444 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -645,7 +645,7 @@ public enum StorageFormat { ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches()), N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches()), HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { - final boolean hasHdf5Extension = Pattern.compile("\\.h(df)5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches(); + final boolean hasHdf5Extension = Pattern.compile("\\.h(df)?5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches(); return hasHdf5Extension || isHDF5(uri.getPath()); }); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java new file mode 100644 index 0000000..a1d432f --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java @@ -0,0 +1,153 @@ +package org.janelia.saalfeldlab.n5.universe; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; + +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; +import org.junit.Test; + +public class N5FactoryTests { + + @Test + public void testStorageFormatGuesses() throws URISyntaxException { + + final URI noExt = new URI("file:///tmp/a"); + final URI h5Ext = new URI("file:///tmp/a.h5"); + final URI hdf5Ext = new URI("file:///tmp/a.hdf5"); + final URI n5Ext = new URI("file:///tmp/a.n5"); + final URI zarrExt = new URI("file:///tmp/a.zarr"); + final URI unknownExt = new URI("file:///tmp/a.abc"); + + assertNull("no extension null", N5Factory.StorageFormat.guessStorageFromUri(noExt)); + + /** + * h5 tests fail now because these test whether the file exists. It + * should not do that, if, for example, we're making a writer. + */ + assertEquals("h5 extension == h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(h5Ext)); + assertNotEquals("h5 extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(h5Ext)); + assertNotEquals("h5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(h5Ext)); + + assertEquals("hdf5 extension == h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(hdf5Ext)); + assertNotEquals("hdf5 extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(hdf5Ext)); + assertNotEquals("hdf5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(hdf5Ext)); + + assertNotEquals("n5 extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + assertEquals("n5 extension == n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + assertNotEquals("n5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + + assertNotEquals("zarr extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + assertNotEquals("zarr extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + assertEquals("zarr extension == zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + + assertNotEquals("unknown extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNotEquals("unknown extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNotEquals("unknown extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + } + + @Test + public void testWriterTypeByExtension() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] ext = new String[]{".h5", ".hdf5", ".n5", ".zarr"}; + final Class[] readerTypes = new Class[]{ + N5HDF5Writer.class, + N5HDF5Writer.class, + N5KeyValueWriter.class, + ZarrKeyValueWriter.class + }; + + for (int i = 0; i < ext.length; i++) { + final File tmpWithExt = new File(tmp, "foo" + i + ext[i]); + final String extUri = new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, extUri, readerTypes[i], " with extension"); + } + + } catch (IOException e) { + e.printStackTrace(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + + } + + @Test + public void testWriterTypeByPrefix() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] prefix = new String[]{"h5", "hdf5", "n5", "zarr"}; + final Class[] readerTypes = new Class[]{ + N5HDF5Writer.class, + N5HDF5Writer.class, + N5KeyValueWriter.class, + ZarrKeyValueWriter.class + }; + + for (int i = 0; i < prefix.length; i++) { + final File tmpNoExt = new File(tmp, "foo"+i); + + final String prefixUri = prefix[i] + ":" + new URI("file", null, tmpNoExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUri, readerTypes[i], " with prefix"); + + final String prefixUriSlashes = prefix[i] + "://" + new URI("file", null, tmpNoExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUriSlashes, readerTypes[i], " with prefix slashes"); + } + + // ensure that prefix is preferred to extensions + final String[] extensions = new String[]{".h5", ".hdf5", ".n5", ".zarr"}; + + for (int i = 0; i < prefix.length; i++) { + for (int j = 0; j < extensions.length; j++) { + + final File tmpWithExt = new File(tmp, "foo"+i+extensions[j]); + + final String prefixUri = prefix[i] + ":" + new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUri, readerTypes[i], " with prefix"); + + final String prefixUriSlashes = prefix[i] + "://" + new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUriSlashes, readerTypes[i], " with prefix slashes"); + } + } + + } catch (IOException e) { + e.printStackTrace(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + } + + private void checkWriterTypeFromFactory(N5Factory factory, String uri, Class expected, String messageSuffix) { + + final N5Writer n5 = factory.openWriter(uri); + assertNotNull("null n5 for " + uri, n5); + assertEquals(expected.getName() + messageSuffix, expected, n5.getClass()); + n5.remove(); + } + +} From 021d529b253edfca7ab64966dcc604a02fe8d71f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 28 Feb 2024 17:08:35 -0500 Subject: [PATCH 17/31] fix: guess storage format matchers should `find` refactor: move hdf5 specific logic to n5-hdf5 --- pom.xml | 2 +- .../saalfeldlab/n5/universe/N5Factory.java | 41 +++++++------------ 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index 609d6fb..0035a24 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,7 @@ sign,deploy-to-scijava 3.1.4-SNAPSHOT - 2.1.0 + 2.1.1-SNAPSHOT 7.0.0 4.0.2 4.0.1-SNAPSHOT diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 642b444..b9fe5f1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -45,6 +45,7 @@ import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; +import org.janelia.saalfeldlab.n5.hdf5.HDF5Utils; import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; @@ -55,14 +56,10 @@ import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; import javax.annotation.Nullable; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.FileSystems; -import java.util.Arrays; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; @@ -84,7 +81,6 @@ public class N5Factory implements Serializable { private static final long serialVersionUID = -6823715427289454617L; private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); - private static byte[] HDF5_SIG = {(byte)137, 72, 68, 70, 13, 10, 26, 10}; private int[] hdf5DefaultBlockSize = {64, 64, 64, 1, 1}; private boolean hdf5OverrideBlockSize = false; private GsonBuilder gsonBuilder = new GsonBuilder(); @@ -100,21 +96,6 @@ public class N5Factory implements Serializable { private String s3Endpoint; private boolean createBucket = false; - private static boolean isHDF5(String path) { - - final File f = new File(path); - if (!f.exists() || !f.isFile()) - return false; - - try (final FileInputStream in = new FileInputStream(f)) { - final byte[] sig = new byte[8]; - in.read(sig); - return Arrays.equals(sig, HDF5_SIG); - } catch (final IOException e) { - return false; - } - } - private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); @@ -475,6 +456,15 @@ public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException { return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, this::openWriter); } + public N5Writer openWriter(final StorageFormat format, final String uri) { + + try { + return openN5Container(format, N5URI.encodeAsUri(uri), this::openWriter); + } catch (URISyntaxException e) { + throw new N5Exception(e); + } + } + public N5Writer openWriter(final StorageFormat format, final URI uri) { createBucket = true; @@ -500,8 +490,7 @@ private N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable fin for (StorageFormat format : StorageFormat.values()) { try { return openWriter(format, access, containerPath); - } - catch (Exception e) {} + } catch (Exception ignored) {} } throw new N5Exception("Unable to open " + containerPath + " as N5Writer"); @@ -642,11 +631,11 @@ enum KeyValueAccessBackend implements Predicate, BiFunction Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches()), - N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches()), + ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), + N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { - final boolean hasHdf5Extension = Pattern.compile("\\.h(df)?5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).matches(); - return hasHdf5Extension || isHDF5(uri.getPath()); + final boolean hasHdf5Extension = Pattern.compile("\\.h(df)?5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find(); + return hasHdf5Extension || HDF5Utils.isHDF5(uri.getPath()); }); static final Pattern STORAGE_SCHEME_PATTERN = Pattern.compile("^(\\s*(?(n5|h(df)?5|zarr)):(//)?)?(?.*)$", Pattern.CASE_INSENSITIVE); From 22dcef35fb25217042b6ffc8e2a34b0c100b92e6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 28 Feb 2024 17:08:50 -0500 Subject: [PATCH 18/31] feat(test): more tests! --- .../n5/universe/N5FactoryTests.java | 164 ++++++++++++++++-- .../n5/universe/N5StorageTests.java | 20 ++- 2 files changed, 168 insertions(+), 16 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java index a1d432f..8124501 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java @@ -1,22 +1,29 @@ package org.janelia.saalfeldlab.n5.universe; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; +import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; +import org.junit.Test; import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; +import java.nio.file.Path; -import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Writer; -import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; -import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; public class N5FactoryTests { @@ -52,9 +59,9 @@ public void testStorageFormatGuesses() throws URISyntaxException { assertNotEquals("zarr extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); assertEquals("zarr extension == zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); - assertNotEquals("unknown extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); - assertNotEquals("unknown extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); - assertNotEquals("unknown extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNull("unknown extension != h5", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNull("unknown extension != n5", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); + assertNull("unknown extension != zarr", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); } @Test @@ -141,13 +148,142 @@ public void testWriterTypeByPrefix() { tmp.delete(); } } + + @Test + public void testDefaultForAmbiguousWriters() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] paths = new String[]{ + "a_non_hdf5_file", + "an_hdf5_file", + "a_zarr_directory", + "an_n5_directory", + "an_empty_directory", + "a_non_existent_path" + }; + + final Path tmpPath = tmp.toPath(); + + final File tmpNonHdf5File = tmpPath.resolve(paths[0]).toFile(); + tmpNonHdf5File.createNewFile(); + tmpNonHdf5File.deleteOnExit(); + + factory.openWriter(StorageFormat.HDF5, tmpPath.resolve(paths[1]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.ZARR, tmpPath.resolve(paths[2]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.N5, tmpPath.resolve(paths[3]).toFile().getCanonicalPath()).close(); + + final File tmpEmptyDir = tmpPath.resolve(paths[4]).toFile(); + tmpEmptyDir.mkdirs(); + tmpEmptyDir.deleteOnExit(); + + + + final Class[] writerTypes = new Class[]{ + null, + N5HDF5Writer.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class, + ZarrKeyValueWriter.class + }; + + for (int i = 0; i < paths.length; i++) { + + final String prefixUri = new URI("file", null, tmpPath.resolve(paths[i]).normalize().toString(), null).toString(); + checkWriterTypeFromFactory( factory, prefixUri, writerTypes[i], " with path " + paths[i]); + } + + } catch (IOException | URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + } + + + @Test + public void testDefaultForAmbiguousReaders() { + + final N5Factory factory = new N5Factory(); + + File tmp = null; + try { + tmp = Files.createTempDirectory("factory-test-").toFile(); + + final String[] paths = new String[]{ + "a_non_hdf5_file", + "an_hdf5_file", + "a_zarr_directory", + "an_n5_directory", + "an_empty_directory", + "a_non_existent_path" + }; + + final Path tmpPath = tmp.toPath(); + + final File tmpNonHdf5File = tmpPath.resolve(paths[0]).toFile(); + tmpNonHdf5File.createNewFile(); + tmpNonHdf5File.deleteOnExit(); + + factory.openWriter(StorageFormat.HDF5, tmpPath.resolve(paths[1]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.ZARR, tmpPath.resolve(paths[2]).toFile().getCanonicalPath()).close(); + factory.openWriter(StorageFormat.N5, tmpPath.resolve(paths[3]).toFile().getCanonicalPath()).close(); + + final File tmpEmptyDir = tmpPath.resolve(paths[4]).toFile(); + tmpEmptyDir.mkdirs(); + tmpEmptyDir.deleteOnExit(); + + + + final Class[] readerTypes = new Class[]{ + null, + N5HDF5Reader.class, + ZarrKeyValueReader.class, + N5KeyValueReader.class, + N5KeyValueReader.class, + null + }; + + for (int i = 0; i < paths.length; i++) { + + final String prefixUri = new URI("file", null, tmpPath.resolve(paths[i]).normalize().toString(), null).toString(); + checkReaderTypeFromFactory( factory, prefixUri, readerTypes[i], " with path " + paths[i]); + } + + } catch (IOException | URISyntaxException e) { + e.printStackTrace(); + } finally { + tmp.delete(); + } + } private void checkWriterTypeFromFactory(N5Factory factory, String uri, Class expected, String messageSuffix) { + if (expected == null) { + assertThrows(N5Exception.class, () -> factory.openWriter(uri)); + return; + } + final N5Writer n5 = factory.openWriter(uri); - assertNotNull("null n5 for " + uri, n5); + assertNotNull( "null n5 for " + uri, n5); assertEquals(expected.getName() + messageSuffix, expected, n5.getClass()); n5.remove(); } + private void checkReaderTypeFromFactory(N5Factory factory, String uri, Class expected, String messageSuffix) { + + if (expected == null) { + assertThrows(N5Exception.class, () -> factory.openReader(uri)); + return; + } + + final N5Reader n5 = factory.openReader(uri); + assertNotNull( "null n5 for " + uri, n5); + assertEquals(expected.getName() + messageSuffix, expected, n5.getClass()); + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index 6b9ce0d..0ac5bb3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -197,8 +197,11 @@ public static abstract class N5GoogleCloudFactoryTest extends N5FactoryTest { @Override Storage createGoogleCloudStorage() { final Storage storage = getOrCreateStorage(); - N5GoogleCloudFactoryTest.storage = storage; - return storage; + if (storage == null) + N5GoogleCloudFactoryTest.storage = super.createGoogleCloudStorage(); + else + N5GoogleCloudFactoryTest.storage = storage; + return N5GoogleCloudFactoryTest.storage; } }; } @@ -238,7 +241,20 @@ public static class N5GoogleCloudMockTest extends N5GoogleCloudFactoryTest { return MockGoogleCloudStorageFactory.getOrCreateStorage(); } + } + + + public static class N5GoogleCloudBackendTest extends N5GoogleCloudFactoryTest { + @Override public Class getBackendTargetClass() { + + return GoogleCloudStorageKeyValueAccess.class; + } + + @Override protected Storage getOrCreateStorage() { + + return factory.createGoogleCloudStorage(); + } } } From 3d770c43b958f18e73ca7a83f04d8754a24e102f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 1 Mar 2024 15:55:15 -0500 Subject: [PATCH 19/31] feat(test): improve remote backend test framework --- .../n5/universe/N5StorageTests.java | 131 +++++++-------- .../n5/universe/ZarrStorageTests.java | 151 ++++++++++++------ 2 files changed, 168 insertions(+), 114 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index 0ac5bb3..a505f61 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -9,12 +9,17 @@ import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; -import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTest; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; +import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.junit.After; import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestWatcher; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -26,6 +31,7 @@ import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; +import static org.junit.Assert.assertTrue; @RunWith(Suite.class) @Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5AmazonS3BackendTest.class}) @@ -122,33 +128,30 @@ public static abstract class N5AmazonS3FactoryTest extends N5FactoryTest { @AfterClass public static void removeTestBucket() { - if (s3.doesBucketExistV2(testBucket)) - s3.deleteBucket(testBucket); - } - } + if (s3 != null && s3.doesBucketExistV2(testBucket)) + N5Factory.createWriter("s3://" + testBucket).remove(); + } - public static class N5AmazonS3MockTest extends N5AmazonS3FactoryTest { @Override public N5Factory getFactory() { - if (factory == null) { - factory = new N5Factory() { + if (factory != null) + return factory; + factory = new N5Factory() { - @Override AmazonS3 createS3(String uri) { + @Override AmazonS3 createS3(String uri) { - AmazonS3 s3 = MockS3Factory.getOrCreateS3(); - N5AmazonS3FactoryTest.s3 = s3; - return s3; - } - }; - } + if (N5AmazonS3FactoryTest.s3 == null) + N5AmazonS3FactoryTest.s3 = super.createS3(uri); + return s3; + } + }; return factory; } @Override protected String tempN5Location() { try { - return new URI("s3", testBucket, tempContainerPath(), null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); @@ -156,69 +159,50 @@ public static class N5AmazonS3MockTest extends N5AmazonS3FactoryTest { } } - public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { - - @Override public Class getBackendTargetClass() { + public static class N5AmazonS3MockTest extends N5AmazonS3FactoryTest { + public N5AmazonS3MockTest() { - return AmazonS3KeyValueAccess.class; + N5AmazonS3FactoryTest.s3 = MockS3Factory.getOrCreateS3(); } @Override protected String tempN5Location() { try { - final String s3ContainerUri = new URI("s3", testBucket, tempContainerPath(), null).toString(); - if (s3 == null) { - s3 = factory.createS3(s3ContainerUri); - } - return s3ContainerUri; + return new URI("http", "localhost:8001", "/" + testBucket + tempContainerPath(), null, null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } } } - public static abstract class N5GoogleCloudFactoryTest extends N5FactoryTest { + public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { - private static String testBucket = N5GoogleCloudStorageTest.tempBucketName(); - private static Storage storage = null; + @BeforeClass + public static void ensureBucketExists() { - protected abstract Storage getOrCreateStorage(); - @Override public Class getBackendTargetClass() { - return GoogleCloudStorageKeyValueAccess.class; + N5Factory.createWriter("s3://" + testBucket); + assertTrue(s3.doesBucketExistV2(testBucket)); } - @Override public N5Factory getFactory() { + @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); - if (factory == null) { - factory = new N5Factory() { - - @Override Storage createGoogleCloudStorage() { - - final Storage storage = getOrCreateStorage(); - if (storage == null) - N5GoogleCloudFactoryTest.storage = super.createGoogleCloudStorage(); - else - N5GoogleCloudFactoryTest.storage = storage; - return N5GoogleCloudFactoryTest.storage; - } - }; - } - return factory; + public N5AmazonS3BackendTest() { + + N5AmazonS3FactoryTest.s3 = null; } + } - @Override - protected String tempN5Location() { + public static abstract class N5GoogleCloudFactoryTest extends N5FactoryTest { - try { - return new URI("gs", testBucket, tempContainerPath(), null).toString(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + protected static String testBucket = N5GoogleCloudStorageTests.tempBucketName(); + protected static Storage storage = null; + @Override public Class getBackendTargetClass() { + return GoogleCloudStorageKeyValueAccess.class; + } @AfterClass public static void removeTestBucket() { @@ -228,33 +212,50 @@ public static void removeTestBucket() { storage.delete(testBucket); } } - } - public static class N5GoogleCloudMockTest extends N5GoogleCloudFactoryTest { + @Override public N5Factory getFactory() { - @Override public Class getBackendTargetClass() { + if (factory != null) + return factory; + factory = new N5Factory() { - return GoogleCloudStorageKeyValueAccess.class; + @Override Storage createGoogleCloudStorage() { + + return storage; + } + }; + return factory; } - @Override protected Storage getOrCreateStorage() { + @Override protected String tempN5Location() { - return MockGoogleCloudStorageFactory.getOrCreateStorage(); + try { + return new URI("gs", testBucket, tempContainerPath(), null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } } + public static class N5GoogleCloudMockTest extends N5GoogleCloudFactoryTest { + public N5GoogleCloudMockTest() { + N5GoogleCloudFactoryTest.storage = MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + } public static class N5GoogleCloudBackendTest extends N5GoogleCloudFactoryTest { - @Override public Class getBackendTargetClass() { + @BeforeClass + public static void ensureBucketExists() { - return GoogleCloudStorageKeyValueAccess.class; + final N5Writer writer = N5Factory.createWriter("gs://" + testBucket); + assertTrue(writer.exists("")); } - @Override protected Storage getOrCreateStorage() { + public N5GoogleCloudBackendTest() { - return factory.createGoogleCloudStorage(); + N5GoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index ce14c05..126cafc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -1,19 +1,25 @@ package org.janelia.saalfeldlab.n5.universe; import com.amazonaws.services.s3.AmazonS3; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.gson.GsonBuilder; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; -import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTest; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; +import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; -import org.janelia.saalfeldlab.n5.s3.backend.BackendS3Factory; +import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; import org.janelia.saalfeldlab.n5.zarr.N5ZarrTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Ignore; +import org.junit.Rule; +import org.junit.rules.TestWatcher; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -24,9 +30,10 @@ import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempBucketName; import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath; +import static org.junit.Assert.assertTrue; @RunWith(Suite.class) -@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3Test.class}) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3MockTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class}) public class ZarrStorageTests { public static abstract class ZarrFactoryTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { @@ -63,7 +70,9 @@ public ZarrFactoryTest() { factory.gsonBuilder(gsonBuilder); factory.zarrDimensionSeparator(dimensionSeparator); factory.zarrMapN5Attributes(mapN5DatasetAttributes); - return getWriter(location); + final N5Writer writer = getWriter(location); + tempWriters.add(writer); + return writer; } @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { @@ -89,10 +98,14 @@ public ZarrFactoryTest() { } @Ignore - @Override public void testReadZarrPython() {} + @Override public void testReadZarrPython() { + + } @Ignore - @Override public void testReadZarrNestedPython() {} + @Override public void testReadZarrNestedPython() { + + } } public static class ZarrFileSystemTest extends ZarrFactoryTest { @@ -112,95 +125,113 @@ public static class ZarrFileSystemTest extends ZarrFactoryTest { } } - public static class ZarrAmazonS3Test extends ZarrFactoryTest { + public static abstract class ZarrAmazonS3FactoryTest extends ZarrFactoryTest { + + public static AmazonS3 s3 = null; + + final static String testBucket = tempBucketName(); @Override public Class getBackendTargetClass() { return AmazonS3KeyValueAccess.class; } + @AfterClass + public static void removeTestBucket() { + + if (s3 != null && s3.doesBucketExistV2(testBucket)) + N5Factory.createWriter("s3://" + testBucket).remove(); + } + @Override public N5Factory getFactory() { - if (factory == null) { - factory = new N5Factory() { + if (factory != null) + return factory; + factory = new N5Factory() { - @Override AmazonS3 createS3(String uri) { + @Override AmazonS3 createS3(String uri) { - return MockS3Factory.getOrCreateS3(); - } - }; - } + if (ZarrAmazonS3FactoryTest.s3 == null) + ZarrAmazonS3FactoryTest.s3 = super.createS3(uri); + return s3; + } + }; return factory; } - final String testBucket = tempBucketName(); - @Override protected String tempN5Location() { try { -// return new URI("s3", testBucket, tempContainerPath(), null).toString(); - return new URI("http", "localhost:8001", "/" + testBucket + tempContainerPath(), null, null).toString(); + return new URI("s3", testBucket, tempContainerPath(), null).toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } } } + public static class ZarrAmazonS3MockTest extends ZarrAmazonS3FactoryTest { + public ZarrAmazonS3MockTest() { + ZarrAmazonS3FactoryTest.s3 = MockS3Factory.getOrCreateS3(); + } - public static class ZarrAmazonS3BackendTest extends N5StorageTests.N5FactoryTest { - - @Override public Class getBackendTargetClass() { + @Override protected String tempN5Location() { - return AmazonS3KeyValueAccess.class; + try { + return new URI("http", "localhost:8001", "/" + testBucket + tempContainerPath(), null, null).toString(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } + } - @Override public N5Factory getFactory() { + public static class ZarrAmazonS3BackendTest extends ZarrAmazonS3FactoryTest { - if (factory == null) { - factory = new N5Factory(); // { -// -// @Override AmazonS3 createS3(String uri) { -// -// return BackendS3Factory.getOrCreateS3(); -// } -// }; - } - return factory; + @BeforeClass + public static void ensureBucketExists() { + + N5Factory.createWriter("s3://" + testBucket); + assertTrue(s3.doesBucketExistV2(testBucket)); } - final String testBucket = tempBucketName(); + @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); - @Override protected String tempN5Location() { + public ZarrAmazonS3BackendTest() { - try { - return new URI("s3", testBucket, tempContainerPath(), null).toString(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } + ZarrAmazonS3FactoryTest.s3 = null; } } - public static class ZarrGoogleCloudTest extends ZarrStorageTests.ZarrFactoryTest { + public static abstract class ZarrGoogleCloudFactoryTest extends ZarrFactoryTest { - private String testBucket = N5GoogleCloudStorageTest.tempBucketName(); + protected static String testBucket = N5GoogleCloudStorageTests.tempBucketName(); + protected static Storage storage = null; @Override public Class getBackendTargetClass() { return GoogleCloudStorageKeyValueAccess.class; } + @AfterClass + public static void removeTestBucket() { + + final Bucket bucket = storage.get(testBucket); + if (bucket != null && bucket.exists()) { + storage.delete(testBucket); + } + } + @Override public N5Factory getFactory() { - if (factory == null) { - factory = new N5Factory() { + if (factory != null) + return factory; + factory = new N5Factory() { - @Override Storage createGoogleCloudStorage() { + @Override Storage createGoogleCloudStorage() { - return MockGoogleCloudStorageFactory.getOrCreateStorage(); - } - }; - } + return storage; + } + }; return factory; } @@ -213,4 +244,26 @@ public static class ZarrGoogleCloudTest extends ZarrStorageTests.ZarrFactoryTest } } } + + public static class ZarrGoogleCloudMockTest extends ZarrGoogleCloudFactoryTest { + public ZarrGoogleCloudMockTest() { + + ZarrGoogleCloudFactoryTest.storage = MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + } + + public static class ZarrGoogleCloudBackendTest extends ZarrGoogleCloudFactoryTest { + + @BeforeClass + public static void ensureBucketExists() { + + final N5Writer writer = N5Factory.createWriter("gs://" + testBucket); + assertTrue(writer.exists("")); + } + + public ZarrGoogleCloudBackendTest() { + + ZarrGoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); + } + } } From c300e5cb5bf1ff77ff2f20e0835015681552ddd9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 1 Mar 2024 16:52:50 -0500 Subject: [PATCH 20/31] feat: add static methods for no-configuration openReader/Writer --- .../saalfeldlab/n5/universe/N5Factory.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index b9fe5f1..e986802 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -78,6 +78,8 @@ */ public class N5Factory implements Serializable { + private static final N5Factory FACTORY = new N5Factory(); + private static final long serialVersionUID = -6823715427289454617L; private final static Pattern HTTPS_SCHEME = Pattern.compile("http(s)?", Pattern.CASE_INSENSITIVE); private final static Pattern FILE_SCHEME = Pattern.compile("file", Pattern.CASE_INSENSITIVE); @@ -686,4 +688,26 @@ public static Pair getStorageFromNestedScheme(String uri) return new ValuePair<>(null, uriGroup); } } + + /** + * Creates an N5 writer for the specified container URI with default N5Factory configuration. + * + * @param containerUri location of the N5 container + * @return an N5Writer instance for the given containerURI + */ + public static N5Writer createWriter(String containerUri) { + + return FACTORY.openWriter(containerUri); + } + + /** + * Creates an N5Reader at containerURI with default N5Factory configuration. + * + * @param containerUri location of the N5 container + * @return an N5Reader instance for the given containerURI + */ + public static N5Reader createReader(String containerUri) { + + return FACTORY.openReader(containerUri); + } } \ No newline at end of file From 0431bbb9b82c84b2e690a8f92b75b73946ba0622 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 1 Mar 2024 16:54:55 -0500 Subject: [PATCH 21/31] feat: more improvements --- .../saalfeldlab/n5/universe/N5Factory.java | 80 +++++++------------ 1 file changed, 30 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index e986802..7e5cdf1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -61,7 +61,6 @@ import java.net.URISyntaxException; import java.nio.file.FileSystems; import java.util.function.BiFunction; -import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -94,14 +93,12 @@ public class N5Factory implements Serializable { private String s3Region = null; private AWSCredentials s3Credentials = null; private boolean s3Anonymous = true; - private boolean s3RetryWithCredentials = false; private String s3Endpoint; - private boolean createBucket = false; private static GoogleCloudStorageKeyValueAccess newGoogleCloudKeyValueAccess(final URI uri, final N5Factory factory) { final GoogleCloudStorageURI googleCloudUri = new GoogleCloudStorageURI(uri); - return new GoogleCloudStorageKeyValueAccess(factory.createGoogleCloudStorage(), googleCloudUri.getBucket(), factory.createBucket); + return new GoogleCloudStorageKeyValueAccess(factory.createGoogleCloudStorage(), googleCloudUri, true); } private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, final N5Factory factory) { @@ -109,7 +106,7 @@ private static AmazonS3KeyValueAccess newAmazonS3KeyValueAccess(final URI uri, f final String uriString = uri.toString(); final AmazonS3 s3 = factory.createS3(uriString); - return new AmazonS3KeyValueAccess(s3, uri.toString(), factory.createBucket); + return new AmazonS3KeyValueAccess(s3, uri, true); } private static FileSystemKeyValueAccess newFileSystemKeyValueAccess(final URI uri, final N5Factory factory) { @@ -183,9 +180,9 @@ public N5Factory s3UseCredentials(final AWSCredentials credentials) { return this; } + @Deprecated public N5Factory s3RetryWithCredentials() { - s3RetryWithCredentials = true; return this; } @@ -205,7 +202,7 @@ AmazonS3 createS3(final String uri) { try { return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region); - } catch (final Exception e) { + } catch (final Throwable e) { throw new N5Exception("Could not create s3 client from uri: " + uri, e); } } @@ -222,7 +219,8 @@ Storage createGoogleCloudStorage() { * @param uri to create a {@link KeyValueAccess} from. * @return the {@link KeyValueAccess} and container path, or null if none are valid */ - Pair getKeyValueAccess(final URI uri) { + @Nullable + KeyValueAccess getKeyValueAccess(final URI uri) { /*NOTE: The order of these tests is very important, as the predicates for each * backend take into account reasonable defaults when possible. @@ -231,14 +229,14 @@ Pair getKeyValueAccess(final URI uri) { for (KeyValueAccessBackend backend : KeyValueAccessBackend.values()) { final KeyValueAccess kva = backend.apply(uri, this); if (kva != null) - return new ValuePair<>(kva, backend.parseContainerPath.apply(uri)); + return kva; } return null; } /** * Open an {@link N5Reader} over an N5 Container. - * + *

* NOTE: The name seems to imply that this will open any N5Reader, over a * {@link FileSystemKeyValueAccess} however that is misleading. Instead * this will open any N5Container that is a valid {@link StorageFormat#N5}. @@ -359,8 +357,8 @@ private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable fin for (StorageFormat format : StorageFormat.values()) { try { return openReader(format, access, containerPath); + } catch (Throwable e) { } - catch (Exception e) {} } throw new N5Exception("Unable to open " + containerPath + " as N5Reader"); @@ -380,7 +378,7 @@ private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable fin /** * Open an {@link N5Writer} for N5 Container. - * + *

* NOTE: The name seems to imply that this will open any N5Writer, over a * {@link FileSystemKeyValueAccess} however that is misleading. Instead * this will open any N5Container that is a valid {@link StorageFormat#N5}. @@ -469,10 +467,7 @@ public N5Writer openWriter(final StorageFormat format, final String uri) { public N5Writer openWriter(final StorageFormat format, final URI uri) { - createBucket = true; - final N5Writer n5Writer = openN5Container(format, uri, this::openWriter); - createBucket = false; - return n5Writer; + return openN5Container(format, uri, this::openWriter); } /** @@ -492,7 +487,8 @@ private N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable fin for (StorageFormat format : StorageFormat.values()) { try { return openWriter(format, access, containerPath); - } catch (Exception ignored) {} + } catch (Throwable ignored) { + } } throw new N5Exception("Unable to open " + containerPath + " as N5Writer"); @@ -526,15 +522,14 @@ private T openN5ContainerWithStorageFormat( private T openN5ContainerWithBackend( final KeyValueAccessBackend backend, - final String uri, + final String containerUri, final TriFunction openWithBackend ) throws URISyntaxException { - final Pair formatAndUri = StorageFormat.parseUri(uri); - final URI asUri = formatAndUri.getB(); - final KeyValueAccess kva = backend.apply(asUri, this); - final String containerPath = backend.parseContainerPath.apply(asUri); - return openWithBackend.apply(formatAndUri.getA(), kva, containerPath); + final Pair formatAndUri = StorageFormat.parseUri(containerUri); + final URI uri = formatAndUri.getB(); + final KeyValueAccess kva = backend.apply(uri, this); + return openWithBackend.apply(formatAndUri.getA(), kva, uri.toString()); } private T openN5Container( @@ -542,37 +537,29 @@ private T openN5Container( final URI uri, final TriFunction openWithKva) { - final Pair accessAndContainerPath = getKeyValueAccess(uri); - if (accessAndContainerPath == null) + final KeyValueAccess kva = getKeyValueAccess(uri); + if (kva == null) throw new N5Exception("Cannot get KeyValueAccess at " + uri); - final KeyValueAccess access = accessAndContainerPath.getA(); - final String containerPath = accessAndContainerPath.getB(); - return openWithKva.apply(storageFormat, access, containerPath); + return openWithKva.apply(storageFormat, kva, uri.toString()); } private T openN5Container( - final String uri, + final String containerUri, final BiFunction openWithFormat, final TriFunction openWithKva) { final Pair storageAndUri; try { - storageAndUri = StorageFormat.parseUri(uri); + storageAndUri = StorageFormat.parseUri(containerUri); } catch (URISyntaxException e) { - throw new N5Exception("Unable to open " + uri + " as N5 Container", e); + throw new N5Exception("Unable to open " + containerUri + " as N5 Container", e); } final StorageFormat format = storageAndUri.getA(); - final URI asUri = storageAndUri.getB(); + final URI uri = storageAndUri.getB(); if (format != null) - return openWithFormat.apply(format, asUri); - - final Pair accessAndContainerPath = getKeyValueAccess(asUri); - if (accessAndContainerPath == null) - throw new N5Exception("Cannot get KeyValueAccess at " + asUri); - final KeyValueAccess access = accessAndContainerPath.getA(); - final String containerPath = accessAndContainerPath.getB(); - - return openWithKva.apply(null, access, containerPath); + return openWithFormat.apply(format, uri); + else + return openN5Container(null, uri, openWithKva); } /** @@ -596,27 +583,20 @@ enum KeyValueAccessBackend implements Predicate, BiFunction { final String scheme = uri.getScheme(); final boolean hasScheme = scheme != null; - return !hasScheme || hasScheme && FILE_SCHEME.asPredicate().test(scheme); + return !hasScheme || FILE_SCHEME.asPredicate().test(scheme); }, N5Factory::newFileSystemKeyValueAccess); private final Predicate backendTest; private final BiFunction backendGenerator; - private final Function parseContainerPath; KeyValueAccessBackend(Predicate test, BiFunction generator) { - this(test, generator, URI::getPath); - } - - KeyValueAccessBackend(Predicate test, BiFunction generator, final Function getContainerPath) { - backendTest = test; backendGenerator = generator; - parseContainerPath = getContainerPath; } @Override public KeyValueAccess apply(final URI uri, final N5Factory factory) { From 799c51e2fd76734ad42e894891a1b868eb139e2c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 1 Mar 2024 17:07:11 -0500 Subject: [PATCH 22/31] chore(test): add missing test classes --- .../org/janelia/saalfeldlab/n5/universe/N5StorageTests.java | 2 +- .../org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index a505f61..e6b43e3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue; @RunWith(Suite.class) -@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5AmazonS3BackendTest.class}) +@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5AmazonS3BackendTest.class, N5StorageTests.N5GoogleCloudMockTest.class, N5StorageTests.N5GoogleCloudBackendTest.class}) public class N5StorageTests { public static abstract class N5FactoryTest extends AbstractN5Test implements StorageSchemeWrappedN5Test { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index 126cafc..535e3cc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue; @RunWith(Suite.class) -@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3MockTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class}) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3MockTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class, ZarrStorageTests.ZarrGoogleCloudMockTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class}) public class ZarrStorageTests { public static abstract class ZarrFactoryTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { From d4982d9d4377b5d8c8bd91072744e15d894c381c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 4 Mar 2024 10:20:46 -0500 Subject: [PATCH 23/31] fix(test): separate backend tests --- pom.xml | 2 +- .../org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java | 2 +- .../org/janelia/saalfeldlab/n5/universe/N5StorageTests.java | 4 +--- .../org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 0035a24..e082718 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ maven-surefire-plugin - **Backend*.java + **Backend** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java index 8124501..813da5c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java @@ -265,7 +265,7 @@ public void testDefaultForAmbiguousReaders() { private void checkWriterTypeFromFactory(N5Factory factory, String uri, Class expected, String messageSuffix) { if (expected == null) { - assertThrows(N5Exception.class, () -> factory.openWriter(uri)); + assertThrows("Should throw exception for " + uri, N5Exception.class, () -> factory.openWriter(uri)); return; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index e6b43e3..b6ae5b4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue; @RunWith(Suite.class) -@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5AmazonS3BackendTest.class, N5StorageTests.N5GoogleCloudMockTest.class, N5StorageTests.N5GoogleCloudBackendTest.class}) +@Suite.SuiteClasses({N5StorageTests.N5FileSystemTest.class, N5StorageTests.N5AmazonS3MockTest.class, N5StorageTests.N5GoogleCloudMockTest.class}) public class N5StorageTests { public static abstract class N5FactoryTest extends AbstractN5Test implements StorageSchemeWrappedN5Test { @@ -180,8 +180,6 @@ public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { @BeforeClass public static void ensureBucketExists() { - - N5Factory.createWriter("s3://" + testBucket); assertTrue(s3.doesBucketExistV2(testBucket)); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index 535e3cc..f005fa8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue; @RunWith(Suite.class) -@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3MockTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class, ZarrStorageTests.ZarrGoogleCloudMockTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class}) +@Suite.SuiteClasses({ZarrStorageTests.ZarrFileSystemTest.class, ZarrStorageTests.ZarrAmazonS3MockTest.class, ZarrStorageTests.ZarrGoogleCloudMockTest.class}) public class ZarrStorageTests { public static abstract class ZarrFactoryTest extends N5ZarrTest implements StorageSchemeWrappedN5Test { From 2e7f5b6dcfec48c8ea41e0569bd635307518fbe3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 4 Mar 2024 10:45:13 -0500 Subject: [PATCH 24/31] fix(test): add test file --- .../n5/universe/BackendStorageFormatTests.java | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java new file mode 100644 index 0000000..2d80d00 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/BackendStorageFormatTests.java @@ -0,0 +1,9 @@ +package org.janelia.saalfeldlab.n5.universe; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({N5StorageTests.N5AmazonS3BackendTest.class, N5StorageTests.N5GoogleCloudBackendTest.class, ZarrStorageTests.ZarrAmazonS3BackendTest.class, ZarrStorageTests.ZarrGoogleCloudBackendTest.class}) +public class BackendStorageFormatTests { +} From 9c6ba21d7273ec4a160831462cd90ccfb4b82849 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 4 Mar 2024 11:32:02 -0500 Subject: [PATCH 25/31] perf(factory): use '/' as default zarr dimension separator --- .../java/org/janelia/saalfeldlab/n5/universe/N5Factory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 7e5cdf1..0c13965 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -86,7 +86,7 @@ public class N5Factory implements Serializable { private boolean hdf5OverrideBlockSize = false; private GsonBuilder gsonBuilder = new GsonBuilder(); private boolean cacheAttributes = true; - private String zarrDimensionSeparator = "."; + private String zarrDimensionSeparator = "/"; private boolean zarrMapN5DatasetAttributes = true; private boolean zarrMergeAttributes = true; private String googleCloudProjectId = null; From 2554090650fed8f2e7f106516f01894b9fd5f388 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 4 Mar 2024 11:54:46 -0500 Subject: [PATCH 26/31] fix(test): NPE on static s3 instances for backend test --- .../saalfeldlab/n5/universe/N5StorageTests.java | 4 ++-- .../saalfeldlab/n5/universe/ZarrStorageTests.java | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index b6ae5b4..3687732 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -180,8 +180,8 @@ public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { @BeforeClass public static void ensureBucketExists() { - N5Factory.createWriter("s3://" + testBucket); - assertTrue(s3.doesBucketExistV2(testBucket)); + final N5Writer writer = N5Factory.createWriter("s3://" + testBucket); + assertTrue(writer.exists("")); } @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index f005fa8..128f877 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -9,7 +9,6 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; -import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; @@ -190,8 +189,8 @@ public static class ZarrAmazonS3BackendTest extends ZarrAmazonS3FactoryTest { @BeforeClass public static void ensureBucketExists() { - N5Factory.createWriter("s3://" + testBucket); - assertTrue(s3.doesBucketExistV2(testBucket)); + final N5Writer writer = N5Factory.createWriter("s3://" + testBucket); + assertTrue(writer.exists("")); } @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); @@ -260,10 +259,5 @@ public static void ensureBucketExists() { final N5Writer writer = N5Factory.createWriter("gs://" + testBucket); assertTrue(writer.exists("")); } - - public ZarrGoogleCloudBackendTest() { - - ZarrGoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); - } } } From d15371e8c0641481564a19034e94332a0f0bc86d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 4 Mar 2024 14:04:53 -0500 Subject: [PATCH 27/31] fix(test): delete `ensureBucketExists` writers --- .../saalfeldlab/n5/universe/N5StorageTests.java | 11 +++++++++-- .../saalfeldlab/n5/universe/ZarrStorageTests.java | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java index 3687732..da0bb5a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5StorageTests.java @@ -180,8 +180,9 @@ public static class N5AmazonS3BackendTest extends N5AmazonS3FactoryTest { @BeforeClass public static void ensureBucketExists() { - final N5Writer writer = N5Factory.createWriter("s3://" + testBucket); + final N5Writer writer = N5Factory.createWriter("s3://" + testBucket + "/" + tempContainerPath()); assertTrue(writer.exists("")); + writer.remove(); } @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); @@ -247,13 +248,19 @@ public static class N5GoogleCloudBackendTest extends N5GoogleCloudFactoryTest { @BeforeClass public static void ensureBucketExists() { - final N5Writer writer = N5Factory.createWriter("gs://" + testBucket); + final N5Writer writer = N5Factory.createWriter("gs://" + testBucket + "/" + tempContainerPath()); assertTrue(writer.exists("")); + writer.remove(); } public N5GoogleCloudBackendTest() { N5GoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); } + + @Override public void testVersion() throws NumberFormatException, IOException, URISyntaxException { + + super.testVersion(); + } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index 128f877..f73e58c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -189,8 +189,9 @@ public static class ZarrAmazonS3BackendTest extends ZarrAmazonS3FactoryTest { @BeforeClass public static void ensureBucketExists() { - final N5Writer writer = N5Factory.createWriter("s3://" + testBucket); + final N5Writer writer = N5Factory.createWriter("s3://" + testBucket + "/" + tempContainerPath()); assertTrue(writer.exists("")); + writer.remove(); } @Rule public TestWatcher skipIfErroneousFailure = new N5AmazonS3Tests.SkipErroneousNoSuchBucketFailure(); @@ -256,8 +257,9 @@ public static class ZarrGoogleCloudBackendTest extends ZarrGoogleCloudFactoryTes @BeforeClass public static void ensureBucketExists() { - final N5Writer writer = N5Factory.createWriter("gs://" + testBucket); + final N5Writer writer = N5Factory.createWriter("gs://" + testBucket + "/" + tempContainerPath()); assertTrue(writer.exists("")); + writer.remove(); } } } From 80dc11fde8c4d2f2e4a182a9ea5afd57afc08e58 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 4 Mar 2024 14:16:25 -0500 Subject: [PATCH 28/31] fix(test): set static `storage` instance --- .../janelia/saalfeldlab/n5/universe/ZarrStorageTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java index f73e58c..a8a6e1b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/ZarrStorageTests.java @@ -9,6 +9,7 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; +import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; @@ -261,5 +262,10 @@ public static void ensureBucketExists() { assertTrue(writer.exists("")); writer.remove(); } + + public ZarrGoogleCloudBackendTest() { + + ZarrGoogleCloudFactoryTest.storage = BackendGoogleCloudStorageFactory.getOrCreateStorage(); + } } } From 42f76f852bd4f0972b424fceecc21553105bd569 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 4 Mar 2024 14:58:22 -0500 Subject: [PATCH 29/31] test(factory): uris with trailing slashes for n5 and zarr --- .../n5/universe/N5FactoryTests.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java index 813da5c..fb26e0b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java @@ -34,7 +34,9 @@ public void testStorageFormatGuesses() throws URISyntaxException { final URI h5Ext = new URI("file:///tmp/a.h5"); final URI hdf5Ext = new URI("file:///tmp/a.hdf5"); final URI n5Ext = new URI("file:///tmp/a.n5"); + final URI n5ExtSlash = new URI("file:///tmp/a.n5/"); final URI zarrExt = new URI("file:///tmp/a.zarr"); + final URI zarrExtSlash = new URI("file:///tmp/a.zarr/"); final URI unknownExt = new URI("file:///tmp/a.abc"); assertNull("no extension null", N5Factory.StorageFormat.guessStorageFromUri(noExt)); @@ -55,10 +57,18 @@ public void testStorageFormatGuesses() throws URISyntaxException { assertEquals("n5 extension == n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); assertNotEquals("n5 extension != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(n5Ext)); + assertNotEquals("n5 extension slash != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(n5ExtSlash)); + assertEquals("n5 extension slash == n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(n5ExtSlash)); + assertNotEquals("n5 extension slash != zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(n5ExtSlash)); + assertNotEquals("zarr extension != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); assertNotEquals("zarr extension != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); assertEquals("zarr extension == zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(zarrExt)); + assertNotEquals("zarr extension slash != h5", StorageFormat.HDF5, N5Factory.StorageFormat.guessStorageFromUri(zarrExtSlash)); + assertNotEquals("zarr extension slash != n5", StorageFormat.N5, N5Factory.StorageFormat.guessStorageFromUri(zarrExtSlash)); + assertEquals("zarr extension slash == zarr", StorageFormat.ZARR, N5Factory.StorageFormat.guessStorageFromUri(zarrExtSlash)); + assertNull("unknown extension != h5", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); assertNull("unknown extension != n5", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); assertNull("unknown extension != zarr", N5Factory.StorageFormat.guessStorageFromUri(unknownExt)); @@ -73,17 +83,23 @@ public void testWriterTypeByExtension() { try { tmp = Files.createTempDirectory("factory-test-").toFile(); - final String[] ext = new String[]{".h5", ".hdf5", ".n5", ".zarr"}; + final String[] ext = new String[]{".h5", ".hdf5", ".n5", ".n5", ".zarr", ".zarr"}; + + // necessary because new File() removes trailing slash + final String[] trailing = new String[]{"", "", "", "/", "", "/"}; + final Class[] readerTypes = new Class[]{ N5HDF5Writer.class, N5HDF5Writer.class, N5KeyValueWriter.class, + N5KeyValueWriter.class, + ZarrKeyValueWriter.class, ZarrKeyValueWriter.class }; for (int i = 0; i < ext.length; i++) { final File tmpWithExt = new File(tmp, "foo" + i + ext[i]); - final String extUri = new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString(); + final String extUri = new URI("file", null, tmpWithExt.toURI().normalize().getPath(), null).toString() + trailing[i]; checkWriterTypeFromFactory( factory, extUri, readerTypes[i], " with extension"); } From 05cb1a2d570448f870359dbd5edc73b8125df9ef Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 4 Mar 2024 15:16:22 -0500 Subject: [PATCH 30/31] fix(test): guess storage format based on extension regardless of trailing separator --- .../java/org/janelia/saalfeldlab/n5/universe/N5Factory.java | 4 ++-- .../org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java index 0c13965..96d79a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java @@ -613,8 +613,8 @@ enum KeyValueAccessBackend implements Predicate, BiFunction Pattern.compile("\\.zarr$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), - N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), + ZARR(Pattern.compile("zarr", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.zarr(" + FileSystems.getDefault().getSeparator() + ")?$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), + N5(Pattern.compile("n5", Pattern.CASE_INSENSITIVE), uri -> Pattern.compile("\\.n5(" + FileSystems.getDefault().getSeparator() + ")?$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find()), HDF5(Pattern.compile("h(df)?5", Pattern.CASE_INSENSITIVE), uri -> { final boolean hasHdf5Extension = Pattern.compile("\\.h(df)?5$", Pattern.CASE_INSENSITIVE).matcher(uri.getPath()).find(); return hasHdf5Extension || HDF5Utils.isHDF5(uri.getPath()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java index fb26e0b..84dc7f5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/universe/N5FactoryTests.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -85,8 +86,9 @@ public void testWriterTypeByExtension() { final String[] ext = new String[]{".h5", ".hdf5", ".n5", ".n5", ".zarr", ".zarr"}; - // necessary because new File() removes trailing slash - final String[] trailing = new String[]{"", "", "", "/", "", "/"}; + // necessary because new File() removes trailing separator + final String separator = FileSystems.getDefault().getSeparator(); + final String[] trailing = new String[]{"", "", "", separator, "", separator}; final Class[] readerTypes = new Class[]{ N5HDF5Writer.class, From 85b6ee04763360550a6fa56e1eabea557fb6825a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 5 Mar 2024 10:21:14 -0500 Subject: [PATCH 31/31] chore: bump all n5-dependency versions --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index e082718..dddbb97 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5-universe - 1.3.2-SNAPSHOT + 1.4.0-SNAPSHOT N5-Universe Utilities spanning all of the N5 repositories @@ -111,14 +111,14 @@ sign,deploy-to-scijava - 3.1.4-SNAPSHOT - 2.1.1-SNAPSHOT + 3.2.0 + 2.2.0 7.0.0 4.0.2 - 4.0.1-SNAPSHOT + 4.1.0 - 1.2.2-SNAPSHOT - 4.0.3-SNAPSHOT + 1.3.0 + 4.1.0 1.0.0-preview.20191208 1.4.1