Merge pull request #61 from PreibischLab/uri-problem

Replace special case handling in URITools.getRelativeCloudPath
PreibischLab · Dec 1, 2024 · 9805791 · 9805791
2 parents 5491ae5 + 870d210
commit 9805791
Showing 1 changed file with 27 additions and 79 deletions.
diff --git a/src/main/java/util/URITools.java b/src/main/java/util/URITools.java
@@ -185,7 +185,7 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
 			// fist make a copy of the XML and save it to not loose it
 			try
 			{
-				final String xmlFile = getRelativeCloudPath( xmlURI );
+				final String xmlFile = toNormalPath( kva, xmlURI );
 
 				if ( kva.exists( xmlFile ) )
 				{
@@ -478,47 +478,43 @@ else if ( SplitViewerImgLoader.class.isInstance( loader ) )
 			return false;
 	}
 
-	/**
-	 * This is an abstraction that is only required because the Google cloud and AWS KeyValueAccesses behave differently
-	 *
-	 * @param uri - the URI for which to create the relative path
-	 * @return the relative path
-	 * @throws URISyntaxException
-	 */
-	public static String getRelativeCloudPath( final URI uri ) throws URISyntaxException
-	{
-		if ( URITools.isGC( uri ) )
-			return new URI( null, null, uri.getPath(), null ).toString();
-		else if ( URITools.isS3( uri ) )
-			return new URI( uri.getScheme(), uri.getHost(), uri.getPath(), null ).toString(); // TODO: this is a bug
-		else
-		{
-			if ( uri.getScheme() != null )
-				throw new RuntimeException( "Unsupported uri scheme: " + uri.getScheme() + " in '" + uri + "'." );
-			else
-				throw new RuntimeException( "Cannot get a relative cloud path for a relative path '" + uri + "'." );
-		}
-	}
-
 	public static BufferedReader openFileReadCloudReader( final KeyValueAccess kva, final URI uri ) throws IOException
 	{
 		return new BufferedReader(new InputStreamReader( openFileReadCloudStream( kva, uri )));
 	}
 
 	public static InputStream openFileReadCloudStream( final KeyValueAccess kva, final URI uri ) throws IOException
 	{
-		final String relativePath;
+		return kva.lockForReading( toNormalPath( kva, uri ) ).newInputStream();
+	}
 
+	/**
+	 * Get the "normalPath" of the given {@code URI} for the given {@code KeyValueAccess}.
+	 * <p>
+	 * The {@code URI} can be absolute or relative.
+	 * Relative {@code URI} are interpreted as relative to the root of the {@code KeyValueAccess}
+	 * (e.g. {@code 'file:/'} for {@code FileSystemKeyValueAccess},
+	 * {@code 's3://bucket-name/'} for {@code AmazonS3KeyValueAccess}, etc).
+	 *
+	 * @param kva a KeyValueAccess
+	 * @param uri absolute or relative URI
+	 * @return normalPath of {@code uri} for the given {@code KeyValueAccess}
+	 * @throws IOException
+	 */
+	static String toNormalPath( final KeyValueAccess kva, final URI uri ) throws IOException
+	{
 		try
 		{
-			relativePath = getRelativeCloudPath( uri );
+			final URI root = kva.uri( "/" );
+			final URI relativeURI = uri.isAbsolute()
+					? new URI( "/" ).resolve( root.relativize( uri ) )
+					: uri;
+			return kva.compose( root, relativeURI.getPath() );
 		}
-		catch (URISyntaxException e)
+		catch ( URISyntaxException e )
 		{
-			throw new IOException( e.getMessage() );
+			throw new IOException( e );
 		}
-
-		return kva.lockForReading( relativePath ).newInputStream();
 	}
 
 	public static PrintWriter openFileWriteCloudWriter( final KeyValueAccess kva, final URI uri ) throws IOException
@@ -528,22 +524,11 @@ public static PrintWriter openFileWriteCloudWriter( final KeyValueAccess kva, fi
 
 	public static OutputStream openFileWriteCloudStream( final KeyValueAccess kva, final URI uri ) throws IOException
 	{
-		final String relativePath;
-
-		try
-		{
-			relativePath = getRelativeCloudPath( uri );
-		}
-		catch (URISyntaxException e)
-		{
-			throw new IOException( e.getMessage() );
-		}
-
-		return kva.lockForWriting( relativePath ).newOutputStream();
+		return kva.lockForWriting( toNormalPath( kva, uri ) ).newOutputStream();
 	}
 
 	/**
-	 * Note: it is up to you to create the correct relative paths using getRelativeCloudPath()
+	 * Note: it is up to you to create the correct relative paths using toNormalPath()
 	 *
 	 * @param kva
 	 * @param relativeSrc
@@ -720,39 +705,6 @@ public static void copyFile( final File inputFile, final File outputFile ) throw
 		}
 	}
 
-	public static void minimalExampleTobiS3GS() throws URISyntaxException, IOException
-	{
-		URI gcURI = URITools.toURI( "gs://janelia-spark-test/I2K-test/dataset.xml" );
-		URI s3URI = URITools.toURI( "s3://janelia-bigstitcher-spark/Stitching/dataset.xml" );
-
-		// assemble scheme + bucket and location for Google Cloud
-		URI gcBucket = new URI( gcURI.getScheme(), gcURI.getHost(), null, null );
-		URI gcLocation = new URI( null, null, gcURI.getPath(), null );
-
-		System.out.println( "Google cloud: Instantiating N5Reader to grab a key-value-access for: '" + gcBucket + "'" );
-		System.out.println( "Google cloud: Lock for reading on: '" + gcLocation  + "'");
-
-		final N5Reader gcN5 = instantiateN5Reader(StorageFormat.N5, gcBucket );
-		final KeyValueAccess gcKVA = ((GsonKeyValueN5Reader)gcN5).getKeyValueAccess();
-		gcKVA.lockForReading( gcLocation.toString()).newInputStream().skip( 1000 );
-
-		// assemble scheme + bucket and location for AWS S3 (using full path for location, which should actually be relative, same as for google cloud above)
-		URI s3Bucket = new URI( s3URI.getScheme(), s3URI.getHost(), null, null );
-		URI s3Location = new URI( s3URI.getScheme(), s3URI.getHost(), s3URI.getPath(), null );
-
-		System.out.println( "AWS: Instantiating N5Reader to grab a key-value-access for: '" + s3Bucket + "'" );
-		System.out.println( "AWS: Lock for reading on: '" + s3Location  + "'");
-
-		final N5Reader s3N5 = instantiateN5Reader(StorageFormat.N5, s3Bucket );
-		final KeyValueAccess s3KVA = ((GsonKeyValueN5Reader)s3N5).getKeyValueAccess();
-		s3KVA.lockForReading( s3Location.toString() ).newInputStream().skip( 1000 );
-
-		// assemble relative location for AWS S3 (which fails)
-		s3Location = new URI( null, null, s3URI.getPath(), null );
-		System.out.println( "AWS: Lock for reading on: '" + s3Location  + "' (fails)");
-		s3KVA.lockForReading( s3Location.toString() ).newInputStream().skip( 1000 );
-	}
-
 	public static void main( String[] args ) throws SpimDataException, IOException, URISyntaxException
 	{
 		URI uri1 = URITools.toURI( "s3://aind-open-data/exaSPIM_708373_2024-04-02_19-49-38/SPIM.ome.zarr/" );
@@ -763,10 +715,6 @@ public static void main( String[] args ) throws SpimDataException, IOException,
 
 		System.exit( 0 );
 
-		minimalExampleTobiS3GS();
-
-		System.exit( 0 );
-
 		URI gcURI = URITools.toURI( "gs://janelia-spark-test/I2K-test/dataset.xml" );
 		System.out.println( "isGC: " + isGC(gcURI) + " [" + gcURI + "]" );
 		SpimData2 sdGC = loadSpimData(gcURI, new XmlIoSpimData2() );