Skip to content

Commit

Permalink
actually fix the issue and always try with credentials first for read…
Browse files Browse the repository at this point in the history 
…er and writer
  • Loading branch information
@StephanPreibisch
StephanPreibisch committed Sep 25, 2024
1 parent 7614704 commit 68a4da7
Showing 1 changed file with 63 additions and 56 deletions.
119 changes: 63 additions & 56 deletions src/main/java/util/URITools.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ public static class ParsedBucket
public String bucket;
public String rootDir;
public String file;

@Override
public String toString()
{
return "protocol: '" + protocol + "', bucket: '" + bucket + "', rootdir: '" + rootDir + "', file: '" + file + "'";
}
}

public static void saveSpimData( final SpimData2 data, final URI xmlURI, final XmlIoSpimData2 io ) throws SpimDataException
Expand All @@ -75,42 +81,38 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
final ParsedBucket pb;
final KeyValueAccess kva;

System.out.println( xmlURI );

try
{
pb = URITools.parseCloudLink( xmlURI.toString() );
kva = URITools.getWriteKeyValueAccessForBucket( pb );
kva = URITools.getKeyValueAccessForBucket( pb );
}
catch ( Exception e )
{
throw new SpimDataException( "Could not parse cloud link and setup KeyValueAccess for '" + xmlURI + "': " + e );
}

System.out.println( pb );

// fist make a copy of the XML and save it to not loose it
try
{

if ( kva.exists( pb.rootDir + "/" + pb.file ) )
final String xmlFile = pb.protocol + pb.bucket + "/" + pb.rootDir + "/" + pb.file;
if ( kva.exists( xmlFile ) )
{
int maxExistingBackup = 0;
for ( int i = 1; i < XmlIoSpimData2.numBackups; ++i )
if ( kva.exists( pb.rootDir + "/" + pb.file + "~" + i ) )
if ( kva.exists( xmlFile + "~" + i ) )
maxExistingBackup = i;
else
break;

// copy the backups
try
{
for ( int i = maxExistingBackup; i >= 1; --i )
URITools.copy(kva, pb.rootDir + "/" + pb.file + "~" + i, pb.rootDir + "/" + pb.file + "~" + (i + 1) );

URITools.copy(kva, pb.rootDir + "/" + pb.file, pb.rootDir + "/" + pb.file + "~1" );
}
catch ( final Exception e )
{
IOFunctions.println( "Could not save backup of XML file: " + e );
e.printStackTrace();
}
for ( int i = maxExistingBackup; i >= 1; --i )
URITools.copy(kva, xmlFile + "~" + i, xmlFile + "~" + (i + 1) );

URITools.copy(kva, xmlFile, xmlFile + "~1" );
}
}
catch ( Exception e )
Expand All @@ -120,16 +122,13 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )

try
{
final Document doc = new Document( io.toXml( data, new File( ".") ) );
final Document doc = new Document( io.toXml( data, getParent( xmlURI ) ) );
final XMLOutputter xout = new XMLOutputter( Format.getPrettyFormat() );
final String xmlString = xout.outputString( doc );
//System.out.println( xmlString );

final OutputStream os = kva.lockForWriting( pb.rootDir + "/" + pb.file ).newOutputStream();
final PrintWriter pw = new PrintWriter( os );

final PrintWriter pw = openFileWriteCloud( kva, pb.protocol + pb.bucket + "/" + pb.rootDir + "/" + pb.file );
pw.println( xmlString );
pw.close();
os.close();
}
catch ( Exception e )
{
Expand Down Expand Up @@ -177,7 +176,7 @@ else if ( format.equals( StorageFormat.HDF5 ))
}
catch ( Exception e )
{
System.out.println( "Anonymous failed; trying writing with credentials ..." );
System.out.println( "With credentials failed; trying anonymous ..." );

n5w = new N5Factory().openWriter( format, uri );
}
Expand All @@ -204,16 +203,16 @@ else if ( format.equals( StorageFormat.ZARR ))

try
{
System.out.println( "Trying anonymous reading ..." );
n5r = new N5Factory().openReader( format, uri );
}
catch ( Exception e )
{
System.out.println( "Anonymous failed; trying reading with credentials ..." );
System.out.println( "Trying reading with credentials ..." );
N5Factory factory = new N5Factory();
factory.s3UseCredentials();
n5r = factory.openReader( format, uri );
}
catch ( Exception e )
{
System.out.println( "With credentials failed; trying anonymous ..." );
n5r = new N5Factory().openReader( format, uri );
}

return n5r;
//return new N5Factory().openReader( format, uri ); // cloud support, avoid dependency hell if it is a local file
Expand All @@ -237,16 +236,16 @@ else if ( uri.toString().toLowerCase().endsWith( ".n5" ) )

try
{
System.out.println( "Trying anonymous reading ..." );
n5r = new N5Factory().openReader( uri.toString() );
}
catch ( Exception e )
{
System.out.println( "Anonymous failed; trying reading with credentials ..." );
System.out.println( "Trying reading with credentials ..." );
N5Factory factory = new N5Factory();
factory.s3UseCredentials();
n5r = factory.openReader( uri.toString() );
}
catch ( Exception e )
{
System.out.println( "With credentials failed; trying anonymous ..." );
n5r = new N5Factory().openReader( uri.toString() );
}

return n5r;
//return new N5Factory().openReader( uri.toString() ); // cloud support, avoid dependency hell if it is a local file
Expand Down Expand Up @@ -298,14 +297,15 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
//super.load(null, xmlURI); // how do I use this?

final ParsedBucket pb = URITools.parseCloudLink( xmlURI.toString() );
final KeyValueAccess kva = URITools.getReadKeyValueAccessForBucket( pb );
final KeyValueAccess kva = URITools.getKeyValueAccessForBucket( pb );

final SAXBuilder sax = new SAXBuilder();
Document doc;
try
{
final InputStream is = kva.lockForReading( pb.rootDir + "/" + pb.file ).newInputStream();
final InputStream is = kva.lockForReading( pb.protocol + pb.bucket + "/" + pb.rootDir + "/" + pb.file ).newInputStream();
doc = sax.build( is );
is.close();
}
catch ( final Exception e )
{
Expand All @@ -330,32 +330,15 @@ else if ( URITools.isS3( xmlURI ) || URITools.isGC( xmlURI ) )
}
}

/*
public static KeyValueAccess getKeyValueAccessForBucket( String bucketUri )
{
final N5Reader n5r = new N5Factory().openReader( StorageFormat.N5, bucketUri );
final KeyValueAccess kva = ((GsonKeyValueN5Reader)n5r).getKeyValueAccess();
return kva;
}
*/

public static KeyValueAccess getReadKeyValueAccessForBucket( final ParsedBucket pb )
public static KeyValueAccess getKeyValueAccessForBucket( final ParsedBucket pb )
{
// we use a reader so it does not create an attributes.json; the KeyValueAccess is able to write anyways
final N5Reader n5r = instantiateN5Reader(StorageFormat.N5, URI.create( pb.protocol + pb.bucket ) );//new N5Factory().openReader( StorageFormat.N5, pb.protocol + pb.bucket );
final KeyValueAccess kva = ((GsonKeyValueN5Reader)n5r).getKeyValueAccess();

return kva;
}

public static KeyValueAccess getWriteKeyValueAccessForBucket( final ParsedBucket pb )
{
final N5Writer n5w = instantiateN5Writer(StorageFormat.N5, URI.create( pb.protocol + pb.bucket ) );//new N5Factory().openReader( StorageFormat.N5, pb.protocol + pb.bucket );
final KeyValueAccess kva = ((GsonKeyValueN5Reader)n5w).getKeyValueAccess();

return kva;
}

public static ParsedBucket parseCloudLink( final String uri )
{
System.out.println( "Parsing link path for '" + uri + "':" );
Expand Down Expand Up @@ -506,4 +489,28 @@ public static void copyFile( final File inputFile, final File outputFile ) throw
}
}

public static void main( String[] args ) throws SpimDataException
{
ParsedBucket pb = URITools.parseCloudLink( "s3://janelia-bigstitcher-spark/Stitching-test/dataset.xml" );
KeyValueAccess kva = URITools.getKeyValueAccessForBucket( pb );

System.out.println( pb );

try
{
BufferedReader reader = openFileReadCloud(kva, "s3://janelia-bigstitcher-spark/Stitching-test/test_1727295175425.txt" );
reader.lines().forEach( s -> System.out.println( s ) );
reader.close();

String path = pb.protocol + pb.bucket + "/" + pb.rootDir + "/" + "test_" + System.currentTimeMillis() + ".txt";

final PrintWriter pw = openFileWriteCloud( kva, path );
pw.println( "hallo " + System.currentTimeMillis() );
pw.close();
}
catch ( Exception e )
{
throw new SpimDataException( "Could not save xml '" + "': " + e );
}
}
}

0 comments on commit 68a4da7

Please sign in to comment.