From c1ae2dbb34b0763cdb579be27db9ce635fc8b1ee Mon Sep 17 00:00:00 2001 From: "arnett, stu" Date: Tue, 3 Nov 2015 14:38:03 -0600 Subject: [PATCH] v2.0.7 --- .../com/emc/object/s3/LargeFileUploader.java | 113 ++++++++++------ .../object/s3/jersey/AuthorizationFilter.java | 2 +- .../emc/object/util/ProgressInputStream.java | 63 +++++++++ .../com/emc/object/util/ProgressListener.java | 47 +++++++ .../emc/object/util/ProgressOutputStream.java | 61 +++++++++ .../java/com/emc/object/util/RestUtil.java | 127 +++++++++++++++++- .../s3/S3EncryptionClientBasicTest.java | 15 +++ .../com/emc/object/s3/S3JerseyClientTest.java | 79 ++++++++++- .../com/emc/object/util/RestUtilTest.java | 19 +++ 9 files changed, 485 insertions(+), 41 deletions(-) mode change 100644 => 100755 src/main/java/com/emc/object/s3/LargeFileUploader.java create mode 100755 src/main/java/com/emc/object/util/ProgressInputStream.java create mode 100755 src/main/java/com/emc/object/util/ProgressListener.java create mode 100644 src/main/java/com/emc/object/util/ProgressOutputStream.java diff --git a/src/main/java/com/emc/object/s3/LargeFileUploader.java b/src/main/java/com/emc/object/s3/LargeFileUploader.java old mode 100644 new mode 100755 index 64e7d5f0..64af590d --- a/src/main/java/com/emc/object/s3/LargeFileUploader.java +++ b/src/main/java/com/emc/object/s3/LargeFileUploader.java @@ -33,6 +33,8 @@ import com.emc.object.s3.bean.MultipartPartETag; import com.emc.object.s3.request.*; import com.emc.object.util.InputStreamSegment; +import com.emc.object.util.ProgressInputStream; +import com.emc.object.util.ProgressListener; import com.emc.rest.util.SizedInputStream; import org.apache.log4j.Logger; @@ -47,6 +49,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; /** * Convenience class to facilitate multipart upload for large files. This class will split the file @@ -74,8 +77,9 @@ public class LargeFileUploader implements Runnable { private Long partSize = DEFAULT_PART_SIZE; private int threads = DEFAULT_THREADS; private ExecutorService executorService; + private AtomicLong bytesTransferred = new AtomicLong(); + private ProgressListener progressListener; - private long bytesTransferred; private String eTag; /** @@ -113,7 +117,6 @@ public void doMultipartUpload() { String uploadId = s3Client.initiateMultipartUpload(initRequest).getUploadId(); List> futures = new ArrayList>(); - List segmentStreams = new ArrayList(); try { // submit all upload tasks int partNumber = 1; @@ -121,14 +124,7 @@ public void doMultipartUpload() { while (offset < fullSize) { if (offset + length > fullSize) length = fullSize - offset; - SizedInputStream segmentStream = file != null - ? new InputStreamSegment(new FileInputStream(file), offset, length) - : new SizedInputStream(stream, length); - segmentStreams.add(segmentStream); - - UploadPartRequest partRequest = new UploadPartRequest(bucket, key, uploadId, partNumber++, segmentStream); - partRequest.setContentLength(length); - futures.add(executorService.submit(new UploadPartTask(partRequest))); + futures.add(executorService.submit(new UploadPartTask(uploadId, partNumber++, offset, length))); offset += length; } @@ -155,10 +151,6 @@ public void doMultipartUpload() { if (e instanceof RuntimeException) throw (RuntimeException) e; throw new RuntimeException("error during upload", e); } finally { - for (SizedInputStream segmentStream : segmentStreams) { - bytesTransferred += segmentStream.getRead(); - } - // make sure all spawned threads are shut down executorService.shutdown(); @@ -184,22 +176,13 @@ public void doByteRangeUpload() { s3Client.putObject(request); List> futures = new ArrayList>(); - List segmentStreams = new ArrayList(); try { // submit all upload tasks - PutObjectRequest rangeRequest; long offset = 0, length = partSize; while (offset < fullSize) { if (offset + length > fullSize) length = fullSize - offset; - Range range = Range.fromOffsetLength(offset, length); - SizedInputStream segmentStream = file != null - ? new InputStreamSegment(new FileInputStream(file), offset, length) - : new SizedInputStream(stream, length); - segmentStreams.add(segmentStream); - - rangeRequest = new PutObjectRequest(bucket, key, segmentStream).withRange(range); - futures.add(executorService.submit(new PutObjectTask(rangeRequest))); + futures.add(executorService.submit(new PutObjectTask(offset, length))); offset += length; } @@ -219,10 +202,6 @@ public void doByteRangeUpload() { if (e instanceof RuntimeException) throw (RuntimeException) e; throw new RuntimeException("error during upload", e); } finally { - for (SizedInputStream segmentStream : segmentStreams) { - bytesTransferred += segmentStream.getRead(); - } - // make sure all spawned threads are shut down executorService.shutdown(); @@ -300,7 +279,7 @@ public long getFullSize() { } public long getBytesTransferred() { - return bytesTransferred; + return bytesTransferred.get(); } public String getETag() { @@ -339,6 +318,14 @@ public void setCloseStream(boolean closeStream) { this.closeStream = closeStream; } + public ProgressListener getProgressListener() { + return progressListener; + } + + public void setProgressListener(ProgressListener progressListener) { + this.progressListener = progressListener; + } + public long getPartSize() { return partSize; } @@ -368,6 +355,14 @@ public ExecutorService getExecutorService() { return executorService; } + private void updateBytesTransferred(long count) { + long totalTransferred = bytesTransferred.addAndGet(count); + + if(progressListener != null) { + progressListener.progress(totalTransferred, fullSize); + } + } + /** * Allows for providing a custom thread executor (i.e. for custom thread factories). Note that if * you set a custom executor service, the threads property will be ignored. @@ -411,29 +406,71 @@ public LargeFileUploader withExecutorService(ExecutorService executorService) { return this; } - protected class UploadPartTask implements Callable { - private UploadPartRequest request; + public LargeFileUploader withProgressListener(ProgressListener progressListener) { + setProgressListener(progressListener); + return this; + } + + private class UploadPartTask implements Callable { + private String uploadId; + private int partNumber; + private long offset; + private long length; - public UploadPartTask(UploadPartRequest request) { - this.request = request; + public UploadPartTask(String uploadId, int partNumber, long offset, long length) { + this.uploadId = uploadId; + this.partNumber = partNumber; + this.offset = offset; + this.length = length; } @Override public MultipartPartETag call() throws Exception { - return s3Client.uploadPart(request); + SizedInputStream segmentStream; + if (file != null) { + segmentStream = new InputStreamSegment(new ProgressInputStream(new FileInputStream(file), progressListener), offset, length); + } else { + segmentStream = new SizedInputStream(new ProgressInputStream(stream, progressListener), length); + } + + UploadPartRequest request = new UploadPartRequest(bucket, key, uploadId, partNumber++, segmentStream); + request.setContentLength(length); + + MultipartPartETag etag = s3Client.uploadPart(request); + updateBytesTransferred(length); + return etag; } } protected class PutObjectTask implements Callable { - private PutObjectRequest request; + private long offset; + private long length; - public PutObjectTask(PutObjectRequest request) { - this.request = request; + public PutObjectTask(long offset, long length) { + this.offset = offset; + this.length = length; } @Override public String call() throws Exception { - return s3Client.putObject(request).getETag(); + Range range = Range.fromOffsetLength(offset, length); + + SizedInputStream segmentStream = file != null + ? new InputStreamSegment(new ProgressInputStream(new FileInputStream(file), progressListener), + offset, length) : new SizedInputStream(new ProgressInputStream(stream, progressListener), + length); + + PutObjectRequest request = new PutObjectRequest(bucket, key, segmentStream).withRange(range); + + String etag = s3Client.putObject(request).getETag(); + long length = 0; + if(request.getRange() != null) { + length = request.getRange().getLast() - request.getRange().getFirst() + 1; + } else if(request.getContentLength() != null) { + length = request.getContentLength(); + } + updateBytesTransferred(length); + return etag; } } } diff --git a/src/main/java/com/emc/object/s3/jersey/AuthorizationFilter.java b/src/main/java/com/emc/object/s3/jersey/AuthorizationFilter.java index 7e066573..1e70343d 100644 --- a/src/main/java/com/emc/object/s3/jersey/AuthorizationFilter.java +++ b/src/main/java/com/emc/object/s3/jersey/AuthorizationFilter.java @@ -53,7 +53,7 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio // if no identity is provided, this is an anonymous client if (s3Config.getIdentity() != null) { - Map parameters = RestUtil.getQueryParameterMap(request.getURI().getQuery()); + Map parameters = RestUtil.getQueryParameterMap(request.getURI().getRawQuery()); String resource = RestUtil.getEncodedPath(request.getURI()); // check if bucket is in hostname diff --git a/src/main/java/com/emc/object/util/ProgressInputStream.java b/src/main/java/com/emc/object/util/ProgressInputStream.java new file mode 100755 index 00000000..c6eeb276 --- /dev/null +++ b/src/main/java/com/emc/object/util/ProgressInputStream.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.object.util; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * InputStream wrapper class that fires events when data is read so realtime performance can be measured. + */ +public class ProgressInputStream extends FilterInputStream { + private final ProgressListener listener; + + public ProgressInputStream(InputStream wrappedStream, ProgressListener listener) { + super(wrappedStream); + this.listener = listener; + } + + @Override + public int read() throws IOException { + // This is a really, really bad idea. + throw new RuntimeException("No, I'm not going to let you kill performance."); + } + + @Override + public int read(byte[] b) throws IOException { + int count = in.read(b); + if(listener != null) listener.transferred(count); + return count; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int count = in.read(b, off, len); + if(listener != null) listener.transferred(count); + return count; + } +} diff --git a/src/main/java/com/emc/object/util/ProgressListener.java b/src/main/java/com/emc/object/util/ProgressListener.java new file mode 100755 index 00000000..fbf23d5b --- /dev/null +++ b/src/main/java/com/emc/object/util/ProgressListener.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.object.util; + +/** + * Interface for reporting progress on large file transfers. + */ +public interface ProgressListener { + /** + * Provides feedback on the number of bytes completed and the total number of bytes to transfer. + * @param completed bytes completely transferred + * @param total total number of bytes to transfer + */ + void progress(long completed, long total); + + /** + * Reports that some bytes have been transferred. This is a raw method that will be called frequently and can + * be used for computing current transfer rate. Note that if data is retried, the sum of this method's events + * may be more than the total object size. For reporting on percent complete, use the progress method instead. + * @param size number of bytes transferred + */ + void transferred(long size); +} diff --git a/src/main/java/com/emc/object/util/ProgressOutputStream.java b/src/main/java/com/emc/object/util/ProgressOutputStream.java new file mode 100644 index 00000000..e5ceabcf --- /dev/null +++ b/src/main/java/com/emc/object/util/ProgressOutputStream.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.object.util; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * OutputStream wrapper class that fires events when data is written so realtime performance can be measured. + */ +public class ProgressOutputStream extends FilterOutputStream { + private final ProgressListener listener; + + public ProgressOutputStream(OutputStream out, ProgressListener listener) { + super(out); + this.listener = listener; + } + + @Override + public void write(int b) throws IOException { + // This is a really, really bad idea. + throw new RuntimeException("No, I'm not going to let you kill performance."); + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + listener.transferred(b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + listener.transferred(len); + } +} diff --git a/src/main/java/com/emc/object/util/RestUtil.java b/src/main/java/com/emc/object/util/RestUtil.java index 51e7c43e..181f634c 100644 --- a/src/main/java/com/emc/object/util/RestUtil.java +++ b/src/main/java/com/emc/object/util/RestUtil.java @@ -26,11 +26,16 @@ */ package com.emc.object.util; +import sun.nio.cs.ThreadLocalCoders; + import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -230,7 +235,7 @@ public static URI buildUri(String scheme, String host, int port, String path, St throws URISyntaxException { URI uri = new URI(scheme, null, host, port, path, query, fragment); - String uriString = uri.toASCIIString(); + String uriString = toASCIIString(uri); // workaround for https://bugs.openjdk.java.net/browse/JDK-8037396 uriString = uriString.replace("[", "%5B").replace("]", "%5D"); @@ -248,6 +253,126 @@ public static URI buildUri(String scheme, String host, int port, String path, St return new URI(uriString); } + /** + * Returns the content of this URI as a US-ASCII string. + * + *

Note: this starts our customized version of URI's toASCIIString. We differ in only one aspect: we do + * NOT normalize Unicode characters. This is because certain Unicode characters may have different compositions + * and normalization may change the UTF-8 sequence represented by a character. We must maintain the same UTF-8 + * sequence in and out and therefore we cannot normalize the sequences. + * + *

If this URI does not contain any characters in the other + * category then an invocation of this method will return the same value as + * an invocation of the {@link #toString() toString} method. Otherwise + * this method works as if by invoking that method and then encoding the result.

+ * + * @return The string form of this URI, encoded as needed + * so that it only contains characters in the US-ASCII + * charset + */ + public static String toASCIIString(URI u) { + String s = defineString(u); + return encode(s); + } + + /** + * Defines a URI string. Provided for our special URI encoder. + * @param u URI to encode + * @return String for the URI + */ + private static String defineString(URI u) { + + StringBuffer sb = new StringBuffer(); + if (u.getScheme() != null) { + sb.append(u.getScheme()); + sb.append(':'); + } + if (u.isOpaque()) { + sb.append(u.getRawSchemeSpecificPart()); + } else { + if (u.getHost() != null) { + sb.append("//"); + if (u.getUserInfo() != null) { + sb.append(u.getUserInfo()); + sb.append('@'); + } + boolean needBrackets = ((u.getHost().indexOf(':') >= 0) + && !u.getHost().startsWith("[") + && !u.getHost().endsWith("]")); + if (needBrackets) sb.append('['); + sb.append(u.getHost()); + if (needBrackets) sb.append(']'); + if (u.getPort() != -1) { + sb.append(':'); + sb.append(u.getPort()); + } + } else if (u.getRawAuthority() != null) { + sb.append("//"); + sb.append(u.getRawAuthority()); + } + if (u.getRawPath() != null) + sb.append(u.getRawPath()); + if (u.getRawQuery() != null) { + sb.append('?'); + sb.append(u.getRawQuery()); + } + } + if (u.getRawFragment() != null) { + sb.append('#'); + sb.append(u.getRawFragment()); + } + return sb.toString(); + } + + /** + * Encodes all characters >= \u0080 into escaped, normalized UTF-8 octets, + * assuming that s is otherwise legal + */ + private static String encode(String s) { + int n = s.length(); + if (n == 0) + return s; + + // First check whether we actually need to encode + for (int i = 0;;) { + if (s.charAt(i) >= '\u0080') + break; + if (++i >= n) + return s; + } + + ByteBuffer bb = null; + try { + bb = ThreadLocalCoders.encoderFor("UTF-8") + .encode(CharBuffer.wrap(s)); + } catch (CharacterCodingException x) { + assert false; + } + + StringBuffer sb = new StringBuffer(); + while (bb.hasRemaining()) { + int b = bb.get() & 0xff; + if (b >= 0x80) + appendEscape(sb, (byte)b); + else + sb.append((char)b); + } + return sb.toString(); + } + + private final static char[] hexDigits = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + private static void appendEscape(StringBuffer sb, byte b) { + sb.append('%'); + sb.append(hexDigits[(b >> 4) & 0x0f]); + sb.append(hexDigits[(b >> 0) & 0x0f]); + } + + public static URI replaceHost(URI uri, String host) throws URISyntaxException { return buildUri(uri.getScheme(), host, uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); } diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java index cb520310..a5176fda 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java @@ -430,4 +430,19 @@ public void testPutObjectWithMd5() throws Exception { @Override public void testPutObjectWithRetentionPeriod() throws Exception { } + + @Ignore + @Override + public void testMpuAbortInMiddle() throws Exception { + } + + @Ignore + @Override + public void testLargeFileUploaderStream() throws Exception { + } + + @Ignore + @Override + public void testLargeFileUploaderProgressListener() throws Exception { + } } diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index e10c63f5..aab1b695 100755 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -31,6 +31,7 @@ import com.emc.object.s3.bean.*; import com.emc.object.s3.jersey.S3JerseyClient; import com.emc.object.s3.request.*; +import com.emc.object.util.ProgressListener; import com.emc.rest.smart.Host; import com.emc.rest.smart.ecs.Vdc; import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config; @@ -49,6 +50,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; public class S3JerseyClientTest extends AbstractS3ClientTest { private static final Logger l4j = Logger.getLogger(S3JerseyClientTest.class); @@ -405,6 +407,30 @@ public void testListObjectsPagingWithPrefix() throws Exception { Assert.assertEquals("should be 4 pages", 4, requestCount); } + @Ignore // blocked by STORAGE-9574 + @Test + public void testListObjectsWithEncoding() throws Exception { + String key = "foo\u001do", content = "Hello List!"; + client.putObject(getTestBucket(), key, content, null); + + try { + ListObjectsRequest request = new ListObjectsRequest(getTestBucket()).withEncodingType(EncodingType.url); + ListObjectsResult result = client.listObjects(request); + Assert.assertNotNull("ListObjectsResult was null, but should NOT have been", result); + + List resultObjects = result.getObjects(); + Assert.assertNotNull("List was null, but should NOT have been", resultObjects); + Assert.assertEquals(1, resultObjects.size()); + + S3Object object = resultObjects.get(0); + Assert.assertEquals(key, object.getKey()); + Assert.assertEquals((long) content.length(), object.getSize().longValue()); + + } finally { + client.deleteObject(getTestBucket(), key); + } + } + @Test public void testListAndReadVersions() throws Exception { // turn on versioning first @@ -667,6 +693,51 @@ public void testLargeFileUploader() throws Exception { uploader.doByteRangeUpload(); } + @Test + public void testLargeFileUploaderProgressListener() throws Exception { + String key = "large-file-uploader.bin"; + int size = 20 * 1024 * 1024 + 123; // > 20MB + byte[] data = new byte[size]; + new Random().nextBytes(data); + File file = File.createTempFile("large-file-uploader-test", null); + file.deleteOnExit(); + OutputStream out = new FileOutputStream(file); + out.write(data); + out.close(); + final AtomicLong completed = new AtomicLong(); + final AtomicLong total = new AtomicLong(); + final AtomicLong transferred = new AtomicLong(); + ProgressListener pl = new ProgressListener() { + + @Override + public void progress(long c, long t) { + completed.set(c); + total.set(t); + } + + @Override + public void transferred(long size) { + transferred.addAndGet(size); + } + }; + + LargeFileUploader uploader = new LargeFileUploader(client, getTestBucket(), key, file).withProgressListener(pl); + uploader.setPartSize(LargeFileUploader.MIN_PART_SIZE); + + // multipart + uploader.doMultipartUpload(); + + Assert.assertEquals(size, uploader.getBytesTransferred()); + Assert.assertTrue(uploader.getETag().contains("-")); // hyphen signifies multipart / updated object + Assert.assertArrayEquals(data, client.readObject(getTestBucket(), key, byte[].class)); + Assert.assertEquals(size, completed.get()); + Assert.assertEquals(size, total.get()); + Assert.assertTrue(String.format("Should transfer at least %d bytes but only got %d", size, transferred.get()), + transferred.get() >= size); + + client.deleteObject(getTestBucket(), key); + } + @Test public void testLargeFileUploaderStream() throws Exception { String key = "large-file-uploader-stream.bin"; @@ -1754,7 +1825,7 @@ public void testPreSignedUrl() throws Exception { .withIdentity("AKIAIOSFODNN7EXAMPLE").withSecretKey("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")); URL url = tempClient.getPresignedUrl("johnsmith", "photos/puppy.jpg", new Date(1175139620000L)); Assert.assertEquals("https://johnsmith.s3.amazonaws.com/photos/puppy.jpg" + - "?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1175139620&Signature=NpgCjnDzrM%2BWFzoENXmpNDUsSn8%3D", + "?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1175139620&Signature=NpgCjnDzrM%2BWFzoENXmpNDUsSn8%3D", url.toString()); } @@ -1811,6 +1882,12 @@ public void run() { Assert.assertEquals(0, client.listMultipartUploads(getTestBucket()).getUploads().size()); } + @Test + public void testListMarkerWithPercent() throws Exception { + String marker = "foo/bar/blah%blah"; + client.listObjects(new ListObjectsRequest(getTestBucket()).withMarker(marker)); + } + protected void assertAclEquals(AccessControlList acl1, AccessControlList acl2) { Assert.assertEquals(acl1.getOwner(), acl2.getOwner()); Assert.assertEquals(acl1.getGrants(), acl2.getGrants()); diff --git a/src/test/java/com/emc/object/util/RestUtilTest.java b/src/test/java/com/emc/object/util/RestUtilTest.java index bd47fa98..28901d1e 100644 --- a/src/test/java/com/emc/object/util/RestUtilTest.java +++ b/src/test/java/com/emc/object/util/RestUtilTest.java @@ -141,6 +141,25 @@ public void testReplacePath() throws Exception { Assert.assertEquals(new URI(post), RestUtil.replacePath(uri, "/" + bucket + "/" + key)); } + // Unicode "OHM SYMBOL" + public static final byte[] OHM_UTF8 = new byte[] { (byte)0xe2, (byte)0x84, (byte)0xa6 }; + + /** + * Tests URI building to make sure that it doesn't modify UTF-8 sequences. The default URI.toAsciiString runs the + * path through Unicode "Normalization" that modifies some Unicode characters. We need to make sure any UTF-8 + * input sequences are the same in and out so object keys are not changed. In this test specifically, the Ohm + * symbol below is normalized to a plain Omega symbol, changing the UTF-8 sequence causing ECS to say the object + * cannot be found. + */ + @Test + public void testUnicodeEncode() throws Exception { + // IntelliJ normalizes Ohm to Omega so you can't paste it as a literal. + String ohm = new String(OHM_UTF8, "UTF-8"); + + URI u = RestUtil.buildUri("http", "www.foo.com", -1, "/100 " + ohm + " Differential impedance 2.rar", null, null); + Assert.assertEquals("http://www.foo.com/100%20%E2%84%A6%20Differential%20impedance%202.rar", u.toString()); + } + private String encodePath(String path) { return RestUtil.urlEncode(path).replace("%2F", "/"); }