v2.1.0

build.gradle

@@ -37,7 +37,7 @@ buildscript {
 apply from: "$commonBuildDir/ecs-publish.gradle"
 
 dependencies {
-    compile 'com.emc.ecs:smart-client:2.0.5',
+    compile 'com.emc.ecs:smart-client:2.0.6',
             'com.emc.ecs:object-transform:1.0.2',
             'org.jdom:jdom2:2.0.6'
     testCompile 'junit:junit:4.12'

src/main/java/com/emc/object/s3/jersey/ChecksumFilter.java

@@ -50,21 +50,26 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio
             ClientResponse response = getNext().handle(request);
 
             // pull etag from response headers
-            String etag = RestUtil.getFirstAsString(response.getHeaders(), RestUtil.HEADER_ETAG);
-            if (etag != null) etag = etag.replaceAll("\"", "");
-            if (etag != null && (etag.length() <= 2 || etag.contains("-"))) etag = null; // look for valid etags
+            String md5Header = RestUtil.getFirstAsString(response.getHeaders(), RestUtil.HEADER_ETAG);
+            if (md5Header != null) md5Header = md5Header.replaceAll("\"", "");
+            if (md5Header != null && (md5Header.length() <= 2 || md5Header.contains("-")))
+                md5Header = null; // look for valid etags
 
-            if (verifyWrite != null && verifyWrite && etag != null) {
+            // also look for content MD5 (this trumps etag if present)
+            String contentMd5 = RestUtil.getFirstAsString(response.getHeaders(), RestUtil.EMC_CONTENT_MD5);
+            if (contentMd5 != null) md5Header = contentMd5;
+
+            if (verifyWrite != null && verifyWrite && md5Header != null) {
                 // verify write checksum
-                if (!adapter.getChecksum().getValue().equals(etag))
-                    throw new ChecksumError("Checksum failure while writing stream", adapter.getChecksum().getValue(), etag);
+                if (!adapter.getChecksum().getValue().equals(md5Header))
+                    throw new ChecksumError("Checksum failure while writing stream", adapter.getChecksum().getValue(), md5Header);
             }
 
             Boolean verifyRead = (Boolean) request.getProperties().get(RestUtil.PROPERTY_VERIFY_READ_CHECKSUM);
-            if (verifyRead != null && verifyRead && etag != null) {
+            if (verifyRead != null && verifyRead && md5Header != null) {
                 // wrap stream to verify read checksum
                 response.setEntityInputStream(new ChecksummedInputStream(response.getEntityInputStream(),
-                        new ChecksumValueImpl(ChecksumAlgorithm.MD5, 0, etag))); // won't have length for chunked responses
+                        new ChecksumValueImpl(ChecksumAlgorithm.MD5, 0, md5Header))); // won't have length for chunked responses
             }
 
             return response;

src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java

@@ -415,7 +415,10 @@ public ListObjectsResult listObjects(String bucketName, String prefix) {
 
     @Override
     public ListObjectsResult listObjects(ListObjectsRequest request) {
-        return executeRequest(client, request, ListObjectsResult.class);
+        ListObjectsResult result = executeRequest(client, request, ListObjectsResult.class);
+        if (result.isTruncated() && result.getNextMarker() == null)
+            result.setNextMarker(result.getObjects().get(result.getObjects().size() - 1).getKey());
+        return result;
     }
 
     @Override

src/main/java/com/emc/object/util/RestUtil.java

@@ -62,6 +62,7 @@ public final class RestUtil {
     public static final String EMC_PREFIX = "x-emc-";
 
     public static final String EMC_APPEND_OFFSET = EMC_PREFIX + "append-offset";
+    public static final String EMC_CONTENT_MD5 = EMC_PREFIX + "content-md5";
     public static final String EMC_FS_ENABLED = EMC_PREFIX + "file-system-access-enabled";
     public static final String EMC_NAMESPACE = EMC_PREFIX + "namespace";
     public static final String EMC_VPOOL = EMC_PREFIX + "vpool";

src/test/java/com/emc/object/s3/ChecksumFilterTest.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2015, EMC Corporation.
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * + Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * + Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ * + The name of EMC Corporation may not be used to endorse or promote
+ *   products derived from this software without specific prior written
+ *   permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.emc.object.s3;
+
+import com.emc.object.s3.jersey.ChecksumFilter;
+import com.emc.object.util.ChecksumError;
+import com.emc.object.util.RestUtil;
+import com.sun.jersey.api.client.*;
+import com.sun.jersey.core.header.InBoundHeaders;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Random;
+
+public class ChecksumFilterTest {
+    @Test
+    public void testContentMd5() throws Exception {
+        byte[] data = new byte[1024];
+        new Random().nextBytes(data);
+
+        MockClientHandler mockHandler = new MockClientHandler();
+
+        Client client = new Client(mockHandler);
+        client.addFilter(new ChecksumFilter());
+
+        // positive test
+        mockHandler.setBadMd5(false);
+        WebResource resource = client.resource("http://foo.com");
+        resource.setProperty(RestUtil.PROPERTY_VERIFY_WRITE_CHECKSUM, Boolean.TRUE);
+        ClientResponse response = resource.put(ClientResponse.class, data);
+        Assert.assertNotNull(response);
+        Assert.assertEquals(200, response.getStatus());
+
+        try {
+            mockHandler.setBadMd5(true);
+            resource = client.resource("http://foo.com");
+            resource.setProperty(RestUtil.PROPERTY_VERIFY_WRITE_CHECKSUM, Boolean.TRUE);
+            resource.put(ClientResponse.class, data);
+            Assert.fail("bad MD5 should throw exception");
+        } catch (ChecksumError e) {
+            // expected
+        }
+    }
+
+    // assumes byte[] entity
+    class MockClientHandler implements ClientHandler {
+        boolean badMd5 = false;
+
+        @Override
+        public ClientResponse handle(ClientRequest cr) throws ClientHandlerException {
+            byte[] content = (byte[]) cr.getEntity();
+
+            // make sure entity is actually written (so digest stream will get real MD5)
+            try {
+                OutputStream out = cr.getAdapter().adapt(cr, new ByteArrayOutputStream());
+                out.write((byte[]) cr.getEntity());
+                out.close();
+            } catch (IOException e) {
+                throw new ClientHandlerException(e);
+            }
+
+            // set content MD5 header in response (bad or real)
+            InBoundHeaders headers = new InBoundHeaders();
+            if (badMd5) headers.add(RestUtil.EMC_CONTENT_MD5, "abcdef0123456789abcdef0123456789");
+            else headers.add(RestUtil.EMC_CONTENT_MD5, DigestUtils.md5Hex(content));
+
+            // return mock response with headers and no data
+            return new ClientResponse(ClientResponse.Status.OK, headers, new ByteArrayInputStream(new byte[0]), null);
+        }
+
+        public void setBadMd5(boolean badMd5) {
+            this.badMd5 = badMd5;
+        }
+    }
+}

src/test/java/com/emc/object/s3/S3JerseyClientTest.java

@@ -1904,6 +1904,35 @@ public void testListMarkerWithSpecialChars() throws Exception {
         Assert.assertEquals(marker, result.getMarker());
     }
 
+    @Test
+    public void testListPagesNoDelimiter() throws Exception {
+        int total = 10, page = 3;
+        for (int i = 0; i < total; i++) {
+            client.putObject(getTestBucket(), "key-" + i, "key-" + i, "text/plain");
+        }
+
+        Set<String> allKeys = new HashSet<String>();
+
+        ListObjectsResult result = null;
+        String nextMarker = null;
+        do {
+            if (result == null) result = client.listObjects(new ListObjectsRequest(getTestBucket()).withMaxKeys(page));
+            else result = client.listMoreObjects(result);
+            Assert.assertNotNull(result);
+            if (result.isTruncated()) {
+                Assert.assertEquals(page, result.getObjects().size());
+                Assert.assertNotNull(result.getNextMarker());
+                Assert.assertNotEquals(nextMarker, result.getNextMarker());
+                nextMarker = result.getNextMarker();
+            }
+            for (S3Object object : result.getObjects()) {
+                allKeys.add(object.getKey());
+            }
+        } while (result.isTruncated());
+
+        Assert.assertEquals(total, allKeys.size());
+    }
+
     @Ignore // TODO: blocked by STORAGE-9574
     @Test
     public void testListMarkerWithIllegalChars() throws Exception {