Merge pull request #78 from rpmoore/master

Metadata Support

build.gradle

@@ -15,7 +15,7 @@
 
 allprojects {
     group = 'com.spectralogic.ds3'
-    version = '1.1.0-RC7'
+    version = '1.1.0-RC8'
 }
 
 subprojects {
@@ -32,14 +32,17 @@ subprojects {
     dependencies {
         compile     'org.slf4j:slf4j-api:1.7.12'
         compile     'org.slf4j:jcl-over-slf4j:1.7.12'
-        testCompile 'org.mockito:mockito-all:1.9.5'
-        testCompile 'junit:junit:4.11'
+        testCompile ('org.mockito:mockito-core:1.9.5') {
+            exclude group: 'org.hamcrest'
+        }
+
+        testCompile 'junit:junit:4.12'
         testCompile 'org.slf4j:slf4j-simple:1.7.12'
     }
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '2.4'
+    gradleVersion = '2.6'
 }
 
 project(':ds3-sdk-integration') {

ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Metadata_Test.java

@@ -0,0 +1,148 @@
+package com.spectralogic.ds3client.integration;
+
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Lists;
+import com.spectralogic.ds3client.Ds3Client;
+import com.spectralogic.ds3client.commands.*;
+import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
+import com.spectralogic.ds3client.models.bulk.Ds3Object;
+import com.spectralogic.ds3client.networking.Metadata;
+import com.spectralogic.ds3client.serializer.XmlProcessingException;
+import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
+import org.apache.commons.io.IOUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.security.SignatureException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class Metadata_Test {
+    private static Ds3Client client;
+
+    @BeforeClass
+    public static void startup() {
+        client = Util.fromEnv();
+    }
+
+    @AfterClass
+    public static void teardown() throws IOException {
+        client.close();
+    }
+
+    @Test
+    public void singleMetadataField() throws IOException, SignatureException, XmlProcessingException {
+
+        final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value"));
+
+        final List<String> values = metadata.get("name");
+        assertThat(values, is(notNullValue()));
+        assertFalse(values.isEmpty());
+        assertThat(values.size(), is(1));
+        assertThat(values.get(0), is("value"));
+
+    }
+
+    @Test
+    public void multipleMetadataFields() throws IOException, SignatureException, XmlProcessingException {
+
+        final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value", "key2", "value2"));
+
+        final Set<String> keys = metadata.keys();
+
+        assertThat(keys.size(), is(2));
+        assertTrue(keys.contains("name"));
+        assertTrue(keys.contains("key2"));
+
+        final List<String> values = metadata.get("name");
+        assertThat(values, is(notNullValue()));
+        assertFalse(values.isEmpty());
+        assertThat(values.size(), is(1));
+        assertThat(values.get(0), is("value"));
+
+    }
+
+    /*
+    //TODO There is currently a limitation in BP where it does not handle metadata values that have the same key.
+    @Test
+    public void multipleMetadataFieldsForSameKey() throws XmlProcessingException, SignatureException, IOException {
+
+        final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value", "name", "value2"));
+
+        final Set<String> keys = metadata.keys();
+
+        assertThat(keys.size(), is(1));
+        assertTrue(keys.contains("name"));
+
+        final List<String> values = metadata.get("name");
+        assertThat(values, is(notNullValue()));
+        assertFalse(values.isEmpty());
+        assertThat(values, hasItem("value"));
+    }
+
+    @Test
+    public void multipleMetadataFieldsForSameKeyDifferentCase() throws XmlProcessingException, SignatureException, IOException {
+
+        final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value", "Name", "value2"));
+
+        final Set<String> keys = metadata.keys();
+
+        assertThat(keys.size(), is(1));
+        assertTrue(keys.contains("name"));
+
+        final List<String> values = metadata.get("name");
+        assertThat(values, is(notNullValue()));
+        assertFalse(values.isEmpty());
+        assertThat(values, hasItem("value"));
+    }
+    */
+
+    private static Metadata processMetadataRequest(final String bucketName, final ImmutableMultimap<String, String> metadata) throws IOException, SignatureException, XmlProcessingException {
+        final Ds3ClientHelpers wrapper = Ds3ClientHelpers.wrap(client);
+        wrapper.ensureBucketExists(bucketName);
+
+        try {
+            final Ds3Object obj = new Ds3Object("obj.txt", 1024);
+            final BulkPutResponse putResponse = client.bulkPut(new BulkPutRequest(bucketName, Lists.newArrayList(obj)));
+            final PutObjectRequest request = new PutObjectRequest(bucketName, obj.getName(), putResponse.getResult().getJobId(), 1024, 0, buildRandomChannel(1024));
+
+            for (final Map.Entry<String, String> entry : metadata.entries()) {
+                request.withMetaData(entry.getKey(), entry.getValue());
+            }
+
+            final PutObjectResponse putObjResponse = client.putObject(request);
+
+            assertThat(putObjResponse.getStatusCode(), is(200));
+
+            final HeadObjectResponse response = client.headObject(new HeadObjectRequest(bucketName, obj.getName()));
+
+            assertThat(response.getStatusCode(), is(200));
+
+            return response.getMetadata();
+
+        } finally {
+            Util.deleteAllContents(client, bucketName);
+        }
+    }
+
+    private static SeekableByteChannel buildRandomChannel(int length) throws IOException {
+
+        final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(System.currentTimeMillis(), length));
+        final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
+
+        final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(length);
+        channel.write(randomBuffer);
+
+        return channel;
+    }
+}

ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java

@@ -144,6 +144,17 @@ GetObjectResponse getObject(GetObjectRequest request)
     PutObjectResponse putObject(PutObjectRequest request)
             throws IOException, SignatureException;
 
+    /**
+     * Performs an HTTP HEAD for an object.  If the object exists then a successful response is returned and any metadata
+     * associated with the object.
+     * @param request The Object to perform the HTTP HEAD for.
+     * @return If successful a response is returned with any metadata that was assigned with the object was created.
+     * @throws IOException
+     * @throws SignatureException
+     */
+    HeadObjectResponse headObject(HeadObjectRequest request)
+            throws IOException, SignatureException;
+
     /**
      * Primes the Ds3 appliance for a Bulk Get.  This does not perform the gets for each individual files.  See
      * {@link #getObject(GetObjectRequest)} for performing the get.

ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java

@@ -90,6 +90,11 @@ public PutObjectResponse putObject(final PutObjectRequest request) throws IOExce
         return new PutObjectResponse(this.netClient.getResponse(request));
     }
 
+    @Override
+    public HeadObjectResponse headObject(final HeadObjectRequest request) throws IOException, SignatureException {
+        return new HeadObjectResponse(this.netClient.getResponse(request));
+    }
+
     @Override
     public BulkGetResponse bulkGet(final BulkGetRequest request) throws IOException, SignatureException {
         return new BulkGetResponse(this.netClient.getResponse(request));

ds3-sdk/src/main/java/com/spectralogic/ds3client/HeadersImpl.java

@@ -15,29 +15,41 @@
 
 package com.spectralogic.ds3client;
 
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import com.spectralogic.ds3client.networking.Headers;
 import org.apache.http.Header;
 
-public class HeadersImpl implements Headers {
+import java.util.List;
+import java.util.Set;
 
-    private final Header[] headers;
+class HeadersImpl implements Headers {
 
-    HeadersImpl(final Header[] allHeaders){
-        this.headers = allHeaders;
-    }
+    private final ImmutableMultimap<String, String> headers;
 
-    @Override
-    public String get(final String key) {
-        return findHeader(key);
+
+    HeadersImpl(final Header[] allHeaders){
+        this.headers = toMultiMap(allHeaders);
     }
 
-    private String findHeader(final String key) {
+    private static ImmutableMultimap<String, String> toMultiMap(final Header[] headers){
+        final ImmutableMultimap.Builder<String, String> builder = ImmutableMultimap.builder();
 
         for (final Header header : headers) {
-            if (header.getName().equalsIgnoreCase(key)) {
-                return header.getValue();
-            }
+            builder.put(header.getName(), header.getValue());
         }
-        return null;
+
+        return builder.build();
+    }
+
+    @Override
+    public List<String> get(final String key) {
+        return Lists.newArrayList(headers.get(key));
+    }
+
+    @Override
+    public Set<String> keys() {
+        return Sets.newHashSet(headers.keySet());
     }
 }

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AbstractResponse.java

@@ -18,16 +18,22 @@
 import com.google.common.collect.ImmutableSet;
 import com.spectralogic.ds3client.models.Error;
 import com.spectralogic.ds3client.networking.FailedRequestException;
+import com.spectralogic.ds3client.networking.Headers;
 import com.spectralogic.ds3client.networking.WebResponse;
 import com.spectralogic.ds3client.serializer.XmlOutput;
 
 import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
+import java.util.List;
 
 public abstract class AbstractResponse implements Ds3Response{
+    final private static Logger LOG = LoggerFactory.getLogger(AbstractResponse.class);
+
     final public static String UTF8 = "UTF-8";
 
     final private WebResponse response;
@@ -36,14 +42,23 @@ public abstract class AbstractResponse implements Ds3Response{
     public AbstractResponse(final WebResponse response) throws IOException {
         this.response = response;
         if (response != null) {
-            this.md5 = this.response.getHeaders().get("Content-MD5");
+            this.md5 = getFirstHeaderValue(this.response.getHeaders(), "Content-MD5");
         }
         else {
             this.md5 = null;
         }
         this.processResponse();
     }
 
+    final protected String getFirstHeaderValue(final Headers headers, final String key) {
+        final List<String> valueList = headers.get(key);
+        if (valueList == null || valueList.isEmpty()) {
+            return null;
+        } else {
+            return valueList.get(0);
+        }
+    }
+
     protected abstract void processResponse() throws IOException;
 
     public  WebResponse getResponse() {
@@ -81,6 +96,7 @@ private static Error parseErrorResponse(final String responseString) {
             return XmlOutput.fromXml(responseString, Error.class);
         } catch (final IOException e) {
             // It's likely the response string is not in a valid error format.
+            LOG.error("Failed to parse error response", e);
             return null;
         }
     }

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java

@@ -73,7 +73,7 @@ private static Objects parseChunk(final WebResponse webResponse) throws IOExcept
     }
 
     private static int parseRetryAfter(final WebResponse webResponse) {
-        final String retryAfter = webResponse.getHeaders().get("Retry-After");
+        final String retryAfter = webResponse.getHeaders().get("Retry-After").get(0);
         if (retryAfter == null) {
             throw new RetryAfterExpectedException();
         }

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java

@@ -74,7 +74,7 @@ private static MasterObjectList parseMasterObjectList(final WebResponse webRespo
     }
 
     private static int parseRetryAfter(final WebResponse webResponse) {
-        final String retryAfter = webResponse.getHeaders().get("Retry-After");
+        final String retryAfter = webResponse.getHeaders().get("Retry-After").get(0);
         if (retryAfter == null) {
             throw new RetryAfterExpectedException();
         }

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectResponse.java

@@ -15,6 +15,7 @@
 
 package com.spectralogic.ds3client.commands;
 
+import com.spectralogic.ds3client.networking.Metadata;
 import com.spectralogic.ds3client.networking.WebResponse;
 import com.spectralogic.ds3client.utils.IOUtils;
 
@@ -28,6 +29,10 @@ public GetObjectResponse(final WebResponse response, final WritableByteChannel d
         download(destinationChannel, bufferSize);
     }
 
+    public Metadata getMetadata() {
+        return new MetadataImpl(this.getResponse().getHeaders());
+    }
+
     @Override
     protected void processResponse() throws IOException {
         this.checkStatusCode(200);

ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectRequest.java

@@ -0,0 +1,24 @@
+package com.spectralogic.ds3client.commands;
+
+import com.spectralogic.ds3client.HttpVerb;
+
+public class HeadObjectRequest extends AbstractRequest {
+
+    private final String bucketName;
+    private final String objectName;
+
+    public HeadObjectRequest(final String bucketName, final String objectName) {
+        this.bucketName = bucketName;
+        this.objectName = objectName;
+    }
+
+    @Override
+    public String getPath() {
+        return "/" + bucketName + "/" + objectName;
+    }
+
+    @Override
+    public HttpVerb getVerb() {
+        return HttpVerb.HEAD;
+    }
+}