Skip to content

Commit

Permalink
Merge pull request #13 from rpmoore/master
Browse files Browse the repository at this point in the history
Adding ensureBucketExists helper
  • Loading branch information
hansdude committed May 8, 2014
2 parents b83086b + 2397975 commit f44571f
Show file tree
Hide file tree
Showing 9 changed files with 322 additions and 145 deletions.
32 changes: 14 additions & 18 deletions src/main/java/com/spectralogic/ds3client/Ds3Client.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,7 @@
import java.net.URISyntaxException;
import java.security.SignatureException;

import com.spectralogic.ds3client.commands.BulkGetRequest;
import com.spectralogic.ds3client.commands.BulkGetResponse;
import com.spectralogic.ds3client.commands.BulkPutRequest;
import com.spectralogic.ds3client.commands.BulkPutResponse;
import com.spectralogic.ds3client.commands.DeleteBucketRequest;
import com.spectralogic.ds3client.commands.DeleteBucketResponse;
import com.spectralogic.ds3client.commands.DeleteObjectRequest;
import com.spectralogic.ds3client.commands.DeleteObjectResponse;
import com.spectralogic.ds3client.commands.GetBucketRequest;
import com.spectralogic.ds3client.commands.GetBucketResponse;
import com.spectralogic.ds3client.commands.GetObjectRequest;
import com.spectralogic.ds3client.commands.GetObjectResponse;
import com.spectralogic.ds3client.commands.GetServiceRequest;
import com.spectralogic.ds3client.commands.GetServiceResponse;
import com.spectralogic.ds3client.commands.PutBucketRequest;
import com.spectralogic.ds3client.commands.PutBucketResponse;
import com.spectralogic.ds3client.commands.PutObjectRequest;
import com.spectralogic.ds3client.commands.PutObjectResponse;
import com.spectralogic.ds3client.commands.*;
import com.spectralogic.ds3client.models.Credentials;
import com.spectralogic.ds3client.networking.NetworkClient;

Expand Down Expand Up @@ -203,6 +186,19 @@ public PutBucketResponse putBucket(final PutBucketRequest request) throws IOExce
return new PutBucketResponse(netClient.getResponse(request));
}

/**
* Performs a HTTP HEAD for a bucket. The HEAD will return information about if the bucket exists, or if the user
* has access to that bucket.
* @param request The Head Bucket Request object used to customize the HTTP request. See {@link HeadBucketRequest}.
* @return The response object is returned and contains the status of the bucket. See {@link HeadBucketResponse} for
* the full list of status that a bucket can be in.
* @throws IOException
* @throws SignatureException
*/
public HeadBucketResponse headBucket(final HeadBucketRequest request) throws IOException, SignatureException {
return new HeadBucketResponse(netClient.getResponse(request));
}

/**
* Deletes a bucket from a DS3 endpoint. <b>Note:</b> all objects must be deleted first before deleteBucket will
* succeed.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.io.InputStream;
import java.io.StringWriter;

import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.methods.CloseableHttpResponse;

Expand All @@ -42,19 +43,32 @@ CloseableHttpResponse getResponse() {
return response;
}

void checkStatusCode(final int expectedStatus) throws IOException {
final int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != expectedStatus) {
void checkStatusCode(final int ... expectedStatuses) throws IOException {
final ImmutableSet<Integer> expectedSet = createExpectedSet(expectedStatuses);
final int statusCode = getStatusCode();
if (!expectedSet.contains(statusCode)) {
final String responseString = readResponseString();
throw new FailedRequestException(
expectedStatus,
expectedStatuses,
statusCode,
parseErrorResponse(responseString),
responseString
);
}
}


int getStatusCode() {
return response.getStatusLine().getStatusCode();
}

private ImmutableSet<Integer> createExpectedSet(final int[] expectedStatuses) {
final ImmutableSet.Builder<Integer> setBuilder = ImmutableSet.builder();
for(final int status: expectedStatuses) {
setBuilder.add(status);
}
return setBuilder.build();
}

private static Error parseErrorResponse(final String responseString) {
try {
return XmlOutput.fromXml(responseString, Error.class);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* ******************************************************************************
* Copyright 2014 Spectra Logic Corporation. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
* this file except in compliance with the License. A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file.
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
* ****************************************************************************
*/

package com.spectralogic.ds3client.commands;

import com.spectralogic.ds3client.HttpVerb;

/**
* {@code HeadBucketRequest} is used to return back information on if a bucket exists, or if a user has access to that bucket.
*/
public class HeadBucketRequest extends AbstractRequest {

private final String bucketName;

public HeadBucketRequest(final String bucketName) {
this.bucketName = bucketName;
}

@Override
public String getPath() {
return "/" + bucketName;
}

@Override
public HttpVerb getVerb() {
return HttpVerb.HEAD;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* ******************************************************************************
* Copyright 2014 Spectra Logic Corporation. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
* this file except in compliance with the License. A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file.
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
* ****************************************************************************
*/

package com.spectralogic.ds3client.commands;

import org.apache.http.client.methods.CloseableHttpResponse;

import java.io.IOException;

public class HeadBucketResponse extends AbstractResponse {

static public enum Status {
EXISTS, DOESNTEXIST, NOTAUTHORIZED
}

private Status status;

public HeadBucketResponse(final CloseableHttpResponse response) throws IOException {
super(response);
}

public Status getStatus() {
return status;
}

@Override
protected void processResponse() throws IOException {
checkStatusCode(200, 403, 404);
final int statusCode = getStatusCode();
setStatus(statusCode);
}

private void setStatus(final int statusCode) {
switch(statusCode) {
case 200: this.status = Status.EXISTS; break;
case 403: this.status = Status.NOTAUTHORIZED; break;
case 404: this.status = Status.DOESNTEXIST; break;
}
}
}
141 changes: 35 additions & 106 deletions src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,38 +17,20 @@

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.SignatureException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import com.google.common.collect.Lists;
import com.spectralogic.ds3client.Ds3Client;
import com.spectralogic.ds3client.commands.BulkGetRequest;
import com.spectralogic.ds3client.commands.BulkGetResponse;
import com.spectralogic.ds3client.commands.BulkPutRequest;
import com.spectralogic.ds3client.commands.BulkPutResponse;
import com.spectralogic.ds3client.commands.GetBucketRequest;
import com.spectralogic.ds3client.commands.GetBucketResponse;
import com.spectralogic.ds3client.models.Contents;
import com.spectralogic.ds3client.models.Ds3Object;
import com.spectralogic.ds3client.models.ListBucketResult;
import com.spectralogic.ds3client.models.MasterObjectList;
import com.spectralogic.ds3client.serializer.XmlProcessingException;

/**
* A wrapper around the {@link com.spectralogic.ds3client.Ds3Client} which automates common tasks.
*/
public class Ds3ClientHelpers {
private static final int DEFAULT_MAX_KEYS = 1000;

private final Ds3Client client;

public abstract class Ds3ClientHelpers {

public interface ObjectGetter {
/**
* Must save the {@code contents} for the given {@code key}.
Expand Down Expand Up @@ -104,135 +86,82 @@ public interface ReadJob extends Job {
/**
* Wraps the given {@link com.spectralogic.ds3client.Ds3Client} with helper methods.
*/
public Ds3ClientHelpers(final Ds3Client client) {
this.client = client;
public static Ds3ClientHelpers wrap(final Ds3Client client) {
return new Ds3ClientHelpersImpl(client);
}

/**
* Performs a bulk put job creation request and returns an {@link WriteJob}.
* See {@link WriteJob} for information on how to write the objects for the job.
*
*
* @throws SignatureException
* @throws IOException
* @throws XmlProcessingException
*/
public WriteJob startWriteJob(final String bucket, final Iterable<Ds3Object> objectsToWrite)
throws SignatureException, IOException, XmlProcessingException {
try(final BulkPutResponse prime = this.client.bulkPut(new BulkPutRequest(bucket, Lists.newArrayList(objectsToWrite)))) {
final MasterObjectList result = prime.getResult();
return new WriteJobImpl(this.client, result.getJobid(), bucket, result.getObjects());
}
}

public abstract Ds3ClientHelpers.WriteJob startWriteJob(final String bucket, final Iterable<Ds3Object> objectsToWrite)
throws SignatureException, IOException, XmlProcessingException;

/**
* Performs a bulk get job creation request and returns an {@link ReadJob}.
* See {@link ReadJob} for information on how to read the objects for the job.
*
*
* @throws SignatureException
* @throws IOException
* @throws XmlProcessingException
*/
public ReadJob startReadJob(final String bucket, final Iterable<Ds3Object> objectsToRead)
throws SignatureException, IOException, XmlProcessingException {
try(final BulkGetResponse prime = this.client.bulkGet(new BulkGetRequest(bucket, Lists.newArrayList(objectsToRead)))) {
final MasterObjectList result = prime.getResult();
return new ReadJobImpl(this.client, result.getJobid(), bucket, result.getObjects());
}
}

public abstract Ds3ClientHelpers.ReadJob startReadJob(final String bucket, final Iterable<Ds3Object> objectsToRead)
throws SignatureException, IOException, XmlProcessingException;

/**
* Performs a bulk get job creation request for all of the objects in the given bucket and returns an {@link ReadJob}.
*
*
* @throws SignatureException
* @throws IOException
* @throws XmlProcessingException
*/
public ReadJob startReadAllJob(final String bucket)
throws SignatureException, IOException, XmlProcessingException {
final Iterable<Contents> contentsList = this.listObjects(bucket);

final List<Ds3Object> ds3Objects = new ArrayList<>();
for (final Contents contents : contentsList) {
ds3Objects.add(new Ds3Object(contents.getKey()));
}

return this.startReadJob(bucket, ds3Objects);
}
public abstract Ds3ClientHelpers.ReadJob startReadAllJob(final String bucket)
throws SignatureException, IOException, XmlProcessingException;

/**
* Ensures that a bucket exists. The the bucket does not exist, it will be created.
* @param bucket The name of the bucket to check that it exists.
* @throws IOException
* @throws SignatureException
*/
public abstract void ensureBucketExists(final String bucket) throws IOException, SignatureException;

/**
* Returns information about all of the objects in the bucket, regardless of how many objects the bucket contains.
*
*
* @throws SignatureException
* @throws IOException
*/
public Iterable<Contents> listObjects(final String bucket) throws SignatureException, IOException {
return this.listObjects(bucket, null);
}
public abstract Iterable<Contents> listObjects(final String bucket) throws SignatureException, IOException;

/**
* Returns information about all of the objects in the bucket, regardless of how many objects the bucket contains.
*
*
* @throws SignatureException
* @throws IOException
*/
public Iterable<Contents> listObjects(final String bucket, final String keyPrefix) throws SignatureException, IOException {
return this.listObjects(bucket, keyPrefix, Integer.MAX_VALUE);
}
public abstract Iterable<Contents> listObjects(final String bucket, final String keyPrefix)
throws SignatureException, IOException;

/**
* Returns information about all of the objects in the bucket, regardless of how many objects the bucket contains.
*
*
* @throws SignatureException
* @throws IOException
*/
public Iterable<Contents> listObjects(final String bucket, final String keyPrefix, final int maxKeys) throws SignatureException, IOException {
final List<Contents> contentList = new ArrayList<>();

int remainingKeys = maxKeys;
boolean isTruncated = false;
String marker = null;

do {
final GetBucketRequest request = new GetBucketRequest(bucket);
request.withMaxKeys(Math.min(remainingKeys, DEFAULT_MAX_KEYS));
if (keyPrefix != null) {
request.withPrefix(keyPrefix);
}
if (isTruncated) {
request.withNextMarker(marker);
}

try (final GetBucketResponse response = this.client.getBucket(request)) {
final ListBucketResult result = response.getResult();

isTruncated = result.isTruncated();
marker = result.getNextMarker();
remainingKeys -= result.getContentsList().size();

for (final Contents contents : result.getContentsList()) {
contentList.add(contents);
}
}
} while (isTruncated && remainingKeys > 0);

return contentList;
}
public abstract Iterable<Contents> listObjects(final String bucket, final String keyPrefix, final int maxKeys)
throws SignatureException, IOException;

/**
* Returns an object list with which you can call {@code startWriteJob} based on the files in a {@code directory}.
* This method traverses the {@code directory} recursively.
*
*
* @throws IOException
*/
public Iterable<Ds3Object> listObjectsForDirectory(final Path directory) throws IOException {
final List<Ds3Object> objects = new ArrayList<>();
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
objects.add(new Ds3Object(directory.relativize(file).toString(), Files.size(file)));
return FileVisitResult.CONTINUE;
}
});
return objects;
}
public abstract Iterable<Ds3Object> listObjectsForDirectory(final Path directory) throws IOException;
}

Loading

0 comments on commit f44571f

Please sign in to comment.