diff --git a/build.gradle b/build.gradle index c06d2510..f560f564 100644 --- a/build.gradle +++ b/build.gradle @@ -37,8 +37,8 @@ buildscript { apply from: "$commonBuildDir/ecs-publish.gradle" dependencies { - compile 'com.emc.ecs:smart-client:1.0.2', - 'com.emc.vipr:vipr-object-transformations:2.0.3', + compile 'com.emc.ecs:smart-client:2.0.0', + 'com.emc.ecs:object-transform:1.0.0', 'org.jdom:jdom2:2.0.5' testCompile 'junit:junit:4.11' } diff --git a/src/main/java/com/emc/object/AbstractJerseyClient.java b/src/main/java/com/emc/object/AbstractJerseyClient.java index 624f8379..daec7735 100644 --- a/src/main/java/com/emc/object/AbstractJerseyClient.java +++ b/src/main/java/com/emc/object/AbstractJerseyClient.java @@ -34,9 +34,6 @@ import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config; import org.apache.log4j.Logger; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.ext.MessageBodyWriter; -import java.io.InputStream; import java.lang.annotation.Annotation; import java.net.URI; import java.util.Map; @@ -71,30 +68,15 @@ protected ClientResponse executeRequest(Client client, ObjectRequest request) { if (entityRequest.getEntity() != null) entity = entityRequest.getEntity(); - // calculate and set content-length - Long size = entityRequest.getContentLength(); - if (size == null) { - // try and find an entity writer that can determine the size - // TODO: can remove when chunked encoding is supported - MediaType mediaType = MediaType.valueOf(contentType); - MessageBodyWriter writer = client.getProviders().getMessageBodyWriter(entity.getClass(), entity.getClass(), - EMPTY_ANNOTATIONS, mediaType); - size = writer.getSize(entity, entity.getClass(), entity.getClass(), EMPTY_ANNOTATIONS, mediaType); - } + // if content-length is set (perhaps by user), force jersey to use it + if (entityRequest.getContentLength() != null) { + SizeOverrideWriter.setEntitySize(entityRequest.getContentLength()); - // if size cannot be determined, enable buffering to let HttpClient set the content-length - // NOTE: if a non-apache client handler is used, this has no effect and chunked encoding will always be enabled - // NOTE2: if the entity is an input stream, it will be streamed into memory to determine its size. this - // may cause OutOfMemoryException if there is not enough memory to hold the data - // TODO: can remove when chunked encoding is supported - if (size < 0) { - l4j.info("entity size cannot be determined; enabling Apache client entity buffering..."); - if (entity instanceof InputStream) - l4j.warn("set a content-length for input streams to save memory"); + // otherwise chunked encoding will be used. if the request does not support it, turn on + // buffering in the apache client (will set content length from buffered write) + } else if (!entityRequest.isChunkable()) { request.property(ApacheHttpClient4Config.PROPERTY_ENABLE_BUFFERING, Boolean.TRUE); } - - SizeOverrideWriter.setEntitySize(size); } else { // no entity, but make sure the apache handler doesn't mess up the content-length somehow diff --git a/src/main/java/com/emc/object/EncryptionConfig.java b/src/main/java/com/emc/object/EncryptionConfig.java index ca643656..7e4a6389 100644 --- a/src/main/java/com/emc/object/EncryptionConfig.java +++ b/src/main/java/com/emc/object/EncryptionConfig.java @@ -26,68 +26,45 @@ */ package com.emc.object; -import com.emc.vipr.transform.TransformConstants; -import com.emc.vipr.transform.TransformException; -import com.emc.vipr.transform.encryption.*; +import com.emc.codec.compression.deflate.DeflateCodec; +import com.emc.codec.encryption.BasicKeyProvider; +import com.emc.codec.encryption.EncryptionCodec; +import com.emc.codec.encryption.KeyProvider; +import com.emc.codec.encryption.KeystoreKeyProvider; -import javax.crypto.NoSuchPaddingException; import java.security.*; +import java.util.HashMap; import java.util.Map; import java.util.Set; /** - * Creates an encryption configuration for use with an object transform client. - * Both keystore keys and bare RSA KeyPairs are supported. + * Creates an encryption configuration for use with {@link com.emc.object.s3.jersey.S3EncryptionClient}. + * Both keystore keys and bare RSA KeyPairs are supported. You can optionally implement your own + * {@link KeyProvider} as well. */ public class EncryptionConfig { - private EncryptionTransformFactory factory; - - public static String getEncryptionMode(Map metadata) { - String transformModes = metadata.get(TransformConstants.META_TRANSFORM_MODE); - - // During decode, we process transforms in reverse order. - String[] modes = transformModes.split("\\|"); - for (int i = modes.length - 1; i >= 0; i--) { - if (modes[i].startsWith(TransformConstants.ENCRYPTION_CLASS)) return modes[i]; - } - - return null; - } - - public static void setEncryptionMode(Map metadata, String encryptionMode) { - metadata.put(TransformConstants.META_TRANSFORM_MODE, encryptionMode); - } + private String encryptionSpec = new EncryptionCodec().getDefaultEncodeSpec(); + private boolean compressionEnabled = false; + private String compressionSpec = new DeflateCodec().getDefaultEncodeSpec(); + private Map codecProperties = new HashMap(); /** * Creates a new EncryptionConfig object that will retrieve keys from a Keystore - * object. + * object. Note that currently, only RSA keys are supported. * * @param keystore the Keystore containing the master encryption key and any * additional decryption key(s). * @param masterKeyPassword password for the master keys. Note that this * implementation assumes that all master keys use the same password. * @param masterKeyAlias name of the master encryption key in the Keystore object. - * @param provider (optional) if not-null, the Provider object to use for all - * encryption operations. If null, the default provider(s) will be used from your - * java.security file. - * @param keySize size of encryption key to use, either 128 or 256. Note that to use - * 256-bit AES keys, you will probably need the unlimited strength jurisdiction files - * installed in your JRE. - * @throws java.security.InvalidKeyException if the master encryption key cannot be loaded. - * @throws java.security.NoSuchAlgorithmException if the AES encryption algorithm is not available. - * @throws javax.crypto.NoSuchPaddingException if PKCS5Padding is not available. - * @throws TransformException if some other error occurred initializing the encryption. + * @throws java.security.KeyStoreException if the keystore has not been initialized properly. + * @throws java.security.NoSuchAlgorithmException if the master key's algorithm is not available. + * @throws java.security.InvalidKeyException if the master key alias is not found in the keystore. + * @throws java.security.UnrecoverableKeyException if the master key is not readable (e.g. bad password) */ - public EncryptionConfig(KeyStore keystore, char[] masterKeyPassword, - String masterKeyAlias, Provider provider, int keySize) - throws InvalidKeyException, NoSuchAlgorithmException, - NoSuchPaddingException, TransformException { - if (provider == null) { - factory = new KeyStoreEncryptionFactory(keystore, masterKeyAlias, masterKeyPassword); - } else { - factory = new KeyStoreEncryptionFactory(keystore, masterKeyAlias, masterKeyPassword, provider); - } - factory.setEncryptionSettings(TransformConstants.DEFAULT_ENCRYPTION_TRANSFORM, keySize, provider); + public EncryptionConfig(KeyStore keystore, char[] masterKeyPassword, String masterKeyAlias) + throws KeyStoreException, NoSuchAlgorithmException, InvalidKeyException, UnrecoverableKeyException { + this(new KeystoreKeyProvider(keystore, masterKeyPassword, masterKeyAlias)); } /** @@ -96,37 +73,113 @@ public EncryptionConfig(KeyStore keystore, char[] masterKeyPassword, * @param masterEncryptionKey the KeyPair to use for encryption. * @param decryptionKeys (optional) additional KeyPair objects available to * decrypt objects. - * @param provider (optional) if not-null, the Provider object to use for all - * encryption operations. If null, the default provider(s) will be used from your - * java.security file. - * @param keySize size of encryption key to use, either 128 or 256. Note that to use - * 256-bit AES keys, you will probably need the unlimited strength jurisdiction files - * installed in your JRE. - * @throws java.security.InvalidKeyException if the master encryption key is not valid - * @throws java.security.NoSuchAlgorithmException if the AES encryption algorithm is not available. - * @throws javax.crypto.NoSuchPaddingException if PKCS5Padding is not available. - * @throws TransformException if some other error occurred initializing the encryption. */ - public EncryptionConfig(KeyPair masterEncryptionKey, Set decryptionKeys, - Provider provider, int keySize) - throws InvalidKeyException, NoSuchAlgorithmException, - NoSuchPaddingException, TransformException { + public EncryptionConfig(KeyPair masterEncryptionKey, Set decryptionKeys) { + this(new BasicKeyProvider(masterEncryptionKey, decryptionKeys.toArray(new KeyPair[decryptionKeys.size()]))); + } + + public EncryptionConfig(KeyProvider keyProvider) { + codecProperties.put(EncryptionCodec.PROP_KEY_PROVIDER, keyProvider); + } + + public KeyProvider getKeyProvider() { + return EncryptionCodec.getKeyProvider(codecProperties); + } + + public int getKeySize() { + return EncryptionCodec.getKeySize(codecProperties); + } + + /** + * Set the size of encryption key to use, either 128 or 256. Note that to use + * 256-bit AES keys, you will probably need the unlimited strength jurisdiction files + * installed in your JRE. + */ + public void setKeySize(int keySize) { + EncryptionCodec.setKeySize(codecProperties, keySize); + } + + public Provider getSecurityProvider() { + return EncryptionCodec.getSecurityProvider(codecProperties); + } + + /** + * Set the Java security provider to use for all encryption operations. If not specified, + * the default provider(s) will be used from your java.security file. + */ + public void setSecurityProvider(Provider provider) { + EncryptionCodec.setPropSecurityProvider(codecProperties, provider); + } + + public String getEncryptionSpec() { + return encryptionSpec; + } + + public void setEncryptionSpec(String encryptionSpec) { + this.encryptionSpec = encryptionSpec; + } + + public boolean isCompressionEnabled() { + return compressionEnabled; + } + + /** + * If set to true, will enable compression for objects created with the encryption client. + */ + public void setCompressionEnabled(boolean compressionEnabled) { + this.compressionEnabled = compressionEnabled; + } - if (provider == null) { - factory = new BasicEncryptionTransformFactory(masterEncryptionKey, decryptionKeys); - } else { - factory = new BasicEncryptionTransformFactory(masterEncryptionKey, decryptionKeys, provider); - } - factory.setEncryptionSettings(TransformConstants.DEFAULT_ENCRYPTION_TRANSFORM, keySize, provider); + public String getCompressionSpec() { + return compressionSpec; } /** - * Returns the configured EncryptionTransformFactory. + * Sets the compression spec to use for compression. Defaults to Deflate, level 5. * - * @return the configured EncryptionTransformFactory. + * @see DeflateCodec#encodeSpec(int) + * @see com.emc.codec.compression.lzma.LzmaCodec#encodeSpec(int) */ - public EncryptionTransformFactory getFactory() { - return factory; + public void setCompressionSpec(String compressionSpec) { + this.compressionSpec = compressionSpec; + } + + public Map getCodecProperties() { + return codecProperties; } + public void setCodecProperties(Map codecProperties) { + this.codecProperties = codecProperties; + } + + /** + * @see #setKeySize(int) + */ + public EncryptionConfig withKeySize(int keySize) { + setKeySize(keySize); + return this; + } + + /** + * @see #setSecurityProvider(Provider) + */ + public EncryptionConfig withSecurityProvider(Provider provider) { + setSecurityProvider(provider); + return this; + } + + public EncryptionConfig withEncryptionSpec(String encryptionSpec) { + setEncryptionSpec(encryptionSpec); + return this; + } + + public EncryptionConfig withCompressionEnabled(boolean compressionEnabled) { + setCompressionEnabled(compressionEnabled); + return this; + } + + public EncryptionConfig withCompressionSpec(String compressionSpec) { + setCompressionSpec(compressionSpec); + return this; + } } diff --git a/src/main/java/com/emc/object/EntityRequest.java b/src/main/java/com/emc/object/EntityRequest.java index 7a077e7b..a5e85468 100644 --- a/src/main/java/com/emc/object/EntityRequest.java +++ b/src/main/java/com/emc/object/EntityRequest.java @@ -32,4 +32,6 @@ public interface EntityRequest { String getContentType(); Long getContentLength(); + + boolean isChunkable(); } diff --git a/src/main/java/com/emc/object/ObjectConfig.java b/src/main/java/com/emc/object/ObjectConfig.java index 371f9390..2ac8adb2 100644 --- a/src/main/java/com/emc/object/ObjectConfig.java +++ b/src/main/java/com/emc/object/ObjectConfig.java @@ -26,15 +26,14 @@ */ package com.emc.object; +import com.emc.rest.smart.Host; import com.emc.rest.smart.SmartConfig; +import com.emc.rest.smart.ecs.Vdc; import org.apache.log4j.Logger; import java.net.URI; import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public abstract class ObjectConfig> { private static final Logger l4j = Logger.getLogger(ObjectConfig.class); @@ -42,7 +41,8 @@ public abstract class ObjectConfig> { public static final String PROPERTY_POLL_PROTOCOL = "com.emc.object.pollProtocol"; public static final String PROPERTY_POLL_PORT = "com.emc.object.pollPort"; public static final String PROPERTY_POLL_INTERVAL = "com.emc.object.pollInterval"; - public static final String PROPERTY_DISABLE_POLLING = "com.emc.object.disablePolling"; + public static final String PROPERTY_DISABLE_HEALTH_CHECK = "com.emc.object.disableHealthCheck"; + public static final String PROPERTY_DISABLE_HOST_UPDATE = "com.emc.object.disableHostUpdate"; public static final String PROPERTY_PROXY_URI = "com.emc.object.proxyUri"; public static final String PROPERTY_PROXY_USER = "com.emc.object.proxyUser"; public static final String PROPERTY_PROXY_PASS = "com.emc.object.proxyPass"; @@ -53,7 +53,7 @@ public abstract class ObjectConfig> { System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch")); private Protocol protocol; - private List hosts; + private List vdcs; private int port = -1; private String rootContext; private String namespace; @@ -65,19 +65,30 @@ public abstract class ObjectConfig> { private Map properties = new HashMap(); - public ObjectConfig(Protocol protocol, int port, String... hostList) { + /** + * Single VDC or virtual host constructor. + */ + public ObjectConfig(Protocol protocol, int port, String... hosts) { + this(protocol, port, new Vdc(hosts)); + } + + /** + * Multiple VDC constructor. + */ + public ObjectConfig(Protocol protocol, int port, Vdc... vdcs) { this.protocol = protocol; this.port = port; - this.hosts = Arrays.asList(hostList); + this.vdcs = Arrays.asList(vdcs); } - public abstract String resolveHost(); + public abstract Host resolveHost(); /** * Resolves a path relative to the API context. The returned URI will be of the format * scheme://host[:port]/[rootContext/]relativePath?query. The scheme and port are pulled from the first endpoint in * the endpoints list. The host to use may be virtual (to be resolved by a load balancer) or calculated in - * implementations as round-robin or single-host. + * implementations as round-robin or single-host. Note this is not to be confused with the client-side load + * balancing provided by the smart client, which overrides any host set here. */ public URI resolvePath(String relativePath, String query) { String path = "/"; @@ -89,7 +100,7 @@ public URI resolvePath(String relativePath, String query) { path += relativePath; try { - URI uri = new URI(protocol.toString(), null, resolveHost(), port, path, query, null); + URI uri = new URI(protocol.toString(), null, resolveHost().getName(), port, path, query, null); l4j.debug("raw path & query: " + path + "?" + query); l4j.debug("resolved URI: " + uri); @@ -102,9 +113,15 @@ public URI resolvePath(String relativePath, String query) { public SmartConfig toSmartConfig() { - SmartConfig smartConfig = new SmartConfig(hosts); + List allHosts = new ArrayList(); + for (Vdc vdc : vdcs) { + allHosts.addAll(vdc.getHosts()); + } + + SmartConfig smartConfig = new SmartConfig(allHosts); - smartConfig.setDisablePolling(Boolean.parseBoolean(propAsString(properties, PROPERTY_DISABLE_POLLING))); + smartConfig.setHealthCheckEnabled(!Boolean.parseBoolean(propAsString(properties, PROPERTY_DISABLE_HEALTH_CHECK))); + smartConfig.setHostUpdateEnabled(!Boolean.parseBoolean(propAsString(properties, PROPERTY_DISABLE_HOST_UPDATE))); if (properties.containsKey(PROPERTY_POLL_INTERVAL)) { try { @@ -117,15 +134,15 @@ public SmartConfig toSmartConfig() { try { if (properties.containsKey(PROPERTY_PROXY_URI)) - smartConfig.setProxyUri(new URI(propAsString(PROPERTY_PROXY_URI))); + smartConfig.setProxyUri(new URI(getPropAsString(PROPERTY_PROXY_URI))); } catch (URISyntaxException e) { throw new IllegalArgumentException("invalid proxy URI", e); } - smartConfig.setProxyUser(propAsString(PROPERTY_PROXY_USER)); - smartConfig.setProxyPass(propAsString(PROPERTY_PROXY_PASS)); + smartConfig.setProxyUser(getPropAsString(PROPERTY_PROXY_USER)); + smartConfig.setProxyPass(getPropAsString(PROPERTY_PROXY_PASS)); for (String prop : properties.keySet()) { - smartConfig.property(prop, properties.get(prop)); + smartConfig.withProperty(prop, properties.get(prop)); } return smartConfig; @@ -140,8 +157,8 @@ public Protocol getProtocol() { return protocol; } - public List getHosts() { - return hosts; + public List getVdcs() { + return vdcs; } public int getPort() { @@ -213,19 +230,19 @@ public Map getProperties() { return properties; } - public Object property(String propName) { + public Object getProperty(String propName) { return properties.get(propName); } - public String propAsString(String propName) { - Object value = property(propName); + public String getPropAsString(String propName) { + Object value = getProperty(propName); return value == null ? null : value.toString(); } /** * Allows custom properties to be set. These will be passed on to other client components (i.e. Jersey ClientConfig) */ - public void property(String propName, Object value) { + public void setProperty(String propName, Object value) { properties.put(propName, value); } @@ -267,7 +284,7 @@ public ObjectConfig withEncryptionConfig(EncryptionConfig encryptionConfig) { @SuppressWarnings("unchecked") public T withProperty(String propName, Object value) { - property(propName, value); + setProperty(propName, value); return (T) this; } @@ -275,7 +292,7 @@ public T withProperty(String propName, Object value) { public String toString() { return "ObjectConfig{" + "protocol=" + protocol + - ", hosts=" + hosts + + ", vdcs=" + vdcs + ", port=" + port + ", rootContext='" + rootContext + '\'' + ", namespace='" + namespace + '\'' + diff --git a/src/main/java/com/emc/object/s3/LargeFileDownloader.java b/src/main/java/com/emc/object/s3/LargeFileDownloader.java index 054a16d6..f65adc51 100644 --- a/src/main/java/com/emc/object/s3/LargeFileDownloader.java +++ b/src/main/java/com/emc/object/s3/LargeFileDownloader.java @@ -48,19 +48,16 @@ public class LargeFileDownloader implements Runnable { public static final Logger l4j = Logger.getLogger(LargeFileDownloader.class); - public static final int MIN_PART_SIZE = 1024 * 1024; // 1MB - public static final int DEFAULT_PART_SIZE = 5 * MIN_PART_SIZE; // 5MB + public static final int MIN_PART_SIZE = 2 * 1024 * 1024; // 2MB + public static final int DEFAULT_PART_SIZE = 4 * 1024 * 1024; // 4MB - public static final int DEFAULT_BUFFER_SIZE = 32 * 1024; // 32KB - - public static final int DEFAULT_THREADS = 6; + public static final int DEFAULT_THREADS = 8; private S3Client s3Client; private String bucket; private String key; private File file; private long partSize = DEFAULT_PART_SIZE; - private int bufferSize = DEFAULT_BUFFER_SIZE; private int threads = DEFAULT_THREADS; private ExecutorService executorService; @@ -103,7 +100,6 @@ public void run() { // submit all download tasks - GetObjectRequest request; long offset = 0, length = partSize; while (offset < objectSize) { if (offset + length > objectSize) length = objectSize - offset; @@ -157,17 +153,6 @@ public void setPartSize(long partSize) { this.partSize = partSize; } - public int getBufferSize() { - return bufferSize; - } - - /** - * Sets the size of the buffer to use when reading object parts. Default is 32K - */ - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - public int getThreads() { return threads; } diff --git a/src/main/java/com/emc/object/s3/LargeFileUploader.java b/src/main/java/com/emc/object/s3/LargeFileUploader.java index d8a5ff07..478702fa 100644 --- a/src/main/java/com/emc/object/s3/LargeFileUploader.java +++ b/src/main/java/com/emc/object/s3/LargeFileUploader.java @@ -54,7 +54,7 @@ public class LargeFileUploader implements Runnable { public static final int DEFAULT_THREADS = 8; - public static final long MIN_PART_SIZE = 5 * 1024 * 1024; // 5MB + public static final long MIN_PART_SIZE = 4 * 1024 * 1024; // 4MB public static final int MAX_PARTS = 10000; private S3Client s3Client; @@ -90,8 +90,11 @@ public void doMultipartUpload() { if (!file.exists() || !file.canRead()) throw new IllegalArgumentException("cannot read file: " + file.getPath()); + // make sure content-length isn't set + if (objectMetadata != null) objectMetadata.setContentLength(null); + long minPartSize = Math.max(MIN_PART_SIZE, file.length() / MAX_PARTS + 1); - l4j.info(String.format("minimum part size calculated as %,dk", minPartSize / 1024)); + l4j.debug(String.format("minimum part size calculated as %,dk", minPartSize / 1024)); if (partSize == null) partSize = minPartSize; if (partSize < minPartSize) { @@ -152,8 +155,11 @@ public void doByteRangeUpload() { if (!file.exists() || !file.canRead()) throw new IllegalArgumentException("cannot read file: " + file.getPath()); + // make sure content-length isn't set + if (objectMetadata != null) objectMetadata.setContentLength(null); + long minPartSize = Math.max(MIN_PART_SIZE, file.length() / MAX_PARTS + 1); - l4j.info(String.format("minimum part size calculated as %,dk", minPartSize / 1024)); + l4j.debug(String.format("minimum part size calculated as %,dk", minPartSize / 1024)); if (partSize == null) partSize = minPartSize; if (partSize < minPartSize) { diff --git a/src/main/java/com/emc/object/s3/S3Config.java b/src/main/java/com/emc/object/s3/S3Config.java index 56a28276..7383c1c2 100644 --- a/src/main/java/com/emc/object/s3/S3Config.java +++ b/src/main/java/com/emc/object/s3/S3Config.java @@ -28,6 +28,8 @@ import com.emc.object.ObjectConfig; import com.emc.object.Protocol; +import com.emc.rest.smart.Host; +import com.emc.rest.smart.ecs.Vdc; /** * By default, the smart client is enabled, which means virtual host-style buckets/namespaces cannot be used. To use @@ -47,17 +49,27 @@ protected static int defaultPort(Protocol protocol) { protected boolean useVHost = false; protected boolean signNamespace = true; + /** + * Single VDC constructor. + */ public S3Config(Protocol protocol, String... hostList) { super(protocol, defaultPort(protocol), hostList); } - public S3Config(Protocol protocol, int port, String... hostList) { - super(protocol, port, hostList); + /** + * Multiple VDC constructor. + */ + public S3Config(Protocol protocol, Vdc... vdcs) { + this(protocol, defaultPort(protocol), vdcs); + } + + public S3Config(Protocol protocol, int port, Vdc... vdcs) { + super(protocol, port, vdcs); } @Override - public String resolveHost() { - return getHosts().get(0); + public Host resolveHost() { + return getVdcs().get(0).getHosts().get(0); } public boolean isUseVHost() { diff --git a/src/main/java/com/emc/object/s3/S3HostListProvider.java b/src/main/java/com/emc/object/s3/S3HostListProvider.java deleted file mode 100644 index 87a72e8b..00000000 --- a/src/main/java/com/emc/object/s3/S3HostListProvider.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2015, EMC Corporation. - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * + Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * + The name of EMC Corporation may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package com.emc.object.s3; - -import com.emc.object.s3.bean.ListDataNode; -import com.emc.rest.smart.HostListProvider; -import com.emc.rest.smart.LoadBalancer; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.WebResource; -import org.apache.commons.codec.binary.Base64; -import org.apache.log4j.Logger; - -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.util.SimpleTimeZone; - -public class S3HostListProvider implements HostListProvider { - private static final Logger l4j = Logger.getLogger(S3HostListProvider.class); - - public static final String DEFAULT_PROTOCOL = "https"; - public static final int DEFAULT_PORT = 9021; - - protected static final SimpleDateFormat rfc822DateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); - - static { - rfc822DateFormat.setTimeZone(new SimpleTimeZone(0, "GMT")); - } - - private Client client; - private LoadBalancer loadBalancer; - private String user; - private String secret; - private String protocol = DEFAULT_PROTOCOL; - private int port = DEFAULT_PORT; - - public S3HostListProvider(Client client, LoadBalancer loadBalancer, String user, String secret) { - this.client = client; - this.loadBalancer = loadBalancer; - this.user = user; - this.secret = secret; - } - - public List getHostList() { - String host = loadBalancer.getTopHost().getName(); - String portStr = (port > -1) ? ":" + port : ""; - String path = "/?endpoint"; - String uri = protocol + "://" + host + portStr + path; - - // format date - String rfcDate; - synchronized (rfc822DateFormat) { - rfcDate = rfc822DateFormat.format(new Date()); - } - - // generate signature - String canonicalString = "GET\n\n\n" + rfcDate + "\n" + path; - String signature; - try { - signature = getSignature(canonicalString, secret); - } catch (Exception e) { - throw new RuntimeException("could not generate signature", e); - } - - // construct request - WebResource.Builder request = client.resource(uri).getRequestBuilder(); - - // add date and auth headers - request.header("Date", rfcDate); - request.header("Authorization", "AWS " + user + ":" + signature); - - // make REST call - return request.get(ListDataNode.class).getDataNodes(); - } - - protected String getSignature(String canonicalString, String secret) throws Exception { - Mac mac = Mac.getInstance("HmacSHA1"); - mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA1")); - String signature = new String(Base64.encodeBase64(mac.doFinal(canonicalString.getBytes("UTF-8")))); - l4j.debug("canonicalString:\n" + canonicalString); - l4j.debug("signature:\n" + signature); - return signature; - } - - public Client getClient() { - return client; - } - - public LoadBalancer getLoadBalancer() { - return loadBalancer; - } - - public String getUser() { - return user; - } - - public String getSecret() { - return secret; - } - - public String getProtocol() { - return protocol; - } - - public void setProtocol(String protocol) { - this.protocol = protocol; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } -} diff --git a/src/main/java/com/emc/object/s3/S3ObjectMetadata.java b/src/main/java/com/emc/object/s3/S3ObjectMetadata.java index a5f80148..fd72ad8d 100644 --- a/src/main/java/com/emc/object/s3/S3ObjectMetadata.java +++ b/src/main/java/com/emc/object/s3/S3ObjectMetadata.java @@ -256,6 +256,11 @@ public S3ObjectMetadata withContentLength(Long contentLength) { return this; } + public S3ObjectMetadata withContentLength(int contentLength) { + return withContentLength((long) contentLength); + } + + public S3ObjectMetadata withContentMd5(String contentMd5) { setContentMd5(contentMd5); return this; diff --git a/src/main/java/com/emc/object/s3/S3VHostConfig.java b/src/main/java/com/emc/object/s3/S3VHostConfig.java index 2b24d0db..335c609b 100644 --- a/src/main/java/com/emc/object/s3/S3VHostConfig.java +++ b/src/main/java/com/emc/object/s3/S3VHostConfig.java @@ -28,6 +28,7 @@ import com.emc.object.ObjectConfig; import com.emc.object.Protocol; +import com.emc.rest.smart.ecs.Vdc; import java.net.URI; @@ -43,14 +44,15 @@ */ public class S3VHostConfig extends S3Config { public S3VHostConfig(URI endpoint) { - super(Protocol.valueOf(endpoint.getScheme().toUpperCase()), endpoint.getPort(), endpoint.getHost()); + super(Protocol.valueOf(endpoint.getScheme().toUpperCase()), endpoint.getPort(), new Vdc(endpoint.getHost())); // standard VHost type signs namespace useVHost = true; signNamespace = true; - // make sure we don't poll for individual nodes - property(ObjectConfig.PROPERTY_DISABLE_POLLING, "true"); + // make sure we disable "smart" features + setProperty(ObjectConfig.PROPERTY_DISABLE_HOST_UPDATE, "true"); + setProperty(ObjectConfig.PROPERTY_DISABLE_HEALTH_CHECK, "true"); } /** diff --git a/src/main/java/com/emc/object/s3/bean/AccessControlList.java b/src/main/java/com/emc/object/s3/bean/AccessControlList.java index b42889ce..2a17ad30 100644 --- a/src/main/java/com/emc/object/s3/bean/AccessControlList.java +++ b/src/main/java/com/emc/object/s3/bean/AccessControlList.java @@ -28,14 +28,12 @@ import com.emc.object.util.RestUtil; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlElementWrapper; -import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlType; +import javax.xml.bind.annotation.*; import java.util.*; @XmlRootElement(name = "AccessControlPolicy") @XmlType(propOrder = {"owner", "grants"}) +@XmlSeeAlso({CanonicalUser.class, Group.class}) public class AccessControlList { private CanonicalUser owner; private Set grants = new LinkedHashSet(); diff --git a/src/main/java/com/emc/object/s3/bean/Group.java b/src/main/java/com/emc/object/s3/bean/Group.java index 8e01f3fd..f5d29d28 100644 --- a/src/main/java/com/emc/object/s3/bean/Group.java +++ b/src/main/java/com/emc/object/s3/bean/Group.java @@ -31,6 +31,9 @@ @XmlType(name = "Group") public class Group extends AbstractGrantee { + public static final Group ALL_USERS = new Group("http://acs.amazonaws.com/groups/global/AllUsers"); + public static final Group AUTHENTICATED_USERS = new Group("http://acs.amazonaws.com/groups/global/AuthenticatedUsers"); + private String uri; public Group() { diff --git a/src/main/java/com/emc/object/s3/jersey/CodecFilter.java b/src/main/java/com/emc/object/s3/jersey/CodecFilter.java index 98159846..eb16bf04 100644 --- a/src/main/java/com/emc/object/s3/jersey/CodecFilter.java +++ b/src/main/java/com/emc/object/s3/jersey/CodecFilter.java @@ -26,30 +26,26 @@ */ package com.emc.object.s3.jersey; -import com.emc.object.EncryptionConfig; +import com.emc.codec.CodecChain; import com.emc.object.s3.S3ObjectMetadata; -import com.emc.object.util.CloseEventOutputStream; import com.emc.object.util.RestUtil; import com.emc.rest.smart.SizeOverrideWriter; -import com.emc.rest.util.SizedInputStream; -import com.emc.vipr.transform.InputTransform; -import com.emc.vipr.transform.OutputTransform; -import com.emc.vipr.transform.TransformException; -import com.emc.vipr.transform.encryption.EncryptionTransformFactory; import com.sun.jersey.api.client.*; import com.sun.jersey.api.client.filter.ClientFilter; -import java.io.File; +import javax.ws.rs.core.MultivaluedMap; import java.io.IOException; import java.io.OutputStream; -import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class CodecFilter extends ClientFilter { - EncryptionTransformFactory factory; + private CodecChain encodeChain; + private Map codecProperties; - public CodecFilter(EncryptionTransformFactory factory) { - this.factory = factory; + public CodecFilter(CodecChain encodeChain) { + this.encodeChain = encodeChain; } @SuppressWarnings("unchecked") @@ -60,10 +56,8 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio Boolean encode = (Boolean) request.getProperties().get(RestUtil.PROPERTY_ENCODE_ENTITY); if (encode != null && encode) { - // alter entity size (if necessary) to reflect encrypted size - Long contentLength = SizeOverrideWriter.getEntitySize(); - if (contentLength != null && contentLength >= 0) - SizeOverrideWriter.setEntitySize((contentLength / 16 + 1) * 16); + // we don't know what the size will be; this will turn on chunked encoding in the apache client + SizeOverrideWriter.setEntitySize(-1L); // wrap output stream with encryptor request.setAdapter(new EncryptAdapter(request.getAdapter(), userMeta)); @@ -72,21 +66,41 @@ public ClientResponse handle(ClientRequest request) throws ClientHandlerExceptio // execute request ClientResponse response = getNext().handle(request); - try { + // get user metadata from response headers + MultivaluedMap headers = response.getHeaders(); + Map storedMeta = S3ObjectMetadata.getUserMetadata(headers); + Set keysToRemove = new HashSet(); + keysToRemove.addAll(storedMeta.keySet()); + + // get encode specs from user metadata + String[] encodeSpecs = CodecChain.getEncodeSpecs(storedMeta); + if (encodeSpecs != null) { + + // create codec chain + CodecChain decodeChain = new CodecChain(encodeSpecs).withProperties(codecProperties); + + // do we need to decode the entity? Boolean decode = (Boolean) request.getProperties().get(RestUtil.PROPERTY_DECODE_ENTITY); if (decode != null && decode) { - // get encryption spec from metadata - Map userMetadata = S3ObjectMetadata.getUserMetadata(response.getHeaders()); - String encryptionMode = EncryptionConfig.getEncryptionMode(userMetadata); + // wrap input stream with decryptor (this will remove any encode metadata from storedMeta) + response.setEntityInputStream(decodeChain.getDecodeStream(response.getEntityInputStream(), storedMeta)); + } else { - // wrap input stream with decryptor - InputTransform transform = factory.getInputTransform(encryptionMode, response.getEntityInputStream(), userMetadata); - response.setEntityInputStream(transform.getDecodedInputStream()); + // need to remove any encode metadata so we can update the headers + decodeChain.removeEncodeMetadata(storedMeta, decodeChain.getEncodeMetadataList(storedMeta)); + } + + // should we keep the encode headers? + Boolean keepHeaders = (Boolean) request.getProperties().get(RestUtil.PROPERTY_KEEP_ENCODE_HEADERS); + if (keepHeaders == null || !keepHeaders) { + + // remove encode metadata from headers (storedMeta now contains only user-defined metadata) + keysToRemove.removeAll(storedMeta.keySet()); // all metadata - user-defined metadata + for (String key : keysToRemove) { + headers.remove(S3ObjectMetadata.getHeaderName(key)); + } } - } catch (Exception e) { - if (e instanceof RuntimeException) throw (RuntimeException) e; - throw new ClientHandlerException("error decrypting object content", e); } return response; @@ -103,19 +117,20 @@ private class EncryptAdapter extends AbstractClientRequestAdapter { @Override public OutputStream adapt(ClientRequest request, OutputStream out) throws IOException { - try { - final OutputTransform transform = factory.getOutputTransform(out, new HashMap()); - out = new CloseEventOutputStream(transform.getEncodedOutputStream(), new Runnable() { - @Override - public void run() { - encMeta.putAll(transform.getEncodedMetadata()); - EncryptionConfig.setEncryptionMode(encMeta, transform.getTransformConfig()); - } - }); - return getAdapter().adapt(request, out); // don't break the chain - } catch (TransformException e) { - throw new ClientHandlerException("error encrypting object content", e); - } + return getAdapter().adapt(request, encodeChain.getEncodeStream(out, encMeta)); // don't break the chain } } + + public Map getCodecProperties() { + return codecProperties; + } + + public void setCodecProperties(Map codecProperties) { + this.codecProperties = codecProperties; + } + + public CodecFilter withCodecProperties(Map codecProperties) { + setCodecProperties(codecProperties); + return this; + } } diff --git a/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java b/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java index bb9bbe40..7fc7012b 100644 --- a/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java +++ b/src/main/java/com/emc/object/s3/jersey/S3EncryptionClient.java @@ -26,15 +26,15 @@ */ package com.emc.object.s3.jersey; +import com.emc.codec.CodecChain; +import com.emc.codec.encryption.DoesNotNeedRekeyException; +import com.emc.codec.encryption.EncryptionCodec; import com.emc.object.EncryptionConfig; import com.emc.object.s3.S3Config; import com.emc.object.s3.S3ObjectMetadata; import com.emc.object.s3.bean.*; import com.emc.object.s3.request.*; import com.emc.object.util.RestUtil; -import com.emc.vipr.transform.TransformException; -import com.emc.vipr.transform.encryption.DoesNotNeedRekeyException; -import com.emc.vipr.transform.encryption.EncryptionTransformFactory; import com.sun.jersey.api.client.ClientHandler; import com.sun.jersey.api.client.filter.ClientFilter; @@ -107,11 +107,17 @@ public class S3EncryptionClient extends S3JerseyClient { private static final String PARTIAL_READ_MSG = "Partial object reads are not " + "supported by the encryption client"; - private EncryptionTransformFactory factory; + private EncryptionConfig encryptionConfig; public S3EncryptionClient(S3Config s3Config, EncryptionConfig encryptionConfig) { super(s3Config); - this.factory = encryptionConfig.getFactory(); + this.encryptionConfig = encryptionConfig; + + // create an encode chain based on parameters + CodecChain encodeChain = encryptionConfig.isCompressionEnabled() + ? new CodecChain(encryptionConfig.getCompressionSpec(), encryptionConfig.getEncryptionSpec()) + : new CodecChain(encryptionConfig.getEncryptionSpec()); + encodeChain.setProperties(encryptionConfig.getCodecProperties()); // insert codec filter into chain before the checksum filter // as usual, Jersey makes this quite hard @@ -123,7 +129,7 @@ public S3EncryptionClient(S3Config s3Config, EncryptionConfig encryptionConfig) ClientFilter filter = (ClientFilter) handler; if (filter instanceof ChecksumFilter) { // insert codec filter before checksum filter - filters.add(new CodecFilter(factory)); + filters.add(new CodecFilter(encodeChain).withCodecProperties(encryptionConfig.getCodecProperties())); } filters.add(filter); handler = filter.getNext(); @@ -149,39 +155,27 @@ public S3EncryptionClient(S3Config s3Config, EncryptionConfig encryptionConfig) * @return true if the object was successfully rekeyed, false if the object already uses the new key * @throws java.lang.IllegalArgumentException if the object is not encrypted */ - public boolean rekey(String bucketName, String key) throws DoesNotNeedRekeyException { + public boolean rekey(String bucketName, String key) { // read the metadata for the object - S3ObjectMetadata objectMetadata = getObjectMetadata(bucketName, key); + GetObjectMetadataRequest request = new GetObjectMetadataRequest(bucketName, key); + request.property(RestUtil.PROPERTY_KEEP_ENCODE_HEADERS, Boolean.TRUE); + S3ObjectMetadata objectMetadata = getObjectMetadata(request); Map userMetadata = objectMetadata.getUserMetadata(); - // get the encryption spec used - String encMode = EncryptionConfig.getEncryptionMode(userMetadata); - if (encMode == null) { - throw new IllegalArgumentException("Object is not encrypted"); - } - - if (factory.canDecode(encMode, userMetadata)) { - try { - // re-sign the object encryption key - Map rekeyMeta = factory.rekey(userMetadata); - for (String name : rekeyMeta.keySet()) { - objectMetadata.addUserMetadata(name, rekeyMeta.get(name)); - } + try { + // re-sign the object encryption key + // note this call will update userMetadata + new EncryptionCodec().rekey(userMetadata, encryptionConfig.getCodecProperties()); - // push the re-signed keys as metadata - AccessControlList acl = getObjectAcl(bucketName, key); - super.copyObject(new CopyObjectRequest(bucketName, key, bucketName, key) - .withAcl(acl).withObjectMetadata(objectMetadata)); + // push the re-signed keys as metadata + AccessControlList acl = getObjectAcl(bucketName, key); + super.copyObject(new CopyObjectRequest(bucketName, key, bucketName, key) + .withAcl(acl).withObjectMetadata(objectMetadata)); - return true; - } catch (DoesNotNeedRekeyException e) { - return false; - } catch (TransformException e) { - throw new RuntimeException("Error rekeying object: " + bucketName + "/" + key, e); - } - } else { - throw new RuntimeException("Cannot handle encryption mode '" + encMode + "'"); + return true; + } catch (DoesNotNeedRekeyException e) { + return false; } } diff --git a/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java b/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java index bbc85905..952d77d1 100644 --- a/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java +++ b/src/main/java/com/emc/object/s3/jersey/S3JerseyClient.java @@ -34,9 +34,11 @@ import com.emc.object.s3.bean.*; import com.emc.object.s3.request.*; import com.emc.object.util.RestUtil; +import com.emc.rest.smart.LoadBalancer; import com.emc.rest.smart.PollingDaemon; import com.emc.rest.smart.SmartClientFactory; import com.emc.rest.smart.SmartConfig; +import com.emc.rest.smart.ecs.EcsHostListProvider; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; @@ -52,6 +54,7 @@ public class S3JerseyClient extends AbstractJerseyClient implements S3Client { protected S3Config s3Config; protected Client client; + protected LoadBalancer loadBalancer; public S3JerseyClient(S3Config s3Config) { super(s3Config); @@ -60,6 +63,7 @@ public S3JerseyClient(S3Config s3Config) { // add Checksum SmartConfig smartConfig = s3Config.toSmartConfig(); + loadBalancer = smartConfig.getLoadBalancer(); // creates a standard (non-load-balancing) jersey client client = SmartClientFactory.createStandardClient(smartConfig); @@ -69,24 +73,28 @@ public S3JerseyClient(S3Config s3Config) { // S.C. - ENDPOINT POLLING // create a host list provider based on the S3 ?endpoint call (will use the standard client we just made) - S3HostListProvider hostListProvider = new S3HostListProvider(client, smartConfig.getLoadBalancer(), + EcsHostListProvider hostListProvider = new EcsHostListProvider(client, loadBalancer, s3Config.getIdentity(), s3Config.getSecretKey()); smartConfig.setHostListProvider(hostListProvider); - if (s3Config.property(S3Config.PROPERTY_POLL_PROTOCOL) != null) - hostListProvider.setProtocol(s3Config.propAsString(S3Config.PROPERTY_POLL_PROTOCOL)); + if (s3Config.getProperty(S3Config.PROPERTY_POLL_PROTOCOL) != null) + hostListProvider.setProtocol(s3Config.getPropAsString(S3Config.PROPERTY_POLL_PROTOCOL)); else hostListProvider.setProtocol(s3Config.getProtocol().toString()); - if (s3Config.property(S3Config.PROPERTY_POLL_PORT) != null) { + if (s3Config.getProperty(S3Config.PROPERTY_POLL_PORT) != null) { try { - hostListProvider.setPort(Integer.parseInt(s3Config.propAsString(S3Config.PROPERTY_POLL_PORT))); + hostListProvider.setPort(Integer.parseInt(s3Config.getPropAsString(S3Config.PROPERTY_POLL_PORT))); } catch (NumberFormatException e) { throw new RuntimeException(String.format("invalid poll port (%s=%s)", - S3Config.PROPERTY_POLL_PORT, s3Config.propAsString(S3Config.PROPERTY_POLL_PORT)), e); + S3Config.PROPERTY_POLL_PORT, s3Config.getPropAsString(S3Config.PROPERTY_POLL_PORT)), e); } - } else + } else { hostListProvider.setPort(s3Config.getPort()); + } + + // S.C. - VDC CONFIGURATION + hostListProvider.setVdcs(s3Config.getVdcs()); // S.C. - CLIENT CREATION // create a load-balancing jersey client @@ -117,6 +125,10 @@ public void shutdown() { if (pollingDaemon != null) pollingDaemon.terminate(); } + public LoadBalancer getLoadBalancer() { + return loadBalancer; + } + @Override public ListDataNode listDataNodes() { return executeRequest(client, new ObjectRequest(Method.GET, "", "endpoint"), ListDataNode.class); diff --git a/src/main/java/com/emc/object/s3/request/CompleteMultipartUploadRequest.java b/src/main/java/com/emc/object/s3/request/CompleteMultipartUploadRequest.java index 1a66592d..c77328a6 100644 --- a/src/main/java/com/emc/object/s3/request/CompleteMultipartUploadRequest.java +++ b/src/main/java/com/emc/object/s3/request/CompleteMultipartUploadRequest.java @@ -66,6 +66,11 @@ public Long getContentLength() { return null; // assume chunked encoding or buffering } + @Override + public boolean isChunkable() { + return false; + } + public String getUploadId() { return uploadId; } diff --git a/src/main/java/com/emc/object/s3/request/DeleteObjectsRequest.java b/src/main/java/com/emc/object/s3/request/DeleteObjectsRequest.java index 5dd61ead..eee1522d 100644 --- a/src/main/java/com/emc/object/s3/request/DeleteObjectsRequest.java +++ b/src/main/java/com/emc/object/s3/request/DeleteObjectsRequest.java @@ -56,6 +56,11 @@ public Long getContentLength() { return null; // assume chunked encoding or buffering } + @Override + public boolean isChunkable() { + return false; + } + public DeleteObjects getDeleteObjects() { return deleteObjects; } diff --git a/src/main/java/com/emc/object/s3/request/GenericBucketEntityRequest.java b/src/main/java/com/emc/object/s3/request/GenericBucketEntityRequest.java index 0ccd55ea..da979c53 100644 --- a/src/main/java/com/emc/object/s3/request/GenericBucketEntityRequest.java +++ b/src/main/java/com/emc/object/s3/request/GenericBucketEntityRequest.java @@ -50,7 +50,12 @@ public String getContentType() { @Override public Long getContentLength() { - return null; // assume chunked encoding or buffering + return null; // assume buffering + } + + @Override + public boolean isChunkable() { + return false; } public void setContentType(String contentType) { diff --git a/src/main/java/com/emc/object/s3/request/PutObjectRequest.java b/src/main/java/com/emc/object/s3/request/PutObjectRequest.java index 2dd51b92..87127bbd 100644 --- a/src/main/java/com/emc/object/s3/request/PutObjectRequest.java +++ b/src/main/java/com/emc/object/s3/request/PutObjectRequest.java @@ -84,6 +84,11 @@ public Long getContentLength() { return objectMetadata != null ? objectMetadata.getContentLength() : null; } + @Override + public boolean isChunkable() { + return true; + } + public S3ObjectMetadata getObjectMetadata() { return objectMetadata; } diff --git a/src/main/java/com/emc/object/s3/request/SetBucketAclRequest.java b/src/main/java/com/emc/object/s3/request/SetBucketAclRequest.java index 39907c6f..52e4179d 100644 --- a/src/main/java/com/emc/object/s3/request/SetBucketAclRequest.java +++ b/src/main/java/com/emc/object/s3/request/SetBucketAclRequest.java @@ -66,6 +66,11 @@ public Long getContentLength() { return null; // assume chunked encoding or buffering } + @Override + public boolean isChunkable() { + return false; + } + public AccessControlList getAcl() { return acl; } diff --git a/src/main/java/com/emc/object/s3/request/SetObjectAclRequest.java b/src/main/java/com/emc/object/s3/request/SetObjectAclRequest.java index e7253f86..9e34cfc7 100644 --- a/src/main/java/com/emc/object/s3/request/SetObjectAclRequest.java +++ b/src/main/java/com/emc/object/s3/request/SetObjectAclRequest.java @@ -71,7 +71,12 @@ public String getContentType() { @Override public Long getContentLength() { - return null; // assume chunked encoding or buffering + return null; // assume buffering + } + + @Override + public boolean isChunkable() { + return false; } public String getVersionId() { diff --git a/src/main/java/com/emc/object/s3/request/UploadPartRequest.java b/src/main/java/com/emc/object/s3/request/UploadPartRequest.java index ffbe0bae..2d352a6a 100644 --- a/src/main/java/com/emc/object/s3/request/UploadPartRequest.java +++ b/src/main/java/com/emc/object/s3/request/UploadPartRequest.java @@ -68,14 +68,19 @@ public Object getEntity() { return getObject(); } + @Override + public String getContentType() { + return null; + } + @Override public Long getContentLength() { return contentLength; } @Override - public String getContentType() { - return null; + public boolean isChunkable() { + return true; } public String getUploadId() { diff --git a/src/main/java/com/emc/object/util/ChecksummedInputStream.java b/src/main/java/com/emc/object/util/ChecksummedInputStream.java index 21693ab8..47eef841 100644 --- a/src/main/java/com/emc/object/util/ChecksummedInputStream.java +++ b/src/main/java/com/emc/object/util/ChecksummedInputStream.java @@ -127,9 +127,11 @@ private void update(byte[] bytes, int offset, int length) { } private void finish() { - String referenceValue = verifyChecksum.getValue(); - String calculatedValue = checksum.getValue(); - if (!referenceValue.equals(calculatedValue)) - throw new ChecksumError("Checksum failure while reading stream", referenceValue, calculatedValue); + if (verifyChecksum != null) { + String referenceValue = verifyChecksum.getValue(); + String calculatedValue = checksum.getValue(); + if (!referenceValue.equals(calculatedValue)) + throw new ChecksumError("Checksum failure while reading stream", referenceValue, calculatedValue); + } } } diff --git a/src/main/java/com/emc/object/util/CloseEventOutputStream.java b/src/main/java/com/emc/object/util/CloseEventOutputStream.java deleted file mode 100644 index b90e4868..00000000 --- a/src/main/java/com/emc/object/util/CloseEventOutputStream.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2015, EMC Corporation. - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * + Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * + The name of EMC Corporation may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package com.emc.object.util; - -import java.io.IOException; -import java.io.OutputStream; - -public class CloseEventOutputStream extends OutputStream { - private OutputStream delegate; - private Runnable runOnCLose; - - public CloseEventOutputStream(OutputStream delegate, Runnable runOnCLose) { - this.delegate = delegate; - this.runOnCLose = runOnCLose; - } - - @Override - public void write(int b) throws IOException { - delegate.write(b); - } - - @Override - public void write(byte[] b) throws IOException { - delegate.write(b); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - delegate.write(b, off, len); - } - - @Override - public void flush() throws IOException { - delegate.flush(); - } - - @Override - public void close() throws IOException { - delegate.close(); - runOnCLose.run(); - } -} diff --git a/src/main/java/com/emc/object/util/RestUtil.java b/src/main/java/com/emc/object/util/RestUtil.java index 925fdc73..be0da255 100644 --- a/src/main/java/com/emc/object/util/RestUtil.java +++ b/src/main/java/com/emc/object/util/RestUtil.java @@ -66,6 +66,7 @@ public final class RestUtil { public static final String PROPERTY_USER_METADATA = "com.emc.object.userMetadata"; public static final String PROPERTY_ENCODE_ENTITY = "com.emc.object.codec.encodeEntity"; public static final String PROPERTY_DECODE_ENTITY = "com.emc.object.codec.decodeEntity"; + public static final String PROPERTY_KEEP_ENCODE_HEADERS = "com.emc.object.codec.keepEncodeHeaders"; public static final String PROPERTY_VERIFY_READ_CHECKSUM = "com.emc.object.verifyReadChecksum"; public static final String PROPERTY_VERIFY_WRITE_CHECKSUM = "com.emc.object.verifyWriteChecksum"; diff --git a/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java b/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java index 0a617743..875369ca 100644 --- a/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java +++ b/src/test/java/com/emc/object/s3/AbstractS3ClientTest.java @@ -32,15 +32,31 @@ import com.emc.object.s3.bean.AbstractVersion; import com.emc.object.s3.bean.S3Object; import com.emc.object.s3.bean.VersioningConfiguration; +import com.emc.object.s3.jersey.S3JerseyClient; import com.emc.object.util.TestProperties; +import com.emc.rest.smart.LoadBalancer; +import com.emc.rest.smart.ecs.Vdc; import com.emc.util.TestConfig; +import org.apache.log4j.Logger; +import org.junit.After; import java.net.URI; +import java.util.Arrays; import java.util.Properties; public abstract class AbstractS3ClientTest extends AbstractClientTest { + private static final Logger l4j = Logger.getLogger(AbstractS3ClientTest.class); + protected S3Client client; + @After + public void dumpLBStats() { + if (client != null) { + LoadBalancer loadBalancer = ((S3JerseyClient) client).getLoadBalancer(); + l4j.info(Arrays.toString(loadBalancer.getHostStats())); + } + } + @Override protected void createBucket(String bucketName) throws Exception { client.createBucket(bucketName); @@ -48,7 +64,7 @@ protected void createBucket(String bucketName) throws Exception { @Override protected void cleanUpBucket(String bucketName) throws Exception { - try { + if (client != null && client.bucketExists(bucketName)) { if (client.getBucketVersioning(bucketName).getStatus() == VersioningConfiguration.Status.Enabled) { for (AbstractVersion version : client.listVersions(bucketName, null).getVersions()) { client.deleteVersion(bucketName, version.getKey(), version.getVersionId()); @@ -59,8 +75,6 @@ protected void cleanUpBucket(String bucketName) throws Exception { } } client.deleteBucket(bucketName); - } catch (S3Exception e) { - if (!"NoSuchBucket".equals(e.getErrorCode())) throw e; } } @@ -77,13 +91,13 @@ protected S3Config createS3Config() throws Exception { if (enableVhost) { s3Config = new S3VHostConfig(endpoint); } else if (endpoint.getPort() > 0) { - s3Config = new S3Config(Protocol.valueOf(endpoint.getScheme().toUpperCase()), endpoint.getPort(), endpoint.getHost()); + s3Config = new S3Config(Protocol.valueOf(endpoint.getScheme().toUpperCase()), endpoint.getPort(), new Vdc(endpoint.getHost())); } else { s3Config = new S3Config(Protocol.valueOf(endpoint.getScheme().toUpperCase()), endpoint.getHost()); } s3Config.withIdentity(accessKey).withSecretKey(secretKey); - if (proxyUri != null) s3Config.property(ObjectConfig.PROPERTY_PROXY_URI, proxyUri); + if (proxyUri != null) s3Config.setProperty(ObjectConfig.PROPERTY_PROXY_URI, proxyUri); // uncomment to hit a single node //s3Config.property(ObjectConfig.PROPERTY_DISABLE_POLLING, true); diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java index 592f7076..e0fc68c0 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientBasicTest.java @@ -26,44 +26,63 @@ */ package com.emc.object.s3; +import com.emc.codec.encryption.BasicKeyProvider; +import com.emc.codec.encryption.EncryptionConstants; +import com.emc.codec.encryption.EncryptionUtil; import com.emc.object.EncryptionConfig; -import com.emc.object.s3.bean.GetObjectResult; +import com.emc.object.s3.bean.*; import com.emc.object.s3.jersey.S3EncryptionClient; -import com.emc.object.s3.request.GetObjectRequest; -import com.emc.vipr.transform.TransformConstants; -import com.emc.vipr.transform.encryption.BasicEncryptionTransformFactory; -import com.emc.vipr.transform.encryption.KeyUtils; +import com.emc.object.s3.jersey.S3JerseyClient; +import com.emc.object.s3.request.PutObjectRequest; +import com.emc.util.RandomInputStream; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.LogMF; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Assume; +import org.junit.Ignore; import org.junit.Test; import java.io.InputStream; import java.security.KeyPair; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; -import java.util.Arrays; -import java.util.HashSet; import java.util.Properties; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; -public class S3EncryptionClientBasicTest extends S3EncryptionClientKeyStoreTest { +public class S3EncryptionClientBasicTest extends S3JerseyClientTest { private static final Logger l4j = Logger.getLogger(S3EncryptionClientBasicTest.class); - private String keyFile = "keys.properties"; + protected int keySize = 128; + protected S3JerseyClient rclient; + protected S3EncryptionClient eclient; + + private BasicKeyProvider _keyProvider; private KeyPair _masterKey; private KeyPair _oldKey; @Override - protected EncryptionConfig createEncryptionConfig() throws Exception { - return new EncryptionConfig(getMasterKey(), new HashSet(Arrays.asList(getMasterKey(), getOldKey())), null, keySize); + protected String getTestBucketPrefix() { + return "s3-encryption-client-test"; } @Override - protected String getMasterKeyFingerprint() throws Exception { - return KeyUtils.getRsaPublicKeyFingerprint((RSAPublicKey) getMasterKey().getPublic(), null); + public void initClient() throws Exception { + rclient = new S3JerseyClient(createS3Config()); + client = eclient = new S3EncryptionClient(createS3Config(), createEncryptionConfig()); + } + + protected EncryptionConfig createEncryptionConfig() throws Exception { + return new EncryptionConfig(getKeyProvider()).withKeySize(keySize); + } + + protected synchronized BasicKeyProvider getKeyProvider() throws Exception { + if (_keyProvider == null) { + _keyProvider = new BasicKeyProvider(getMasterKey(), getOldKey()); + } + return _keyProvider; } protected synchronized KeyPair getMasterKey() throws Exception { @@ -78,21 +97,101 @@ protected synchronized KeyPair getOldKey() throws Exception { protected void loadKeys() throws Exception { Properties keyprops = new Properties(); - InputStream keystream = getClass().getClassLoader().getResourceAsStream(keyFile); + InputStream keystream = getClass().getClassLoader().getResourceAsStream("keys.properties"); Assume.assumeNotNull(keystream); keyprops.load(keystream); - _masterKey = KeyUtils.rsaKeyPairFromBase64(keyprops.getProperty("masterkey.public"), keyprops.getProperty("masterkey.private")); + _masterKey = EncryptionUtil.rsaKeyPairFromBase64(keyprops.getProperty("masterkey.public"), keyprops.getProperty("masterkey.private")); LogMF.debug(l4j, "Master key sizes: public: {} private: {}", ((RSAPublicKey) _masterKey.getPublic()).getModulus().bitLength(), ((RSAPrivateKey) _masterKey.getPrivate()).getModulus().bitLength()); - _oldKey = KeyUtils.rsaKeyPairFromBase64(keyprops.getProperty("oldkey.public"), keyprops.getProperty("oldkey.private")); + _oldKey = EncryptionUtil.rsaKeyPairFromBase64(keyprops.getProperty("oldkey.public"), keyprops.getProperty("oldkey.private")); LogMF.debug(l4j, "Old key sizes: public: {} private: {}", ((RSAPublicKey) _oldKey.getPublic()).getModulus().bitLength(), ((RSAPrivateKey) _oldKey.getPrivate()).getModulus().bitLength()); } - @Override + @Test + public void testEncodeMeta() throws Exception { + String key = "hello.txt"; + String content = "Hello World!"; + + client.putObject(getTestBucket(), key, content, null); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + + Assert.assertEquals("unencrypted size incorrect", "12", + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE)); + Assert.assertEquals("encrypted size incorrect", 16, objectMetadata.getContentLength().longValue()); + Assert.assertEquals("unencrypted sha1 incorrect", "2ef7bde608ce5404e97d5f042f95f89f1c232871", + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + Assert.assertEquals("master key ID incorrect", getKeyProvider().getMasterKeyFingerprint(), + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + Assert.assertNotNull("IV null", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_IV)); + Assert.assertNotNull("Object key", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_OBJECT_KEY)); + Assert.assertNotNull("Missing metadata signature", + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_META_SIG)); + } + + @Test + public void testWithUserMeta() throws Exception { + String key = "metadata-test"; + String content = "Hello Metadata!!"; + String m1 = "meta1", v1 = "value1", m2 = "meta2", v2 = "value2"; + S3ObjectMetadata metadata = new S3ObjectMetadata().addUserMetadata(m1, v1).addUserMetadata(m2, v2); + client.putObject(new PutObjectRequest(getTestBucket(), key, content).withObjectMetadata(metadata)); + + metadata = client.getObjectMetadata(getTestBucket(), key); + Assert.assertEquals(2, metadata.getUserMetadata().size()); + Assert.assertNotNull(metadata.getUserMetadata(m1)); + Assert.assertNotNull(metadata.getUserMetadata(m2)); + Assert.assertEquals(v1, metadata.getUserMetadata(m1)); + Assert.assertEquals(v2, metadata.getUserMetadata(m2)); + } + + @Test + public void testStream() throws Exception { + String key = "test-file.txt"; + InputStream rawInput = getClass().getClassLoader().getResourceAsStream("uncompressed.txt"); + Assume.assumeNotNull(rawInput); + + client.putObject(new PutObjectRequest(getTestBucket(), key, rawInput) + .withObjectMetadata(new S3ObjectMetadata().withContentLength(2516125L))); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + + Assert.assertEquals("unencrypted size incorrect", "2516125", + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE)); + Assert.assertEquals("encrypted size incorrect", 2516128L, objectMetadata.getContentLength().longValue()); + Assert.assertEquals("unencrypted sha1 incorrect", "027e997e6b1dfc97b93eb28dc9a6804096d85873", + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + Assert.assertEquals("master key ID incorrect", getKeyProvider().getMasterKeyFingerprint(), + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + Assert.assertNotNull("IV null", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_IV)); + Assert.assertNotNull("Object key", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_OBJECT_KEY)); + Assert.assertNotNull("Missing metadata signature", + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_META_SIG)); + } + + // Test a stream > 4MB. + @Test + public void testLargeStream() throws Exception { + String key = "big-stream.obj"; + int size = 5 * 1024 * 1024 + 13; + RandomInputStream rs = new RandomInputStream(size); + + client.putObject(new PutObjectRequest(getTestBucket(), key, rs) + .withObjectMetadata(new S3ObjectMetadata().withContentLength((long) size))); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + + // Make sure the checksum matches + String sha1hex = DigestUtils.sha1Hex(client.readObject(getTestBucket(), key, byte[].class)); + + assertNotNull("Missing SHA1 meta", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + assertEquals("SHA1 incorrect", sha1hex, + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + assertEquals("Stream length incorrect", size, + Integer.parseInt(objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE))); + } + @Test public void testRekey() throws Exception { String key = "rekey-test.txt"; @@ -104,27 +203,211 @@ public void testRekey() throws Exception { Assert.assertFalse(eclient.rekey(getTestBucket(), key)); // change master key - EncryptionConfig encryptionConfig = createEncryptionConfig(); - ((BasicEncryptionTransformFactory) encryptionConfig.getFactory()).setMasterEncryptionKey(getOldKey()); - S3EncryptionClient eclient2 = new S3EncryptionClient(createS3Config(), encryptionConfig); + getKeyProvider().setMasterKey(getOldKey()); // now actually rekey - Assert.assertTrue(eclient2.rekey(getTestBucket(), key)); + Assert.assertTrue(eclient.rekey(getTestBucket(), key)); // Read back and test - GetObjectResult result = client.getObject(new GetObjectRequest(getTestBucket(), key), String.class); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); - assertEquals("Content differs", content, result.getObject()); + assertEquals("Content differs", content, client.readObject(getTestBucket(), key, String.class)); assertEquals("unencrypted size incorrect", "12", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SIZE)); - assertEquals("encrypted size incorrect", 16, result.getObjectMetadata().getContentLength().longValue()); + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE)); + assertEquals("encrypted size incorrect", 16, objectMetadata.getContentLength().longValue()); assertEquals("unencrypted sha1 incorrect", "2ef7bde608ce5404e97d5f042f95f89f1c232871", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SHA1)); - assertEquals("master key ID incorrect", KeyUtils.getRsaPublicKeyFingerprint((RSAPublicKey) getOldKey().getPublic(), null), - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_KEY_ID)); - Assert.assertNotNull("IV null", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_IV)); - Assert.assertNotNull("Object key", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_OBJECT_KEY)); + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + assertEquals("master key ID incorrect", EncryptionUtil.getRsaPublicKeyFingerprint((RSAPublicKey) getOldKey().getPublic()), + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + Assert.assertNotNull("IV null", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_IV)); + Assert.assertNotNull("Object key", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_OBJECT_KEY)); Assert.assertNotNull("Missing metadata signature", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_META_SIG)); + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_META_SIG)); + } + + @Test + public void testRekeyAcl() throws Exception { + String key = "rekey-with-acl-test.txt"; + String content = "hello rekey with acl!"; + + // custom ACL + String identity = createS3Config().getIdentity(); + AccessControlList acl = new AccessControlList(); + acl.addGrants(new Grant(new CanonicalUser(identity, identity), Permission.FULL_CONTROL)); + acl.addGrants(new Grant(Group.ALL_USERS, Permission.FULL_CONTROL)); + + PutObjectRequest request = new PutObjectRequest(getTestBucket(), key, content).withAcl(acl); + client.putObject(request); + + // verify custom ACL + Assert.assertTrue(client.getObjectAcl(getTestBucket(), key).getGrants() + .contains(new Grant(Group.ALL_USERS, Permission.FULL_CONTROL))); + + // change master key + getKeyProvider().setMasterKey(getOldKey()); + + // now actually rekey + Assert.assertTrue(eclient.rekey(getTestBucket(), key)); + + // Read back and test + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + + // verify rekey + assertEquals("master key ID incorrect", EncryptionUtil.getRsaPublicKeyFingerprint((RSAPublicKey) getOldKey().getPublic()), + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + + // verify ACL + acl = client.getObjectAcl(getTestBucket(), key); + Assert.assertTrue(acl.getGrants().contains(new Grant(Group.ALL_USERS, Permission.FULL_CONTROL))); + } + + // the following methods aren't supported in the encryption client + + @Ignore + @Override + public void testReadObjectStreamRange() throws Exception { + } + + @Ignore + @Override + public void testInitiateListAbortMultipartUploads() throws Exception { + } + + @Ignore + @Override + public void testUpdateObjectWithRange() throws Exception { + } + + @Ignore + @Override + public void testSingleMultipartUploadMostSimpleOnePart() throws Exception { + } + + @Ignore + @Override + public void testSingleMultipartUploadMostSimple() throws Exception { + } + + @Ignore + @Override + public void testSingleMultipartUploadSimple() throws Exception { + } + + @Ignore + @Override + public void testMultiThreadMultipartUploadMostSimple() throws Exception { + } + + @Ignore + @Override + public void testLargeObjectContentLength() throws Exception { + } + + @Ignore + @Override + public void testSingleMultipartUploadListParts() throws Exception { + } + + @Ignore + @Override + public void testLargeFileUploader() throws Exception { + } + + @Ignore + @Override + public void testMultiThreadMultipartUploadListPartsPagination() throws Exception { + } + + @Ignore + @Override + public void testLargeFileDownloader() throws Exception { + } + + @Ignore + @Override + public void testAppendObject() throws Exception { + } + + // the following methods are unnecessary and/or do not test anything related to encryption + + + @Ignore + @Override + public void testCreateExistingBucket() throws Exception { + } + + @Ignore + @Override + public void testListBuckets() throws Exception { + } + + @Ignore + @Override + public void testListBucketsReq() { + } + + @Ignore + @Override + public void testBucketExists() throws Exception { + } + + @Ignore + @Override + public void testCreateBucketRequest() throws Exception { + } + + @Ignore + @Override + public void testDeleteBucket() throws Exception { + } + + @Ignore + @Override + public void testDeleteBucketWithObjects() throws Exception { + } + + @Ignore + @Override + public void testSetBucketAcl() throws Exception { + } + + @Ignore + @Override + public void testSetBucketAclCanned() { + } + + @Ignore + @Override + public void testGetBucketCors() throws Exception { + } + + @Ignore + @Override + public void testDeleteBucketCors() throws Exception { + } + + @Ignore + @Override + public void testDeleteBucketLifecycle() throws Exception { + } + + @Ignore + @Override + public void testCreateBucket() throws Exception { + } + + @Ignore + @Override + public void testBucketLocation() throws Exception { + } + + @Ignore + @Override + public void testSetBucketVersioning() throws Exception { + } + + @Ignore + @Override + public void testBucketVersions() throws Exception { } } diff --git a/src/test/java/com/emc/object/s3/S3EncryptionClientKeyStoreTest.java b/src/test/java/com/emc/object/s3/S3EncryptionClientKeyStoreTest.java index 3a886128..c0bc1258 100644 --- a/src/test/java/com/emc/object/s3/S3EncryptionClientKeyStoreTest.java +++ b/src/test/java/com/emc/object/s3/S3EncryptionClientKeyStoreTest.java @@ -26,165 +26,54 @@ */ package com.emc.object.s3; +import com.emc.codec.encryption.BasicKeyProvider; +import com.emc.codec.encryption.EncryptionConstants; +import com.emc.codec.encryption.EncryptionUtil; +import com.emc.codec.encryption.KeystoreKeyProvider; import com.emc.object.EncryptionConfig; -import com.emc.object.s3.bean.GetObjectResult; import com.emc.object.s3.jersey.S3EncryptionClient; -import com.emc.object.s3.request.GetObjectRequest; -import com.emc.object.s3.request.PutObjectRequest; -import com.emc.util.RandomInputStream; -import com.emc.vipr.transform.TransformConstants; -import com.emc.vipr.transform.encryption.KeyStoreEncryptionFactory; -import com.emc.vipr.transform.encryption.KeyUtils; import org.apache.log4j.Logger; import org.junit.Assert; -import org.junit.Assume; -import org.junit.Ignore; import org.junit.Test; import java.io.FileNotFoundException; import java.io.InputStream; +import java.security.KeyPair; import java.security.KeyStore; -import java.security.MessageDigest; import java.security.interfaces.RSAPublicKey; import java.util.Enumeration; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -public class S3EncryptionClientKeyStoreTest extends S3JerseyClientTest { +public class S3EncryptionClientKeyStoreTest extends S3EncryptionClientBasicTest { private static final Logger l4j = Logger.getLogger(S3JerseyClientTest.class); - protected int keySize = 128; - protected S3EncryptionClient eclient; - - private String keystorePassword = "viprviprvipr"; private String keyAlias = "masterkey"; private String oldKeyAlias = "oldkey"; - private String keystoreFile = "keystore.jks"; - private KeyStore _keystore; - - @Override - protected String getTestBucketPrefix() { - return "s3-encryption-client-test"; - } + private BasicKeyProvider _keyProvider; @Override - public void initClient() throws Exception { - client = eclient = new S3EncryptionClient(createS3Config(), createEncryptionConfig()); - } - - protected EncryptionConfig createEncryptionConfig() throws Exception { - return new EncryptionConfig(getKeystore(), keystorePassword.toCharArray(), keyAlias, null, keySize); - } - - protected String getMasterKeyFingerprint() throws Exception { - return getKeyFingerprint(keyAlias); - } - - private synchronized KeyStore getKeystore() throws Exception { - if (_keystore == null) { - _keystore = KeyStore.getInstance("jks"); + protected synchronized BasicKeyProvider getKeyProvider() throws Exception { + if (_keyProvider == null) { + String keystoreFile = "keystore.jks"; + String keystorePassword = "viprviprvipr"; + KeyStore keyStore = KeyStore.getInstance("jks"); InputStream in = this.getClass().getClassLoader().getResourceAsStream(keystoreFile); if (in == null) throw new FileNotFoundException(keystoreFile); - _keystore.load(in, keystorePassword.toCharArray()); + keyStore.load(in, keystorePassword.toCharArray()); l4j.debug("Keystore Loaded"); - for (Enumeration aliases = _keystore.aliases(); aliases.hasMoreElements(); ) { + for (Enumeration aliases = keyStore.aliases(); aliases.hasMoreElements(); ) { l4j.debug("Found key: " + aliases.nextElement()); } - } - return _keystore; - } - - protected String getKeyFingerprint(String keyAlias) throws Exception { - return KeyUtils.getRsaPublicKeyFingerprint((RSAPublicKey) getKeystore().getCertificate(keyAlias).getPublicKey(), null); - } - - @Test - public void testEncryption() throws Exception { - String key = "hello.txt"; - String content = "Hello World!"; - - client.putObject(getTestBucket(), key, content, null); - GetObjectResult result = client.getObject(new GetObjectRequest(getTestBucket(), key), byte[].class); - Assert.assertEquals("unencrypted size incorrect", "12", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SIZE)); - Assert.assertEquals("encrypted size incorrect", 16, result.getObjectMetadata().getContentLength().longValue()); - Assert.assertEquals("unencrypted sha1 incorrect", "2ef7bde608ce5404e97d5f042f95f89f1c232871", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SHA1)); - Assert.assertEquals("master key ID incorrect", getMasterKeyFingerprint(), - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_KEY_ID)); - Assert.assertNotNull("IV null", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_IV)); - Assert.assertNotNull("Object key", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_OBJECT_KEY)); - Assert.assertNotNull("Missing metadata signature", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_META_SIG)); - } - - @Test - public void testWithMetadata() throws Exception { - String key = "metadata-test"; - String content = "Hello Metadata!!"; - String m1 = "meta1", v1 = "value1", m2 = "meta2", v2 = "value2"; - S3ObjectMetadata metadata = new S3ObjectMetadata().addUserMetadata(m1, v1).addUserMetadata(m2, v2); - client.putObject(new PutObjectRequest(getTestBucket(), key, content).withObjectMetadata(metadata)); - - metadata = client.getObjectMetadata(getTestBucket(), key); - Assert.assertNotNull(metadata.getUserMetadata(m1)); - Assert.assertNotNull(metadata.getUserMetadata(m2)); - Assert.assertEquals(v1, metadata.getUserMetadata(m1)); - Assert.assertEquals(v2, metadata.getUserMetadata(m2)); - } - - @Test - public void testStream() throws Exception { - String key = "test-file.txt"; - InputStream rawInput = getClass().getClassLoader().getResourceAsStream("uncompressed.txt"); - Assume.assumeNotNull(rawInput); - - client.putObject(new PutObjectRequest(getTestBucket(), key, rawInput) - .withObjectMetadata(new S3ObjectMetadata().withContentLength(2516125L))); - S3ObjectMetadata objectMetadata = client.getObjectMetadata(getTestBucket(), key); - - Assert.assertEquals("unencrypted size incorrect", "2516125", - objectMetadata.getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SIZE)); - Assert.assertEquals("encrypted size incorrect", 2516128L, objectMetadata.getContentLength().longValue()); - Assert.assertEquals("unencrypted sha1 incorrect", "027e997e6b1dfc97b93eb28dc9a6804096d85873", - objectMetadata.getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SHA1)); - Assert.assertEquals("master key ID incorrect", getMasterKeyFingerprint(), - objectMetadata.getUserMetadata(TransformConstants.META_ENCRYPTION_KEY_ID)); - Assert.assertNotNull("IV null", objectMetadata.getUserMetadata(TransformConstants.META_ENCRYPTION_IV)); - Assert.assertNotNull("Object key", objectMetadata.getUserMetadata(TransformConstants.META_ENCRYPTION_OBJECT_KEY)); - Assert.assertNotNull("Missing metadata signature", - objectMetadata.getUserMetadata(TransformConstants.META_ENCRYPTION_META_SIG)); - } - - // Test a stream > 4MB. - @Test - public void testLargeStream() throws Exception { - String key = "big-stream.obj"; - int size = 5 * 1024 * 1024 + 13; - RandomInputStream rs = new RandomInputStream(size); - - client.putObject(new PutObjectRequest(getTestBucket(), key, rs) - .withObjectMetadata(new S3ObjectMetadata().withContentLength((long) size))); - GetObjectResult result = client.getObject(new GetObjectRequest(getTestBucket(), key), byte[].class); - - // Make sure the checksum matches - MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); - byte[] sha1Digest = sha1.digest(result.getObject()); - - // Hex Encode it - String sha1hex = KeyUtils.toHexPadded(sha1Digest); - - assertNotNull("Missing SHA1 meta", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SHA1)); - assertEquals("SHA1 incorrect", sha1hex, - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SHA1)); - assertEquals("Stream length incorrect", size, - Integer.parseInt(result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SIZE))); + _keyProvider = new KeystoreKeyProvider(keyStore, keystorePassword.toCharArray(), keyAlias); + } + return _keyProvider; } + @Override @Test public void testRekey() throws Exception { String key = "rekey-test.txt"; @@ -197,98 +86,28 @@ public void testRekey() throws Exception { // change master key EncryptionConfig encryptionConfig = createEncryptionConfig(); - ((KeyStoreEncryptionFactory) encryptionConfig.getFactory()).setMasterEncryptionKeyAlias(oldKeyAlias); + ((KeystoreKeyProvider) getKeyProvider()).setMasterKeyAlias(oldKeyAlias); S3EncryptionClient eclient2 = new S3EncryptionClient(createS3Config(), encryptionConfig); // now actually rekey Assert.assertTrue(eclient2.rekey(getTestBucket(), key)); // Read back and test - GetObjectResult result = client.getObject(new GetObjectRequest(getTestBucket(), key), String.class); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); - assertEquals("Content differs", content, result.getObject()); + KeyPair oldKeyPair = ((KeystoreKeyProvider) getKeyProvider()).getKeyFromAlias(oldKeyAlias); + String oldKeyFingerprint = EncryptionUtil.getRsaPublicKeyFingerprint((RSAPublicKey) oldKeyPair.getPublic()); + assertEquals("Content differs", content, client.readObject(getTestBucket(), key, String.class)); assertEquals("unencrypted size incorrect", "12", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SIZE)); - assertEquals("encrypted size incorrect", 16, result.getObjectMetadata().getContentLength().longValue()); + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE)); + assertEquals("encrypted size incorrect", 16, objectMetadata.getContentLength().longValue()); assertEquals("unencrypted sha1 incorrect", "2ef7bde608ce5404e97d5f042f95f89f1c232871", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_UNENC_SHA1)); - assertEquals("master key ID incorrect", getKeyFingerprint(oldKeyAlias), - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_KEY_ID)); - Assert.assertNotNull("IV null", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_IV)); - Assert.assertNotNull("Object key", result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_OBJECT_KEY)); + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + assertEquals("master key ID incorrect", oldKeyFingerprint, + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + Assert.assertNotNull("IV null", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_IV)); + Assert.assertNotNull("Object key", objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_OBJECT_KEY)); Assert.assertNotNull("Missing metadata signature", - result.getObjectMetadata().getUserMetadata(TransformConstants.META_ENCRYPTION_META_SIG)); - } - - // the following methods aren't supported in the encryption client - - @Ignore - @Override - public void testReadObjectStreamRange() throws Exception { - } - - @Ignore - @Override - public void testInitiateListAbortMultipartUploads() throws Exception { - } - - @Ignore - @Override - public void testUpdateObjectWithRange() throws Exception { - } - - @Ignore - @Override - public void testSingleMultipartUploadMostSimpleOnePart() throws Exception { - } - - @Ignore - @Override - public void testSingleMultipartUploadMostSimple() throws Exception { - } - - @Ignore - @Override - public void testSingleMultipartUploadSimple() throws Exception { - } - - @Ignore - @Override - public void testBucketVersions() throws Exception { - } - - @Ignore - @Override - public void testMultiThreadMultipartUploadMostSimple() throws Exception { - } - - @Ignore - @Override - public void testLargeObjectContentLength() throws Exception { - } - - @Ignore - @Override - public void testSingleMultipartUploadListParts() throws Exception { - } - - @Ignore - @Override - public void testLargeFileUploader() throws Exception { - } - - @Ignore - @Override - public void testMultiThreadMultipartUploadListPartsPagination() throws Exception { - } - - @Ignore - @Override - public void testLargeFileDownloader() throws Exception { - } - - @Ignore - @Override - public void testAppendObject() throws Exception { + objectMetadata.getUserMetadata(EncryptionConstants.META_ENCRYPTION_META_SIG)); } } diff --git a/src/test/java/com/emc/object/s3/S3EncryptionWithCompressionTest.java b/src/test/java/com/emc/object/s3/S3EncryptionWithCompressionTest.java new file mode 100644 index 00000000..c51b32ae --- /dev/null +++ b/src/test/java/com/emc/object/s3/S3EncryptionWithCompressionTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2015, EMC Corporation. + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * + Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + The name of EMC Corporation may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package com.emc.object.s3; + +import com.emc.codec.CodecChain; +import com.emc.codec.compression.CompressionConstants; +import com.emc.codec.encryption.EncryptionConstants; +import com.emc.object.EncryptionConfig; +import com.emc.object.s3.request.PutObjectRequest; +import com.emc.object.util.ChecksumAlgorithm; +import com.emc.object.util.ChecksummedInputStream; +import com.emc.object.util.RunningChecksum; +import com.emc.rest.util.StreamUtil; +import com.emc.util.RandomInputStream; +import org.apache.commons.codec.digest.DigestUtils; +import org.junit.Assume; +import org.junit.Ignore; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.Map; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class S3EncryptionWithCompressionTest extends S3EncryptionClientBasicTest { + @Override + protected EncryptionConfig createEncryptionConfig() throws Exception { + return super.createEncryptionConfig().withCompressionEnabled(true); + } + + @Override + public void testEncodeMeta() throws Exception { + String key = "hello.txt"; + String content = "Hello World!"; + + client.putObject(getTestBucket(), key, content, null); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + Map encodedMetadata = objectMetadata.getUserMetadata(); + + // manually deflate + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DeflaterOutputStream dos = new DeflaterOutputStream(baos, new Deflater(CompressionConstants.DEFAULT_COMPRESSION_LEVEL)); + dos.write(content.getBytes("UTF-8")); + dos.close(); + byte[] deflatedData = baos.toByteArray(); + + assertEquals(32, objectMetadata.getContentLength().longValue()); + + assertEquals("original digest incorrect", DigestUtils.sha1Hex(content.getBytes("UTF-8")), + encodedMetadata.get(CompressionConstants.META_COMPRESSION_UNCOMP_SHA1)); + assertEquals("Uncompressed size incorrect", 12, + Long.parseLong(encodedMetadata.get(CompressionConstants.META_COMPRESSION_UNCOMP_SIZE))); + assertEquals("Compression ratio incorrect", "-66.7%", + encodedMetadata.get(CompressionConstants.META_COMPRESSION_COMP_RATIO)); + assertEquals("Compressed size incorrect", deflatedData.length, + Long.parseLong(encodedMetadata.get(CompressionConstants.META_COMPRESSION_COMP_SIZE))); + + assertEquals("Unencrypted digest incorrect", DigestUtils.sha1Hex(deflatedData), + encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + assertEquals("Unencrypted size incorrect", deflatedData.length, + Long.parseLong(encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE))); + assertNotNull("Missing IV", encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_IV)); + assertEquals("Incorrect master encryption key ID", getKeyProvider().getMasterKeyFingerprint(), + encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + assertNotNull("Missing object key", encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_OBJECT_KEY)); + assertNotNull("Missing metadata signature", encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_META_SIG)); + + assertEquals("Transform config string incorrect", "COMP:Deflate/5,ENC:AES/CBC/PKCS5Padding", + encodedMetadata.get(CodecChain.META_TRANSFORM_MODE)); + } + + @Override + public void testStream() throws Exception { + String key = "test-file.txt"; + InputStream rawInput = getClass().getClassLoader().getResourceAsStream("uncompressed.txt"); + Assume.assumeNotNull(rawInput); + + client.putObject(new PutObjectRequest(getTestBucket(), key, rawInput) + .withObjectMetadata(new S3ObjectMetadata().withContentLength(2516125L))); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + Map encodedMetadata = objectMetadata.getUserMetadata(); + + // manually deflate + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DeflaterOutputStream dos = new DeflaterOutputStream(baos, new Deflater(CompressionConstants.DEFAULT_COMPRESSION_LEVEL)); + StreamUtil.copy(getClass().getClassLoader().getResourceAsStream("uncompressed.txt"), dos, 2516125); + dos.close(); + byte[] deflatedData = baos.toByteArray(); + + assertEquals(223552, objectMetadata.getContentLength().longValue()); + + assertEquals("original digest incorrect", "027e997e6b1dfc97b93eb28dc9a6804096d85873", + encodedMetadata.get(CompressionConstants.META_COMPRESSION_UNCOMP_SHA1)); + assertEquals("Uncompressed size incorrect", 2516125, + Long.parseLong(encodedMetadata.get(CompressionConstants.META_COMPRESSION_UNCOMP_SIZE))); + assertEquals("Compression ratio incorrect", "91.1%", + encodedMetadata.get(CompressionConstants.META_COMPRESSION_COMP_RATIO)); + assertEquals("Compressed size incorrect", deflatedData.length, + Long.parseLong(encodedMetadata.get(CompressionConstants.META_COMPRESSION_COMP_SIZE))); + + assertEquals("Unencrypted digest incorrect", DigestUtils.sha1Hex(deflatedData), + encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_UNENC_SHA1)); + assertEquals("Unencrypted size incorrect", deflatedData.length, + Long.parseLong(encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_UNENC_SIZE))); + assertNotNull("Missing IV", encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_IV)); + assertEquals("Incorrect master encryption key ID", getKeyProvider().getMasterKeyFingerprint(), + encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_KEY_ID)); + assertNotNull("Missing object key", encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_OBJECT_KEY)); + assertNotNull("Missing metadata signature", encodedMetadata.get(EncryptionConstants.META_ENCRYPTION_META_SIG)); + + assertEquals("Transform config string incorrect", "COMP:Deflate/5,ENC:AES/CBC/PKCS5Padding", + encodedMetadata.get(CodecChain.META_TRANSFORM_MODE)); + } + + @Override + public void testLargeStream() throws Exception { + String key = "big-stream.obj"; + int size = 5 * 1024 * 1024 + 13; + RandomInputStream rs = new RandomInputStream(size); + ChecksummedInputStream cis = new ChecksummedInputStream(rs, new RunningChecksum(ChecksumAlgorithm.SHA1)); + + client.putObject(new PutObjectRequest(getTestBucket(), key, cis) + .withObjectMetadata(new S3ObjectMetadata().withContentLength((long) size))); + S3ObjectMetadata objectMetadata = rclient.getObjectMetadata(getTestBucket(), key); + + // Make sure the checksum matches + String sha1hex = cis.getChecksum().getValue(); + + assertNotNull("Missing SHA1 meta", objectMetadata.getUserMetadata(CompressionConstants.META_COMPRESSION_UNCOMP_SHA1)); + assertEquals("SHA1 incorrect", sha1hex, + objectMetadata.getUserMetadata(CompressionConstants.META_COMPRESSION_UNCOMP_SHA1)); + assertEquals("Stream length incorrect", size, + Integer.parseInt(objectMetadata.getUserMetadata(CompressionConstants.META_COMPRESSION_UNCOMP_SIZE))); + } + + @Ignore + @Override + public void testRekey() throws Exception { + } +} diff --git a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java index f546bda0..e636ae71 100644 --- a/src/test/java/com/emc/object/s3/S3JerseyClientTest.java +++ b/src/test/java/com/emc/object/s3/S3JerseyClientTest.java @@ -26,13 +26,17 @@ */ package com.emc.object.s3; +import com.emc.object.ObjectConfig; import com.emc.object.Range; import com.emc.object.s3.bean.*; import com.emc.object.s3.jersey.S3JerseyClient; import com.emc.object.s3.request.*; +import com.emc.rest.smart.Host; +import com.emc.rest.smart.ecs.Vdc; import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config; import org.apache.log4j.Logger; import org.junit.Assert; +import org.junit.Assume; import org.junit.Ignore; import org.junit.Test; @@ -54,6 +58,45 @@ public void initClient() throws Exception { client = new S3JerseyClient(createS3Config()); } + @Test + public void testListDataNodes() throws Exception { + ListDataNode listDataNode = client.listDataNodes(); + Assert.assertNotNull(listDataNode.getVersionInfo()); + Assert.assertNotNull(listDataNode.getDataNodes()); + Assert.assertFalse(listDataNode.getDataNodes().isEmpty()); + } + + @Test + public void testMultipleVdcs() throws Exception { + S3Config config = createS3Config(); + + Assume.assumeFalse(config.isUseVHost()); + + try { + client.listDataNodes(); + } catch (Exception e) { + Assume.assumeNoException(e); + } + + // just going to use the same VDC twice for lack of a geo env. + List hosts = config.getVdcs().get(0).getHosts(); + Vdc vdc1 = new Vdc("vdc1", hosts), vdc2 = new Vdc("vdc2", new ArrayList(hosts)); + + String proxyUri = config.getPropAsString(ObjectConfig.PROPERTY_PROXY_URI); + config = new S3Config(config.getProtocol(), config.getPort(), vdc1, vdc2) + .withIdentity(config.getIdentity()).withSecretKey(config.getSecretKey()); + if (proxyUri != null) config.setProperty(ObjectConfig.PROPERTY_PROXY_URI, proxyUri); + + S3JerseyClient tempClient = new S3JerseyClient(config); + + Thread.sleep(500); // wait for polling daemon to finish initial poll + + Assert.assertTrue(vdc1.getHosts().size() > 1); + Assert.assertTrue(vdc2.getHosts().size() > 1); + Assert.assertEquals(vdc1.getHosts().size() + vdc2.getHosts().size(), + tempClient.getLoadBalancer().getAllHosts().size()); + } + @Test public void testCreateExistingBucket() throws Exception { try { @@ -94,8 +137,6 @@ public void testBucketExists() throws Exception { l4j.debug("JMC testBucketExists succeeded!!!!!!!!!!!!!!!!!!!!!!!!!"); } - //this is tested by default in the @Before method - //void createBucket(String bucketName); @Test public void testCreateBucketRequest() throws Exception { String bucketName = getTestBucket() + "-x"; @@ -113,13 +154,7 @@ public void testDeleteBucket() throws Exception { Assert.assertTrue("failed to create bucket " + bucketName, client.bucketExists(bucketName)); client.deleteBucket(bucketName); - client.bucketExists(bucketName); // workaround for STORAGE-3299 - client.bucketExists(bucketName); // workaround for STORAGE-3299 - client.bucketExists(bucketName); // workaround for STORAGE-3299 Assert.assertFalse("failed to delete bucket " + bucketName, client.bucketExists(bucketName)); - - //JMC need to note that the @After cleanup will fail - l4j.debug("JMC - deleteBucket seemed to work"); } @@ -163,6 +198,7 @@ public void testSetBucketAcl() throws Exception { this.assertSameAcl(acl, client.getBucketAcl(getTestBucket())); } + @Ignore // TODO: blocked by STORAGE-7422 @Test public void testSetBucketAclCanned() { client.setBucketAcl(getTestBucket(), CannedAcl.BucketOwnerFullControl); @@ -532,6 +568,35 @@ public void testUpdateObjectWithRange() throws Exception { Assert.assertEquals(content.substring(0, offset) + contentPart, client.readObject(getTestBucket(), key, String.class)); } + @Test + public void testCreateObjectByteArray() throws Exception { + byte[] data; + Random random = new Random(); + + data = new byte[15]; + random.nextBytes(data); + client.putObject(getTestBucket(), "hello-bytes-small", data, null); + Assert.assertArrayEquals(data, client.readObject(getTestBucket(), "hello-bytes-small", byte[].class)); + + data = new byte[32 * 1024 - 1]; + random.nextBytes(data); + client.putObject(getTestBucket(), "hello-bytes-less", data, null); + Assert.assertArrayEquals(data, client.readObject(getTestBucket(), "hello-bytes-less", byte[].class)); + + data = new byte[32 * 1024 + 1]; + random.nextBytes(data); + client.putObject(getTestBucket(), "hello-bytes-more", data, null); + Assert.assertArrayEquals(data, client.readObject(getTestBucket(), "hello-bytes-more", byte[].class)); + } + + @Test + public void testCreateObjectString() throws Exception { + String key = "string-test"; + String content = "Hello Strings!"; + client.putObject(getTestBucket(), key, content, "text/plain"); + Assert.assertEquals(content, client.readObject(getTestBucket(), key, String.class)); + } + @Test public void testCreateObjectWithRequest() throws Exception { PutObjectRequest request = new PutObjectRequest(getTestBucket(), "/objectPrefix/testObject1", "object content"); @@ -623,6 +688,13 @@ public void testLargeFileUploader() throws Exception { Assert.assertArrayEquals(data, client.readObject(getTestBucket(), key, byte[].class)); Assert.assertEquals(objectMetadata.getUserMetadata(), client.getObjectMetadata(getTestBucket(), key).getUserMetadata()); + + // test issue 1 (https://github.com/emcvipr/ecs-object-client-java/issues/1) + objectMetadata = new S3ObjectMetadata(); + objectMetadata.withContentLength(size); + uploader = new LargeFileUploader(client, getTestBucket(), key + ".2", file); + uploader.setObjectMetadata(objectMetadata); + uploader.doByteRangeUpload(); } @Test @@ -1337,7 +1409,7 @@ public void testGetObjectAcl() throws Exception { Assert.assertNotNull(grant.getPermission()); } } - + @Test public void testSetObjectAcl() throws Exception { String testObject = "/objectPrefix/testObject1"; diff --git a/src/test/java/com/emc/object/util/S3HostListProviderTest.java b/src/test/java/com/emc/object/util/S3HostListProviderTest.java deleted file mode 100644 index 67b74901..00000000 --- a/src/test/java/com/emc/object/util/S3HostListProviderTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2015, EMC Corporation. - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * + Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * + Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * + The name of EMC Corporation may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package com.emc.object.util; - -import com.emc.object.s3.S3HostListProvider; -import com.emc.rest.smart.SmartConfig; -import com.emc.util.TestConfig; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.apache4.ApacheHttpClient4; -import com.sun.jersey.client.apache4.config.ApacheHttpClient4Config; -import org.junit.Assert; -import org.junit.Test; - -import java.net.URI; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -public class S3HostListProviderTest { - @Test - public void testS3HostListProvider() throws Exception { - Properties properties = TestConfig.getProperties(); - - URI serverURI = new URI(TestConfig.getPropertyNotEmpty(properties, TestProperties.S3_ENDPOINT)); - String user = TestConfig.getPropertyNotEmpty(properties, TestProperties.S3_ACCESS_KEY); - String secret = TestConfig.getPropertyNotEmpty(properties, TestProperties.S3_SECRET_KEY); - String proxyUri = properties.getProperty(TestProperties.PROXY_URI); - - ClientConfig clientConfig = new DefaultClientConfig(); - if (proxyUri != null) clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri); - Client client = ApacheHttpClient4.create(clientConfig); - - SmartConfig smartConfig = new SmartConfig(Collections.singletonList(serverURI.getHost())); - - S3HostListProvider hostListProvider = new S3HostListProvider(client, smartConfig.getLoadBalancer(), user, secret); - hostListProvider.setProtocol(serverURI.getScheme()); - hostListProvider.setPort(serverURI.getPort()); - - List hostList = hostListProvider.getHostList(); - - Assert.assertTrue("server list is empty", hostList.size() > 0); - } -} diff --git a/src/test/java/com/emc/util/ConcurrentJunitRunner.java b/src/test/java/com/emc/util/ConcurrentJunitRunner.java index b0ce6695..ee152429 100644 --- a/src/test/java/com/emc/util/ConcurrentJunitRunner.java +++ b/src/test/java/com/emc/util/ConcurrentJunitRunner.java @@ -65,7 +65,7 @@ public void finished() { } catch (ExecutionException e) { e.printStackTrace(); // (JUnit *should* fail the test) } finally { - executorService.shutdownNow(); + executorService.shutdown(); } } }); diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml index fd5a2973..b62305f1 100644 --- a/src/test/resources/log4j.xml +++ b/src/test/resources/log4j.xml @@ -12,7 +12,7 @@ - +