Skip to content

Commit

Permalink
Add storage class, bucket acceleration and bucket encryption options …
Browse files Browse the repository at this point in the history 
…to storage profiles (#44).
  • Loading branch information
@chenkins
chenkins committed Feb 19, 2024
1 parent 5061eaf commit c9e3483
Show file tree
Hide file tree
Showing 8 changed files with 161 additions and 36 deletions.
6 changes: 5 additions & 1 deletion backend/setup/aws_sts/createbucketpermissionpolicy.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
"s3:CreateBucket",
"s3:GetBucketPolicy",
"s3:PutBucketVersioning",
"s3:GetBucketVersioning"
"s3:GetBucketVersioning",
"s3:GetBucketAccelerateConfiguration",
"s3:PutBucketAccelerateConfiguration",
"s3:GetBucketEncryption",
"s3:PutBucketEncryption"
],
"Resource": "arn:aws:s3:::cipherduck*"
},
Expand Down
20 changes: 17 additions & 3 deletions backend/src/main/java/org/cryptomator/hub/api/cipherduck/StorageProfileS3Dto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@

public sealed class StorageProfileS3Dto extends StorageProfileDto permits StorageProfileS3STSDto {

public enum S3_STORAGE_CLASSES {
STANDARD, INTELLIGENT_TIERING, STANDARD_IA, ONEZONE_IA, REDUCED_REDUNDANCY, GLACIER, GLACIER_IR, DEEP_ARCHIVE
}

//======================================================================
// (1) STS and permanent:
// - bucket creation frontend/desktop client (STS)
Expand All @@ -31,20 +35,25 @@ public sealed class StorageProfileS3Dto extends StorageProfileDto permits Storag
@Schema(description = "Whether to use path style for S3 endpoint for template upload/bucket creation.", example = "false", defaultValue = "false")
Boolean withPathStyleAccessEnabled = false;

public StorageProfileS3Dto(){
@JsonProperty(value = "storageClass")
@Schema(description = "Storage class for upload. Defaults to STANDARD", example = "STANDARD", required = true)
S3_STORAGE_CLASSES storageClass = S3_STORAGE_CLASSES.STANDARD;

public StorageProfileS3Dto() {
// jackson
}

public StorageProfileS3Dto(final UUID id, final String name, final Protocol protocol, final boolean archived, final String scheme, final String hostname, final Integer port, final Boolean withPathStyleAccessEnabled) {
public StorageProfileS3Dto(final UUID id, final String name, final Protocol protocol, final boolean archived, final String scheme, final String hostname, final Integer port, final boolean withPathStyleAccessEnabled, final S3_STORAGE_CLASSES storageClass) {
super(id, name, protocol, archived);
this.scheme = scheme;
this.hostname = hostname;
this.port = port;
this.withPathStyleAccessEnabled = withPathStyleAccessEnabled;
this.storageClass = storageClass;
}

static StorageProfileS3Dto fromEntity(final StorageProfileS3 storageProfile) {
return new StorageProfileS3Dto(storageProfile.id, storageProfile.name, Protocol.s3, storageProfile.archived, storageProfile.scheme, storageProfile.hostname, storageProfile.port, storageProfile.withPathStyleAccessEnabled);
return new StorageProfileS3Dto(storageProfile.id, storageProfile.name, Protocol.s3, storageProfile.archived, storageProfile.scheme, storageProfile.hostname, storageProfile.port, storageProfile.withPathStyleAccessEnabled, S3_STORAGE_CLASSES.valueOf(storageProfile.storageClass));
}

public StorageProfileS3 toEntity() {
Expand All @@ -56,6 +65,7 @@ public StorageProfileS3 toEntity() {
storageProfile.hostname = this.hostname;
storageProfile.port = this.port;
storageProfile.withPathStyleAccessEnabled = this.withPathStyleAccessEnabled;
storageProfile.storageClass = this.storageClass.name();
return storageProfile;
}

Expand All @@ -74,4 +84,8 @@ public Integer port() {
public Boolean withPathStyleAccessEnabled() {
return withPathStyleAccessEnabled;
}

public S3_STORAGE_CLASSES storageClass() {
return storageClass;
}
}
40 changes: 34 additions & 6 deletions backend/src/main/java/org/cryptomator/hub/api/cipherduck/StorageProfileS3STSDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@

public final class StorageProfileS3STSDto extends StorageProfileS3Dto {

public enum S3_SERVERSIDE_ENCRYPTION {
NONE, SSE_AES256, SSE_KMS_DEFAULT
}

//======================================================================
// (2) STS only: bucket creation
//======================================================================
Expand Down Expand Up @@ -38,19 +42,27 @@ public final class StorageProfileS3STSDto extends StorageProfileS3Dto {
@Schema(description = "STS endpoint to use for AssumeRoleWithWebIdentity and AssumeRole for getting a temporary access token passed to the backend. Defaults to AWS SDK default.", nullable = true)
String stsEndpoint;

@JsonProperty(value = "bucketVersioning", defaultValue = "true")
@Schema(description = "Enable bucket versioning upon bucket creation", defaultValue = "true")
@JsonProperty(value = "bucketVersioning", defaultValue = "true", required = true)
@Schema(description = "Enable bucket versioning upon bucket creation", defaultValue = "true", required = true)
Boolean bucketVersioning = true;

@JsonProperty(value = "bucketAcceleration", defaultValue = "true")
@Schema(description = "Enable bucket versioning upon bucket creation", defaultValue = "true", required = true)
Boolean bucketAcceleration = true;

@JsonProperty(value = "bucketEncryption", required = true)
@Schema(description = "Enable bucket versioning upon bucket creation", required = true)
S3_SERVERSIDE_ENCRYPTION bucketEncryption = S3_SERVERSIDE_ENCRYPTION.NONE;

//----------------------------------------------------------------------
// (3b) STS client profile custom properties
//----------------------------------------------------------------------
@JsonProperty(value = "stsRoleArn", required = true)
@Schema(description = "roleArn to for STS AssumeRoleWithWebIdentity (AWS and MinIO)", example = "arn:aws:iam::930717317329:role/cipherduck_chain_01")
String stsRoleArn;

@JsonProperty(value = "stsRoleArn2", required = false)
@Schema(description = "roleArn to assume for STS AssumeRole in role chaining (AWS only, not MinIO)", example = "arn:aws:iam::930717317329:role/cipherduck_chain_02")
@JsonProperty(value = "stsRoleArn2")
@Schema(description = "roleArn to assume for STS AssumeRole in role chaining (AWS only, not MinIO)", example = "arn:aws:iam::930717317329:role/cipherduck_chain_02", nullable = true)
String stsRoleArn2;


Expand All @@ -62,15 +74,17 @@ public StorageProfileS3STSDto() {
// jackson
}

public StorageProfileS3STSDto(final UUID id, final String name, final Protocol protocol, final boolean archived, final String scheme, final String hostname, final Integer port, final Boolean withPathStyleAccessEnabled, final String region, final List<String> regions, final String bucketPrefix, final String stsRoleArnClient, final String stsRoleArnHub, final String stsEndpoint, final Boolean bucketVersioning, final String stsRoleArn, final String stsRoleArn2, final Integer stsDurationSeconds) {
super(id, name, protocol, archived, scheme, hostname, port, withPathStyleAccessEnabled);
public StorageProfileS3STSDto(final UUID id, final String name, final Protocol protocol, final boolean archived, final String scheme, final String hostname, final Integer port, final boolean withPathStyleAccessEnabled, final S3_STORAGE_CLASSES storageClass, final String region, final List<String> regions, final String bucketPrefix, final String stsRoleArnClient, final String stsRoleArnHub, final String stsEndpoint, final boolean bucketVersioning, final boolean bucketAcceleration, final S3_SERVERSIDE_ENCRYPTION bucketEncryption, final String stsRoleArn, final String stsRoleArn2, final Integer stsDurationSeconds) {
super(id, name, protocol, archived, scheme, hostname, port, withPathStyleAccessEnabled, storageClass);
this.region = region;
this.regions = regions;
this.bucketPrefix = bucketPrefix;
this.stsRoleArnClient = stsRoleArnClient;
this.stsRoleArnHub = stsRoleArnHub;
this.stsEndpoint = stsEndpoint;
this.bucketVersioning = bucketVersioning;
this.bucketAcceleration = bucketAcceleration;
this.bucketEncryption = bucketEncryption;
this.stsRoleArn = stsRoleArn;
this.stsRoleArn2 = stsRoleArn2;
this.stsDurationSeconds = stsDurationSeconds;
Expand All @@ -86,13 +100,16 @@ static StorageProfileS3STSDto fromEntity(final StorageProfileS3STS storageProfil
storageProfile.hostname,
storageProfile.port,
storageProfile.withPathStyleAccessEnabled,
S3_STORAGE_CLASSES.valueOf(storageProfile.storageClass),
storageProfile.region,
storageProfile.regions,
storageProfile.bucketPrefix,
storageProfile.stsRoleArnClient,
storageProfile.stsRoleArnHub,
storageProfile.stsEndpoint,
storageProfile.bucketVersioning,
storageProfile.bucketAcceleration,
S3_SERVERSIDE_ENCRYPTION.valueOf(storageProfile.bucketEncryption),
storageProfile.stsRoleArn,
storageProfile.stsRoleArn2,
storageProfile.stsDurationSeconds
Expand All @@ -108,13 +125,16 @@ public StorageProfileS3STS toEntity() {
storageProfile.hostname = this.hostname;
storageProfile.port = this.port;
storageProfile.withPathStyleAccessEnabled = this.withPathStyleAccessEnabled;
storageProfile.storageClass = this.storageClass.toString();
storageProfile.region = this.region;
storageProfile.regions = this.regions;
storageProfile.bucketPrefix = this.bucketPrefix;
storageProfile.stsRoleArnClient = this.stsRoleArnClient;
storageProfile.stsRoleArnHub = this.stsRoleArnHub;
storageProfile.stsEndpoint = this.stsEndpoint;
storageProfile.bucketVersioning = this.bucketVersioning;
storageProfile.bucketAcceleration = this.bucketAcceleration;
storageProfile.bucketEncryption = this.bucketEncryption.name();
storageProfile.stsRoleArn = this.stsRoleArn;
storageProfile.stsRoleArn2 = this.stsRoleArn2;
storageProfile.stsDurationSeconds = this.stsDurationSeconds;
Expand Down Expand Up @@ -149,6 +169,14 @@ public Boolean bucketVersioning() {
return bucketVersioning;
}

public Boolean bucketAcceleration() {
return bucketAcceleration;
}

public S3_SERVERSIDE_ENCRYPTION bucketEncryption() {
return bucketEncryption;
}

public String stsRoleArn() {
return stsRoleArn;
}
Expand Down
3 changes: 1 addition & 2 deletions backend/src/main/java/org/cryptomator/hub/api/cipherduck/VaultJWEBackendDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ public record VaultJWEBackendDto(
String defaultPath,
@JsonProperty(value = "nickname", required = true)
String nickname,
@JsonProperty(value = "uuid", required = true)
String uuid, // vault UUID, will be used as bookmark UUID
@JsonProperty(value = "region", required = true)
String region,

@JsonProperty(value = "username")
// for non-STS
String username,
Expand Down
91 changes: 82 additions & 9 deletions backend/src/main/java/org/cryptomator/hub/api/cipherduck/storage/S3StorageHelper.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
package org.cryptomator.hub.api.cipherduck.storage;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.BucketAccelerateConfiguration;
import com.amazonaws.services.s3.model.BucketAccelerateStatus;
import com.amazonaws.services.s3.model.BucketVersioningConfiguration;
import com.amazonaws.services.s3.model.GetBucketEncryptionResult;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.SSEAlgorithm;
import com.amazonaws.services.s3.model.ServerSideEncryptionByDefault;
import com.amazonaws.services.s3.model.ServerSideEncryptionConfiguration;
import com.amazonaws.services.s3.model.ServerSideEncryptionRule;
import com.amazonaws.services.s3.model.SetBucketAccelerateConfigurationRequest;
import com.amazonaws.services.s3.model.SetBucketEncryptionRequest;
import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest;
import jakarta.ws.rs.ClientErrorException;
import jakarta.ws.rs.core.Response;
Expand All @@ -18,6 +29,7 @@

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Collections;

public class S3StorageHelper {
private static final Logger log = Logger.getLogger(S3StorageHelper.class);
Expand Down Expand Up @@ -68,23 +80,84 @@ public static void makeS3Bucket(
com.amazonaws.services.s3.model.PutObjectRequest request2 = new PutObjectRequest(bucketName, String.format("d/%s/%s/", dto.rootDirHash().substring(0, 2), dto.rootDirHash().substring(2)), emptyContent, metadata);
s3.putObject(request2);

// TODO review - should we allow for not setting (because of permissions)?
// enable versioning on the bucket.
if (storageConfig.bucketVersioning()) {
{
if (log.isInfoEnabled()) {
log.info(String.format("Enable bucket versioning on %s (%s, %s)", bucketName, dto, storageConfig));
log.info(String.format("Enable/disable bucket versioning on %s (%s, %s)", bucketName, dto, storageConfig));
}
BucketVersioningConfiguration configuration =
new BucketVersioningConfiguration().withStatus("Enabled");
s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(bucketName, new BucketVersioningConfiguration().withStatus(storageConfig.bucketVersioning() ? BucketVersioningConfiguration.ENABLED : BucketVersioningConfiguration.OFF)));
final BucketVersioningConfiguration conf = s3.getBucketVersioningConfiguration(bucketName);
if (log.isInfoEnabled()) {
log.info(String.format("Enabled/disabled bucket versioning on %s (%s, %s) with status %s", bucketName, dto, storageConfig, conf.getStatus()));
}
}

SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest =
new SetBucketVersioningConfigurationRequest(bucketName, configuration);
// enable/disable bucket acceleration on the bucket
{
if (log.isInfoEnabled()) {
log.info(String.format("Enable/disable bucket acceleration on %s (%s, %s)", bucketName, dto, storageConfig));
}
try {
s3.setBucketAccelerateConfiguration(new SetBucketAccelerateConfigurationRequest(bucketName, new BucketAccelerateConfiguration(storageConfig.bucketAcceleration() ? BucketAccelerateStatus.Enabled : BucketAccelerateStatus.Suspended)));
}
catch (AmazonS3Exception e){
if(!storageConfig.bucketAcceleration() && e.getMessage().contains("MalformedXML")){
// MinIO does not support bucket acceleration nor encryption -> TODO https://github.com/shift7-ch/cipherduck-hub/issues/44 should we make bucketAcceleration attribute in storage profile nullable instead?
// https://min.io/docs/minio/linux/administration/identity-access-management/policy-based-access-control.html
// https://github.com/minio/minio/blob/master/cmd/api-router.go
// https://github.com/minio/minio/issues/14586
log.warn(String.format("Ignoring failed SetBucketAccelerateConfiguration call - MinIO does not support it. Details: %s", e));
}
}

s3.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest);
final BucketAccelerateConfiguration conf = s3.getBucketAccelerateConfiguration(bucketName);
if (log.isInfoEnabled()) {
log.info(String.format("Enabled/disabled bucket acceleration on %s (%s, %s) with status %s", bucketName, dto, storageConfig, conf.getStatus()));
}
}

BucketVersioningConfiguration conf = s3.getBucketVersioningConfiguration(bucketName);
// enable/disable bucket encryption on the bucket
{
if (log.isInfoEnabled()) {
log.info(String.format("Enabled bucket versioning on %s (%s, %s) with status %s", bucketName, dto, storageConfig, conf.getStatus()));
log.info(String.format("Enable/disable bucket encryption on %s (%s, %s)", bucketName, dto, storageConfig));
}
switch (storageConfig.bucketEncryption()) {
case NONE -> {}
case SSE_AES256 -> s3.setBucketEncryption(
new SetBucketEncryptionRequest()
.withBucketName(bucketName)
.withServerSideEncryptionConfiguration(new ServerSideEncryptionConfiguration()
.withRules(Collections.singleton(
new ServerSideEncryptionRule()
.withApplyServerSideEncryptionByDefault(new ServerSideEncryptionByDefault().withSSEAlgorithm(SSEAlgorithm.AES256))))
));
case SSE_KMS_DEFAULT -> s3.setBucketEncryption(
new SetBucketEncryptionRequest()
.withBucketName(bucketName)
.withServerSideEncryptionConfiguration(new ServerSideEncryptionConfiguration()
.withRules(Collections.singleton(
new ServerSideEncryptionRule()
.withApplyServerSideEncryptionByDefault(new ServerSideEncryptionByDefault().withSSEAlgorithm(SSEAlgorithm.KMS))))
));
}
switch (storageConfig.bucketEncryption()) {
case NONE:
// MinIO does not support bucket acceleration nor encryption -> TODO https://github.com/shift7-ch/cipherduck-hub/issues/44 should we make bucketEncryption attribute in storage profile nullable instead?
// https://min.io/docs/minio/linux/administration/identity-access-management/policy-based-access-control.html
// https://github.com/minio/minio/blob/master/cmd/api-router.go
// https://github.com/minio/minio/issues/14586
break;
case SSE_AES256:
case SSE_KMS_DEFAULT:
final GetBucketEncryptionResult conf = s3.getBucketEncryption(bucketName);
if (log.isInfoEnabled()) {
log.info(String.format("Enabled/disabled bucket encryption on %s (%s, %s) with configuration %s", bucketName, dto, storageConfig, conf.getServerSideEncryptionConfiguration()));
}
}
}
// TODO https://github.com/shift7-ch/cipherduck-hub/issues/44 CORS?
//s3.setBucketCrossOriginConfiguration();

}
}
3 changes: 3 additions & 0 deletions backend/src/main/java/org/cryptomator/hub/entities/cipherduck/StorageProfileS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,7 @@ public class StorageProfileS3 extends StorageProfile {// TODO make sealed?

@Column
public Boolean withPathStyleAccessEnabled = false;

@Column
public String storageClass = "STANDARD";
}
20 changes: 9 additions & 11 deletions backend/src/main/java/org/cryptomator/hub/entities/cipherduck/StorageProfileS3STS.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package org.cryptomator.hub.entities.cipherduck;

import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.persistence.Column;
import jakarta.persistence.DiscriminatorValue;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import org.cryptomator.hub.api.cipherduck.StorageProfileS3STSDto;
import org.eclipse.microprofile.openapi.annotations.media.Schema;

import java.util.List;
import java.util.UUID;
Expand Down Expand Up @@ -37,6 +40,12 @@ public class StorageProfileS3STS extends StorageProfileS3 { // TODO make sealed/
@Column
public Boolean bucketVersioning = true;

@Column
public Boolean bucketAcceleration = true;

@Column
public String bucketEncryption;

//----------------------------------------------------------------------
// (3b) STS client profile custom properties
//----------------------------------------------------------------------
Expand All @@ -49,15 +58,4 @@ public class StorageProfileS3STS extends StorageProfileS3 { // TODO make sealed/
@Column
public Integer stsDurationSeconds = null;

public StorageProfileS3STS toEntity() {
final StorageProfileS3STS storageProfileS3 = new StorageProfileS3STS();
storageProfileS3.id = this.id;
storageProfileS3.name = this.name;
storageProfileS3.scheme = this.scheme;
storageProfileS3.hostname = this.hostname;
storageProfileS3.port = this.port;
storageProfileS3.withPathStyleAccessEnabled = this.withPathStyleAccessEnabled;
storageProfileS3.region = this.region;
return storageProfileS3;
}
}
Loading

0 comments on commit c9e3483

Please sign in to comment.