diff --git a/build/aws-sdk-java/src/FunctionalTests.java b/build/aws-sdk-java/src/FunctionalTests.java index 2005a780..caa5cf55 100644 --- a/build/aws-sdk-java/src/FunctionalTests.java +++ b/build/aws-sdk-java/src/FunctionalTests.java @@ -124,7 +124,7 @@ public static void initTests() throws IOException { sseKey3 = new SSECustomerKey(secretKey3); // Create bucket - s3Client.createBucket(new CreateBucketRequest(bucketName)); + s3Client.createBucket(new CreateBucketRequest(bucketName, region)); } public static void teardown() throws IOException { @@ -582,12 +582,11 @@ public static void runTests() throws Exception { public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException { endpoint = System.getenv("SERVER_ENDPOINT"); + region = System.getenv("SERVER_REGION"); accessKey = System.getenv("ACCESS_KEY"); secretKey = System.getenv("SECRET_KEY"); enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1"); - region = "us-east-1"; - if (enableHTTPS) { endpoint = "https://" + endpoint; } else { diff --git a/build/versioning/bucket.go b/build/versioning/bucket.go index f2a68748..e07baf0e 100644 --- a/build/versioning/bucket.go +++ b/build/versioning/bucket.go @@ -31,8 +31,6 @@ import ( // Tests bucket versioned bucket and get its versioning configuration to check func testMakeBucket() { - s3Client.Config.Region = aws.String("us-east-1") - // initialize logging params startTime := time.Now() function := "testCreateVersioningBucket" @@ -42,12 +40,15 @@ func testMakeBucket() { } _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(bucketName), + CreateBucketConfiguration: &s3.CreateBucketConfiguration{ + LocationConstraint: s3Client.Config.Region, + }, }) if err != nil { failureLog(function, args, startTime, "", "Versioning CreateBucket Failed", err).Fatal() return } - defer cleanupBucket(bucketName, function, args, startTime) + defer cleanupBucket(bucketName, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucketName), diff --git a/build/versioning/delete.go b/build/versioning/delete.go index a40d2ac8..c6ecad86 100644 --- a/build/versioning/delete.go +++ b/build/versioning/delete.go @@ -51,7 +51,7 @@ func testDeleteObject() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), @@ -194,7 +194,7 @@ func testDeleteObjects() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), diff --git a/build/versioning/get.go b/build/versioning/get.go index e930488e..96d30bbb 100644 --- a/build/versioning/get.go +++ b/build/versioning/get.go @@ -52,7 +52,7 @@ func testGetObject() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), diff --git a/build/versioning/legalhold.go b/build/versioning/legalhold.go index cf765ee3..953c5a9c 100644 --- a/build/versioning/legalhold.go +++ b/build/versioning/legalhold.go @@ -56,7 +56,7 @@ func testLockingLegalhold() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, true) type uploadedObject struct { legalhold string @@ -207,7 +207,7 @@ func testLockingLegalhold() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucketWithoutLock, function, args, startTime) + defer cleanupBucket(bucketWithoutLock, function, args, startTime, false) input = &s3.GetObjectLegalHoldInput{ Bucket: aws.String(bucketWithoutLock), @@ -255,30 +255,18 @@ func testLockingLegalhold() { } // object-handlers.go > PutObjectLegalHoldHandler > objectlock.ParseObjectLegalHold + // LegalHold has to be removed in the end, otherwise cleanup fails. putInput := &s3.PutObjectInput{ Body: aws.ReadSeekCloser(strings.NewReader("content")), Bucket: aws.String(bucket), Key: aws.String(object), - ObjectLockLegalHoldStatus: aws.String("test"), + ObjectLockLegalHoldStatus: aws.String("OFF"), } - output, err := s3Client.PutObject(putInput) + _, err = s3Client.PutObject(putInput) if err != nil { failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal() return } - uploads[0].versionId = *output.VersionId - - polhInput := &s3.PutObjectLegalHoldInput{ - Bucket: aws.String(bucket), - Key: aws.String(object), - VersionId: aws.String(uploads[0].versionId), - } - // We encountered an internal error, please try again.: cause(EOF) - _, err = s3Client.PutObjectLegalHold(polhInput) - if err == nil { - failureLog(function, args, startTime, "", fmt.Sprintf("PutObjectLegalHold expected to fail but got %v", err), err).Fatal() - return - } successLogger(function, args, startTime).Info() } diff --git a/build/versioning/list.go b/build/versioning/list.go index 0542ea63..c9390c0a 100644 --- a/build/versioning/list.go +++ b/build/versioning/list.go @@ -34,8 +34,9 @@ import ( ) // Test regular listing result with simple use cases: -// Upload an object ten times, delete it once (delete marker) -// and check listing result +// +// Upload an object ten times, delete it once (delete marker) +// and check listing result func testListObjectVersionsSimple() { startTime := time.Now() function := "testListObjectVersionsSimple" @@ -55,7 +56,7 @@ func testListObjectVersionsSimple() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), @@ -241,7 +242,7 @@ func testListObjectVersionsWithPrefixAndDelimiter() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), @@ -382,7 +383,7 @@ func testListObjectVersionsKeysContinuation() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), @@ -488,7 +489,7 @@ func testListObjectVersionsVersionIDContinuation() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), @@ -597,7 +598,7 @@ func testListObjectsVersionsWithEmptyDirObject() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), diff --git a/build/versioning/main.go b/build/versioning/main.go index 7d9f8845..9d130697 100644 --- a/build/versioning/main.go +++ b/build/versioning/main.go @@ -16,6 +16,7 @@ package main import ( + "fmt" "os" "time" @@ -29,7 +30,8 @@ import ( // S3 client for testing var s3Client *s3.S3 -func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time) { +// bypassGovernanceRetention is necessary as always setting BypassGovernanceRetention results in API errors on buckets without Object Locking enabled. +func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time, bypassGovernanceRetention bool) { start := time.Now() input := &s3.ListObjectVersionsInput{ @@ -41,35 +43,53 @@ func cleanupBucket(bucket string, function string, args map[string]interface{}, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { for _, v := range page.Versions { input := &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: v.Key, - VersionId: v.VersionId, - BypassGovernanceRetention: aws.Bool(true), + Bucket: &bucket, + Key: v.Key, + VersionId: v.VersionId, } + // Set BypassGovernanceRetention in separate step. Setting the value in the DeleteObjectInput may lead to the header being present with a value of false. + // This is not allowed by the S3 API and will result in a 409 error, if Object Locking is disabled. + if bypassGovernanceRetention { + input.SetBypassGovernanceRetention(true) + } + _, err := s3Client.DeleteObject(input) if err != nil { - return true + fmt.Printf("Unable to delete object version %s in %s, retrying after 30 seconds: %s\n", *v.Key, bucket, err.Error()) + return false } } for _, v := range page.DeleteMarkers { input := &s3.DeleteObjectInput{ - Bucket: &bucket, - Key: v.Key, - VersionId: v.VersionId, - BypassGovernanceRetention: aws.Bool(true), + Bucket: &bucket, + Key: v.Key, + VersionId: v.VersionId, } + // Set BypassGovernanceRetention in separate step. Setting the value in the DeleteObjectInput may lead to the header being present with a value of false. + // This is not allowed by the S3 API and will result in a 409 error, if Object Locking is disabled. + if bypassGovernanceRetention { + input.SetBypassGovernanceRetention(true) + } + _, err := s3Client.DeleteObject(input) if err != nil { - return true + fmt.Printf("Unable to remove delete marker %s in %s, retrying after 30 seconds: %s\n", *v.Key, bucket, err.Error()) + return false } } return true }) + if err != nil { + fmt.Printf("Unable to iterate bucket %s, retrying after 30 seconds: %s\n", bucket, err.Error()) + time.Sleep(30 * time.Second) + continue + } _, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{ Bucket: aws.String(bucket), }) if err != nil { + fmt.Printf("Unable to delete bucket %s, retrying after 30 seconds: %s\n", bucket, err.Error()) time.Sleep(30 * time.Second) continue } @@ -82,6 +102,7 @@ func cleanupBucket(bucket string, function string, args map[string]interface{}, func main() { endpoint := os.Getenv("SERVER_ENDPOINT") + region := os.Getenv("SERVER_REGION") accessKey := os.Getenv("ACCESS_KEY") secretKey := os.Getenv("SECRET_KEY") secure := os.Getenv("ENABLE_HTTPS") @@ -95,7 +116,7 @@ func main() { s3Config := &aws.Config{ Credentials: creds, Endpoint: aws.String(sdkEndpoint), - Region: aws.String("us-east-1"), + Region: aws.String(region), S3ForcePathStyle: aws.Bool(true), } diff --git a/build/versioning/put.go b/build/versioning/put.go index ca517a2b..366a10e5 100644 --- a/build/versioning/put.go +++ b/build/versioning/put.go @@ -54,7 +54,7 @@ func testPutObject() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), @@ -165,7 +165,7 @@ func testPutObjectWithTaggingAndMetadata() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), diff --git a/build/versioning/retention.go b/build/versioning/retention.go index 98c3ff92..8d72337f 100644 --- a/build/versioning/retention.go +++ b/build/versioning/retention.go @@ -54,7 +54,7 @@ func testLockingRetentionGovernance() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, true) type uploadedObject struct { retention string @@ -154,7 +154,7 @@ func testLockingRetentionCompliance() { return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, true) type uploadedObject struct { retention string @@ -262,7 +262,7 @@ func testPutGetDeleteLockingRetention(function, retentionMode string) { return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, true) oneMinuteRetention := time.Now().UTC().Add(time.Minute) twoMinutesRetention := oneMinuteRetention.Add(time.Minute) diff --git a/build/versioning/stat.go b/build/versioning/stat.go index b9f2c4b0..c5836792 100644 --- a/build/versioning/stat.go +++ b/build/versioning/stat.go @@ -49,7 +49,7 @@ func testStatObject() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), diff --git a/build/versioning/tagging.go b/build/versioning/tagging.go index f41a3583..ae3ccace 100644 --- a/build/versioning/tagging.go +++ b/build/versioning/tagging.go @@ -50,7 +50,7 @@ func testTagging() { failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal() return } - defer cleanupBucket(bucket, function, args, startTime) + defer cleanupBucket(bucket, function, args, startTime, false) putVersioningInput := &s3.PutBucketVersioningInput{ Bucket: aws.String(bucket), diff --git a/run/core/aws-sdk-go/main.go b/run/core/aws-sdk-go/main.go index 5746c442..9a53d264 100644 --- a/run/core/aws-sdk-go/main.go +++ b/run/core/aws-sdk-go/main.go @@ -821,11 +821,6 @@ func testObjectTaggingErrors(s3Client *s3.S3) { // Tests bucket re-create errors. func testCreateBucketError(s3Client *s3.S3) { - region := s3Client.Config.Region - // Amazon S3 returns error in all AWS Regions except in the North Virginia Region. - // More details in https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#S3.CreateBucket - s3Client.Config.Region = aws.String("us-west-1") - // initialize logging params startTime := time.Now() function := "testMakeBucketError" @@ -835,16 +830,11 @@ func testCreateBucketError(s3Client *s3.S3) { } _, err := s3Client.CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(bucketName), + CreateBucketConfiguration: &s3.CreateBucketConfiguration{ + LocationConstraint: s3Client.Config.Region, + }, }) if err != nil { - // InvalidRegion is a valid error if the endpoint doesn't support - // different 'regions', we simply skip this test in such scenarios. - if err.(s3.RequestFailure).Code() == "InvalidRegion" { - // Restore region in s3Client - s3Client.Config.Region = region - successLogger(function, args, startTime).Info() - return - } failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal() return } @@ -852,6 +842,9 @@ func testCreateBucketError(s3Client *s3.S3) { _, errCreating := s3Client.CreateBucket(&s3.CreateBucketInput{ Bucket: aws.String(bucketName), + CreateBucketConfiguration: &s3.CreateBucketConfiguration{ + LocationConstraint: s3Client.Config.Region, + }, }) if errCreating == nil { failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Should Return Error for Existing bucket", err).Fatal() @@ -864,8 +857,6 @@ func testCreateBucketError(s3Client *s3.S3) { return } - // Restore region in s3Client - s3Client.Config.Region = region successLogger(function, args, startTime).Info() } @@ -1077,6 +1068,7 @@ func testSSECopyObject(s3Client *s3.S3) { func main() { endpoint := os.Getenv("SERVER_ENDPOINT") + region := os.Getenv("SERVER_REGION") accessKey := os.Getenv("ACCESS_KEY") secretKey := os.Getenv("SECRET_KEY") secure := os.Getenv("ENABLE_HTTPS") @@ -1096,7 +1088,7 @@ func main() { s3Config := &aws.Config{ Credentials: creds, Endpoint: aws.String(sdkEndpoint), - Region: aws.String("us-east-1"), + Region: aws.String(region), S3ForcePathStyle: aws.Bool(true), } diff --git a/run/core/aws-sdk-ruby/aws-stub-tests.rb b/run/core/aws-sdk-ruby/aws-stub-tests.rb index c6a931f4..06878846 100755 --- a/run/core/aws-sdk-ruby/aws-stub-tests.rb +++ b/run/core/aws-sdk-ruby/aws-stub-tests.rb @@ -728,8 +728,7 @@ def presignedPutObjectTest(data_dir, file_name) # Generate presigned Put URL and parse it uri = URI.parse(presignedPutWrapper(bucket_name, file_name, log_output)) - request = Net::HTTP::Put.new(uri.request_uri, 'content-type' => 'application/octet-stream', - 'x-amz-acl' => 'public-read') + request = Net::HTTP::Put.new(uri.request_uri, 'content-type' => 'application/octet-stream') request.body = IO.read(File.join(data_dir, file_name)) http = Net::HTTP.new(uri.host, uri.port) diff --git a/run/core/awscli/test.sh b/run/core/awscli/test.sh index 5c4a53fe..a448a335 100755 --- a/run/core/awscli/test.sh +++ b/run/core/awscli/test.sh @@ -53,7 +53,7 @@ function log_alert() { function make_bucket() { # Make bucket bucket_name="awscli-mint-test-bucket-$RANDOM" - function="${AWS} s3api create-bucket --bucket ${bucket_name}" + function="${AWS} s3api create-bucket --bucket ${bucket_name} --create-bucket-configuration LocationConstraint=${SERVER_REGION}" # execute the test out=$($function 2>&1)