Respect SERVER_REGION in more targets and fix various bugs

build/aws-sdk-java/src/FunctionalTests.java

@@ -124,7 +124,7 @@ public static void initTests() throws IOException {
         sseKey3 = new SSECustomerKey(secretKey3);
 
         // Create bucket
-        s3Client.createBucket(new CreateBucketRequest(bucketName));
+        s3Client.createBucket(new CreateBucketRequest(bucketName, region));
     }
 
     public static void teardown() throws IOException {
@@ -582,12 +582,11 @@ public static void runTests() throws Exception {
     public static void main(String[] args) throws Exception, IOException, NoSuchAlgorithmException {
 
         endpoint = System.getenv("SERVER_ENDPOINT");
+        region = System.getenv("SERVER_REGION");
         accessKey = System.getenv("ACCESS_KEY");
         secretKey = System.getenv("SECRET_KEY");
         enableHTTPS = System.getenv("ENABLE_HTTPS").equals("1");
 
-        region = "us-east-1";
-
         if (enableHTTPS) {
             endpoint = "https://" + endpoint;
         } else {

build/versioning/bucket.go

@@ -31,8 +31,6 @@ import (
 
 // Tests bucket versioned bucket and get its versioning configuration to check
 func testMakeBucket() {
-	s3Client.Config.Region = aws.String("us-east-1")
-
 	// initialize logging params
 	startTime := time.Now()
 	function := "testCreateVersioningBucket"
@@ -42,12 +40,15 @@ func testMakeBucket() {
 	}
 	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
 		Bucket: aws.String(bucketName),
+		CreateBucketConfiguration: &s3.CreateBucketConfiguration{
+			LocationConstraint: s3Client.Config.Region,
+		},
 	})
 	if err != nil {
 		failureLog(function, args, startTime, "", "Versioning CreateBucket Failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucketName, function, args, startTime)
+	defer cleanupBucket(bucketName, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucketName),

build/versioning/delete.go

@@ -51,7 +51,7 @@ func testDeleteObject() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),
@@ -194,7 +194,7 @@ func testDeleteObjects() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),

build/versioning/get.go

@@ -52,7 +52,7 @@ func testGetObject() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),

build/versioning/legalhold.go

@@ -56,7 +56,7 @@ func testLockingLegalhold() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, true)
 
 	type uploadedObject struct {
 		legalhold        string
@@ -207,7 +207,7 @@ func testLockingLegalhold() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucketWithoutLock, function, args, startTime)
+	defer cleanupBucket(bucketWithoutLock, function, args, startTime, false)
 
 	input = &s3.GetObjectLegalHoldInput{
 		Bucket: aws.String(bucketWithoutLock),
@@ -255,30 +255,18 @@ func testLockingLegalhold() {
 	}
 
 	// object-handlers.go > PutObjectLegalHoldHandler > objectlock.ParseObjectLegalHold
+	// LegalHold has to be removed in the end, otherwise cleanup fails.
 	putInput := &s3.PutObjectInput{
 		Body:                      aws.ReadSeekCloser(strings.NewReader("content")),
 		Bucket:                    aws.String(bucket),
 		Key:                       aws.String(object),
-		ObjectLockLegalHoldStatus: aws.String("test"),
+		ObjectLockLegalHoldStatus: aws.String("OFF"),
 	}
-	output, err := s3Client.PutObject(putInput)
+	_, err = s3Client.PutObject(putInput)
 	if err != nil {
 		failureLog(function, args, startTime, "", fmt.Sprintf("PUT expected to succeed but got %v", err), err).Fatal()
 		return
 	}
-	uploads[0].versionId = *output.VersionId
-
-	polhInput := &s3.PutObjectLegalHoldInput{
-		Bucket:    aws.String(bucket),
-		Key:       aws.String(object),
-		VersionId: aws.String(uploads[0].versionId),
-	}
-	// We encountered an internal error, please try again.: cause(EOF)
-	_, err = s3Client.PutObjectLegalHold(polhInput)
-	if err == nil {
-		failureLog(function, args, startTime, "", fmt.Sprintf("PutObjectLegalHold expected to fail but got %v", err), err).Fatal()
-		return
-	}
 
 	successLogger(function, args, startTime).Info()
 }

build/versioning/list.go

@@ -34,8 +34,9 @@ import (
 )
 
 // Test regular listing result with simple use cases:
-//   Upload an object ten times, delete it once (delete marker)
-//   and check listing result
+//
+//	Upload an object ten times, delete it once (delete marker)
+//	and check listing result
 func testListObjectVersionsSimple() {
 	startTime := time.Now()
 	function := "testListObjectVersionsSimple"
@@ -55,7 +56,7 @@ func testListObjectVersionsSimple() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),
@@ -241,7 +242,7 @@ func testListObjectVersionsWithPrefixAndDelimiter() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),
@@ -382,7 +383,7 @@ func testListObjectVersionsKeysContinuation() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),
@@ -488,7 +489,7 @@ func testListObjectVersionsVersionIDContinuation() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),
@@ -597,7 +598,7 @@ func testListObjectsVersionsWithEmptyDirObject() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),

build/versioning/main.go

@@ -16,6 +16,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"time"
 
@@ -29,7 +30,8 @@ import (
 // S3 client for testing
 var s3Client *s3.S3
 
-func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time) {
+// bypassGovernanceRetention is necessary as always setting BypassGovernanceRetention results in API errors on buckets without Object Locking enabled.
+func cleanupBucket(bucket string, function string, args map[string]interface{}, startTime time.Time, bypassGovernanceRetention bool) {
 	start := time.Now()
 
 	input := &s3.ListObjectVersionsInput{
@@ -41,35 +43,53 @@ func cleanupBucket(bucket string, function string, args map[string]interface{},
 			func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
 				for _, v := range page.Versions {
 					input := &s3.DeleteObjectInput{
-						Bucket:                    &bucket,
-						Key:                       v.Key,
-						VersionId:                 v.VersionId,
-						BypassGovernanceRetention: aws.Bool(true),
+						Bucket:    &bucket,
+						Key:       v.Key,
+						VersionId: v.VersionId,
 					}
+					// Set BypassGovernanceRetention in separate step. Setting the value in the DeleteObjectInput may lead to the header being present with a value of false.
+					// This is not allowed by the S3 API and will result in a 409 error, if Object Locking is disabled.
+					if bypassGovernanceRetention {
+						input.SetBypassGovernanceRetention(true)
+					}
+
 					_, err := s3Client.DeleteObject(input)
 					if err != nil {
-						return true
+						fmt.Printf("Unable to delete object version %s in %s, retrying after 30 seconds: %s\n", *v.Key, bucket, err.Error())
+						return false
 					}
 				}
 				for _, v := range page.DeleteMarkers {
 					input := &s3.DeleteObjectInput{
-						Bucket:                    &bucket,
-						Key:                       v.Key,
-						VersionId:                 v.VersionId,
-						BypassGovernanceRetention: aws.Bool(true),
+						Bucket:    &bucket,
+						Key:       v.Key,
+						VersionId: v.VersionId,
 					}
+					// Set BypassGovernanceRetention in separate step. Setting the value in the DeleteObjectInput may lead to the header being present with a value of false.
+					// This is not allowed by the S3 API and will result in a 409 error, if Object Locking is disabled.
+					if bypassGovernanceRetention {
+						input.SetBypassGovernanceRetention(true)
+					}
+
 					_, err := s3Client.DeleteObject(input)
 					if err != nil {
-						return true
+						fmt.Printf("Unable to remove delete marker %s in %s, retrying after 30 seconds: %s\n", *v.Key, bucket, err.Error())
+						return false
 					}
 				}
 				return true
 			})
+		if err != nil {
+			fmt.Printf("Unable to iterate bucket %s, retrying after 30 seconds: %s\n", bucket, err.Error())
+			time.Sleep(30 * time.Second)
+			continue
+		}
 
 		_, err = s3Client.DeleteBucket(&s3.DeleteBucketInput{
 			Bucket: aws.String(bucket),
 		})
 		if err != nil {
+			fmt.Printf("Unable to delete bucket %s, retrying after 30 seconds: %s\n", bucket, err.Error())
 			time.Sleep(30 * time.Second)
 			continue
 		}
@@ -82,6 +102,7 @@ func cleanupBucket(bucket string, function string, args map[string]interface{},
 
 func main() {
 	endpoint := os.Getenv("SERVER_ENDPOINT")
+	region := os.Getenv("SERVER_REGION")
 	accessKey := os.Getenv("ACCESS_KEY")
 	secretKey := os.Getenv("SECRET_KEY")
 	secure := os.Getenv("ENABLE_HTTPS")
@@ -95,7 +116,7 @@ func main() {
 	s3Config := &aws.Config{
 		Credentials:      creds,
 		Endpoint:         aws.String(sdkEndpoint),
-		Region:           aws.String("us-east-1"),
+		Region:           aws.String(region),
 		S3ForcePathStyle: aws.Bool(true),
 	}
 

build/versioning/put.go

@@ -54,7 +54,7 @@ func testPutObject() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),
@@ -165,7 +165,7 @@ func testPutObjectWithTaggingAndMetadata() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),

build/versioning/retention.go

@@ -54,7 +54,7 @@ func testLockingRetentionGovernance() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, true)
 
 	type uploadedObject struct {
 		retention        string
@@ -154,7 +154,7 @@ func testLockingRetentionCompliance() {
 		return
 	}
 
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, true)
 
 	type uploadedObject struct {
 		retention        string
@@ -262,7 +262,7 @@ func testPutGetDeleteLockingRetention(function, retentionMode string) {
 		return
 	}
 
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, true)
 
 	oneMinuteRetention := time.Now().UTC().Add(time.Minute)
 	twoMinutesRetention := oneMinuteRetention.Add(time.Minute)

build/versioning/stat.go

@@ -49,7 +49,7 @@ func testStatObject() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),

build/versioning/tagging.go

@@ -50,7 +50,7 @@ func testTagging() {
 		failureLog(function, args, startTime, "", "CreateBucket failed", err).Fatal()
 		return
 	}
-	defer cleanupBucket(bucket, function, args, startTime)
+	defer cleanupBucket(bucket, function, args, startTime, false)
 
 	putVersioningInput := &s3.PutBucketVersioningInput{
 		Bucket: aws.String(bucket),