From 83d9f7971b2b02ffc6f7ee39b9926159015bb060 Mon Sep 17 00:00:00 2001 From: Paul Meyer <49727155+katexochen@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:25:27 +0200 Subject: [PATCH] aws: fix issue with creating bucket in us-east-1 Region us-east-1 isn't a valid parameter to a createBucket request. This caused the following error when uploading: Error: uploading variants: uploading image: ensuring bucket exists: creating bucket xxx: operation error S3: CreateBucket, https response error StatusCode: 400, RequestID: xxx, HostID: xx, api error InvalidLocationConstraint: The specified location-constraint is not valid Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com> --- README.md | 7 +++++++ aws/uploader.go | 22 +++++++++++++++------- config/config.go | 17 +++++++++-------- config/validation.rego | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 70d67d9..2030f56 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,13 @@ The description of the AMI. The bucket to upload the image to during the upload process. +### `base.aws.bucketRegionConstraint` / `variant..aws.bucketRegionConstraint` + +- Default: none (defaults to `us-east-1`) +- Required: no +- Template: no + +The region where the buckets exist or should be created. ### `base.aws.blobName` / `variant..aws.blobName` diff --git a/aws/uploader.go b/aws/uploader.go index 3785f6a..669c661 100644 --- a/aws/uploader.go +++ b/aws/uploader.go @@ -163,12 +163,17 @@ func (u *Uploader) ensureBucket(ctx context.Context) error { return nil } u.log.Printf("Bucket %s doesn't exist. Creating.", bucket) - _, err = s3C.CreateBucket(ctx, &s3.CreateBucketInput{ - Bucket: &bucket, - CreateBucketConfiguration: &s3types.CreateBucketConfiguration{ - LocationConstraint: s3types.BucketLocationConstraint(u.config.AWS.Region), - }, - }) + var createBucketConfig *s3types.CreateBucketConfiguration + if u.config.AWS.BucketLocationConstraint != "" { + createBucketConfig = &s3types.CreateBucketConfiguration{ + LocationConstraint: s3types.BucketLocationConstraint(u.config.AWS.BucketLocationConstraint), + } + } + req := &s3.CreateBucketInput{ + Bucket: &bucket, + CreateBucketConfiguration: createBucketConfig, + } + _, err = s3C.CreateBucket(ctx, req) if err != nil { return fmt.Errorf("creating bucket %s: %w", bucket, err) } @@ -253,6 +258,7 @@ func (u *Uploader) importSnapshot(ctx context.Context) (string, error) { }, }) if err != nil { + log.Println(bucketPermissionHelpText) return "", fmt.Errorf("importing snapshot: %w", err) } if importResp.ImportTaskId == nil { @@ -550,6 +556,8 @@ func (u *Uploader) sts(ctx context.Context) (stsAPI, error) { return sts.NewFromConfig(cfg), nil } +const bucketPermissionHelpText = "Importing snapshot failed with \"deleted\" status. This may indicate a missing service role for the AWS service \"vmie.amazonaws.com\" to access the snapshot. See https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#vmimport-role for details." + func waitForSnapshotImport(ctx context.Context, ec2C ec2API, importTaskID string) (string, error) { start := time.Now() for { @@ -586,7 +594,7 @@ func waitForSnapshotImport(ctx context.Context, ec2C ec2API, importTaskID string case string(ec2types.SnapshotStateError): return "", fmt.Errorf("importing snapshot: task failed with message %q", statusMessage) case string("deleted"): - log.Printf("Importing snapshot failed with \"deleted\" status. This may indicate a missing service role for the AWS service \"vmie.amazonaws.com\" to access the snapshot. See https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#vmimport-role for details.") + log.Println(bucketPermissionHelpText) return "", fmt.Errorf("importing snapshot: import state deleted with message %q", statusMessage) default: return "", fmt.Errorf("importing snapshot: status %s with message %q", diff --git a/config/config.go b/config/config.go index 6840b55..6f61744 100644 --- a/config/config.go +++ b/config/config.go @@ -163,14 +163,15 @@ type fieldTemplateData struct { } type AWSConfig struct { - Region string `toml:"region,omitempty"` - ReplicationRegions []string `toml:"replicationRegions,omitempty"` - AMIName string `toml:"amiName,omitempty" template:"true"` - AMIDescription string `toml:"amiDescription,omitempty" template:"true"` - Bucket string `toml:"bucket,omitempty" template:"true"` - BlobName string `toml:"blobName,omitempty" template:"true"` - SnapshotName string `toml:"snapshotName,omitempty" template:"true"` - Publish Option[bool] `toml:"publish,omitempty"` + Region string `toml:"region,omitempty"` + ReplicationRegions []string `toml:"replicationRegions,omitempty"` + AMIName string `toml:"amiName,omitempty" template:"true"` + AMIDescription string `toml:"amiDescription,omitempty" template:"true"` + Bucket string `toml:"bucket,omitempty" template:"true"` + BucketLocationConstraint string `toml:"bucketLocationConstraint,omitempty" template:"false"` + BlobName string `toml:"blobName,omitempty" template:"true"` + SnapshotName string `toml:"snapshotName,omitempty" template:"true"` + Publish Option[bool] `toml:"publish,omitempty"` } type AzureConfig struct { diff --git a/config/validation.rego b/config/validation.rego index 2ef81e4..f62a90d 100644 --- a/config/validation.rego +++ b/config/validation.rego @@ -110,6 +110,42 @@ deny[msg] { msg = sprintf("bucket name %q must not end with the suffix --ol-s3", [input.AWS.Bucket]) } +deny[msg] { + input.Provider == "aws" + input.AWS.BucketLocationConstraint in [ + "af-south-1", + "ap-east-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ca-central-1", + "cn-north-1", + "cn-northwest-1", + "EU", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "me-south-1", + "sa-east-1", + "us-east-2", + "us-gov-east-1", + "us-gov-west-1", + "us-west-1", + "us-west-2", + "ap-south-2", + "eu-south-2", + ] + + msg = sprintf("%q is not a valid bucket location constraint", [ input.AWS.BucketLocationConstraint ] ) +} + deny[msg] { input.Provider == "aws" not is_boolean(input.AWS.Publish)