diff --git a/deploy/crds/planetscale.com_vitessbackupstorages.yaml b/deploy/crds/planetscale.com_vitessbackupstorages.yaml index 46207b67..0981865c 100644 --- a/deploy/crds/planetscale.com_vitessbackupstorages.yaml +++ b/deploy/crds/planetscale.com_vitessbackupstorages.yaml @@ -129,6 +129,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string diff --git a/deploy/crds/planetscale.com_vitessclusters.yaml b/deploy/crds/planetscale.com_vitessclusters.yaml index f04b0e5e..cefa2884 100644 --- a/deploy/crds/planetscale.com_vitessclusters.yaml +++ b/deploy/crds/planetscale.com_vitessclusters.yaml @@ -138,6 +138,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string diff --git a/deploy/crds/planetscale.com_vitesskeyspaces.yaml b/deploy/crds/planetscale.com_vitesskeyspaces.yaml index 674e4588..2d91c067 100644 --- a/deploy/crds/planetscale.com_vitesskeyspaces.yaml +++ b/deploy/crds/planetscale.com_vitesskeyspaces.yaml @@ -136,6 +136,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string diff --git a/deploy/crds/planetscale.com_vitessshards.yaml b/deploy/crds/planetscale.com_vitessshards.yaml index ea951365..dce756a2 100644 --- a/deploy/crds/planetscale.com_vitessshards.yaml +++ b/deploy/crds/planetscale.com_vitessshards.yaml @@ -136,6 +136,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string diff --git a/docs/api.md b/docs/api.md index 5786fb55..acb8ad03 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1926,6 +1926,20 @@ If set, this must point to a file in the format expected for the Default: Use the default credentials of the Node.

+ + +minPartSize
+ +int64 + + + +

MinPartSize is optional and is only needed if we want to set a minimum part size +to be used by the S3 uploader, mainly used to avoid too many small requests to S3 +when there are too many small shards. +Default: 5242880 (5MiB)

+ +

SecretSource diff --git a/docs/api/index.html b/docs/api/index.html index 9d8c90db..f9974877 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -1928,6 +1928,20 @@

S3BackupLocation Default: Use the default credentials of the Node.

+ + +minPartSize
+ +int64 + + + +

MinPartSize is optional and is only needed if we want to set a minimum part size +to be used by the S3 uploader, mainly used to avoid too many small requests to S3 +when there are too many small shards. +Default: 5242880 (5MiB)

+ +

SecretSource diff --git a/pkg/apis/planetscale/v2/vitessbackupstorage_types.go b/pkg/apis/planetscale/v2/vitessbackupstorage_types.go index e396fbbc..1bfdd726 100644 --- a/pkg/apis/planetscale/v2/vitessbackupstorage_types.go +++ b/pkg/apis/planetscale/v2/vitessbackupstorage_types.go @@ -143,6 +143,11 @@ type S3BackupLocation struct { // `~/.aws/credentials` file. // Default: Use the default credentials of the Node. AuthSecret *SecretSource `json:"authSecret,omitempty"` + // MinPartSize is optional and is only needed if we want to set a minimum part size + // to be used by the S3 uploader, mainly used to avoid too many small requests to S3 + // when there are too many small shards. + // Default: 5242880 (5MiB) + MinPartSize int64 `json:"minPartSize,omitempty"` } // AzblobBackupLocation specifies a backup location in Azure Blob Storage. diff --git a/pkg/operator/vitessbackup/storage_s3.go b/pkg/operator/vitessbackup/storage_s3.go index 8a20160e..83df76f6 100644 --- a/pkg/operator/vitessbackup/storage_s3.go +++ b/pkg/operator/vitessbackup/storage_s3.go @@ -31,6 +31,7 @@ func s3BackupFlags(s3 *planetscalev2.S3BackupLocation, clusterName string) vites "s3_backup_storage_bucket": s3.Bucket, "s3_backup_storage_root": rootKeyPrefix(s3.KeyPrefix, clusterName), "s3_backup_force_path_style": s3.ForcePathStyle, + "s3_backup_aws_min_partsize": s3.MinPartSize, } if len(s3.Endpoint) > 0 { flags["s3_backup_aws_endpoint"] = s3.Endpoint diff --git a/test/endtoend/operator/operator-latest.yaml b/test/endtoend/operator/operator-latest.yaml index e7e3d6b9..a7ab8a87 100644 --- a/test/endtoend/operator/operator-latest.yaml +++ b/test/endtoend/operator/operator-latest.yaml @@ -679,6 +679,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string @@ -1995,6 +1998,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string @@ -5249,6 +5255,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string @@ -6696,6 +6705,9 @@ spec: maxLength: 256 pattern: ^[^\r\n]*$ type: string + minPartSize: + format: int64 + type: integer region: minLength: 1 type: string