From 4f308083373dd8a5b409e42fbe3e7fdd561853c8 Mon Sep 17 00:00:00 2001 From: Thomas MacLean Date: Tue, 31 Oct 2023 13:00:06 -0400 Subject: [PATCH] Object lock feature implementation for Storage (#9363) * Early implementation of bucket object lock setting. * Complete implementation of object lock for buckets. * Compelete implementation, waiting for allowlist. * Tests pass against test environment. * Fixes bug in object test. * Update mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go Co-authored-by: Stephen Lewis (Burrows) * Update mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go Co-authored-by: Stephen Lewis (Burrows) * Update mmv1/third_party/terraform/services/storage/resource_storage_bucket_test.go.erb Co-authored-by: Stephen Lewis (Burrows) * Update mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go Co-authored-by: Stephen Lewis (Burrows) * Update mmv1/third_party/terraform/services/storage/resource_storage_bucket.go.erb Co-authored-by: Stephen Lewis (Burrows) * Updates from review comments * Update mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go Co-authored-by: Stephen Lewis (Burrows) * Update mmv1/third_party/terraform/website/docs/r/storage_bucket_object.html.markdown Co-authored-by: Stephen Lewis (Burrows) * Removing non-deterministic date from test * Removing unused variable * fixing nested_customer_encryption anchor link --------- Co-authored-by: Stephen Lewis (Burrows) --- .../storage/resource_storage_bucket.go.erb | 26 +++++- .../storage/resource_storage_bucket_object.go | 80 ++++++++++++++++++- .../resource_storage_bucket_object_test.go | 80 +++++++++++++++++++ .../resource_storage_bucket_test.go.erb | 74 ++++++++++++++--- .../docs/r/storage_bucket.html.markdown | 3 + .../r/storage_bucket_object.html.markdown | 12 ++- 6 files changed, 259 insertions(+), 16 deletions(-) diff --git a/mmv1/third_party/terraform/services/storage/resource_storage_bucket.go.erb b/mmv1/third_party/terraform/services/storage/resource_storage_bucket.go.erb index 63e53fc334e9..7703e96c17f2 100644 --- a/mmv1/third_party/terraform/services/storage/resource_storage_bucket.go.erb +++ b/mmv1/third_party/terraform/services/storage/resource_storage_bucket.go.erb @@ -256,6 +256,13 @@ func ResourceStorageBucket() *schema.Resource { Description: `The bucket's Lifecycle Rules configuration.`, }, + "enable_object_retention": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + Description: `Enables each object in the bucket to have its own retention policy, which prevents deletion until stored for a specific length of time.`, + }, + "versioning": { Type: schema.TypeList, Optional: true, @@ -594,7 +601,11 @@ func resourceStorageBucketCreate(d *schema.ResourceData, meta interface{}) error err = transport_tpg.Retry(transport_tpg.RetryOptions{ RetryFunc: func() error { - res, err = config.NewStorageClient(userAgent).Buckets.Insert(project, sb).Do() + insertCall := config.NewStorageClient(userAgent).Buckets.Insert(project, sb) + if d.Get("enable_object_retention").(bool) { + insertCall.EnableObjectRetention(true) + } + res, err = insertCall.Do() return err }, }) @@ -1121,6 +1132,16 @@ func flattenBucketRetentionPolicy(bucketRetentionPolicy *storage.BucketRetention return bucketRetentionPolicies } +func flattenBucketObjectRetention(bucketObjectRetention *storage.BucketObjectRetention) bool { + if bucketObjectRetention == nil { + return false + } + if bucketObjectRetention.Mode == "Enabled" { + return true + } + return false +} + func expandBucketVersioning(configured interface{}) *storage.BucketVersioning { versionings := configured.([]interface{}) if len(versionings) == 0 { @@ -1620,6 +1641,9 @@ func setStorageBucket(d *schema.ResourceData, config *transport_tpg.Config, res if err := d.Set("logging", flattenBucketLogging(res.Logging)); err != nil { return fmt.Errorf("Error setting logging: %s", err) } + if err := d.Set("enable_object_retention", flattenBucketObjectRetention(res.ObjectRetention)); err != nil { + return fmt.Errorf("Error setting object retention: %s", err) + } if err := d.Set("versioning", flattenBucketVersioning(res.Versioning)); err != nil { return fmt.Errorf("Error setting versioning: %s", err) } diff --git a/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go b/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go index 4ec802ea0e5e..926e16c160bd 100644 --- a/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go +++ b/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object.go @@ -207,10 +207,33 @@ func ResourceStorageBucketObject() *schema.Resource { }, }, + "retention": { + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + ConflictsWith: []string{"event_based_hold"}, + Description: `Object level retention configuration.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "retain_until_time": { + Type: schema.TypeString, + Required: true, + Description: `Time in RFC 3339 (e.g. 2030-01-01T02:03:04Z) until which object retention protects this object.`, + }, + "mode": { + Type: schema.TypeString, + Required: true, + Description: `The object retention mode. Supported values include: "Unlocked", "Locked".`, + }, + }, + }, + }, + "event_based_hold": { - Type: schema.TypeBool, - Optional: true, - Description: `Whether an object is under event-based hold. Event-based hold is a way to retain objects until an event occurs, which is signified by the hold's release (i.e. this value is set to false). After being released (set to false), such objects will be subject to bucket-level retention (if any).`, + Type: schema.TypeBool, + Optional: true, + ConflictsWith: []string{"retention"}, + Description: `Whether an object is under event-based hold. Event-based hold is a way to retain objects until an event occurs, which is signified by the hold's release (i.e. this value is set to false). After being released (set to false), such objects will be subject to bucket-level retention (if any).`, }, "temporary_hold": { @@ -312,6 +335,10 @@ func resourceStorageBucketObjectCreate(d *schema.ResourceData, meta interface{}) object.KmsKeyName = v.(string) } + if v, ok := d.GetOk("retention"); ok { + object.Retention = expandObjectRetention(v) + } + if v, ok := d.GetOk("event_based_hold"); ok { object.EventBasedHold = v.(bool) } @@ -357,6 +384,16 @@ func resourceStorageBucketObjectUpdate(d *schema.ResourceData, meta interface{}) return fmt.Errorf("Error retrieving object during update %s: %s", name, err) } + hasRetentionChanges := d.HasChange("retention") + if hasRetentionChanges { + if v, ok := d.GetOk("retention"); ok { + res.Retention = expandObjectRetention(v) + } else { + res.Retention = nil + res.NullFields = append(res.NullFields, "Retention") + } + } + if d.HasChange("event_based_hold") { v := d.Get("event_based_hold") res.EventBasedHold = v.(bool) @@ -368,6 +405,9 @@ func resourceStorageBucketObjectUpdate(d *schema.ResourceData, meta interface{}) } updateCall := objectsService.Update(bucket, name, res) + if hasRetentionChanges { + updateCall.OverrideUnlockedRetention(true) + } _, err = updateCall.Do() if err != nil { @@ -443,6 +483,9 @@ func resourceStorageBucketObjectRead(d *schema.ResourceData, meta interface{}) e if err := d.Set("media_link", res.MediaLink); err != nil { return fmt.Errorf("Error setting media_link: %s", err) } + if err := d.Set("retention", flattenObjectRetention(res.Retention)); err != nil { + return fmt.Errorf("Error setting retention: %s", err) + } if err := d.Set("event_based_hold", res.EventBasedHold); err != nil { return fmt.Errorf("Error setting event_based_hold: %s", err) } @@ -513,3 +556,34 @@ func expandCustomerEncryption(input []interface{}) map[string]string { } return expanded } + +func expandObjectRetention(configured interface{}) *storage.ObjectRetention { + retentions := configured.([]interface{}) + if len(retentions) == 0 { + return nil + } + retention := retentions[0].(map[string]interface{}) + + objectRetention := &storage.ObjectRetention{ + RetainUntilTime: retention["retain_until_time"].(string), + Mode: retention["mode"].(string), + } + + return objectRetention +} + +func flattenObjectRetention(objectRetention *storage.ObjectRetention) []map[string]interface{} { + retentions := make([]map[string]interface{}, 0, 1) + + if objectRetention == nil { + return retentions + } + + retention := map[string]interface{}{ + "mode": objectRetention.Mode, + "retain_until_time": objectRetention.RetainUntilTime, + } + + retentions = append(retentions, retention) + return retentions +} diff --git a/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object_test.go b/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object_test.go index cb6a59d0b8a3..54d0dbd78dfc 100644 --- a/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object_test.go +++ b/mmv1/third_party/terraform/services/storage/resource_storage_bucket_object_test.go @@ -415,6 +415,48 @@ func TestAccStorageObject_holds(t *testing.T) { }) } +func TestAccStorageObject_retention(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + data := []byte(content) + h := md5.New() + if _, err := h.Write(data); err != nil { + t.Errorf("error calculating md5: %v", err) + } + dataMd5 := base64.StdEncoding.EncodeToString(h.Sum(nil)) + testFile := getNewTmpTestFile(t, "tf-test") + if err := ioutil.WriteFile(testFile.Name(), data, 0644); err != nil { + t.Errorf("error writing file: %v", err) + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccStorageObjectDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testGoogleStorageBucketsObjectRetention(bucketName, "2040-01-01T02:03:04.000Z"), + Check: resource.ComposeTestCheckFunc( + testAccCheckGoogleStorageObject(t, bucketName, objectName, dataMd5), + ), + }, + { + Config: testGoogleStorageBucketsObjectRetention(bucketName, "2040-01-02T02:03:04.000Z"), + Check: resource.ComposeTestCheckFunc( + testAccCheckGoogleStorageObject(t, bucketName, objectName, dataMd5), + ), + }, + { + Config: testGoogleStorageBucketsObjectRetentionDisabled(bucketName), + Check: resource.ComposeTestCheckFunc( + testAccCheckGoogleStorageObject(t, bucketName, objectName, dataMd5), + ), + }, + }, + }) +} + func testAccCheckGoogleStorageObject(t *testing.T, bucket, object, md5 string) resource.TestCheckFunc { return testAccCheckGoogleStorageObjectWithEncryption(t, bucket, object, md5, "") } @@ -646,6 +688,44 @@ resource "google_storage_bucket_object" "object" { `, bucketName, objectName, content, customerEncryptionKey) } +func testGoogleStorageBucketsObjectRetention(bucketName string, retainUntilTime string) string { + return fmt.Sprintf(` +resource "google_storage_bucket" "bucket" { + name = "%s" + location = "US" + force_destroy = true + enable_object_retention = true +} + +resource "google_storage_bucket_object" "object" { + name = "%s" + bucket = google_storage_bucket.bucket.name + content = "%s" + retention { + mode = "Unlocked" + retain_until_time = "%s" + } +} +`, bucketName, objectName, content, retainUntilTime) +} + +func testGoogleStorageBucketsObjectRetentionDisabled(bucketName string) string { + return fmt.Sprintf(` +resource "google_storage_bucket" "bucket" { + name = "%s" + location = "US" + force_destroy = true + enable_object_retention = true +} + +resource "google_storage_bucket_object" "object" { + name = "%s" + bucket = google_storage_bucket.bucket.name + content = "%s" +} +`, bucketName, objectName, content) +} + func testGoogleStorageBucketsObjectHolds(bucketName string, eventBasedHold bool, temporaryHold bool) string { return fmt.Sprintf(` resource "google_storage_bucket" "bucket" { diff --git a/mmv1/third_party/terraform/services/storage/resource_storage_bucket_test.go.erb b/mmv1/third_party/terraform/services/storage/resource_storage_bucket_test.go.erb index f3bf7f4f8e25..9cbd43fe8aa7 100644 --- a/mmv1/third_party/terraform/services/storage/resource_storage_bucket_test.go.erb +++ b/mmv1/third_party/terraform/services/storage/resource_storage_bucket_test.go.erb @@ -691,6 +691,47 @@ func TestAccStorageBucket_forceDestroyObjectDeleteError(t *testing.T) { }) } +func TestAccStorageBucket_enable_object_retention(t *testing.T) { + t.Parallel() + + var bucket storage.Bucket + bucketName := acctest.TestBucketName(t) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccStorageBucketDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccStorageBucket_enable_object_retention(bucketName, "true"), + Check: resource.ComposeTestCheckFunc( + testAccCheckStorageBucketExists( + t, "google_storage_bucket.bucket", bucketName, &bucket), + ), + }, + { + ResourceName: "google_storage_bucket.bucket", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"force_destroy"}, + }, + { + Config: testAccStorageBucket_enable_object_retention(bucketName, "false"), + Check: resource.ComposeTestCheckFunc( + testAccCheckStorageBucketExists( + t, "google_storage_bucket.bucket", bucketName, &bucket), + ), + }, + { + ResourceName: "google_storage_bucket.bucket", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"force_destroy"}, + }, + }, + }) +} + func TestAccStorageBucket_versioning(t *testing.T) { t.Parallel() @@ -698,9 +739,9 @@ func TestAccStorageBucket_versioning(t *testing.T) { bucketName := fmt.Sprintf("tf-test-acl-bucket-%d", acctest.RandInt(t)) acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, + PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - CheckDestroy: testAccStorageBucketDestroyProducer(t), + CheckDestroy: testAccStorageBucketDestroyProducer(t), Steps: []resource.TestStep{ { Config: testAccStorageBucket_versioning(bucketName, "true"), @@ -719,7 +760,7 @@ func TestAccStorageBucket_versioning(t *testing.T) { ImportStateVerify: true, ImportStateVerifyIgnore: []string{"force_destroy"}, }, - { + { Config: testAccStorageBucket_versioning_empty(bucketName), Check: resource.ComposeTestCheckFunc( testAccCheckStorageBucketExists( @@ -736,7 +777,7 @@ func TestAccStorageBucket_versioning(t *testing.T) { ImportStateVerify: true, ImportStateVerifyIgnore: []string{"force_destroy"}, }, - { + { Config: testAccStorageBucket_versioning(bucketName, "false"), Check: resource.ComposeTestCheckFunc( testAccCheckStorageBucketExists( @@ -753,7 +794,7 @@ func TestAccStorageBucket_versioning(t *testing.T) { ImportStateVerify: true, ImportStateVerifyIgnore: []string{"force_destroy"}, }, - { + { Config: testAccStorageBucket_versioning_empty(bucketName), Check: resource.ComposeTestCheckFunc( testAccCheckStorageBucketExists( @@ -779,9 +820,9 @@ func TestAccStorageBucket_logging(t *testing.T) { bucketName := fmt.Sprintf("tf-test-acl-bucket-%d", acctest.RandInt(t)) acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, + PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - CheckDestroy: testAccStorageBucketDestroyProducer(t), + CheckDestroy: testAccStorageBucketDestroyProducer(t), Steps: []resource.TestStep{ { Config: testAccStorageBucket_logging(bucketName, "log-bucket"), @@ -840,9 +881,9 @@ func TestAccStorageBucket_cors(t *testing.T) { bucketName := fmt.Sprintf("tf-test-acl-bucket-%d", acctest.RandInt(t)) acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, + PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - CheckDestroy: testAccStorageBucketDestroyProducer(t), + CheckDestroy: testAccStorageBucketDestroyProducer(t), Steps: []resource.TestStep{ { Config: testGoogleStorageBucketsCors(bucketName), @@ -872,9 +913,9 @@ func TestAccStorageBucket_defaultEventBasedHold(t *testing.T) { bucketName := fmt.Sprintf("tf-test-acl-bucket-%d", acctest.RandInt(t)) acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, + PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - CheckDestroy: testAccStorageBucketDestroyProducer(t), + CheckDestroy: testAccStorageBucketDestroyProducer(t), Steps: []resource.TestStep{ { Config: testAccStorageBucket_defaultEventBasedHold(bucketName), @@ -1495,6 +1536,17 @@ resource "google_storage_bucket" "bucket" { `, bucketName) } +func testAccStorageBucket_enable_object_retention(bucketName string, enabled string) string { + return fmt.Sprintf(` +resource "google_storage_bucket" "bucket" { + name = "%s" + location = "US" + force_destroy = "true" + enable_object_retention = "%s" +} +`, bucketName, enabled) +} + func testAccStorageBucket_versioning(bucketName, enabled string) string { return fmt.Sprintf(` resource "google_storage_bucket" "bucket" { diff --git a/mmv1/third_party/terraform/website/docs/r/storage_bucket.html.markdown b/mmv1/third_party/terraform/website/docs/r/storage_bucket.html.markdown index 005f1d76328a..0c2123fa25bf 100644 --- a/mmv1/third_party/terraform/website/docs/r/storage_bucket.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/storage_bucket.html.markdown @@ -120,6 +120,9 @@ The following arguments are supported: * `encryption` - (Optional) The bucket's encryption configuration. Structure is [documented below](#nested_encryption). +* `enable_object_retention` - (Optional, Default: false) Enables [object retention](https://cloud.google.com/storage/docs/object-lock) on a storage bucket. + + * `requester_pays` - (Optional, Default: false) Enables [Requester Pays](https://cloud.google.com/storage/docs/requester-pays) on a storage bucket. * `uniform_bucket_level_access` - (Optional, Default: false) Enables [Uniform bucket-level access](https://cloud.google.com/storage/docs/uniform-bucket-level-access) access to a bucket. diff --git a/mmv1/third_party/terraform/website/docs/r/storage_bucket_object.html.markdown b/mmv1/third_party/terraform/website/docs/r/storage_bucket_object.html.markdown index 57995a5d90b1..4832d1bdcf12 100644 --- a/mmv1/third_party/terraform/website/docs/r/storage_bucket_object.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/storage_bucket_object.html.markdown @@ -67,7 +67,9 @@ One of the following is required: * `content_type` - (Optional) [Content-Type](https://tools.ietf.org/html/rfc7231#section-3.1.1.5) of the object data. Defaults to "application/octet-stream" or "text/plain; charset=utf-8". * `customer_encryption` - (Optional) Enables object encryption with Customer-Supplied Encryption Key (CSEK). [Google [documentation about](#nested_customer_encryption) CSEK.](https://cloud.google.com/storage/docs/encryption/customer-supplied-keys) - Structure is documented below. + Structure is [documented below](#nested_customer_encryption). + +* `retention` - (Optional) The [object retention](http://cloud.google.com/storage/docs/object-lock) settings for the object. The retention settings allow an object to be retained until a provided date. Structure is [documented below](#nested_retention). * `event_based_hold` - (Optional) Whether an object is under [event-based hold](https://cloud.google.com/storage/docs/object-holds#hold-types). Event-based hold is a way to retain objects until an event occurs, which is signified by the hold's release (i.e. this value is set to false). After being released (set to false), such objects will be subject to bucket-level retention (if any). @@ -89,6 +91,14 @@ One of the following is required: * `encryption_key` - (Required) Base64 encoded Customer-Supplied Encryption Key. +The `retention` block supports: + +* `mode` - (Required) The retention policy mode. Either `Locked` or `Unlocked`. + +* `retain_until_time` - (Required) The time to retain the object until in RFC 3339 format, for example 2012-11-15T16:19:00.094Z. + + + ## Attributes Reference In addition to the arguments listed above, the following computed attributes are