diff --git a/mmv1/products/storage/Folder.yaml b/mmv1/products/storage/Folder.yaml new file mode 100644 index 000000000000..96627f0747bb --- /dev/null +++ b/mmv1/products/storage/Folder.yaml @@ -0,0 +1,88 @@ +# Copyright 2024 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: 'Folder' +kind: 'storage#folder' +base_url: 'b/{{bucket}}/folders' +self_link: 'b/{{bucket}}/folders/{{%name}}' +id_format: '{{bucket}}/{{name}}' +delete_url: 'b/{{bucket}}/folders/{{%name}}' +create_url: 'b/{{bucket}}/folders' +has_self_link: true +timeouts: + insert_minutes: 20 + update_minutes: 20 + delete_minutes: 20 +exclude_sweeper: true +import_format: + - '{{bucket}}/folders/{{%name}}' + - '{{bucket}}/{{%name}}' +examples: + - name: 'storage_folder_basic' + primary_resource_id: 'folder' + vars: + bucket_name: 'my-bucket' + ignore_read_extra: + - 'force_destroy' +description: | + A Google Cloud Storage Folder. + + The Folder resource represents a folder in a Cloud Storage bucket with hierarchical namespace enabled +references: + guides: + 'Official Documentation': 'https://cloud.google.com/storage/docs/folders-overview' + api: 'https://cloud.google.com/storage/docs/json_api/v1/folders' +custom_code: + custom_import: templates/terraform/custom_import/storage_folder.go.tmpl + custom_update: templates/terraform/custom_update/storage_folder_update.go.tmpl + custom_delete: templates/terraform/custom_delete/storage_folder_delete.go.tmpl +virtual_fields: + - name: 'force_destroy' + description: + If set to true, folder will be force destroyed. + type: Boolean + default_value: false +parameters: + - name: 'bucket' + resource: 'Bucket' + imports: 'name' + description: 'The name of the bucket that contains the folder.' + required: true + immutable: true + url_param_only: true + - name: 'name' + description: | + The name of the folder expressed as a path. Must include + trailing '/'. For example, `example_dir/example_dir2/`. + required: true + # The API returns values with trailing slashes, even if not + # provided. Enforcing trailing slashes prevents diffs and ensures + # consistent output. + validation: + regex: '/$' +properties: + - name: createTime + type: String + description: | + The timestamp at which this folder was created. + output: true + - name: updateTime + type: String + description: | + The timestamp at which this folder was most recently updated. + output: true + - name: metageneration + type: String + description: | + The metadata generation of the folder. + output: true diff --git a/mmv1/templates/terraform/custom_delete/storage_folder_delete.go.tmpl b/mmv1/templates/terraform/custom_delete/storage_folder_delete.go.tmpl new file mode 100644 index 000000000000..d3607ca82091 --- /dev/null +++ b/mmv1/templates/terraform/custom_delete/storage_folder_delete.go.tmpl @@ -0,0 +1,88 @@ +bucket := d.Get("bucket").(string) +name := d.Get("name").(string) + +var listError, deleteObjectError error +for deleteObjectError == nil { + res, err := config.NewStorageClient(userAgent).Objects.List(bucket).Prefix(name).Do() + if err != nil { + log.Printf("Error listing contents of bucket %s: %v", bucket, err) + listError = err + break + } + + if len(res.Items) == 0 { + break // 0 items, folder empty + } + + if !d.Get("force_destroy").(bool) { + deleteErr := fmt.Errorf("Error trying to delete folder %s containing objects without force_destroy set to true", bucket) + log.Printf("Error! %s : %s\n\n", bucket, deleteErr) + return deleteErr + } + // GCS requires that a folder be empty (have no objects or object + // versions) before it can be deleted. + log.Printf("[DEBUG] GCS Folder attempting to forceDestroy\n\n") + + // Create a workerpool for parallel deletion of resources. In the + // future, it would be great to expose Terraform's global parallelism + // flag here, but that's currently reserved for core use. Testing + // shows that NumCPUs-1 is the most performant on average networks. + // + // The challenge with making this user-configurable is that the + // configuration would reside in the Terraform configuration file, + // decreasing its portability. Ideally we'd want this to connect to + // Terraform's top-level -parallelism flag, but that's not plumbed nor + // is it scheduled to be plumbed to individual providers. + wp := workerpool.New(runtime.NumCPU() - 1) + + for _, object := range res.Items { + log.Printf("[DEBUG] Found %s", object.Name) + object := object + + wp.Submit(func() { + log.Printf("[TRACE] Attempting to delete %s", object.Name) + if err := config.NewStorageClient(userAgent).Objects.Delete(bucket, object.Name).Generation(object.Generation).Do(); err != nil { + deleteObjectError = err + log.Printf("[ERR] Failed to delete storage object %s: %s", object.Name, err) + } else { + log.Printf("[TRACE] Successfully deleted %s", object.Name) + } + }) + } + + // Wait for everything to finish. + wp.StopWait() +} + +if listError != nil { + return fmt.Errorf("could not delete non-empty folder due to error when listing contents: %v", listError) +} +log.Printf("[DEBUG] force_destroy value: %#v", d.Get("force_destroy").(bool)) +foldersList, err := config.NewStorageClient(userAgent).Folders.List(bucket).Prefix(name).Do() +if err != nil { + return err +} +if len(foldersList.Items) == 1 || d.Get("force_destroy").(bool) { + log.Printf("[DEBUG] folder names to delete: %#v", name) + items := foldersList.Items + for i := len(items) - 1; i >= 0; i-- { + err = transport_tpg.Retry(transport_tpg.RetryOptions{ + RetryFunc: func() error { + err = config.NewStorageClient(userAgent).Folders.Delete(bucket, items[i].Name).Do() + return err + }, + Timeout: d.Timeout(schema.TimeoutDelete), + ErrorRetryPredicates: []transport_tpg.RetryErrorPredicateFunc{transport_tpg.Is429RetryableQuotaError}, + }) + if err != nil { + return err + } + } + + log.Printf("[DEBUG] Finished deleting Folder %q: %#v", d.Id(), name) +} else { + deleteErr := fmt.Errorf("Error trying to delete folder without force_destroy set to true") + log.Printf("Error! %s : %s\n\n", name, deleteErr) + return deleteErr +} +return nil \ No newline at end of file diff --git a/mmv1/templates/terraform/custom_import/storage_folder.go.tmpl b/mmv1/templates/terraform/custom_import/storage_folder.go.tmpl new file mode 100644 index 000000000000..3117bf3eb879 --- /dev/null +++ b/mmv1/templates/terraform/custom_import/storage_folder.go.tmpl @@ -0,0 +1,19 @@ +config := meta.(*transport_tpg.Config) +if err := tpgresource.ParseImportId([]string{ + "^(?P[^/]+)/folders/(?P.+)$", + "^(?P[^/]+)/(?P.+)$", +}, d, config); err != nil { + return nil, err +} + +// Replace import id for the resource id +id, err := tpgresource.ReplaceVars(d, config, "{{"{{"}}bucket{{"}}"}}/{{"{{"}}name{{"}}"}}") +if err != nil { + return nil, fmt.Errorf("Error constructing id: %s", err) +} +d.SetId(id) +if err := d.Set("force_destroy", false); err != nil { + return nil, fmt.Errorf("Error setting force_destroy: %s", err) +} + +return []*schema.ResourceData{d}, nil \ No newline at end of file diff --git a/mmv1/templates/terraform/custom_update/storage_folder_update.go.tmpl b/mmv1/templates/terraform/custom_update/storage_folder_update.go.tmpl new file mode 100644 index 000000000000..0e3beee0aaf3 --- /dev/null +++ b/mmv1/templates/terraform/custom_update/storage_folder_update.go.tmpl @@ -0,0 +1,10 @@ +_ = config +// we can only get here if force_destroy was updated +if d.Get("force_destroy") != nil { + if err := d.Set("force_destroy", d.Get("force_destroy")); err != nil { + return fmt.Errorf("Error updating force_destroy: %s", err) + } +} + +// all other fields are immutable, don't do anything else +return nil \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/storage_folder_basic.tf.tmpl b/mmv1/templates/terraform/examples/storage_folder_basic.tf.tmpl new file mode 100644 index 000000000000..48873c753532 --- /dev/null +++ b/mmv1/templates/terraform/examples/storage_folder_basic.tf.tmpl @@ -0,0 +1,15 @@ +resource "google_storage_bucket" "bucket" { + name = "{{index $.Vars "bucket_name"}}" + location = "EU" + uniform_bucket_level_access = true + hierarchical_namespace { + enabled = true + } + force_destroy = true +} + +resource "google_storage_folder" "{{$.PrimaryResourceId}}" { + bucket = google_storage_bucket.bucket.name + name = "parent-folder/" + force_destroy = true +} diff --git a/mmv1/third_party/terraform/services/storage/resource_storage_folder_test.go b/mmv1/third_party/terraform/services/storage/resource_storage_folder_test.go new file mode 100644 index 000000000000..d275806f510d --- /dev/null +++ b/mmv1/third_party/terraform/services/storage/resource_storage_folder_test.go @@ -0,0 +1,238 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 +package storage_test + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "regexp" + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/terraform" + "google.golang.org/api/storage/v1" + + "github.com/hashicorp/terraform-provider-google/google/acctest" +) + +func TestAccStorageFolder_storageFolderBasic(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + Steps: []resource.TestStep{ + { + Config: testAccStorageFolder_storageBucket(bucketName, true, true) + testAccStorageFolder_storageFolder(true), + }, + { + ResourceName: "google_storage_folder.folder", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"bucket", "recursive", "force_destroy"}, + }, + }, + }) +} + +func TestAccStorageFolder_hnsDisabled(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + Steps: []resource.TestStep{ + { + Config: testAccStorageFolder_storageBucket(bucketName, false, true) + testAccStorageFolder_storageFolder(true), + ExpectError: regexp.MustCompile("Error creating Folder: googleapi: Error 409: The bucket does not support hierarchical namespace., conflict"), + }, + }, + }) +} + +func TestAccStorageFolder_FolderForceDestroy(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + + data := []byte("data data data") + + testFile := getNewTmpTestFile(t, "tf-test") + if err := ioutil.WriteFile(testFile.Name(), data, 0644); err != nil { + t.Errorf("error writing file: %v", err) + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + Steps: []resource.TestStep{ + { + Config: testAccStorageFolder_storageBucketObject(bucketName, true, true, testFile.Name()), + Check: resource.ComposeTestCheckFunc( + testAccCheckStorageBucketUploadItem(t, bucketName), + ), + }, + }, + }) +} + +func TestAccStorageFolder_DeleteEmptyFolderWithForceDestroyDefault(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + Steps: []resource.TestStep{ + { + Config: testAccStorageFolder_storageBucket(bucketName, true, true) + testAccStorageFolder_storageOneFolder(false), + }, + }, + }) +} + +func TestAccStorageFolder_FailDeleteNonEmptyFolder(t *testing.T) { + t.Parallel() + + bucketName := acctest.TestBucketName(t) + folderName := "folder/" + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + Steps: []resource.TestStep{ + { + Config: testAccStorageFolder_storageBucket(bucketName, true, true) + testAccStorageFolder_storageOneFolder(false), + Check: resource.ComposeTestCheckFunc( + testAccStorageCreatSubFolder(t, bucketName, folderName), + testAccStorageDeleteFolder(t, bucketName, folderName), + ), + ExpectError: regexp.MustCompile("googleapi: Error 409: The folder you tried to delete is not empty"), + }, + { + Config: testAccStorageFolder_storageBucket(bucketName, true, true) + testAccStorageFolder_storageOneFolder(true), + }, + }, + }) +} + +func testAccStorageFolder_storageBucket(bucketName string, hnsFlag bool, forceDestroy bool) string { + return fmt.Sprintf(` +resource "google_storage_bucket" "bucket" { + name = "%s" + location = "EU" + uniform_bucket_level_access = true + hierarchical_namespace { + enabled = %t + } + force_destroy = %t +} +`, bucketName, hnsFlag, forceDestroy) +} + +func testAccStorageFolder_storageBucketObject(bucketName string, hnsFlag bool, forceDestroy bool, fileName string) string { + return fmt.Sprintf(` +resource "google_storage_bucket" "bucket" { + name = "%s" + location = "EU" + uniform_bucket_level_access = true + hierarchical_namespace { + enabled = %t + } + force_destroy = true +} +resource "google_storage_folder" "folder" { + bucket = google_storage_bucket.bucket.name + name = "folder/" + force_destroy = %t +} +resource "google_storage_folder" "subfolder" { + bucket = google_storage_bucket.bucket.name + name = "${google_storage_folder.folder.name}subfolder/" + force_destroy = %t +} +resource "google_storage_bucket_object" "object" { + name = "${google_storage_folder.subfolder.name}tffile" + bucket = google_storage_bucket.bucket.name + source = "%s" +} +`, bucketName, hnsFlag, forceDestroy, forceDestroy, fileName) +} + +func testAccStorageFolder_storageFolder(forceDestroy bool) string { + return fmt.Sprintf(` +resource "google_storage_folder" "folder" { + bucket = google_storage_bucket.bucket.name + name = "folder/" + force_destroy = %t +} +resource "google_storage_folder" "subfolder" { + bucket = google_storage_bucket.bucket.name + name = "${google_storage_folder.folder.name}name/" + force_destroy = %t +} +`, forceDestroy, forceDestroy) +} + +func testAccStorageFolder_storageOneFolder(forceDestroy bool) string { + return fmt.Sprintf(` +resource "google_storage_folder" "folder" { + bucket = google_storage_bucket.bucket.name + name = "folder/" + force_destroy = %t +} +`, forceDestroy) +} + +func testAccStorageCreatSubFolder(t *testing.T, bucketName, parentFolder string) resource.TestCheckFunc { + return func(s *terraform.State) error { + config := acctest.GoogleProviderConfig(t) + subFolder := &storage.Folder{ + Name: parentFolder + "subfolder/", + } + if res, err := config.NewStorageClient(config.UserAgent).Folders.Insert(bucketName, subFolder).Do(); err == nil { + log.Printf("sub folder created: %s", res.Name) + } else { + log.Printf("failed to create sub folder: %s", subFolder.Name) + } + return nil + } +} + +func testAccStorageDeleteFolder(t *testing.T, bucketName, parentFolder string) resource.TestCheckFunc { + return func(s *terraform.State) error { + config := acctest.GoogleProviderConfig(t) + var deleteError error + if err := config.NewStorageClient(config.UserAgent).Folders.Delete(bucketName, parentFolder).Do(); err == nil { + log.Printf("successfully deleted folder: %s", err) + } else { + deleteError = fmt.Errorf("failed to deleted folder: %s", err) + } + return deleteError + } +} + +func testAccCheckStorageBucketUploadItem(t *testing.T, bucketName string) resource.TestCheckFunc { + return func(s *terraform.State) error { + config := acctest.GoogleProviderConfig(t) + + data := bytes.NewBufferString("test") + dataReader := bytes.NewReader(data.Bytes()) + object := &storage.Object{Name: "folder/" + "bucketDestroyTestFile"} + + if res, err := config.NewStorageClient(config.UserAgent).Objects.Insert(bucketName, object).Media(dataReader).Do(); err == nil { + log.Printf("[INFO] Created object %v at location %v\n\n", res.Name, res.SelfLink) + } else { + return fmt.Errorf("Objects.Insert failed: %v", err) + } + + return nil + } +}