From 07c60c86e7620da377759d6287bc67a9073ef189 Mon Sep 17 00:00:00 2001 From: Salome Papiashvili Date: Mon, 6 Nov 2023 22:10:05 +0100 Subject: [PATCH] Storage config (#9151) Co-authored-by: Riley Karson --- .../resource_composer_environment.go.erb | 61 +++++++++++++- .../resource_composer_environment_test.go.erb | 79 +++++++++++++++++++ .../docs/r/composer_environment.html.markdown | 14 +++- 3 files changed, 149 insertions(+), 5 deletions(-) diff --git a/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb b/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb index 01f803757b46..5f8a44f1842d 100644 --- a/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb +++ b/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb @@ -73,7 +73,7 @@ var ( "config.0.workloads_config", "config.0.environment_size", "config.0.master_authorized_networks_config", - "config.0.resilience_mode", + "config.0.resilience_mode", } recoveryConfigKeys = []string{ @@ -878,7 +878,7 @@ func ResourceComposerEnvironment() *schema.Resource { Optional: true, Elem: &schema.Schema{Type: schema.TypeString}, Description: `User-defined labels for this environment. The labels map can contain no more than 64 entries. Entries of the labels map are UTF8 strings that comply with the following restrictions: Label keys must be between 1 and 63 characters long and must conform to the following regular expression: [a-z]([-a-z0-9]*[a-z0-9])?. Label values must be between 0 and 63 characters long and must conform to the regular expression ([a-z]([-a-z0-9]*[a-z0-9])?)?. No more than 64 labels can be associated with a given environment. Both keys and values must be <= 128 bytes in size. - + **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field 'effective_labels' for all of the labels present on the resource.`, }, @@ -896,6 +896,24 @@ func ResourceComposerEnvironment() *schema.Resource { Description: `All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Terraform, other clients and services.`, Elem: &schema.Schema{Type: schema.TypeString}, }, + + "storage_config": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Description: `Configuration options for storage used by Composer environment.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "bucket": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: `Optional. Name of an existing Cloud Storage bucket to be used by the environment.`, + }, + }, + }, + }, }, UseJSONNumber: true, } @@ -918,10 +936,16 @@ func resourceComposerEnvironmentCreate(d *schema.ResourceData, meta interface{}) return err } + transformedStorageConfig, err := expandComposerStorageConfig(d.Get("storage_config"), d, config) + if err != nil { + return err + } + env := &composer.Environment{ Name: envName.ResourceName(), Labels: tpgresource.ExpandEffectiveLabels(d), Config: transformedConfig, + StorageConfig: transformedStorageConfig, } // Some fields cannot be specified during create and must be updated post-creation. @@ -1007,6 +1031,9 @@ func resourceComposerEnvironmentRead(d *schema.ResourceData, meta interface{}) e if err := d.Set("effective_labels", res.Labels); err != nil { return fmt.Errorf("Error setting Environment effective_labels: %s", err) } + if err := d.Set("storage_config", flattenComposerStorageConfig(res.StorageConfig)); err != nil { + return fmt.Errorf("Error setting Storage: %s", err) + } return nil } @@ -1350,6 +1377,17 @@ func resourceComposerEnvironmentImport(d *schema.ResourceData, meta interface{}) return []*schema.ResourceData{d}, nil } +func flattenComposerStorageConfig(storageConfig *composer.StorageConfig) interface{} { + if storageConfig == nil { + return nil + } + + transformed := make(map[string]interface{}) + transformed["bucket"] = storageConfig.Bucket + + return []interface{}{transformed} +} + func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interface{} { if envCfg == nil { return nil @@ -1734,9 +1772,9 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf if transformedResilienceMode == "STANDARD_RESILIENCE" { transformed.ResilienceMode = "RESILIENCE_MODE_UNSPECIFIED" } else { - transformed.ResilienceMode = transformedResilienceMode + transformed.ResilienceMode = transformedResilienceMode } - + transformedMasterAuthorizedNetworksConfig, err := expandComposerEnvironmentConfigMasterAuthorizedNetworksConfig(original["master_authorized_networks_config"], d, config) if err != nil { @@ -2143,7 +2181,22 @@ func expandComposerEnvironmentIPAllocationPolicy(v interface{}, d *schema.Resour transformed.ServicesSecondaryRangeName = v.(string) } return transformed, nil +} +func expandComposerStorageConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (*composer.StorageConfig, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := &composer.StorageConfig{} + + if v, ok := original["bucket"]; ok { + transformed.Bucket = v.(string) + } + + return transformed, nil } func expandComposerEnvironmentServiceAccount(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (string, error) { diff --git a/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb b/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb index 83725a9bb8f7..5df49c724ac3 100644 --- a/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb +++ b/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb @@ -20,6 +20,7 @@ import ( const testComposerEnvironmentPrefix = "tf-test-composer-env" const testComposerNetworkPrefix = "tf-test-composer-net" +const testComposerBucketPrefix = "tf-test-composer-bucket" func allComposerServiceAgents() []string { return []string{ @@ -1081,6 +1082,84 @@ func testAccComposerEnvironmentDestroyProducer(t *testing.T) func(s *terraform.S } } +// Checks environment creation with custom bucket +func TestAccComposerEnvironment_customBucket(t *testing.T) { + t.Parallel() + + bucketName := fmt.Sprintf("%s-%d", testComposerBucketPrefix, acctest.RandInt(t)) + envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, acctest.RandInt(t)) + network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, acctest.RandInt(t)) + subnetwork := network + "-1" + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccComposerEnvironmentDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComposerEnvironment_customBucket(bucketName, envName, network, subnetwork), + }, + { + ResourceName: "google_composer_environment.test", + ImportState: true, + ImportStateVerify: true, + }, + // This is a terrible clean-up step in order to get destroy to succeed, + // due to dangling firewall rules left by the Composer Environment blocking network deletion. + // TODO: Remove this check if firewall rules bug gets fixed by Composer. + { + PlanOnly: true, + ExpectNonEmptyPlan: false, + Config: testAccComposerEnvironment_customBucket(bucketName, envName, network, subnetwork), + Check: testAccCheckClearComposerEnvironmentFirewalls(t, network), + }, + }, + }) +} + +func testAccComposerEnvironment_customBucket(bucketName, envName, network, subnetwork string) string { + return fmt.Sprintf(` +resource "google_storage_bucket" "test" { + name = "%s" + location = "us-central1" + force_destroy = true +} + +resource "google_composer_environment" "test" { + name = "%s" + region = "us-central1" + config { + node_config { + network = google_compute_network.test.self_link + subnetwork = google_compute_subnetwork.test.self_link + ip_allocation_policy { + cluster_ipv4_cidr_block = "10.0.0.0/16" + } + } + software_config { + image_version = "composer-2.4.2-airflow-2" + } + } + storage_config { + bucket = google_storage_bucket.test.name + } +} + +// use a separate network to avoid conflicts with other tests running in parallel +// that use the default network/subnet +resource "google_compute_network" "test" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "test" { + name = "%s" + ip_cidr_range = "10.2.0.0/16" + region = "us-central1" + network = google_compute_network.test.self_link +} +`, bucketName, envName, network, subnetwork) +} + func testAccComposerEnvironment_basic(name, network, subnetwork string) string { return fmt.Sprintf(` resource "google_composer_environment" "test" { diff --git a/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown b/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown index 3ddb514dd449..907ec29f293e 100644 --- a/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/composer_environment.html.markdown @@ -622,7 +622,7 @@ The `web_server_network_access_control` supports: * `cidr_block` - (Required) - `cidr_block< must be specified in CIDR notation. + `cidr_block` must be specified in CIDR notation. ## Argument Reference - Cloud Composer 2 @@ -656,6 +656,11 @@ The following arguments are supported: (Optional) The ID of the project in which the resource belongs. If it is not provided, the provider project is used. +* `storage_config` - + (Optional) + Configuration options for storage used by Composer environment. Structure is documented below. + + The `config` block supports: * `node_config` - @@ -708,6 +713,13 @@ The `config` block supports: Google Compute Engine Public IPs and Google Prod IPs. Structure is documented below. +The `storage_config` block supports: + +* `bucket` - + (Required) + Name of an existing Cloud Storage bucket to be used by the environment. + + The `node_config` block supports: * `network` -