From 2ba9e22aa0cd744800bd97beac0ec9f033a26731 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Wed, 21 Feb 2024 21:24:13 +0000 Subject: [PATCH] Add volume replication support for Google Cloud NetApp Volumes (#9816) * Initial replication commit * Cleanup work - Renamed a lot of files to make clear which resource the belong to - Updated documentation for resource fields - Renamed a few resource fields and changed some types - Disabled the custom code for now. Needs to be discussed first * Update example file * Updated example file * Major updates - Reorganisation of block - Reorganisation of fields to match API documentation - Updated example parameters - Added missing API fields - Improved descriptions - * For replication deletion, stop replication first * Add support for deleting destination volume on replication delete * Make volumes deletable in presence of snapshots. This change will be PRed for volume resource independently. Adding it here while it is not in main. * Improving debug error message * yaml check and format fix * Add wait for mirror to initialize. Required to run destroy shortly after create. * Wait on destroy, not on create * Make deleting a replication more robust - doc improvements - started to implement stop/resume. More work required. - renamed a few files to better reflect what they are good for * adding support for stop/resume * yamlformat and lint * Add force delete to delete volumes with nested snapshots * resource test first version * More changes to make tests solid - Introduced new parameter to wait for mirror_status==MIRRORED - more mirror state reconciliation * Test updates * few cleanups * Make virtual field verifies happy * Minor test improvements * More fine tuning - Remove merge conflict in volume.yaml - make generated test work - make output field work - ignore_read for virtual fields * Resource name change as suggested by @slevenick * Remove snapshot code block and fix typo * Detect manual stop/resume actions * Remove ignore_read for deletion_policy * - Made destinationVolumeParameters immutable. It still requires ignore_read. - removed ignore_read from virtual_fields * destinationVolumeParameters are only evaluated at create. Make the immutable. * Name cleanups and comment improvements * removed comment Co-authored-by: Shuya Ma <87669292+shuyama1@users.noreply.github.com> * tabs to spaces in resource block Co-authored-by: Shuya Ma <87669292+shuyama1@users.noreply.github.com> * Updates to address review comments - make wait_for_mirror also work for stop/resume, additionally to create - convert tabs in test resource blocks to spaces - fix typos * Rewording of comments Co-authored-by: Shuya Ma <87669292+shuyama1@users.noreply.github.com> --------- Co-authored-by: G-NamanGupta Co-authored-by: Shuya Ma <87669292+shuyama1@users.noreply.github.com> [upstream:37fb2ebf10afb245649e86f17bc344bf7b10ef8b] Signed-off-by: Modular Magician --- go.mod | 2 +- go.sum | 4 +- .../netapp/netapp_volume_replication.go | 215 ++++++++++++++++++ 3 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 tfplan2cai/converters/google/resources/services/netapp/netapp_volume_replication.go diff --git a/go.mod b/go.mod index 398ea7424..6ffcec637 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/hcl/v2 v2.19.1 github.com/hashicorp/terraform-json v0.18.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.31.0 - github.com/hashicorp/terraform-provider-google-beta v1.20.1-0.20240221190706-d4c89b3aba13 + github.com/hashicorp/terraform-provider-google-beta v1.20.1-0.20240221192239-22da542ba88e github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 410c1cac3..c45e8cb7e 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,8 @@ github.com/hashicorp/terraform-plugin-log v0.9.0/go.mod h1:rKL8egZQ/eXSyDqzLUuwU github.com/hashicorp/terraform-plugin-mux v0.13.0 h1:79U401/3nd8CWwDGtTHc8F3miSCAS9XGtVarxSTDgwA= github.com/hashicorp/terraform-plugin-sdk/v2 v2.31.0 h1:Bl3e2ei2j/Z3Hc2HIS15Gal2KMKyLAZ2om1HCEvK6es= github.com/hashicorp/terraform-plugin-sdk/v2 v2.31.0/go.mod h1:i2C41tszDjiWfziPQDL5R/f3Zp0gahXe5No/MIO9rCE= -github.com/hashicorp/terraform-provider-google-beta v1.20.1-0.20240221190706-d4c89b3aba13 h1:ZidAcWK/N2f0DqDydKex+/VpMFMxFsv91POuYZ5hVp4= -github.com/hashicorp/terraform-provider-google-beta v1.20.1-0.20240221190706-d4c89b3aba13/go.mod h1:iQJGQHx40qba9wE8pOGbItSL5G+XRunVeL6RFRJTNcE= +github.com/hashicorp/terraform-provider-google-beta v1.20.1-0.20240221192239-22da542ba88e h1:ipgLIQsEdGqzcUrBq6m2d+j94qnnTlVivCNwAsoOdW0= +github.com/hashicorp/terraform-provider-google-beta v1.20.1-0.20240221192239-22da542ba88e/go.mod h1:iQJGQHx40qba9wE8pOGbItSL5G+XRunVeL6RFRJTNcE= github.com/hashicorp/terraform-registry-address v0.2.3 h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI= github.com/hashicorp/terraform-registry-address v0.2.3/go.mod h1:lFHA76T8jfQteVfT7caREqguFrW3c4MFSPhZB7HHgUM= github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= diff --git a/tfplan2cai/converters/google/resources/services/netapp/netapp_volume_replication.go b/tfplan2cai/converters/google/resources/services/netapp/netapp_volume_replication.go new file mode 100644 index 000000000..a4c71f18f --- /dev/null +++ b/tfplan2cai/converters/google/resources/services/netapp/netapp_volume_replication.go @@ -0,0 +1,215 @@ +// ---------------------------------------------------------------------------- +// +// *** AUTO GENERATED CODE *** Type: MMv1 *** +// +// ---------------------------------------------------------------------------- +// +// This file is automatically generated by Magic Modules and manual +// changes will be clobbered when the file is regenerated. +// +// Please read more about how to change this file in +// .github/CONTRIBUTING.md. +// +// ---------------------------------------------------------------------------- + +package netapp + +import ( + "fmt" + "log" + "reflect" + "time" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + + "github.com/GoogleCloudPlatform/terraform-google-conversion/v5/tfplan2cai/converters/google/resources/cai" + "github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgresource" + transport_tpg "github.com/hashicorp/terraform-provider-google-beta/google-beta/transport" +) + +// Custom function to wait for mirrorState target states +func NetAppVolumeReplicationWaitForMirror(d *schema.ResourceData, meta interface{}, targetState string) error { + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + url, err := tpgresource.ReplaceVars(d, config, "{{NetappBasePath}}projects/{{project}}/locations/{{location}}/volumes/{{volume_name}}/replications/{{name}}") + if err != nil { + return err + } + + billingProject := "" + + project, err := tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for volume replication: %s", err) + } + billingProject = project + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + for { + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "GET", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, + }) + if err != nil { + return transport_tpg.HandleNotFoundError(err, d, fmt.Sprintf("NetappVolumeReplication %q", d.Id())) + } + + log.Printf("[DEBUG] waiting for mirrorState. actual: %v, target: %v", res["mirrorState"], targetState) + + if res["mirrorState"] == targetState { + break + } + + time.Sleep(30 * time.Second) + // This method can potentially run for days, e.g. when setting up a replication for a source volume + // with dozens of TiB of data. Timeout handling yes/no? + } + + return nil +} + +const NetappVolumeReplicationAssetType string = "netapp.googleapis.com/VolumeReplication" + +func ResourceConverterNetappVolumeReplication() cai.ResourceConverter { + return cai.ResourceConverter{ + AssetType: NetappVolumeReplicationAssetType, + Convert: GetNetappVolumeReplicationCaiObject, + } +} + +func GetNetappVolumeReplicationCaiObject(d tpgresource.TerraformResourceData, config *transport_tpg.Config) ([]cai.Asset, error) { + name, err := cai.AssetName(d, config, "//netapp.googleapis.com/projects/{{project}}/locations/{{location}}/volumes/{{volume_name}}/replications/{{name}}") + if err != nil { + return []cai.Asset{}, err + } + if obj, err := GetNetappVolumeReplicationApiObject(d, config); err == nil { + return []cai.Asset{{ + Name: name, + Type: NetappVolumeReplicationAssetType, + Resource: &cai.AssetResource{ + Version: "v1beta1", + DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/netapp/v1beta1/rest", + DiscoveryName: "VolumeReplication", + Data: obj, + }, + }}, nil + } else { + return []cai.Asset{}, err + } +} + +func GetNetappVolumeReplicationApiObject(d tpgresource.TerraformResourceData, config *transport_tpg.Config) (map[string]interface{}, error) { + obj := make(map[string]interface{}) + replicationScheduleProp, err := expandNetappVolumeReplicationReplicationSchedule(d.Get("replication_schedule"), d, config) + if err != nil { + return nil, err + } else if v, ok := d.GetOkExists("replication_schedule"); !tpgresource.IsEmptyValue(reflect.ValueOf(replicationScheduleProp)) && (ok || !reflect.DeepEqual(v, replicationScheduleProp)) { + obj["replicationSchedule"] = replicationScheduleProp + } + destinationVolumeParametersProp, err := expandNetappVolumeReplicationDestinationVolumeParameters(d.Get("destination_volume_parameters"), d, config) + if err != nil { + return nil, err + } else if v, ok := d.GetOkExists("destination_volume_parameters"); !tpgresource.IsEmptyValue(reflect.ValueOf(destinationVolumeParametersProp)) && (ok || !reflect.DeepEqual(v, destinationVolumeParametersProp)) { + obj["destinationVolumeParameters"] = destinationVolumeParametersProp + } + descriptionProp, err := expandNetappVolumeReplicationDescription(d.Get("description"), d, config) + if err != nil { + return nil, err + } else if v, ok := d.GetOkExists("description"); !tpgresource.IsEmptyValue(reflect.ValueOf(descriptionProp)) && (ok || !reflect.DeepEqual(v, descriptionProp)) { + obj["description"] = descriptionProp + } + labelsProp, err := expandNetappVolumeReplicationEffectiveLabels(d.Get("effective_labels"), d, config) + if err != nil { + return nil, err + } else if v, ok := d.GetOkExists("effective_labels"); !tpgresource.IsEmptyValue(reflect.ValueOf(labelsProp)) && (ok || !reflect.DeepEqual(v, labelsProp)) { + obj["labels"] = labelsProp + } + + return obj, nil +} + +func expandNetappVolumeReplicationReplicationSchedule(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} + +func expandNetappVolumeReplicationDestinationVolumeParameters(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedStoragePool, err := expandNetappVolumeReplicationDestinationVolumeParametersStoragePool(original["storage_pool"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedStoragePool); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["storagePool"] = transformedStoragePool + } + + transformedVolumeId, err := expandNetappVolumeReplicationDestinationVolumeParametersVolumeId(original["volume_id"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedVolumeId); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["volumeId"] = transformedVolumeId + } + + transformedShareName, err := expandNetappVolumeReplicationDestinationVolumeParametersShareName(original["share_name"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedShareName); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["shareName"] = transformedShareName + } + + transformedDescription, err := expandNetappVolumeReplicationDestinationVolumeParametersDescription(original["description"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedDescription); val.IsValid() && !tpgresource.IsEmptyValue(val) { + transformed["description"] = transformedDescription + } + + return transformed, nil +} + +func expandNetappVolumeReplicationDestinationVolumeParametersStoragePool(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} + +func expandNetappVolumeReplicationDestinationVolumeParametersVolumeId(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} + +func expandNetappVolumeReplicationDestinationVolumeParametersShareName(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} + +func expandNetappVolumeReplicationDestinationVolumeParametersDescription(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} + +func expandNetappVolumeReplicationDescription(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} + +func expandNetappVolumeReplicationEffectiveLabels(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (map[string]string, error) { + if v == nil { + return map[string]string{}, nil + } + m := make(map[string]string) + for k, val := range v.(map[string]interface{}) { + m[k] = val.(string) + } + return m, nil +}