Cai2Hcl: refactor of converter_map and split of utils.go (GoogleCloud…

…Platform#9378)

* Add cai2hcl generated converters for 3 Compute resources.

These are: BackendService, GlobalBackendService, ForwardingRule.

This is the output of cai2hcl provider, commited as is
without the generator Ruby code itself, as agreed with
library owners.
Reason: limited capacity of the supporting team to
perform code reviews and the future need to port this
solution to golang as a part of migration to go. As soon
the staffing and/or migration is done, we will come back
to implement the generated converters as
originally planned.

* Split current PR into 3 chunks:
1. Refactor of shared files (Uber Converter)
2. Add ForwardingRuleConverter (generated)
3. Add BackendServiceConverters (generated)
+ ability to match converters by name

This commit removes part2 and part3, leaving only part1 in this branch.

* Fix versions after merge conflict

* Remove auto-generated code header from FR

* Remove unused util function

* Fix merge conflict resolution issue

* Remove unnecessary result map normalization (will needed later with generated converters)

* Simplify cai2hcl by reworking UberConverter's as a single ConverterMap.

1. Rename UberConverter to ConverterMap.
2. Remove ConverterMap's from each API folder and replace them with a single top-level ConverterMap

* Simplify converters

* Flatten converter_map.go and use strings as converter names instead of constants.

mmv1/third_party/cai2hcl/common/hcl_write.go

@@ -3,10 +3,26 @@ package common
 import (
 	"fmt"
 
+	"github.com/hashicorp/hcl/hcl/printer"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/zclconf/go-cty/cty"
 )
 
+// HclWriteBlocks prints HCLResourceBlock objects as string.
+func HclWriteBlocks(blocks []*HCLResourceBlock) ([]byte, error) {
+	f := hclwrite.NewFile()
+	rootBody := f.Body()
+
+	for _, resourceBlock := range blocks {
+		hclBlock := rootBody.AppendNewBlock("resource", resourceBlock.Labels)
+		if err := hclWriteBlock(resourceBlock.Value, hclBlock.Body()); err != nil {
+			return nil, err
+		}
+	}
+
+	return printer.Format(f.Bytes())
+}
+
 func hclWriteBlock(val cty.Value, body *hclwrite.Body) error {
 	if val.IsNull() {
 		return nil

mmv1/third_party/cai2hcl/common/utils.go

@@ -5,16 +5,13 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/GoogleCloudPlatform/terraform-google-conversion/v5/caiasset"
 	hashicorpcty "github.com/hashicorp/go-cty/cty"
-	"github.com/hashicorp/hcl/hcl/printer"
-	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
 )
 
-// Extracts named part from resource url.
+// ParseFieldValue extracts named part from resource url.
 func ParseFieldValue(url string, name string) string {
 	fragments := strings.Split(url, "/")
 	for ix, item := range fragments {
@@ -25,7 +22,7 @@ func ParseFieldValue(url string, name string) string {
 	return ""
 }
 
-// Decodes the map object into the target struct.
+// DecodeJSON decodes the map object into the target struct.
 func DecodeJSON(data map[string]interface{}, v interface{}) error {
 	b, err := json.Marshal(data)
 	if err != nil {
@@ -37,12 +34,13 @@ func DecodeJSON(data map[string]interface{}, v interface{}) error {
 	return nil
 }
 
-// Converts resource from untyped map format to TF JSON.
+// MapToCtyValWithSchema converts resource from untyped map format to TF JSON.
 func MapToCtyValWithSchema(m map[string]interface{}, s map[string]*schema.Schema) (cty.Value, error) {
 	b, err := json.Marshal(&m)
 	if err != nil {
 		return cty.NilVal, fmt.Errorf("error marshaling map as JSON: %v", err)
 	}
+
 	ty, err := hashicorpCtyTypeToZclconfCtyType(schema.InternalMap(s).CoreConfigSchema().ImpliedType())
 	if err != nil {
 		return cty.NilVal, fmt.Errorf("error casting type: %v", err)
@@ -54,44 +52,6 @@ func MapToCtyValWithSchema(m map[string]interface{}, s map[string]*schema.Schema
 	return ret, nil
 }
 
-func Convert(assets []*caiasset.Asset, converterNames map[string]string, converterMap map[string]Converter) ([]byte, error) {
-	// Group resources from the same tf resource type for convert.
-	// tf -> cai has 1:N mappings occasionally
-	groups := make(map[string][]*caiasset.Asset)
-	for _, asset := range assets {
-		name, ok := converterNames[asset.Type]
-		if !ok {
-			continue
-		}
-		groups[name] = append(groups[name], asset)
-	}
-
-	f := hclwrite.NewFile()
-	rootBody := f.Body()
-	for name, v := range groups {
-		converter, ok := converterMap[name]
-		if !ok {
-			continue
-		}
-		items, err := converter.Convert(v)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, resourceBlock := range items {
-			hclBlock := rootBody.AppendNewBlock("resource", resourceBlock.Labels)
-			if err := hclWriteBlock(resourceBlock.Value, hclBlock.Body()); err != nil {
-				return nil, err
-			}
-		}
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return printer.Format(f.Bytes())
-}
-
 func hashicorpCtyTypeToZclconfCtyType(t hashicorpcty.Type) (cty.Type, error) {
 	b, err := json.Marshal(t)
 	if err != nil {

mmv1/third_party/cai2hcl/common/utils_test.go

@@ -0,0 +1,42 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	tpg_provider "github.com/hashicorp/terraform-provider-google-beta/google-beta/provider"
+)
+
+func TestSubsetOfFieldsMapsToCtyValue(t *testing.T) {
+	schema := createSchema("google_compute_forwarding_rule")
+
+	outputMap := map[string]interface{}{
+		"name": "forwarding-rule-1",
+	}
+
+	val, err := MapToCtyValWithSchema(outputMap, schema)
+
+	assert.Nil(t, err)
+	assert.Equal(t, "forwarding-rule-1", val.GetAttr("name").AsString())
+}
+
+func TestWrongFieldTypeBreaksConversion(t *testing.T) {
+	resourceSchema := createSchema("google_compute_backend_service")
+	outputMap := map[string]interface{}{
+		"name":        "fr-1",
+		"description": []string{"unknownValue"}, // string is required, not array.
+	}
+
+	val, err := MapToCtyValWithSchema(outputMap, resourceSchema)
+
+	assert.True(t, val.IsNull())
+	assert.Contains(t, err.Error(), "string is required")
+}
+
+func createSchema(name string) map[string]*schema.Schema {
+	provider := tpg_provider.Provider()
+
+	return provider.ResourcesMap[name].Schema
+}

mmv1/third_party/cai2hcl/convert.go

@@ -15,13 +15,38 @@ type Options struct {
 	ErrorLogger *zap.Logger
 }
 
-// Converts CAI Assets into HCL.
+// Converts CAI Assets into HCL string.
 func Convert(assets []*caiasset.Asset, options *Options) ([]byte, error) {
 	if options == nil || options.ErrorLogger == nil {
 		return nil, fmt.Errorf("logger is not initialized")
 	}
 
-	t, err := common.Convert(assets, ConverterNames, ConverterMap)
+	// Group resources from the same TF resource type for convert.
+	// tf -> cai has 1:N mappings occasionally
+	groups := make(map[string][]*caiasset.Asset)
+	for _, asset := range assets {
+
+		name, _ := AssetTypeToConverter[asset.Type]
+		if name != "" {
+			groups[name] = append(groups[name], asset)
+		}
+	}
+
+	allBlocks := []*common.HCLResourceBlock{}
+	for name, assets := range groups {
+		converter, ok := ConverterMap[name]
+		if !ok {
+			continue
+		}
+		newBlocks, err := converter.Convert(assets)
+		if err != nil {
+			return nil, err
+		}
+
+		allBlocks = append(allBlocks, newBlocks...)
+	}
+
+	t, err := common.HclWriteBlocks(allBlocks)
 
 	options.ErrorLogger.Debug(string(t))
 

mmv1/third_party/cai2hcl/convert_test.go

@@ -1,17 +1,14 @@
-package cai2hcl
+package cai2hcl_test
 
 import (
 	"testing"
 
-	"github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/common"
-	"github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/services/compute"
 	cai2hclTesting "github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/testing"
 )
 
 func TestConvertCompute(t *testing.T) {
 	cai2hclTesting.AssertTestFiles(
 		t,
-		ConverterNames, ConverterMap,
 		"./services/compute/testdata",
 		[]string{
 			"full_compute_instance",
@@ -21,40 +18,8 @@ func TestConvertCompute(t *testing.T) {
 func TestConvertResourcemanager(t *testing.T) {
 	cai2hclTesting.AssertTestFiles(
 		t,
-		ConverterNames, ConverterMap,
 		"./services/resourcemanager/testdata",
 		[]string{
 			"project_create",
 		})
 }
-
-func TestConvertPanicsOnConverterNamesConflict(t *testing.T) {
-	assertPanic(t, func() {
-		joinConverterNames([]map[string]string{
-			{"compute.googleapis.com/Instance": "compute_instance_1"},
-			{"compute.googleapis.com/Instance": "compute_instance_2"},
-		})
-	})
-}
-
-func TestConvertPanicsOnConverterMapConflict(t *testing.T) {
-	assertPanic(t, func() {
-		joinConverterMaps([]map[string]common.Converter{
-			common.CreateConverterMap(map[string]common.ConverterFactory{
-				"google_compute_instance": compute.NewComputeInstanceConverter,
-			}),
-			common.CreateConverterMap(map[string]common.ConverterFactory{
-				"google_compute_instance": compute.NewComputeForwardingRuleConverter,
-			}),
-		})
-	})
-}
-
-func assertPanic(t *testing.T, f func()) {
-	defer func() {
-		if r := recover(); r == nil {
-			t.Errorf("The code did not panic")
-		}
-	}()
-	f()
-}

mmv1/third_party/cai2hcl/converter_map.go

@@ -4,49 +4,25 @@ import (
 	"github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/common"
 	"github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/services/compute"
 	"github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/services/resourcemanager"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	tpg_provider "github.com/hashicorp/terraform-provider-google-beta/google-beta/provider"
 )
 
-var allConverterNames = []map[string]string{
-	compute.ConverterNames,
-	resourcemanager.ConverterNames,
-}
-
-var allConverterMaps = []map[string]common.Converter{
-	compute.ConverterMap,
-	resourcemanager.ConverterMap,
-}
-
-var ConverterNames = joinConverterNames(allConverterNames)
-var ConverterMap = joinConverterMaps(allConverterMaps)
+var provider *schema.Provider = tpg_provider.Provider()
 
-func joinConverterNames(arr []map[string]string) map[string]string {
-	result := make(map[string]string)
+// AssetTypeToConverter is a mapping from Asset Type to converter instance.
+var AssetTypeToConverter = map[string]string{
+	compute.ComputeInstanceAssetType:       "google_compute_instance",
+	compute.ComputeForwardingRuleAssetType: "google_compute_forwarding_rule",
 
-	for _, m := range arr {
-		for key, value := range m {
-			if _, hasKey := result[key]; hasKey {
-				panic("Converters from different services are not unique")
-			}
-
-			result[key] = value
-		}
-	}
-
-	return result
+	resourcemanager.ProjectAssetType:        "google_project",
+	resourcemanager.ProjectBillingAssetType: "google_project",
 }
 
-func joinConverterMaps(arr []map[string]common.Converter) map[string]common.Converter {
-	result := make(map[string]common.Converter)
-
-	for _, m := range arr {
-		for key, value := range m {
-			if _, hasKey := result[key]; hasKey {
-				panic("Converters from different services are not unique")
-			}
-
-			result[key] = value
-		}
-	}
+// ConverterMap is a collection of converters instances, indexed by name.
+var ConverterMap = map[string]common.Converter{
+	"google_compute_instance":        compute.NewComputeInstanceConverter(provider),
+	"google_compute_forwarding_rule": compute.NewComputeForwardingRuleConverter(provider),
 
-	return result
+	"google_project": resourcemanager.NewProjectConverter(provider),
 }

mmv1/third_party/cai2hcl/services/compute/compute_forwarding_rule.go

@@ -10,19 +10,24 @@ import (
 	computeV1 "google.golang.org/api/compute/v1"
 )
 
-// ComputeForwardingRuleAssetType is the CAI asset type name for compute instance.
+// ComputeForwardingRuleAssetType is a CAI asset type name.
 const ComputeForwardingRuleAssetType string = "compute.googleapis.com/ForwardingRule"
 
+// ComputeForwardingRuleSchemaName is a TF resource schema name.
+const ComputeForwardingRuleSchemaName string = "google_compute_forwarding_rule"
+
 // ComputeForwardingRuleConverter for regional forwarding rule.
 type ComputeForwardingRuleConverter struct {
 	name   string
 	schema map[string]*tfschema.Schema
 }
 
 // NewComputeForwardingRuleConverter returns an HCL converter for compute instance.
-func NewComputeForwardingRuleConverter(name string, schema map[string]*tfschema.Schema) common.Converter {
+func NewComputeForwardingRuleConverter(provider *tfschema.Provider) common.Converter {
+	schema := provider.ResourcesMap[ComputeForwardingRuleSchemaName].Schema
+
 	return &ComputeForwardingRuleConverter{
-		name:   name,
+		name:   ComputeForwardingRuleSchemaName,
 		schema: schema,
 	}
 }

mmv1/third_party/cai2hcl/services/compute/compute_forwarding_rule_test.go

@@ -1,17 +1,14 @@
-package compute
+package compute_test
 
 import (
 	"testing"
 
-	cai2hclTesting "github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/testing"
+	cai2hcl_testing "github.com/GoogleCloudPlatform/terraform-google-conversion/v5/cai2hcl/testing"
 )
 
 func TestComputeForwardingRule(t *testing.T) {
-	cai2hclTesting.AssertTestFiles(
+	cai2hcl_testing.AssertTestFiles(
 		t,
-		ConverterNames, ConverterMap,
 		"./testdata",
-		[]string{
-			"full_compute_forwarding_rule",
-		})
+		[]string{"compute_forwarding_rule"})
 }