diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..bbe71e73 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,101 @@ +## v2.0.0-beta + +Target azurerm version: v4.0.0 + +FEATURES: +- The new migration flow uses `import` and `removed` block instead of importing resources and removing resources from terraform state directly. +- Support `working-dir` flag to specify the working directory + +## v1.15.0 +Target azurerm version: v3.114.0 + +## v1.14.0 +Target azurerm version: v3.110.0 + +## v1.13.0 +Target azurerm version: v3.99.0 +Target azapi version: v1.13.0 + +## v1.12.0 +Target azurerm version: v3.83.0 + +## v1.11.0 +Target azurerm version: v3.83.0 + +## v1.10.0 +Target azurerm version: v3.79.0 + +## v1.9.0 +Target azurerm version: v3.71.0 + +## v1.8.0 +Target azurerm version: v3.66.0 + +## v1.7.0 +Target azurerm version: v3.61.0 + +ENHANCEMENTS: +- Refactor: use `tfadd` to generate config from state + +BUG FIXES: +- Fix import with `for_each` statement + +## v1.6.0 +Target azurerm version: v3.55.0 + +ENHANCEMENTS: +- Refactor: use aztft to get resource type & upgrade to go 1.19 + +BUG FIXES: +- Fix import with `count` statement + +## v1.5.0 +Target azurerm version: v3.50.0 + +## v1.4.0 +Target azurerm version: v3.45.0 + +## v1.3.0 +Target azurerm version: v3.41.0 + +## v1.2.0 +Target azurerm version: v3.37.0 + +## v1.1.0 +Target azurerm version: v3.31.0 + +## v1.0.0 +Target azurerm version: v3.24.0 + +## v0.6.0 +Target azurerm version: v3.22.0 + +## v0.5.0 +Target azurerm version: v3.18.0 + +FEATURES: +- Refresh state after migrating update resources. + +## v0.4.0 +Target azurerm version: v3.11.0 + +## v0.3.0 +Target azurerm version: v3.1.0 + +## v0.2.0 +Target azurerm version: v3.0.2 + +## v0.1.0 +Target azurerm version: v2.99.0 + +FEATURES: +- Support resource `azapi_resource` migration +- Support resource `azapi_update_resource` migration +- Support meta-argument `for_each` +- Support meta-argument `count` +- Support meta-argument `depends_on`, `lifecycle` and `provisioner` +- Support dependency injection in array and primitive value. +- Support dependency injection in Map and other complicated struct value. +- Support user input when there are multiple/none `azurerm` resource match for the resource id +- Support migration based on `azurerm` provider's property coverage +- Support ignore terraform addresses listed in file `azapi2azurerm.ignore` \ No newline at end of file diff --git a/GNUmakefile b/GNUmakefile index 84b8d1ea..07f3c91e 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -16,4 +16,8 @@ test: lint: @echo "==> Checking source code against linters..." - @if command -v golangci-lint; then (golangci-lint run ./...); else ($(GOPATH)/bin/golangci-lint run ./...); fi \ No newline at end of file + @if command -v golangci-lint; then (golangci-lint run ./...); else ($(GOPATH)/bin/golangci-lint run ./...); fi + +terrafmt: + @echo "==> Fixing acceptance test terraform blocks code with terrafmt..." + @find . | egrep "_test.go" | sort | while read f; do terrafmt fmt -f $$f; done \ No newline at end of file diff --git a/cmd/migrate_command.go b/cmd/migrate_command.go index 3ca60794..1e7b4843 100644 --- a/cmd/migrate_command.go +++ b/cmd/migrate_command.go @@ -1,12 +1,12 @@ package cmd import ( - "context" "flag" "fmt" "log" "os" "path/filepath" + "strconv" "strings" "github.com/Azure/azapi2azurerm/azurerm" @@ -17,26 +17,31 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/mitchellh/cli" + "github.com/zclconf/go-cty/cty" ) const filenameImport = "imports.tf" const providerConfig = ` provider "azurerm" { features {} + subscription_id = "%s" } ` const tempDir = "temp" type MigrateCommand struct { - Ui cli.Ui - Verbose bool - Strict bool + Ui cli.Ui + Verbose bool + Strict bool + workingDir string } func (c *MigrateCommand) flags() *flag.FlagSet { fs := defaultFlagSet("plan") fs.BoolVar(&c.Verbose, "v", false, "whether show terraform logs") fs.BoolVar(&c.Strict, "strict", false, "strict mode: API versions must be matched") + fs.StringVar(&c.workingDir, "working-dir", "", "path to Terraform configuration files") + fs.Usage = func() { c.Ui.Error(c.Help()) } return fs } @@ -51,9 +56,13 @@ func (c MigrateCommand) Run(args []string) int { return 1 } + if c.workingDir == "" { + c.workingDir, _ = os.Getwd() + } + log.Printf("[INFO] working directory: %s", c.workingDir) + log.Printf("[INFO] initializing terraform...") - workingDirectory, _ := os.Getwd() - terraform, err := tf.NewTerraform(workingDirectory, c.Verbose) + terraform, err := tf.NewTerraform(c.workingDir, c.Verbose) if err != nil { log.Fatal(err) } @@ -83,27 +92,11 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource log.Printf("[INFO] -----------------------------------------------") log.Printf("[INFO] task: migrate azapi_resource") - // generate import config - config := "" - for _, resource := range resources { - config += resource.EmptyImportConfig() - } workingDirectory := terraform.GetWorkingDirectory() - if err := os.WriteFile(filepath.Join(workingDirectory, filenameImport), []byte(config), 0600); err != nil { - log.Fatal(err) - } // import and generate config for index, r := range resources { log.Printf("[INFO] migrating resource %s (%d instances) to resource %s...", r.OldAddress(nil), len(r.Instances), r.NewAddress(nil)) - // import into real state - for _, instance := range r.Instances { - address := r.NewAddress(instance.Index) - log.Printf("[INFO] importing %s to %s...", instance.ResourceId, address) - if err := terraform.Import(address, instance.ResourceId); err != nil { - log.Printf("[Error] error importing %s : %s", address, instance.ResourceId) - } - } // write empty config to temp dir for import tempDirectoryCreate(workingDirectory) @@ -113,7 +106,14 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource log.Fatal(err) } - config := providerConfig + subscriptionId := "" + for _, instance := range r.Instances { + if strings.HasPrefix(instance.ResourceId, "/subscriptions/") { + subscriptionId = strings.Split(instance.ResourceId, "/")[2] + break + } + } + config := fmt.Sprintf(providerConfig, subscriptionId) for _, instance := range r.Instances { if !r.IsMultipleResources() { config += fmt.Sprintf("resource \"%s\" \"%s\" {}\n", r.ResourceType, r.Label) @@ -195,17 +195,6 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource } tempDirectoryCleanup(workingDirectory) - // remove from state - for _, r := range resources { - if r.Migrated { - log.Printf("[INFO] removing %s from state", r.OldAddress(nil)) - exec := terraform.GetExec() - if err := exec.StateRm(context.TODO(), r.OldAddress(nil)); err != nil { - log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(nil), err) - } - } - } - // migrate depends_on, lifecycle, provisioner for index, r := range resources { if existingBlock, err := helper.GetResourceBlock(workingDirectory, r.OldAddress(nil)); err == nil && existingBlock != nil { @@ -221,15 +210,40 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource } // remove from config - if err := os.RemoveAll(filepath.Join(workingDirectory, filenameImport)); err != nil { - log.Fatal(err) - } for _, r := range resources { if r.Migrated { log.Printf("[INFO] removing %s from config", r.OldAddress(nil)) - if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(nil), r.Block); err != nil { + + importBlock := hclwrite.NewBlock("import", nil) + if r.IsMultipleResources() { + forEachMap := make(map[string]cty.Value) + for _, instance := range r.Instances { + switch v := instance.Index.(type) { + case string: + forEachMap[instance.ResourceId] = cty.StringVal(v) + default: + value, _ := strconv.ParseInt(fmt.Sprintf("%v", v), 10, 64) + forEachMap[instance.ResourceId] = cty.NumberIntVal(value) + } + } + importBlock.Body().SetAttributeValue("for_each", cty.MapVal(forEachMap)) + importBlock.Body().SetAttributeTraversal("id", hcl.Traversal{hcl.TraverseRoot{Name: "each"}, hcl.TraverseAttr{Name: "key"}}) + importBlock.Body().SetAttributeTraversal("to", hcl.Traversal{hcl.TraverseRoot{Name: r.ResourceType}, hcl.TraverseAttr{Name: fmt.Sprintf("%s[each.value]", r.Label)}}) + } else { + importBlock.Body().SetAttributeValue("id", cty.StringVal(r.Instances[0].ResourceId)) + importBlock.Body().SetAttributeTraversal("to", hcl.Traversal{hcl.TraverseRoot{Name: r.ResourceType}, hcl.TraverseAttr{Name: r.Label}}) + } + + removedBlock := hclwrite.NewBlock("removed", nil) + removedBlock.Body().SetAttributeTraversal("from", hcl.Traversal{hcl.TraverseRoot{Name: "azapi_resource"}, hcl.TraverseAttr{Name: r.Label}}) + removedLifecycleBlock := hclwrite.NewBlock("lifecycle", nil) + removedLifecycleBlock.Body().SetAttributeValue("destroy", cty.BoolVal(false)) + removedBlock.Body().AppendBlock(removedLifecycleBlock) + + if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(nil), []*hclwrite.Block{removedBlock, importBlock, r.Block}); err != nil { log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(nil), err) } + } } @@ -259,7 +273,14 @@ func (c MigrateCommand) MigrateGenericUpdateResource(terraform *tf.Terraform, re log.Printf("[INFO] task: migrate azapi_update_resource") // generate import config - config := providerConfig + subscriptionId := "" + for _, instance := range resources { + if strings.HasPrefix(instance.Id, "/subscriptions/") { + subscriptionId = strings.Split(instance.Id, "/")[2] + break + } + } + config := fmt.Sprintf(providerConfig, subscriptionId) for _, resource := range resources { config += resource.EmptyImportConfig() } @@ -312,33 +333,22 @@ func (c MigrateCommand) MigrateGenericUpdateResource(terraform *tf.Terraform, re } } - // remove from state - for _, r := range resources { - if r.Migrated { - log.Printf("[INFO] removing %s from state", r.OldAddress()) - exec := terraform.GetExec() - if err := exec.StateRm(context.TODO(), r.OldAddress()); err != nil { - log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(), err) - } - } - } - // remove from config for _, r := range resources { if r.Migrated { log.Printf("[INFO] removing %s from config", r.OldAddress()) - if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(), nil); err != nil { + removedBlock := hclwrite.NewBlock("removed", nil) + removedBlock.Body().SetAttributeTraversal("from", hcl.Traversal{hcl.TraverseRoot{Name: "azapi_update_resource"}, hcl.TraverseAttr{Name: r.OldLabel}}) + removedLifecycleBlock := hclwrite.NewBlock("lifecycle", nil) + removedLifecycleBlock.Body().SetAttributeValue("destroy", cty.BoolVal(false)) + removedBlock.Body().AppendBlock(removedLifecycleBlock) + + if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(), []*hclwrite.Block{removedBlock}); err != nil { log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(), err) } } } - log.Println("[INFO] refreshing state for migrated resources...") - err = terraform.RefreshState(newAddrs) - if err != nil { - log.Printf("refreshing state: %+v", err) - } - tempDirectoryCleanup(workingDirectory) } diff --git a/cmd/migrate_command_test.go b/cmd/migrate_command_test.go index bee6bcb8..8e0eef7f 100644 --- a/cmd/migrate_command_test.go +++ b/cmd/migrate_command_test.go @@ -275,27 +275,27 @@ resource "azapi_resource" "test" { type = "SystemAssigned" } - body = jsonencode({ + body = { properties = { sku = { name = local.AutomationSku } } - }) + } } resource "azapi_resource" "test2" { - name = "${var.AutomationName}another" - parent_id = azurerm_resource_group.test.id - type = "Microsoft.Automation/automationAccounts@2020-01-13-preview" - location = azurerm_resource_group.test.location - body = jsonencode({ + name = "${var.AutomationName}another" + parent_id = azurerm_resource_group.test.id + type = "Microsoft.Automation/automationAccounts@2020-01-13-preview" + location = azurerm_resource_group.test.location + body = { properties = { sku = { - name = jsondecode(azapi_resource.test.output).properties.sku.name + name = azapi_resource.test.output.properties.sku.name } } - }) + } } resource "azurerm_automation_account" "test1" { @@ -309,19 +309,19 @@ resource "azapi_update_resource" "test" { resource_id = azurerm_automation_account.test1.id type = "Microsoft.Automation/automationAccounts@2020-01-13-preview" response_export_values = ["properties.sku"] - body = jsonencode({ + body = { tags = { key = var.Label } - }) + } } output "accountName" { - value = jsondecode(azapi_resource.test.output).name + value = azapi_resource.test.output.name } output "patchAccountSKU" { - value = jsondecode(azapi_update_resource.test.output).properties.sku.name + value = azapi_update_resource.test.output.properties.sku.name } `, template(), randomResourceName(), randomResourceName()) } @@ -350,22 +350,22 @@ variable "accounts" { resource "azapi_resource" "test" { - name = "henglu${each.value.name}" - parent_id = azurerm_resource_group.test.id - type = "Microsoft.Automation/automationAccounts@2020-01-13-preview" + name = "henglu${each.value.name}" + parent_id = azurerm_resource_group.test.id + type = "Microsoft.Automation/automationAccounts@2020-01-13-preview" location = azurerm_resource_group.test.location identity { type = "SystemAssigned" } - body = jsonencode({ + body = { properties = { sku = { name = each.value.sku } } - }) + } for_each = var.accounts } @@ -395,31 +395,29 @@ variable "defName" { } resource "azapi_resource" "test" { - name = "%s" - parent_id = azurerm_resource_group.test.id - type = "Microsoft.Network/serviceEndpointPolicies@2020-11-01" - - body = < azapi2azurerm.exe +Usage: azapi2azurerm [--version] [--help] [] + +Available commands are: + migrate Migrate azapi resources to azurerm resources in current working directory + plan Show azapi resources which can migrate to azurerm resources in current working directory + version Displays the version of the migration tool +``` + +1. Run `azapi2azurerm plan` under your terraform working directory, + it will list all resources that can be migrated from `azapi` provider to `azurerm` provider. + The Terraform addresses listed in file `azapi2azurerm.ignore` will be ignored during migration. +``` +2022/01/25 14:34:46 [INFO] searching azapi_resource & azapi_update_resource... +2022/01/25 14:34:55 [INFO] + +The tool will perform the following actions: + +The following resources will be migrated: +azapi_resource.test2 will be replaced with azurerm_automation_account +azapi_update_resource.test will be replaced with azurerm_automation_account + +The following resources can't be migrated: +azapi_resource.test: input properties not supported: [], output properties not supported: [identity.principalId, identity.type, identity.tenantId] + +The following resources will be ignored in migration: + ``` +2. Run `azapi2azurerm migrate` under your terraform working directory, + it will migrate above resources from `azapi` provider to `azurerm` provider, + both terraform configuration and state. + The Terraform addresses listed in file `azapi2azurerm.ignore` will be ignored during migration. + +## Examples +There're some examples to show the migration results. +1. [case1 - basic](https://github.com/Azure/azapi2azurerm/tree/master/examples/case1%20-%20basic) +2. [case2 - for_each](https://github.com/Azure/azapi2azurerm/tree/master/examples/case2%20-%20for_each) +3. [case3 - nested block](https://github.com/Azure/azapi2azurerm/tree/master/examples/case3%20-%20nested%20block) +4. [case4 - count](https://github.com/Azure/azapi2azurerm/tree/master/examples/case4%20-%20count) +5. [case5 - nested block patch](https://github.com/Azure/azapi2azurerm/tree/master/examples/case5%20-%20nested%20block%20patch) +6. [case6 - meta argument](https://github.com/Azure/azapi2azurerm/tree/master/examples/case6%20-%20meta%20arguments) +7. [case7 - ignore](https://github.com/Azure/azapi2azurerm/tree/master/examples/case7%20-%20ignore) + + ## Install ### From Release @@ -73,50 +119,6 @@ Supported versions: ```bash yay -S azapi2azurerm ``` - -## Command Usage -``` -PS C:\Users\henglu\go\src\github.com\Azure\azapi2azurerm> azapi2azurerm.exe -Usage: azapi2azurerm [--version] [--help] [] - -Available commands are: - migrate Migrate azapi resources to azurerm resources in current working directory - plan Show azapi resources which can migrate to azurerm resources in current working directory - version Displays the version of the migration tool -``` - -1. Run `azapi2azurerm plan` under your terraform working directory, - it will list all resources that can be migrated from `azapi` provider to `azurerm` provider. - The Terraform addresses listed in file `azapi2azurerm.ignore` will be ignored during migration. -``` -2022/01/25 14:34:46 [INFO] searching azapi_resource & azapi_update_resource... -2022/01/25 14:34:55 [INFO] - -The tool will perform the following actions: - -The following resources will be migrated: -azapi_resource.test2 will be replaced with azurerm_automation_account -azapi_update_resource.test will be replaced with azurerm_automation_account - -The following resources can't be migrated: -azapi_resource.test: input properties not supported: [], output properties not supported: [identity.principalId, identity.type, identity.tenantId] - -The following resources will be ignored in migration: - ``` -2. Run `azapi2azurerm migrate` under your terraform working directory, - it will migrate above resources from `azapi` provider to `azurerm` provider, - both terraform configuration and state. - The Terraform addresses listed in file `azapi2azurerm.ignore` will be ignored during migration. - -## Examples -There're some examples to show the migration results. -1. [case1 - basic](https://github.com/Azure/azapi2azurerm/tree/master/examples/case1%20-%20basic) -2. [case2 - for_each](https://github.com/Azure/azapi2azurerm/tree/master/examples/case2%20-%20for_each) -3. [case3 - nested block](https://github.com/Azure/azapi2azurerm/tree/master/examples/case3%20-%20nested%20block) -4. [case4 - count](https://github.com/Azure/azapi2azurerm/tree/master/examples/case4%20-%20count) -5. [case5 - nested block patch](https://github.com/Azure/azapi2azurerm/tree/master/examples/case5%20-%20nested%20block%20patch) -6. [case6 - meta argument](https://github.com/Azure/azapi2azurerm/tree/master/examples/case6%20-%20meta%20arguments) -7. [case7 - ignore](https://github.com/Azure/azapi2azurerm/tree/master/examples/case7%20-%20ignore) ## Features - [x] Support resource `azapi_resource` migration diff --git a/tf/terraform.go b/tf/terraform.go index 7ad1841f..38cf021d 100644 --- a/tf/terraform.go +++ b/tf/terraform.go @@ -225,15 +225,6 @@ func (t *Terraform) Apply() error { return t.exec.Apply(context.TODO()) } -func (t *Terraform) RefreshState(resources []string) error { - // TODO: replace refresh command with apply -refresh-only - opts := make([]tfexec.RefreshCmdOption, 0) - for _, res := range resources { - opts = append(opts, tfexec.Target(res)) - } - return t.exec.Refresh(context.TODO(), opts...) -} - func (t *Terraform) Destroy() error { return t.exec.Destroy(context.TODO()) }