Skip to content

Commit

Permalink
implement a new migration flow with import and removed blocks (#128)
Browse files Browse the repository at this point in the history 
* implement a new migration flow with import and removed blocks

* fix golint

* revert test codes
  • Loading branch information
@ms-henglu
ms-henglu authored Sep 19, 2024
1 parent bee21f2 commit 934828f
Show file tree
Hide file tree
Showing 22 changed files with 770 additions and 345 deletions.
101 changes: 101 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
## v2.0.0-beta

Target azurerm version: v4.0.0

FEATURES:
- The new migration flow uses `import` and `removed` block instead of importing resources and removing resources from terraform state directly.
- Support `working-dir` flag to specify the working directory

## v1.15.0
Target azurerm version: v3.114.0

## v1.14.0
Target azurerm version: v3.110.0

## v1.13.0
Target azurerm version: v3.99.0
Target azapi version: v1.13.0

## v1.12.0
Target azurerm version: v3.83.0

## v1.11.0
Target azurerm version: v3.83.0

## v1.10.0
Target azurerm version: v3.79.0

## v1.9.0
Target azurerm version: v3.71.0

## v1.8.0
Target azurerm version: v3.66.0

## v1.7.0
Target azurerm version: v3.61.0

ENHANCEMENTS:
- Refactor: use `tfadd` to generate config from state

BUG FIXES:
- Fix import with `for_each` statement

## v1.6.0
Target azurerm version: v3.55.0

ENHANCEMENTS:
- Refactor: use aztft to get resource type & upgrade to go 1.19

BUG FIXES:
- Fix import with `count` statement

## v1.5.0
Target azurerm version: v3.50.0

## v1.4.0
Target azurerm version: v3.45.0

## v1.3.0
Target azurerm version: v3.41.0

## v1.2.0
Target azurerm version: v3.37.0

## v1.1.0
Target azurerm version: v3.31.0

## v1.0.0
Target azurerm version: v3.24.0

## v0.6.0
Target azurerm version: v3.22.0

## v0.5.0
Target azurerm version: v3.18.0

FEATURES:
- Refresh state after migrating update resources.

## v0.4.0
Target azurerm version: v3.11.0

## v0.3.0
Target azurerm version: v3.1.0

## v0.2.0
Target azurerm version: v3.0.2

## v0.1.0
Target azurerm version: v2.99.0

FEATURES:
- Support resource `azapi_resource` migration
- Support resource `azapi_update_resource` migration
- Support meta-argument `for_each`
- Support meta-argument `count`
- Support meta-argument `depends_on`, `lifecycle` and `provisioner`
- Support dependency injection in array and primitive value.
- Support dependency injection in Map and other complicated struct value.
- Support user input when there are multiple/none `azurerm` resource match for the resource id
- Support migration based on `azurerm` provider's property coverage
- Support ignore terraform addresses listed in file `azapi2azurerm.ignore`
6 changes: 5 additions & 1 deletion GNUmakefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,8 @@ test:

lint:
@echo "==> Checking source code against linters..."
@if command -v golangci-lint; then (golangci-lint run ./...); else ($(GOPATH)/bin/golangci-lint run ./...); fi
@if command -v golangci-lint; then (golangci-lint run ./...); else ($(GOPATH)/bin/golangci-lint run ./...); fi

terrafmt:
@echo "==> Fixing acceptance test terraform blocks code with terrafmt..."
@find . | egrep "_test.go" | sort | while read f; do terrafmt fmt -f $$f; done
126 changes: 67 additions & 59 deletions cmd/migrate_command.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package cmd

import (
"context"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"strconv"
"strings"

"github.com/Azure/azapi2azurerm/azurerm"
Expand All @@ -17,26 +17,31 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/mitchellh/cli"
"github.com/zclconf/go-cty/cty"
)

const filenameImport = "imports.tf"
const providerConfig = `
provider "azurerm" {
features {}
subscription_id = "%s"
}
`
const tempDir = "temp"

type MigrateCommand struct {
Ui cli.Ui
Verbose bool
Strict bool
Ui cli.Ui
Verbose bool
Strict bool
workingDir string
}

func (c *MigrateCommand) flags() *flag.FlagSet {
fs := defaultFlagSet("plan")
fs.BoolVar(&c.Verbose, "v", false, "whether show terraform logs")
fs.BoolVar(&c.Strict, "strict", false, "strict mode: API versions must be matched")
fs.StringVar(&c.workingDir, "working-dir", "", "path to Terraform configuration files")

fs.Usage = func() { c.Ui.Error(c.Help()) }
return fs
}
Expand All @@ -51,9 +56,13 @@ func (c MigrateCommand) Run(args []string) int {
return 1
}

if c.workingDir == "" {
c.workingDir, _ = os.Getwd()
}
log.Printf("[INFO] working directory: %s", c.workingDir)

log.Printf("[INFO] initializing terraform...")
workingDirectory, _ := os.Getwd()
terraform, err := tf.NewTerraform(workingDirectory, c.Verbose)
terraform, err := tf.NewTerraform(c.workingDir, c.Verbose)
if err != nil {
log.Fatal(err)
}
Expand Down Expand Up @@ -83,27 +92,11 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource
log.Printf("[INFO] -----------------------------------------------")
log.Printf("[INFO] task: migrate azapi_resource")

// generate import config
config := ""
for _, resource := range resources {
config += resource.EmptyImportConfig()
}
workingDirectory := terraform.GetWorkingDirectory()
if err := os.WriteFile(filepath.Join(workingDirectory, filenameImport), []byte(config), 0600); err != nil {
log.Fatal(err)
}

// import and generate config
for index, r := range resources {
log.Printf("[INFO] migrating resource %s (%d instances) to resource %s...", r.OldAddress(nil), len(r.Instances), r.NewAddress(nil))
// import into real state
for _, instance := range r.Instances {
address := r.NewAddress(instance.Index)
log.Printf("[INFO] importing %s to %s...", instance.ResourceId, address)
if err := terraform.Import(address, instance.ResourceId); err != nil {
log.Printf("[Error] error importing %s : %s", address, instance.ResourceId)
}
}

// write empty config to temp dir for import
tempDirectoryCreate(workingDirectory)
Expand All @@ -113,7 +106,14 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource
log.Fatal(err)
}

config := providerConfig
subscriptionId := ""
for _, instance := range r.Instances {
if strings.HasPrefix(instance.ResourceId, "/subscriptions/") {
subscriptionId = strings.Split(instance.ResourceId, "/")[2]
break
}
}
config := fmt.Sprintf(providerConfig, subscriptionId)
for _, instance := range r.Instances {
if !r.IsMultipleResources() {
config += fmt.Sprintf("resource \"%s\" \"%s\" {}\n", r.ResourceType, r.Label)
Expand Down Expand Up @@ -195,17 +195,6 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource
}
tempDirectoryCleanup(workingDirectory)

// remove from state
for _, r := range resources {
if r.Migrated {
log.Printf("[INFO] removing %s from state", r.OldAddress(nil))
exec := terraform.GetExec()
if err := exec.StateRm(context.TODO(), r.OldAddress(nil)); err != nil {
log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(nil), err)
}
}
}

// migrate depends_on, lifecycle, provisioner
for index, r := range resources {
if existingBlock, err := helper.GetResourceBlock(workingDirectory, r.OldAddress(nil)); err == nil && existingBlock != nil {
Expand All @@ -221,15 +210,40 @@ func (c MigrateCommand) MigrateGenericResource(terraform *tf.Terraform, resource
}

// remove from config
if err := os.RemoveAll(filepath.Join(workingDirectory, filenameImport)); err != nil {
log.Fatal(err)
}
for _, r := range resources {
if r.Migrated {
log.Printf("[INFO] removing %s from config", r.OldAddress(nil))
if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(nil), r.Block); err != nil {

importBlock := hclwrite.NewBlock("import", nil)
if r.IsMultipleResources() {
forEachMap := make(map[string]cty.Value)
for _, instance := range r.Instances {
switch v := instance.Index.(type) {
case string:
forEachMap[instance.ResourceId] = cty.StringVal(v)
default:
value, _ := strconv.ParseInt(fmt.Sprintf("%v", v), 10, 64)
forEachMap[instance.ResourceId] = cty.NumberIntVal(value)
}
}
importBlock.Body().SetAttributeValue("for_each", cty.MapVal(forEachMap))
importBlock.Body().SetAttributeTraversal("id", hcl.Traversal{hcl.TraverseRoot{Name: "each"}, hcl.TraverseAttr{Name: "key"}})
importBlock.Body().SetAttributeTraversal("to", hcl.Traversal{hcl.TraverseRoot{Name: r.ResourceType}, hcl.TraverseAttr{Name: fmt.Sprintf("%s[each.value]", r.Label)}})
} else {
importBlock.Body().SetAttributeValue("id", cty.StringVal(r.Instances[0].ResourceId))
importBlock.Body().SetAttributeTraversal("to", hcl.Traversal{hcl.TraverseRoot{Name: r.ResourceType}, hcl.TraverseAttr{Name: r.Label}})
}

removedBlock := hclwrite.NewBlock("removed", nil)
removedBlock.Body().SetAttributeTraversal("from", hcl.Traversal{hcl.TraverseRoot{Name: "azapi_resource"}, hcl.TraverseAttr{Name: r.Label}})
removedLifecycleBlock := hclwrite.NewBlock("lifecycle", nil)
removedLifecycleBlock.Body().SetAttributeValue("destroy", cty.BoolVal(false))
removedBlock.Body().AppendBlock(removedLifecycleBlock)

if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(nil), []*hclwrite.Block{removedBlock, importBlock, r.Block}); err != nil {
log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(nil), err)
}

}
}

Expand Down Expand Up @@ -259,7 +273,14 @@ func (c MigrateCommand) MigrateGenericUpdateResource(terraform *tf.Terraform, re
log.Printf("[INFO] task: migrate azapi_update_resource")

// generate import config
config := providerConfig
subscriptionId := ""
for _, instance := range resources {
if strings.HasPrefix(instance.Id, "/subscriptions/") {
subscriptionId = strings.Split(instance.Id, "/")[2]
break
}
}
config := fmt.Sprintf(providerConfig, subscriptionId)
for _, resource := range resources {
config += resource.EmptyImportConfig()
}
Expand All @@ -277,7 +298,6 @@ func (c MigrateCommand) MigrateGenericUpdateResource(terraform *tf.Terraform, re
}

// import and generate config
newAddrs := make([]string, 0)
for index, r := range resources {
log.Printf("[INFO] migrating resource %s to resource %s", r.OldAddress(), r.NewAddress())
if block, err := importAndGenerateConfig(tempTerraform, r.NewAddress(), r.Id, r.ResourceType, true); err == nil {
Expand All @@ -288,7 +308,6 @@ func (c MigrateCommand) MigrateGenericUpdateResource(terraform *tf.Terraform, re
}
resources[index].Block = helper.InjectReference(resources[index].Block, resources[index].References)
resources[index].Migrated = true
newAddrs = append(newAddrs, r.NewAddress())
log.Printf("[INFO] %s has migrated to %s", r.OldAddress(), r.NewAddress())
} else {
log.Printf("[ERROR] %+v", err)
Expand All @@ -312,33 +331,22 @@ func (c MigrateCommand) MigrateGenericUpdateResource(terraform *tf.Terraform, re
}
}

// remove from state
for _, r := range resources {
if r.Migrated {
log.Printf("[INFO] removing %s from state", r.OldAddress())
exec := terraform.GetExec()
if err := exec.StateRm(context.TODO(), r.OldAddress()); err != nil {
log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(), err)
}
}
}

// remove from config
for _, r := range resources {
if r.Migrated {
log.Printf("[INFO] removing %s from config", r.OldAddress())
if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(), nil); err != nil {
removedBlock := hclwrite.NewBlock("removed", nil)
removedBlock.Body().SetAttributeTraversal("from", hcl.Traversal{hcl.TraverseRoot{Name: "azapi_update_resource"}, hcl.TraverseAttr{Name: r.OldLabel}})
removedLifecycleBlock := hclwrite.NewBlock("lifecycle", nil)
removedLifecycleBlock.Body().SetAttributeValue("destroy", cty.BoolVal(false))
removedBlock.Body().AppendBlock(removedLifecycleBlock)

if err := helper.ReplaceResourceBlock(workingDirectory, r.OldAddress(), []*hclwrite.Block{removedBlock}); err != nil {
log.Printf("[ERROR] error removing %s from state: %+v", r.OldAddress(), err)
}
}
}

log.Println("[INFO] refreshing state for migrated resources...")
err = terraform.RefreshState(newAddrs)
if err != nil {
log.Printf("refreshing state: %+v", err)
}

tempDirectoryCleanup(workingDirectory)
}

Expand Down
Loading

0 comments on commit 934828f

Please sign in to comment.