diff --git a/cli/provider_cache.go b/cli/provider_cache.go index 64c13d60f..712b52718 100644 --- a/cli/provider_cache.go +++ b/cli/provider_cache.go @@ -39,7 +39,7 @@ const ( ) var ( - // HTTPStatusCacheProviderReg is regular expression to determine the success result of the command `terraform init`. + // httpStatusCacheProviderReg is a regular expression to determine the success result of the command `terraform init`. // The reg matches if the text contains "423 Locked", for example: // // - registry.terraform.io/hashicorp/template: could not query provider registry for registry.terraform.io/hashicorp/template: 423 Locked. @@ -53,7 +53,7 @@ var ( // │ provider registry for registry.terraform.io/snowflake-labs/snowflake: 423 // │ Locked // ╵ - HTTPStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(cacheProviderHTTPStatusCode) + `.*` + http.StatusText(cacheProviderHTTPStatusCode)) + httpStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(cacheProviderHTTPStatusCode) + `.*` + http.StatusText(cacheProviderHTTPStatusCode)) ) type ProviderCache struct { @@ -138,17 +138,29 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti // To prevent a loop ctx = shell.ContextWithTerraformCommandHook(ctx, nil) + var ( + cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename) + cacheRequestID = uuid.New().String() + env = providerCacheEnvironment(opts, cliConfigFilename) + commandsArgs [][]string + skipRunTargetCommand bool + ) + // Use Hook only for the `terraform init` command, which can be run explicitly by the user or Terragrunt's `auto-init` feature. - if util.FirstArg(args) != terraform.CommandNameInit { + switch { + case util.FirstArg(args) == terraform.CommandNameInit: + // Provider caching for `terraform init` command. + commandsArgs = [][]string{args} + case util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock: + // Provider caching for `terraform providers lock` command. + commandsArgs = convertToMultipleCommandsByPlatforms(args) + // Since the Terragrunt provider cache server creates the cache and generates the lock file, we don't need to run the `terraform providers lock ...` command at all. + skipRunTargetCommand = true + default: + // skip cache creation for all other commands return shell.RunTerraformCommandWithOutput(ctx, opts, args...) } - var ( - cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename) - cacheRequestID = uuid.New().String() - env = providerCacheEnvironment(opts, cliConfigFilename) - ) - // Create terraform cli config file that enables provider caching and does not use provider cache dir if err := cache.createLocalCLIConfig(opts, cliConfigFilename, cacheRequestID); err != nil { return nil, err @@ -158,8 +170,11 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti // Before each init, we warm up the global cache to ensure that all necessary providers are cached. // To do this we are using 'terraform providers lock' to force TF to request all the providers from our TG cache, and that's how we know what providers TF needs, and can load them into the cache. // It's low cost operation, because it does not cache the same provider twice, but only new previously non-existent providers. - if output, err := runTerraformCommand(ctx, opts, args, env); err != nil { - return output, err + + for _, args := range commandsArgs { + if output, err := runTerraformCommand(ctx, opts, args, env); err != nil { + return output, err + } } caches := cache.providerService.WaitForCacheReady(cacheRequestID) @@ -176,6 +191,13 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti cloneOpts.WorkingDir = opts.WorkingDir maps.Copy(cloneOpts.Env, env) + if util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock { + return &shell.CmdOutput{}, nil + } + + if skipRunTargetCommand { + return &shell.CmdOutput{}, nil + } return shell.RunTerraformCommandWithOutput(ctx, cloneOpts, args...) } @@ -245,7 +267,7 @@ func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, args []string, envs map[string]string) (*shell.CmdOutput, error) { // We use custom writer in order to trap the log from `terraform providers lock -platform=provider-cache` command, which terraform considers an error, but to us a success. - errWriter := util.NewTrapWriter(opts.ErrWriter, HTTPStatusCacheProviderReg) + errWriter := util.NewTrapWriter(opts.ErrWriter, httpStatusCacheProviderReg) // add -no-color flag to args if it was set in Terragrunt arguments if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) { @@ -262,7 +284,7 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a maps.Copy(cloneOpts.Env, envs) } - // If the Terraform error matches `HTTPStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. + // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is. if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 { return output, err } @@ -290,3 +312,32 @@ func providerCacheEnvironment(opts *options.TerragruntOptions, cliConfigFile str return envs } + +// convertToMultipleCommandsByPlatforms converts `providers lock -platform=.. -platform=..` command into multiple commands that include only one platform. +// for example: +// `providers lock -platform=linux_amd64 -platform=darwin_arm64 -platform=freebsd_amd64` +// to +// `providers lock -platform=linux_amd64`, +// `providers lock -platform=darwin_arm64`, +// `providers lock -platform=freebsd_amd64` +func convertToMultipleCommandsByPlatforms(args []string) [][]string { + var ( + filteredArgs []string + platformArgs []string + commandsArgs [][]string + ) + + for _, arg := range args { + if strings.HasPrefix(arg, terraform.FlagNamePlatform) { + platformArgs = append(platformArgs, arg) + } else { + filteredArgs = append(filteredArgs, arg) + } + } + + for _, platformArg := range platformArgs { + commandsArgs = append(commandsArgs, append(filteredArgs, platformArg)) + } + + return commandsArgs +} diff --git a/docs/_docs/02_features/provider-cache.md b/docs/_docs/02_features/provider-cache.md index 70c99a5e7..b2d4eeacf 100644 --- a/docs/_docs/02_features/provider-cache.md +++ b/docs/_docs/02_features/provider-cache.md @@ -106,6 +106,15 @@ Some plugins for some operating systems may not be available in the remote regis Terraform has an official documented setting [network_mirror](https://developer.hashicorp.com/terraform/cli/config/config-file#network_mirror), that works great, but has one major drawback for the local cache server - the need to use https connection with a trusted certificate. Fortunately, there is another way - using the undocumented [host](https://github.com/hashicorp/terraform/issues/28309) setting, which allows Terraform to create connections to the caching server over HTTP. +#### Provider Cache with `providers lock` command + +If you run `providers lock` with enabled Terragrunt Provider Cache, Terragrunt creates the provider cache and generates the lock file on its own, without running `terraform providers lock` at all. + +```shell +terragrunt providers lock -platform=linux_amd64 -platform=darwin_arm64 -platform=freebsd_amd64 \ +--terragrunt-provider-cache +``` + ### Configure the Terragrunt Cache Provider Since Terragrunt Provider Cache is essentially a Private Registry server that accepts requests from Terraform, downloads and saves providers to the cache directory, there are a few more flags that are unlikely to be needed, but are useful to know about: diff --git a/terraform/getproviders/lock.go b/terraform/getproviders/lock.go index cba13a917..6ab3ad78f 100644 --- a/terraform/getproviders/lock.go +++ b/terraform/getproviders/lock.go @@ -3,11 +3,14 @@ package getproviders import ( + "bytes" "context" + "encoding/json" "os" "path/filepath" "sort" "strings" + "unicode" "github.com/gruntwork-io/go-commons/errors" "github.com/gruntwork-io/terragrunt/pkg/log" @@ -59,8 +62,7 @@ func updateLockfile(ctx context.Context, file *hclwrite.File, providers []Provid providerBlock := file.Body().FirstMatchingBlock("provider", []string{provider.Address()}) if providerBlock != nil { // update the existing provider block - err := updateProviderBlock(ctx, providerBlock, provider) - if err != nil { + if err := updateProviderBlock(ctx, providerBlock, provider); err != nil { return err } } else { @@ -68,8 +70,7 @@ func updateLockfile(ctx context.Context, file *hclwrite.File, providers []Provid file.Body().AppendNewline() providerBlock = file.Body().AppendNewBlock("provider", []string{provider.Address()}) - err := updateProviderBlock(ctx, providerBlock, provider) - if err != nil { + if err := updateProviderBlock(ctx, providerBlock, provider); err != nil { return err } } @@ -80,15 +81,9 @@ func updateLockfile(ctx context.Context, file *hclwrite.File, providers []Provid // updateProviderBlock updates the provider block in the dependency lock file. func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, provider Provider) error { - versionAttr := providerBlock.Body().GetAttribute("version") - if versionAttr != nil { - // a version attribute found - versionVal := getAttributeValueAsUnquotedString(versionAttr) - log.Debugf("Check provider version in lock file: address = %s, lock = %s, config = %s", provider.Address(), versionVal, provider.Version()) - if versionVal == provider.Version() { - // Avoid unnecessary recalculations if no version change - return nil - } + hashes, err := getExistingHashes(providerBlock, provider) + if err != nil { + return err } providerBlock.Body().SetAttributeValue("version", cty.StringVal(provider.Version())) @@ -100,7 +95,7 @@ func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, pro if err != nil { return err } - hashes := []Hash{h1Hash} + newHashes := []Hash{h1Hash} documentSHA256Sums, err := provider.DocumentSHA256Sums(ctx) if err != nil { @@ -108,7 +103,14 @@ func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, pro } if documentSHA256Sums != nil { zipHashes := DocumentHashes(documentSHA256Sums) - hashes = append(hashes, zipHashes...) + newHashes = append(newHashes, zipHashes...) + } + + // merge with existing hashes + for _, newHashe := range newHashes { + if !util.ListContainsElement(hashes, newHashe) { + hashes = append(hashes, newHashe) + } } slices.Sort(hashes) @@ -117,6 +119,35 @@ func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, pro return nil } +func getExistingHashes(providerBlock *hclwrite.Block, provider Provider) ([]Hash, error) { + versionAttr := providerBlock.Body().GetAttribute("version") + if versionAttr == nil { + return nil, nil + } + + var hashes []Hash + + // a version attribute found + versionVal := getAttributeValueAsUnquotedString(versionAttr) + log.Debugf("Check provider version in lock file: address = %s, lock = %s, config = %s", provider.Address(), versionVal, provider.Version()) + + if versionVal == provider.Version() { + // if version is equal, get already existing hashes from lock file to merge. + if attr := providerBlock.Body().GetAttribute("hashes"); attr != nil { + vals, err := getAttributeValueAsSlice(attr) + if err != nil { + return nil, err + } + + for _, val := range vals { + hashes = append(hashes, Hash(val)) + } + } + } + + return hashes, nil +} + // getAttributeValueAsString returns a value of Attribute as string. There is no way to get value as string directly, so we parses tokens of Attribute and build string representation. func getAttributeValueAsUnquotedString(attr *hclwrite.Attribute) string { // find TokenEqual @@ -132,6 +163,27 @@ func getAttributeValueAsUnquotedString(attr *hclwrite.Attribute) string { return value } +// getAttributeValueAsSlice returns a value of Attribute as slice. +func getAttributeValueAsSlice(attr *hclwrite.Attribute) ([]string, error) { + expr := attr.Expr() + exprTokens := expr.BuildTokens(nil) + + valBytes := bytes.TrimFunc(exprTokens.Bytes(), func(r rune) bool { + if unicode.IsSpace(r) || r == ']' || r == ',' { + return true + } + return false + }) + valBytes = append(valBytes, ']') + + var val []string + + if err := json.Unmarshal(valBytes, &val); err != nil { + return nil, errors.WithStackTrace(err) + } + return val, nil +} + // tokensForListPerLine builds a hclwrite.Tokens for a given hashes, but breaks the line for each element. func tokensForListPerLine(hashes []Hash) hclwrite.Tokens { // The original TokensForValue implementation does not break line by line for hashes, so we build a token sequence by ourselves. diff --git a/terraform/terraform.go b/terraform/terraform.go index 8b07ca907..5e6dbb3d0 100644 --- a/terraform/terraform.go +++ b/terraform/terraform.go @@ -28,6 +28,9 @@ const ( // `apply -destroy` is alias for `destroy` FlagNameDestroy = "-destroy" + // `platform` is a flag used with the `providers lock` command. + FlagNamePlatform = "-platform" + EnvNameTFCLIConfigFile = "TF_CLI_CONFIG_FILE" EnvNameTFPluginCacheDir = "TF_PLUGIN_CACHE_DIR" EnvNameTFPluginCacheMayBreakDependencyLockFile = "TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE" diff --git a/test/fixture-provider-cache/multiple-platforms/app1/main.tf b/test/fixture-provider-cache/multiple-platforms/app1/main.tf new file mode 100644 index 000000000..f4634f4e3 --- /dev/null +++ b/test/fixture-provider-cache/multiple-platforms/app1/main.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.36.0" + } + azure = { + source = "hashicorp/azurerm" + version = "3.95.0" + } + } +} diff --git a/test/fixture-provider-cache/multiple-platforms/app1/terragrunt.hcl b/test/fixture-provider-cache/multiple-platforms/app1/terragrunt.hcl new file mode 100644 index 000000000..bb7b160de --- /dev/null +++ b/test/fixture-provider-cache/multiple-platforms/app1/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-provider-cache/multiple-platforms/app2/main.tf b/test/fixture-provider-cache/multiple-platforms/app2/main.tf new file mode 100644 index 000000000..f4634f4e3 --- /dev/null +++ b/test/fixture-provider-cache/multiple-platforms/app2/main.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.36.0" + } + azure = { + source = "hashicorp/azurerm" + version = "3.95.0" + } + } +} diff --git a/test/fixture-provider-cache/multiple-platforms/app2/terragrunt.hcl b/test/fixture-provider-cache/multiple-platforms/app2/terragrunt.hcl new file mode 100644 index 000000000..bb7b160de --- /dev/null +++ b/test/fixture-provider-cache/multiple-platforms/app2/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/fixture-provider-cache/multiple-platforms/app3/main.tf b/test/fixture-provider-cache/multiple-platforms/app3/main.tf new file mode 100644 index 000000000..f4634f4e3 --- /dev/null +++ b/test/fixture-provider-cache/multiple-platforms/app3/main.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.36.0" + } + azure = { + source = "hashicorp/azurerm" + version = "3.95.0" + } + } +} diff --git a/test/fixture-provider-cache/multiple-platforms/app3/terragrunt.hcl b/test/fixture-provider-cache/multiple-platforms/app3/terragrunt.hcl new file mode 100644 index 000000000..bb7b160de --- /dev/null +++ b/test/fixture-provider-cache/multiple-platforms/app3/terragrunt.hcl @@ -0,0 +1 @@ +# Intentionally empty diff --git a/test/integration_test.go b/test/integration_test.go index 69fbceeaf..ef6c32697 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -32,6 +32,8 @@ import ( terraws "github.com/gruntwork-io/terratest/modules/aws" "github.com/gruntwork-io/terratest/modules/git" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/api/iterator" @@ -60,6 +62,7 @@ const ( TERRAFORM_REMOTE_STATE_GCP_REGION = "eu" TEST_FIXTURE_PATH = "fixture/" TEST_FIXTURE_INIT_ONCE = "fixture-init-once" + TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS = "fixture-provider-cache/multiple-platforms" TEST_FIXTURE_PROVIDER_CACHE_DIRECT = "fixture-provider-cache/direct" TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR = "fixture-provider-cache/filesystem-mirror" TEST_FIXTURE_DESTROY_ORDER = "fixture-destroy-order" @@ -207,6 +210,65 @@ const ( fixtureDownload = "fixture-download" ) +func TestTerragruntProviderCacheMultiplePlatforms(t *testing.T) { + cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS) + tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS) + + cacheDir, err := util.GetCacheDir() + require.NoError(t, err) + providerCacheDir := filepath.Join(cacheDir, "provider-cache-multiple-platforms") + + var ( + platforms = []string{"linux_amd64", "darwin_arm64"} + platformsArgs []string + ) + + for _, platform := range platforms { + platformsArgs = append(platformsArgs, fmt.Sprintf("-platform=%s", platform)) + } + + runTerragrunt(t, fmt.Sprintf("terragrunt run-all providers lock %s --terragrunt-no-auto-init --terragrunt-provider-cache --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", strings.Join(platformsArgs, " "), providerCacheDir, rootPath)) + + providers := []string{ + "hashicorp/aws/5.36.0", + "hashicorp/azurerm/3.95.0", + } + + registryName := "registry.opentofu.org" + if isTerraform() { + registryName = "registry.terraform.io" + } + + for _, appName := range []string{"app1", "app2", "app3"} { + appPath := filepath.Join(rootPath, appName) + assert.True(t, util.FileExists(appPath)) + + lockfilePath := filepath.Join(appPath, ".terraform.lock.hcl") + lockfileContent, err := os.ReadFile(lockfilePath) + require.NoError(t, err) + + lockfile, diags := hclwrite.ParseConfig(lockfileContent, lockfilePath, hcl.Pos{Line: 1, Column: 1}) + require.False(t, diags.HasErrors()) + require.NotNil(t, lockfile) + + for _, provider := range providers { + provider := path.Join(registryName, provider) + + providerBlock := lockfile.Body().FirstMatchingBlock("provider", []string{filepath.Dir(provider)}) + assert.NotNil(t, providerBlock) + + providerPath := filepath.Join(providerCacheDir, provider) + assert.True(t, util.FileExists(providerPath)) + + for _, platform := range platforms { + platformPath := filepath.Join(providerPath, platform) + assert.True(t, util.FileExists(platformPath)) + } + } + } +} + func TestTerragruntInitOnce(t *testing.T) { t.Parallel()