Terragrunt Provider Cache with providers lock command (#3176)

* feat: caching `providers lock`

* chore: update docs

* fix: markdown

* fix: tests and sonar checks

* chore: code improvements

cli/provider_cache.go

@@ -39,7 +39,7 @@ const (
 )
 
 var (
- // HTTPStatusCacheProviderReg is regular expression to determine the success result of the command `terraform init`.
+ // httpStatusCacheProviderReg is a regular expression to determine the success result of the command `terraform init`.
  // The reg matches if the text contains "423 Locked", for example:
  //
  // - registry.terraform.io/hashicorp/template: could not query provider registry for registry.terraform.io/hashicorp/template: 423 Locked.
@@ -53,7 +53,7 @@ var (
  // │ provider registry for registry.terraform.io/snowflake-labs/snowflake: 423
  // │ Locked
  // ╵
- HTTPStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(cacheProviderHTTPStatusCode) + `.*` + http.StatusText(cacheProviderHTTPStatusCode))
+ httpStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(cacheProviderHTTPStatusCode) + `.*` + http.StatusText(cacheProviderHTTPStatusCode))
 )
 
 type ProviderCache struct {
@@ -138,17 +138,29 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  // To prevent a loop
  ctx = shell.ContextWithTerraformCommandHook(ctx, nil)
 
+ var (
+ cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename)
+ cacheRequestID = uuid.New().String()
+ env = providerCacheEnvironment(opts, cliConfigFilename)
+ commandsArgs [][]string
+ skipRunTargetCommand bool
+ )
+
  // Use Hook only for the `terraform init` command, which can be run explicitly by the user or Terragrunt's `auto-init` feature.
- if util.FirstArg(args) != terraform.CommandNameInit {
+ switch {
+ case util.FirstArg(args) == terraform.CommandNameInit:
+ // Provider caching for `terraform init` command.
+ commandsArgs = [][]string{args}
+ case util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock:
+ // Provider caching for `terraform providers lock` command.
+ commandsArgs = convertToMultipleCommandsByPlatforms(args)
+ // Since the Terragrunt provider cache server creates the cache and generates the lock file, we don't need to run the `terraform providers lock ...` command at all.
+ skipRunTargetCommand = true
+ default:
+ // skip cache creation for all other commands
  return shell.RunTerraformCommandWithOutput(ctx, opts, args...)
  }
 
- var (
- cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename)
- cacheRequestID = uuid.New().String()
- env = providerCacheEnvironment(opts, cliConfigFilename)
- )
-
  // Create terraform cli config file that enables provider caching and does not use provider cache dir
  if err := cache.createLocalCLIConfig(opts, cliConfigFilename, cacheRequestID); err != nil {
  return nil, err
@@ -158,8 +170,11 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  // Before each init, we warm up the global cache to ensure that all necessary providers are cached.
  // To do this we are using 'terraform providers lock' to force TF to request all the providers from our TG cache, and that's how we know what providers TF needs, and can load them into the cache.
  // It's low cost operation, because it does not cache the same provider twice, but only new previously non-existent providers.
- if output, err := runTerraformCommand(ctx, opts, args, env); err != nil {
- return output, err
+
+ for _, args := range commandsArgs {
+ if output, err := runTerraformCommand(ctx, opts, args, env); err != nil {
+ return output, err
+ }
  }
 
  caches := cache.providerService.WaitForCacheReady(cacheRequestID)
@@ -176,6 +191,13 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  cloneOpts.WorkingDir = opts.WorkingDir
  maps.Copy(cloneOpts.Env, env)
 
+ if util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock {
+ return &shell.CmdOutput{}, nil
+ }
+
+ if skipRunTargetCommand {
+ return &shell.CmdOutput{}, nil
+ }
  return shell.RunTerraformCommandWithOutput(ctx, cloneOpts, args...)
 }
 
@@ -245,7 +267,7 @@ func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions
 
 func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, args []string, envs map[string]string) (*shell.CmdOutput, error) {
  // We use custom writer in order to trap the log from `terraform providers lock -platform=provider-cache` command, which terraform considers an error, but to us a success.
- errWriter := util.NewTrapWriter(opts.ErrWriter, HTTPStatusCacheProviderReg)
+ errWriter := util.NewTrapWriter(opts.ErrWriter, httpStatusCacheProviderReg)
 
  // add -no-color flag to args if it was set in Terragrunt arguments
  if util.ListContainsElement(opts.TerraformCliArgs, terraform.FlagNameNoColor) {
@@ -262,7 +284,7 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a
  maps.Copy(cloneOpts.Env, envs)
  }
 
- // If the Terraform error matches `HTTPStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is.
+ // If the Terraform error matches `httpStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is.
  if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 {
  return output, err
  }
@@ -290,3 +312,32 @@ func providerCacheEnvironment(opts *options.TerragruntOptions, cliConfigFile str
 
  return envs
 }
+
+// convertToMultipleCommandsByPlatforms converts `providers lock -platform=.. -platform=..` command into multiple commands that include only one platform.
+// for example:
+// `providers lock -platform=linux_amd64 -platform=darwin_arm64 -platform=freebsd_amd64`
+// to
+// `providers lock -platform=linux_amd64`,
+// `providers lock -platform=darwin_arm64`,
+// `providers lock -platform=freebsd_amd64`
+func convertToMultipleCommandsByPlatforms(args []string) [][]string {
+ var (
+ filteredArgs []string
+ platformArgs []string
+ commandsArgs [][]string
+ )
+
+ for _, arg := range args {
+ if strings.HasPrefix(arg, terraform.FlagNamePlatform) {
+ platformArgs = append(platformArgs, arg)
+ } else {
+ filteredArgs = append(filteredArgs, arg)
+ }
+ }
+
+ for _, platformArg := range platformArgs {
+ commandsArgs = append(commandsArgs, append(filteredArgs, platformArg))
+ }
+
+ return commandsArgs
+}

docs/_docs/02_features/provider-cache.md

@@ -106,6 +106,15 @@ Some plugins for some operating systems may not be available in the remote regis
 
 Terraform has an official documented setting [network_mirror](https://developer.hashicorp.com/terraform/cli/config/config-file#network_mirror), that works great, but has one major drawback for the local cache server - the need to use https connection with a trusted certificate. Fortunately, there is another way - using the undocumented [host](https://github.com/hashicorp/terraform/issues/28309) setting, which allows Terraform to create connections to the caching server over HTTP.
 
+#### Provider Cache with `providers lock` command
+
+If you run `providers lock` with enabled Terragrunt Provider Cache, Terragrunt creates the provider cache and generates the lock file on its own, without running `terraform providers lock` at all.
+
+```shell
+terragrunt providers lock -platform=linux_amd64 -platform=darwin_arm64 -platform=freebsd_amd64 \
+--terragrunt-provider-cache
+```
+
 ### Configure the Terragrunt Cache Provider
 
 Since Terragrunt Provider Cache is essentially a Private Registry server that accepts requests from Terraform, downloads and saves providers to the cache directory, there are a few more flags that are unlikely to be needed, but are useful to know about:

terraform/getproviders/lock.go

@@ -3,11 +3,14 @@
 package getproviders
 
 import (
+ "bytes"
  "context"
+ "encoding/json"
  "os"
  "path/filepath"
  "sort"
  "strings"
+ "unicode"
 
  "github.com/gruntwork-io/go-commons/errors"
  "github.com/gruntwork-io/terragrunt/pkg/log"
@@ -59,17 +62,15 @@ func updateLockfile(ctx context.Context, file *hclwrite.File, providers []Provid
  providerBlock := file.Body().FirstMatchingBlock("provider", []string{provider.Address()})
  if providerBlock != nil {
  // update the existing provider block
- err := updateProviderBlock(ctx, providerBlock, provider)
- if err != nil {
+ if err := updateProviderBlock(ctx, providerBlock, provider); err != nil {
  return err
  }
  } else {
  // create a new provider block
  file.Body().AppendNewline()
  providerBlock = file.Body().AppendNewBlock("provider", []string{provider.Address()})
 
- err := updateProviderBlock(ctx, providerBlock, provider)
- if err != nil {
+ if err := updateProviderBlock(ctx, providerBlock, provider); err != nil {
  return err
  }
  }
@@ -80,15 +81,9 @@ func updateLockfile(ctx context.Context, file *hclwrite.File, providers []Provid
 
 // updateProviderBlock updates the provider block in the dependency lock file.
 func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, provider Provider) error {
- versionAttr := providerBlock.Body().GetAttribute("version")
- if versionAttr != nil {
- // a version attribute found
- versionVal := getAttributeValueAsUnquotedString(versionAttr)
- log.Debugf("Check provider version in lock file: address = %s, lock = %s, config = %s", provider.Address(), versionVal, provider.Version())
- if versionVal == provider.Version() {
- // Avoid unnecessary recalculations if no version change
- return nil
- }
+ hashes, err := getExistingHashes(providerBlock, provider)
+ if err != nil {
+ return err
  }
 
  providerBlock.Body().SetAttributeValue("version", cty.StringVal(provider.Version()))
@@ -100,15 +95,22 @@ func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, pro
  if err != nil {
  return err
  }
- hashes := []Hash{h1Hash}
+ newHashes := []Hash{h1Hash}
 
  documentSHA256Sums, err := provider.DocumentSHA256Sums(ctx)
  if err != nil {
  return err
  }
  if documentSHA256Sums != nil {
  zipHashes := DocumentHashes(documentSHA256Sums)
- hashes = append(hashes, zipHashes...)
+ newHashes = append(newHashes, zipHashes...)
+ }
+
+ // merge with existing hashes
+ for _, newHashe := range newHashes {
+ if !util.ListContainsElement(hashes, newHashe) {
+ hashes = append(hashes, newHashe)
+ }
  }
 
  slices.Sort(hashes)
@@ -117,6 +119,35 @@ func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, pro
  return nil
 }
 
+func getExistingHashes(providerBlock *hclwrite.Block, provider Provider) ([]Hash, error) {
+ versionAttr := providerBlock.Body().GetAttribute("version")
+ if versionAttr == nil {
+ return nil, nil
+ }
+
+ var hashes []Hash
+
+ // a version attribute found
+ versionVal := getAttributeValueAsUnquotedString(versionAttr)
+ log.Debugf("Check provider version in lock file: address = %s, lock = %s, config = %s", provider.Address(), versionVal, provider.Version())
+
+ if versionVal == provider.Version() {
+ // if version is equal, get already existing hashes from lock file to merge.
+ if attr := providerBlock.Body().GetAttribute("hashes"); attr != nil {
+ vals, err := getAttributeValueAsSlice(attr)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, val := range vals {
+ hashes = append(hashes, Hash(val))
+ }
+ }
+ }
+
+ return hashes, nil
+}
+
 // getAttributeValueAsString returns a value of Attribute as string. There is no way to get value as string directly, so we parses tokens of Attribute and build string representation.
 func getAttributeValueAsUnquotedString(attr *hclwrite.Attribute) string {
  // find TokenEqual
@@ -132,6 +163,27 @@ func getAttributeValueAsUnquotedString(attr *hclwrite.Attribute) string {
  return value
 }
 
+// getAttributeValueAsSlice returns a value of Attribute as slice.
+func getAttributeValueAsSlice(attr *hclwrite.Attribute) ([]string, error) {
+ expr := attr.Expr()
+ exprTokens := expr.BuildTokens(nil)
+
+ valBytes := bytes.TrimFunc(exprTokens.Bytes(), func(r rune) bool {
+ if unicode.IsSpace(r) || r == ']' || r == ',' {
+ return true
+ }
+ return false
+ })
+ valBytes = append(valBytes, ']')
+
+ var val []string
+
+ if err := json.Unmarshal(valBytes, &val); err != nil {
+ return nil, errors.WithStackTrace(err)
+ }
+ return val, nil
+}
+
 // tokensForListPerLine builds a hclwrite.Tokens for a given hashes, but breaks the line for each element.
 func tokensForListPerLine(hashes []Hash) hclwrite.Tokens {
  // The original TokensForValue implementation does not break line by line for hashes, so we build a token sequence by ourselves.

terraform/terraform.go

@@ -28,6 +28,9 @@ const (
  // `apply -destroy` is alias for `destroy`
  FlagNameDestroy = "-destroy"
 
+ // `platform` is a flag used with the `providers lock` command.
+ FlagNamePlatform = "-platform"
+
  EnvNameTFCLIConfigFile = "TF_CLI_CONFIG_FILE"
  EnvNameTFPluginCacheDir = "TF_PLUGIN_CACHE_DIR"
  EnvNameTFPluginCacheMayBreakDependencyLockFile = "TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE"

test/fixture-provider-cache/multiple-platforms/app1/main.tf

@@ -0,0 +1,12 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "5.36.0"
+ }
+ azure = {
+ source = "hashicorp/azurerm"
+ version = "3.95.0"
+ }
+ }
+}

test/fixture-provider-cache/multiple-platforms/app1/terragrunt.hcl

@@ -0,0 +1 @@
+# Intentionally empty

test/fixture-provider-cache/multiple-platforms/app2/main.tf

@@ -0,0 +1,12 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "5.36.0"
+ }
+ azure = {
+ source = "hashicorp/azurerm"
+ version = "3.95.0"
+ }
+ }
+}

test/fixture-provider-cache/multiple-platforms/app2/terragrunt.hcl

@@ -0,0 +1 @@
+# Intentionally empty

test/fixture-provider-cache/multiple-platforms/app3/main.tf

@@ -0,0 +1,12 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "5.36.0"
+ }
+ azure = {
+ source = "hashicorp/azurerm"
+ version = "3.95.0"
+ }
+ }
+}

test/fixture-provider-cache/multiple-platforms/app3/terragrunt.hcl

@@ -0,0 +1 @@
+# Intentionally empty