Fixing registry host discovery (Terragrunt Provider Cache) (#3432)

* fix: host discovery

* fix: tests

* fix: grammar

* chore: code improvements

* fix: strict-lint

.strict.golangci.yml

@@ -10,6 +10,9 @@ output:
  print-linter-name: true
 
 linters-settings:
+ wrapcheck:
+ ignorePackageGlobs:
+ - github.com/gruntwork-io/terragrunt/*
  lll:
  line-length: 140
  staticcheck:

cli/provider_cache.go

@@ -2,11 +2,9 @@ package cli
 
 import (
  "context"
- liberrors "errors"
  "fmt"
  "io"
  "net/http"
- "net/url"
  "os"
  "path/filepath"
  "regexp"
@@ -16,6 +14,7 @@ import (
  "github.com/google/uuid"
  "github.com/gruntwork-io/go-commons/errors"
  "github.com/gruntwork-io/terragrunt/options"
+ "github.com/gruntwork-io/terragrunt/pkg/cli"
  "github.com/gruntwork-io/terragrunt/shell"
  "github.com/gruntwork-io/terragrunt/terraform"
  "github.com/gruntwork-io/terragrunt/terraform/cache"
@@ -147,23 +146,29 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
  }, nil
 }
 
-func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *options.TerragruntOptions, args []string) (*util.CmdOutput, error) {
+// TerraformCommandHook warms up the providers cache, creates `.terraform.lock.hcl` and runs the `tofu/terraform init`
+// command with using this cache. Used as a hook function that is called after running the target tofu/terraform command.
+// For example, if the target command is `tofu plan`, it will be intercepted before it is run in the `/shell` package,
+// then control will be passed to this function to init the working directory using cached providers.
+func (cache *ProviderCache) TerraformCommandHook(
+ ctx context.Context,
+ opts *options.TerragruntOptions,
+ args cli.Args,
+) (*util.CmdOutput, error) {
  // To prevent a loop
  ctx = shell.ContextWithTerraformCommandHook(ctx, nil)
 
  var (
  cliConfigFilename = filepath.Join(opts.WorkingDir, localCLIFilename)
- cacheRequestID = uuid.New().String()
- envs = providerCacheEnvironment(opts, cliConfigFilename)
- commandsArgs = convertToMultipleCommandsByPlatforms(args)
+ env = providerCacheEnvironment(opts, cliConfigFilename)
  skipRunTargetCommand bool
  )
 
  // Use Hook only for the `terraform init` command, which can be run explicitly by the user or Terragrunt's `auto-init` feature.
  switch {
- case util.FirstArg(args) == terraform.CommandNameInit:
+ case args.CommandName() == terraform.CommandNameInit:
  // Provider caching for `terraform init` command.
- case util.FirstArg(args) == terraform.CommandNameProviders && util.SecondArg(args) == terraform.CommandNameLock:
+ case args.CommandName() == terraform.CommandNameProviders && args.SubCommandName() == terraform.CommandNameLock:
  // Provider caching for `terraform providers lock` command.
  // Since the Terragrunt provider cache server creates the cache and generates the lock file, we don't need to run the `terraform providers lock ...` command at all.
  skipRunTargetCommand = true
@@ -172,6 +177,29 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  return shell.RunTerraformCommandWithOutput(ctx, opts, args...)
  }
 
+ if output, err := cache.warmUpCache(ctx, opts, cliConfigFilename, args, env); err != nil {
+ return output, err
+ }
+
+ if skipRunTargetCommand {
+ return &util.CmdOutput{}, nil
+ }
+
+ return cache.runTerraformWithCache(ctx, opts, cliConfigFilename, args, env)
+}
+
+func (cache *ProviderCache) warmUpCache(
+ ctx context.Context,
+ opts *options.TerragruntOptions,
+ cliConfigFilename string,
+ args cli.Args,
+ env map[string]string,
+) (*util.CmdOutput, error) {
+ var (
+ cacheRequestID = uuid.New().String()
+ commandsArgs = convertToMultipleCommandsByPlatforms(args)
+ )
+
  // Create terraform cli config file that enables provider caching and does not use provider cache dir
  if err := cache.createLocalCLIConfig(ctx, opts, cliConfigFilename, cacheRequestID); err != nil {
  return nil, err
@@ -183,7 +211,7 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  // It's low cost operation, because it does not cache the same provider twice, but only new previously non-existent providers.
 
  for _, args := range commandsArgs {
- if output, err := runTerraformCommand(ctx, opts, args, envs); err != nil {
+ if output, err := runTerraformCommand(ctx, opts, args, env); err != nil {
  return output, err
  }
  }
@@ -193,10 +221,18 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  return nil, err
  }
 
- if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil {
- return nil, err
- }
+ err = getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches)
+
+ return nil, err
+}
 
+func (cache *ProviderCache) runTerraformWithCache(
+ ctx context.Context,
+ opts *options.TerragruntOptions,
+ cliConfigFilename string,
+ args cli.Args,
+ env map[string]string,
+) (*util.CmdOutput, error) {
  // Create terraform cli config file that uses provider cache dir
  if err := cache.createLocalCLIConfig(ctx, opts, cliConfigFilename, ""); err != nil {
  return nil, err
@@ -208,11 +244,7 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
  }
 
  cloneOpts.WorkingDir = opts.WorkingDir
- cloneOpts.Env = envs
-
- if skipRunTargetCommand {
- return &util.CmdOutput{}, nil
- }
+ cloneOpts.Env = env
 
  return shell.RunTerraformCommandWithOutput(ctx, cloneOpts, args...)
 }
@@ -255,22 +287,15 @@ func (cache *ProviderCache) createLocalCLIConfig(ctx context.Context, opts *opti
  for _, registryName := range opts.ProviderCacheRegistryNames {
  providerInstallationIncludes = append(providerInstallationIncludes, registryName+"/*/*")
 
- urls, err := DiscoveryURL(ctx, registryName)
+ apiURLs, err := cache.Server.DiscoveryURL(ctx, registryName)
  if err != nil {
- if !liberrors.As(err, &NotFoundWellKnownURL{}) {
- return err
- }
-
- urls = DefaultRegistryURLs
- opts.Logger.Debugf("Unable to discover %q registry URLs, reason: %q, use default URLs: %s", registryName, err, urls)
- } else {
- opts.Logger.Debugf("Discovered %q registry URLs: %s", registryName, urls)
+ return err
  }
 
  cfg.AddHost(registryName, map[string]string{
- "providers.v1": fmt.Sprintf("%s/%s/%s/%s/", cache.ProviderController.URL(), cacheRequestID, url.PathEscape(urls.ProvidersV1), registryName),
+ "providers.v1": fmt.Sprintf("%s/%s/%s/", cache.ProviderController.URL(), cacheRequestID, registryName),
  // Since Terragrunt Provider Cache only caches providers, we need to route module requests to the original registry.
- "modules.v1": fmt.Sprintf("https://%s%s", registryName, urls.ModulesV1),
+ "modules.v1": fmt.Sprintf("https://%s%s", registryName, apiURLs.ModulesV1),
  })
  }
 

cli/provider_cache_test.go

@@ -5,7 +5,6 @@ import (
  "fmt"
  "io"
  "net/http"
- "net/url"
  "os"
  "path/filepath"
  "regexp"
@@ -50,53 +49,52 @@ func TestProviderCache(t *testing.T) {
 
  opts := []cache.Option{cache.WithToken(token)}
 
- registryPrefix := url.PathEscape("/v1/providers/")
-
  testCases := []struct {
  opts []cache.Option
- urlPath string
+ fullURLPath string
+ relURLPath string
  expectedStatusCode int
  expectedBodyReg *regexp.Regexp
  expectedCachePath string
  }{
  {
  opts: opts,
- urlPath:  "/.well-known/terraform.json",
+ fullURLPath: "/.well-known/terraform.json",
  expectedStatusCode: http.StatusOK,
  expectedBodyReg: regexp.MustCompile(regexp.QuoteMeta(`{"providers.v1":"/v1/providers"}`)),
  },
  {
  opts: append(opts, cache.WithToken("")),
- urlPath:  "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/aws/versions",
+ relURLPath: "/cache/registry.terraform.io/hashicorp/aws/versions",
  expectedStatusCode: http.StatusUnauthorized,
  },
  {
  opts: opts,
- urlPath:  "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/aws/versions",
+ relURLPath: "/cache/registry.terraform.io/hashicorp/aws/versions",
  expectedStatusCode: http.StatusOK,
  expectedBodyReg: regexp.MustCompile(regexp.QuoteMeta(`"version":"5.36.0","protocols":["5.0"],"platforms"`)),
  },
  {
  opts: opts,
- urlPath:  "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
+ relURLPath: "/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
  expectedStatusCode: http.StatusLocked,
  expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
  },
  {
  opts: opts,
- urlPath:  "/v1/providers/cache/" + registryPrefix + "/registry.terraform.io/hashicorp/template/2.2.0/download/linux/amd64",
+ relURLPath: "/cache/registry.terraform.io/hashicorp/template/2.2.0/download/linux/amd64",
  expectedStatusCode: http.StatusLocked,
  expectedCachePath: "registry.terraform.io/hashicorp/template/2.2.0/linux_amd64/terraform-provider-template_v2.2.0_x4",
  },
  {
  opts: opts,
- urlPath:  fmt.Sprintf("/v1/providers/cache/%s/registry.terraform.io/hashicorp/template/1234.5678.9/download/%s/%s", registryPrefix, runtime.GOOS, runtime.GOARCH),
+ relURLPath: fmt.Sprintf("/cache/registry.terraform.io/hashicorp/template/1234.5678.9/download/%s/%s", runtime.GOOS, runtime.GOARCH),
  expectedStatusCode: http.StatusLocked,
  expectedCachePath: createFakeProvider(t, pluginCacheDir, fmt.Sprintf("registry.terraform.io/hashicorp/template/1234.5678.9/%s_%s/terraform-provider-template_1234.5678.9_x5", runtime.GOOS, runtime.GOARCH)),
  },
  {
  opts: opts,
- urlPath:  "/v1/providers//" + registryPrefix + "/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
+ relURLPath: "//registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
  expectedStatusCode: http.StatusOK,
  expectedBodyReg: regexp.MustCompile(`\{.*` + regexp.QuoteMeta(`"download_url":"http://127.0.0.1:`) + `\d+` + regexp.QuoteMeta(`/downloads/releases.hashicorp.com/terraform-provider-aws/5.36.0/terraform-provider-aws_5.36.0_darwin_arm64.zip"`) + `.*\}`),
  },
@@ -129,7 +127,11 @@ func TestProviderCache(t *testing.T) {
  })
 
  urlPath := server.ProviderController.URL()
- urlPath.Path = testCase.urlPath
+ urlPath.Path += testCase.relURLPath
+
+ if testCase.fullURLPath != "" {
+ urlPath.Path = testCase.fullURLPath
+ }
 
  req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlPath.String(), nil)
  require.NoError(t, err)

pkg/cli/args.go

@@ -36,13 +36,18 @@ func (args Args) First() string {
  return args.Get(0)
 }
 
-// Last returns the last argument or a blank string
+// Second returns the first argument or a blank string.
+func (args Args) Second() string {
+ return args.Get(1)
+}
+
+// Last returns the last argument or a blank string.
 func (args Args) Last() string {
  return args.Get(len(args) - 1)
 }
 
 // Tail returns the rest of the arguments (not the first one)
-// or else an empty string slice
+// or else an empty string slice.
 func (args Args) Tail() Args {
  if args.Len() < tailMinArgsLen {
  return []string{}
@@ -101,7 +106,8 @@ func (args Args) Normalize(acts ...NormalizeActsType) Args {
  return strArgs
 }
 
-// CommandName returns the first value if it starts without a dash `-`, otherwise that means the args do not consist any command and an empty string is returned.
+// CommandName returns the first value if it starts without a dash `-`,
+// otherwise that means the args do not consist any command and an empty string is returned.
 func (args Args) CommandName() string {
  name := args.First()
 
@@ -112,6 +118,18 @@ func (args Args) CommandName() string {
  return ""
 }
 
+// SubCommandName returns the second value if it starts without a dash `-`,
+// otherwise that means the args do not consist a subcommand and an empty string is returned.
+func (args Args) SubCommandName() string {
+ name := args.Second()
+
+ if !strings.HasPrefix(name, "-") {
+ return name
+ }
+
+ return ""
+}
+
 // Contains returns true if args contains the given `target` arg.
 func (args Args) Contains(target string) bool {
  for _, arg := range args {

shell/context.go

@@ -4,6 +4,7 @@ import (
  "context"
 
  "github.com/gruntwork-io/terragrunt/internal/cache"
+ "github.com/gruntwork-io/terragrunt/pkg/cli"
  "github.com/gruntwork-io/terragrunt/util"
 
  "github.com/gruntwork-io/terragrunt/options"
@@ -18,13 +19,15 @@ const (
 
 type ctxKey byte
 
-type RunShellCommandFunc func(ctx context.Context, opts *options.TerragruntOptions, args []string) (*util.CmdOutput, error)
+// RunShellCommandFunc is a context value for `TerraformCommandContextKey` key, used to intercept shell commands.
+type RunShellCommandFunc func(ctx context.Context, opts *options.TerragruntOptions, args cli.Args) (*util.CmdOutput, error)
 
 func ContextWithTerraformCommandHook(ctx context.Context, fn RunShellCommandFunc) context.Context {
  ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName))
  return context.WithValue(ctx, TerraformCommandContextKey, fn)
 }
 
+// TerraformCommandHookFromContext returns `RunShellCommandFunc` from the context if it has been set, otherwise returns nil.
 func TerraformCommandHookFromContext(ctx context.Context) RunShellCommandFunc {
  if val := ctx.Value(TerraformCommandContextKey); val != nil {
  if val, ok := val.(RunShellCommandFunc); ok {

terraform/cache/config.go

@@ -72,7 +72,7 @@ type Config struct {
  shutdownTimeout time.Duration
 
  services []services.Service
- providerHandlers []handlers.ProviderHandler
+ providerHandlers handlers.ProviderHandlers
 
  logger log.Logger
 }