Skip to content

Commit

Permalink
Support all provider_installation methods with provider cache (#3133)
Browse files Browse the repository at this point in the history 
* feat: generating .terraform.lock.hcl

* chore: code improvements

* chore: test improvements

* fix: unit test

* chore: UpdateLockfile unit test

* chore: update TestTerragruntProviderCache

* fix: unit test

* chore: test data

* fix: test

* fix: linter tip

* fix: TestTerragruntProviderCache test

* chore: update docs

* chore: update comment

* chore: comments update

* chore: comment update

* chore: comment update

* chore: added network_mirror handler

* chore: cliconfig package improvements

* chore: cache provider handlers implementation

* chore: CLI config merge improvements

* fix: tests

* fix: tests

* fix: tests

* chore: terraform command output

* chore: temp dir for archvies and lock files

* chore: log level

* chore: don't fail if there are no signatures

* chore: remove commented code

* chore: not fail on first error

* chore: filesystem handler implementation

* chore: default method

* fix: unit test

* chore: add tests

* fix: duplicating

* fix: tests

* fix: tests

* chore: code optimization

* chore: code optimization

* fix: invalid memory address

* fix: invalid memory address

* fix: invalid memory address

* fix: invalid memory address

* chore: testdata

* chore: rename test

* chore: code improvements

* fix: misspelling
  • Loading branch information
@levkohimins
levkohimins authored May 18, 2024
1 parent a549c3b commit 7e283d7
Show file tree
Hide file tree
Showing 85 changed files with 1,483 additions and 634 deletions.
90 changes: 63 additions & 27 deletions cli/provider_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ import (
"github.com/gruntwork-io/terragrunt/shell"
"github.com/gruntwork-io/terragrunt/terraform"
"github.com/gruntwork-io/terragrunt/terraform/cache"
"github.com/gruntwork-io/terragrunt/terraform/cache/controllers"
"github.com/gruntwork-io/terragrunt/terraform/cache/handlers"
"github.com/gruntwork-io/terragrunt/terraform/cache/services"
"github.com/gruntwork-io/terragrunt/terraform/cliconfig"
"github.com/gruntwork-io/terragrunt/terraform/getproviders"
"github.com/gruntwork-io/terragrunt/util"
Expand All @@ -29,6 +29,13 @@ import (
const (
// The paths to the automatically generated local CLI configs
localCLIFilename = ".terraformrc"

// The status returned when making a request to the caching provider.
// It is needed to prevent further loading of providers by terraform, and at the same time make sure that the request was processed successfully.
cacheProviderHTTPStatusCode = http.StatusLocked

// Authentication type on the Terragrunt Provider Cache server.
apiKeyAuth = "x-api-key"
)

var (
Expand All @@ -46,11 +53,13 @@ var (
// │ provider registry for registry.terraform.io/snowflake-labs/snowflake: 423
// │ Locked
// ╵
HTTPStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(controllers.HTTPStatusCacheProvider) + `.*` + http.StatusText(controllers.HTTPStatusCacheProvider))
HTTPStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(cacheProviderHTTPStatusCode) + `.*` + http.StatusText(cacheProviderHTTPStatusCode))
)

type ProviderCache struct {
*cache.Server
cliCfg *cliconfig.Config
providerService *services.ProviderService
}

func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, error) {
Expand All @@ -73,24 +82,56 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
opts.ProviderCacheToken = uuid.New().String()
}
// Currently, the cache server only supports the `x-api-key` token.
if !strings.HasPrefix(strings.ToLower(opts.ProviderCacheToken), handlers.AuthorizationApiKeyHeaderName+":") {
opts.ProviderCacheToken = fmt.Sprintf("%s:%s", handlers.AuthorizationApiKeyHeaderName, opts.ProviderCacheToken)
if !strings.HasPrefix(strings.ToLower(opts.ProviderCacheToken), apiKeyAuth+":") {
opts.ProviderCacheToken = fmt.Sprintf("%s:%s", apiKeyAuth, opts.ProviderCacheToken)
}

userProviderDir, err := cliconfig.UserProviderDir()
if err != nil {
return nil, err
}
providerService := services.NewProviderService(opts.ProviderCacheDir, userProviderDir)

cliCfg, err := cliconfig.LoadUserConfig()
if err != nil {
return nil, err
}

var (
providerHandlers []handlers.ProviderHandler
excludeAddrs []string
)

for _, registryName := range opts.ProviderCacheRegistryNames {
excludeAddrs = append(excludeAddrs, fmt.Sprintf("%s/*/*", registryName))
}

for _, method := range cliCfg.ProviderInstallation.Methods {
switch method := method.(type) {
case *cliconfig.ProviderInstallationFilesystemMirror:
providerHandlers = append(providerHandlers, handlers.NewProviderFilesystemMirrorHandler(providerService, cacheProviderHTTPStatusCode, method))
case *cliconfig.ProviderInstallationNetworkMirror:
providerHandlers = append(providerHandlers, handlers.NewProviderNetworkMirrorHandler(providerService, cacheProviderHTTPStatusCode, method))
case *cliconfig.ProviderInstallationDirect:
providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, method))
}
method.AppendExclude(excludeAddrs)
}
providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect)))

cache := cache.NewServer(
cache.WithHostname(opts.ProviderCacheHostname),
cache.WithPort(opts.ProviderCachePort),
cache.WithToken(opts.ProviderCacheToken),
cache.WithUserProviderDir(userProviderDir),
cache.WithProviderCacheDir(opts.ProviderCacheDir),
cache.WithServices(providerService),
cache.WithProviderHandlers(providerHandlers...),
)

return &ProviderCache{Server: cache}, nil
return &ProviderCache{
Server: cache,
cliCfg: cliCfg,
providerService: providerService,
}, nil
}

func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *options.TerragruntOptions, args []string) (*shell.CmdOutput, error) {
Expand All @@ -117,11 +158,11 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
// Before each init, we warm up the global cache to ensure that all necessary providers are cached.
// To do this we are using 'terraform providers lock' to force TF to request all the providers from our TG cache, and that's how we know what providers TF needs, and can load them into the cache.
// It's low cost operation, because it does not cache the same provider twice, but only new previously non-existent providers.
if err := runTerraformCommand(ctx, opts, args, env); err != nil {
return nil, err
if output, err := runTerraformCommand(ctx, opts, args, env); err != nil {
return output, err
}

caches := cache.Provider.WaitForCacheReady(cacheRequestID)
caches := cache.providerService.WaitForCacheReady(cacheRequestID)
if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil {
return nil, err
}
Expand Down Expand Up @@ -168,10 +209,7 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
// 1. If `cacheRequestID` is set, `terraform init` does _not_ use the provider cache directory, the cache server creates a cache for requested providers and returns HTTP status 423. Since for each module we create the CLI config, using `cacheRequestID` we have the opportunity later retrieve from the cache server exactly those cached providers that were requested by `terraform init` using this configuration.
// 2. If `cacheRequestID` is empty, 'terraform init` uses provider cache directory, the cache server acts as a proxy.
func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions, filename string, cacheRequestID string) error {
cfg, err := cliconfig.LoadUserConfig()
if err != nil {
return err
}
cfg := cache.cliCfg.Clone()
cfg.PluginCacheDir = ""

var providerInstallationIncludes []string
Expand All @@ -180,24 +218,22 @@ func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions
providerInstallationIncludes = append(providerInstallationIncludes, fmt.Sprintf("%s/*/*", registryName))

cfg.AddHost(registryName, map[string]string{
"providers.v1": fmt.Sprintf("%s/%s/%s/", cache.ProviderURL(), cacheRequestID, registryName),
"providers.v1": fmt.Sprintf("%s/%s/%s/", cache.ProviderController.URL(), cacheRequestID, registryName),
// Since Terragrunt Provider Cache only caches providers, we need to route module requests to the original registry.
"modules.v1": fmt.Sprintf("https://%s/v1/modules", registryName),
})
}

if cacheRequestID != "" {
cfg.SetProviderInstallation(
nil,
cliconfig.NewProviderInstallationDirect(nil, nil),
)
} else {
cfg.SetProviderInstallation(
if cacheRequestID == "" {
cfg.AddProviderInstallationMethods(
cliconfig.NewProviderInstallationFilesystemMirror(opts.ProviderCacheDir, providerInstallationIncludes, nil),
cliconfig.NewProviderInstallationDirect(providerInstallationIncludes, nil),
)
}

cfg.AddProviderInstallationMethods(
cliconfig.NewProviderInstallationDirect(nil, nil),
)

if cfgDir := filepath.Dir(filename); !util.FileExists(cfgDir) {
if err := os.MkdirAll(cfgDir, os.ModePerm); err != nil {
return errors.WithStackTrace(err)
Expand All @@ -207,7 +243,7 @@ func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions
return cfg.Save(filename)
}

func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, args []string, envs map[string]string) error {
func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, args []string, envs map[string]string) (*shell.CmdOutput, error) {
// We use custom writer in order to trap the log from `terraform providers lock -platform=provider-cache` command, which terraform considers an error, but to us a success.
errWriter := util.NewTrapWriter(opts.ErrWriter, HTTPStatusCacheProviderReg)

Expand All @@ -227,11 +263,11 @@ func runTerraformCommand(ctx context.Context, opts *options.TerragruntOptions, a
}

// If the Terraform error matches `HTTPStatusCacheProviderReg` we ignore it and hide the log from users, otherwise we process the error as is.
if err := shell.RunTerraformCommand(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 {
return err
if output, err := shell.RunTerraformCommandWithOutput(ctx, cloneOpts, cloneOpts.TerraformCliArgs...); err != nil && len(errWriter.Msgs()) == 0 {
return output, err
}

return nil
return nil, nil
}

// providerCacheEnvironment returns TF_* name/value ENVs, which we use to force terraform processes to make requests through our cache server (proxy) instead of making direct requests to the origin servers.
Expand Down
71 changes: 18 additions & 53 deletions terraform/cache/server_test.go → cli/provider_cache_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package cache
package cli

import (
"context"
Expand All @@ -12,7 +12,10 @@ import (
"testing"

"github.com/google/uuid"
"github.com/gruntwork-io/terragrunt/terraform/cache"
"github.com/gruntwork-io/terragrunt/terraform/cache/handlers"
"github.com/gruntwork-io/terragrunt/terraform/cache/services"
"github.com/gruntwork-io/terragrunt/terraform/cliconfig"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
Expand All @@ -32,21 +35,21 @@ func createFakeProvider(t *testing.T, cacheDir, relativePath string) string {
return relativePath
}

func TestServer(t *testing.T) {
func TestProviderCache(t *testing.T) {
t.Parallel()

token := fmt.Sprintf("%s:%s", handlers.AuthorizationApiKeyHeaderName, uuid.New().String())
token := fmt.Sprintf("%s:%s", apiKeyAuth, uuid.New().String())

providerCacheDir, err := os.MkdirTemp("", "*")
require.NoError(t, err)

pluginCacheDir, err := os.MkdirTemp("", "*")
require.NoError(t, err)

opts := []Option{WithToken(token), WithProviderCacheDir(providerCacheDir), WithUserProviderDir(pluginCacheDir)}
opts := []cache.Option{cache.WithToken(token)}

testCases := []struct {
opts []Option
opts []cache.Option
urlPath string
expectedStatusCode int
expectedBodyReg *regexp.Regexp
Expand All @@ -59,7 +62,7 @@ func TestServer(t *testing.T) {
expectedBodyReg: regexp.MustCompile(regexp.QuoteMeta(`{"providers.v1":"/v1/providers"}`)),
},
{
opts: append(opts, WithToken("")),
opts: append(opts, cache.WithToken("")),
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/versions",
expectedStatusCode: http.StatusUnauthorized,
},
Expand All @@ -75,48 +78,6 @@ func TestServer(t *testing.T) {
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusLocked,
expectedCachePath: "registry.terraform.io/hashicorp/aws/5.36.0/darwin_arm64/terraform-provider-aws_v5.36.0_x5",
},
{
opts: opts,
urlPath: "/v1/providers/cache/registry.terraform.io/hashicorp/template/2.2.0/download/linux/amd64",
Expand All @@ -133,7 +94,7 @@ func TestServer(t *testing.T) {
opts: opts,
urlPath: "/v1/providers//registry.terraform.io/hashicorp/aws/5.36.0/download/darwin/arm64",
expectedStatusCode: http.StatusOK,
expectedBodyReg: regexp.MustCompile(`\{.*` + regexp.QuoteMeta(`"download_url":"http://127.0.0.1:`) + `\d+` + regexp.QuoteMeta(`/downloads/provider/releases.hashicorp.com/terraform-provider-aws/5.36.0/terraform-provider-aws_5.36.0_darwin_arm64.zip"`) + `.*\}`),
expectedBodyReg: regexp.MustCompile(`\{.*` + regexp.QuoteMeta(`"download_url":"http://127.0.0.1:`) + `\d+` + regexp.QuoteMeta(`/downloads/releases.hashicorp.com/terraform-provider-aws/5.36.0/terraform-provider-aws_5.36.0_darwin_arm64.zip"`) + `.*\}`),
},
}
//
Expand All @@ -149,7 +110,12 @@ func TestServer(t *testing.T) {

errGroup, ctx := errgroup.WithContext(ctx)

server := NewServer(testCase.opts...)
providerService := services.NewProviderService(providerCacheDir, pluginCacheDir)
providerHandler := handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect))

testCase.opts = append(testCase.opts, cache.WithServices(providerService), cache.WithProviderHandlers(providerHandler))

server := cache.NewServer(testCase.opts...)
ln, err := server.Listen()
require.NoError(t, err)
defer ln.Close()
Expand All @@ -158,8 +124,7 @@ func TestServer(t *testing.T) {
return server.Run(ctx, ln)
})

urlPath := server.ProviderURL()

urlPath := server.ProviderController.URL()
urlPath.Path = testCase.urlPath

req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlPath.String(), nil)
Expand All @@ -178,7 +143,7 @@ func TestServer(t *testing.T) {
assert.Regexp(t, testCase.expectedBodyReg, string(body))
}

server.Provider.WaitForCacheReady("")
providerService.WaitForCacheReady("")

if testCase.expectedCachePath != "" {
assert.FileExists(t, filepath.Join(providerCacheDir, testCase.expectedCachePath))
Expand Down
15 changes: 14 additions & 1 deletion main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package main

import (
"os"
"strings"

"github.com/gruntwork-io/go-commons/errors"
"github.com/gruntwork-io/terragrunt/cli"
"github.com/gruntwork-io/terragrunt/shell"
"github.com/gruntwork-io/terragrunt/util"
"github.com/hashicorp/go-multierror"
)

// The main entrypoint for Terragrunt
Expand All @@ -24,7 +26,7 @@ func checkForErrorsAndExit(err error) {
if err == nil {
os.Exit(0)
} else {
util.GlobalFallbackLogEntry.Debugf(errors.PrintErrorWithStackTrace(err))
util.GlobalFallbackLogEntry.Debugf(printErrorWithStackTrace(err))
util.GlobalFallbackLogEntry.Errorf(err.Error())

// exit with the underlying error code
Expand All @@ -40,3 +42,14 @@ func checkForErrorsAndExit(err error) {
}

}

func printErrorWithStackTrace(err error) string {
if err, ok := err.(*multierror.Error); ok {
var errsStr []string
for _, err := range err.Errors {
errsStr = append(errsStr, errors.PrintErrorWithStackTrace(err))
}
return strings.Join(errsStr, "\n")
}
return errors.PrintErrorWithStackTrace(err)
}
6 changes: 6 additions & 0 deletions pkg/log/exported.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,3 +131,9 @@ func Logf(level logrus.Level, format string, args ...interface{}) {
logger.Log(level, fmt.Sprintf(format, args...))
}
}

// WithField allocates a new entry and adds a field to it.
func WithField(key string, value interface{}) *logrus.Entry {
return logger.WithField(key, value)

}
Loading

0 comments on commit 7e283d7

Please sign in to comment.