Skip to content

Commit

Permalink
Engine log level propagation (#3443)
Browse files Browse the repository at this point in the history 
* Add passing of engine log level

* Add log level passing

* Setting log level

* Updated log level

* Engine docs update

* Add engine test

* Tests update

* Logs update

* Path update

* Docs update

* Add log disable

* Added flag to disable log

* Engine configuration through cli flags

* Updated engine ctx

* Tests cleanup

* tests cleanup

* Updated imports

* Lint update

* Strict lint update

* Strict lint update
  • Loading branch information
@denis256
denis256 authored Oct 1, 2024
1 parent 58e2c20 commit e3a1cf8
Show file tree
Hide file tree
Showing 11 changed files with 153 additions and 95 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ env: &env
environment:
GRUNTWORK_INSTALLER_VERSION: v0.0.39
MODULE_CI_VERSION: v0.57.0
TOFU_ENGINE_VERSION: "v0.0.4"
TOFU_ENGINE_VERSION: "v0.0.9"

defaults: &defaults
docker:
Expand Down
9 changes: 3 additions & 6 deletions cli/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,11 @@ func (app *App) RunContext(ctx context.Context, args []string) error {
}(ctx)

ctx = config.WithConfigValues(ctx)

// init engine if required
if engine.IsEngineEnabled() {
ctx = engine.WithEngineValues(ctx)
}
// configure engine context
ctx = engine.WithEngineValues(ctx)

defer func(ctx context.Context) {
if err := engine.Shutdown(ctx); err != nil {
if err := engine.Shutdown(ctx, app.opts); err != nil {
_, _ = app.ErrWriter.Write([]byte(err.Error()))
}
}(ctx)
Expand Down
43 changes: 40 additions & 3 deletions cli/commands/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,13 @@ const (
TerragruntProviderCacheRegistryNamesFlagName = "terragrunt-provider-cache-registry-names"
TerragruntProviderCacheRegistryNamesEnvName = "TERRAGRUNT_PROVIDER_CACHE_REGISTRY_NAMES"

// Engine related environment variables.

TerragruntEngineEnableEnvName = "TG_EXPERIMENTAL_ENGINE"
TerragruntEngineCachePathEnv = "TG_ENGINE_CACHE_PATH"
TerragruntEngineSkipCheckEnv = "TG_ENGINE_SKIP_CHECK"
TerragruntEngineLogLevelEnv = "TG_ENGINE_LOG_LEVEL"

HelpFlagName = "help"
VersionFlagName = "version"
)
Expand Down Expand Up @@ -350,9 +357,10 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
},
},
&cli.BoolFlag{
Name: TerragruntLogDisableFlagName,
EnvVar: TerragruntLogDisableEnvName,
Usage: "Disable logging",
Name: TerragruntLogDisableFlagName,
EnvVar: TerragruntLogDisableEnvName,
Usage: "Disable logging",
Destination: &opts.DisableLog,
Action: func(ctx *cli.Context, _ bool) error {
opts.ForwardTFStdout = true
opts.Logger.SetOptions(log.WithFormatter(&format.SilentFormatter{}))
Expand Down Expand Up @@ -492,6 +500,35 @@ func NewGlobalFlags(opts *options.TerragruntOptions) cli.Flags {
EnvVar: TerragruntAuthProviderCmdEnvName,
Usage: "The command and arguments that can be used to fetch authentication configurations.",
},
// Terragrunt engine flags
&cli.BoolFlag{
Name: TerragruntEngineEnableEnvName,
EnvVar: TerragruntEngineEnableEnvName,
Destination: &opts.EngineEnabled,
Usage: "Enable Terragrunt experimental engine.",
Hidden: true,
},
&cli.GenericFlag[string]{
Name: TerragruntEngineCachePathEnv,
EnvVar: TerragruntEngineCachePathEnv,
Destination: &opts.EngineCachePath,
Usage: "Cache path for Terragrunt engine files.",
Hidden: true,
},
&cli.BoolFlag{
Name: TerragruntEngineSkipCheckEnv,
EnvVar: TerragruntEngineSkipCheckEnv,
Destination: &opts.EngineSkipChecksumCheck,
Usage: "Skip checksum check for Terragrunt engine files.",
Hidden: true,
},
&cli.GenericFlag[string]{
Name: TerragruntEngineLogLevelEnv,
EnvVar: TerragruntEngineLogLevelEnv,
Destination: &opts.EngineLogLevel,
Usage: "Terragrunt engine log level.",
Hidden: true,
},
}

flags.Sort()
Expand Down
6 changes: 6 additions & 0 deletions docs/_docs/02_features/engine.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ To disable this feature, set the environment variable:
export TG_ENGINE_SKIP_CHECK=0
```

To configure a custom log level for the engine, set the `TG_ENGINE_LOG_LEVEL` environment variable to one of: `debug`, `info`, `warn`, `error`.

```sh
export TG_ENGINE_LOG_LEVEL=debug
```

### Engine Metadata

The `meta` block is used to pass metadata to the engine. This metadata can be used to configure the engine or pass additional information to the engine.
Expand Down
112 changes: 54 additions & 58 deletions engine/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"

Expand All @@ -38,21 +37,20 @@ import (
)

const (
engineVersion = 1
engineCookieKey = "engine"
engineCookieValue = "terragrunt"
EnableExperimentalEngineEnvName = "TG_EXPERIMENTAL_ENGINE"
DefaultCacheDir = ".cache"
EngineCacheDir = "terragrunt/plugins/iac-engine"
PrefixTrim = "terragrunt-"
FileNameFormat = "terragrunt-iac-%s_%s_%s_%s_%s"
ChecksumFileNameFormat = "terragrunt-iac-%s_%s_%s_SHA256SUMS"
EngineCachePathEnv = "TG_ENGINE_CACHE_PATH"
EngineSkipCheckEnv = "TG_ENGINE_SKIP_CHECK"
defaultEngineRepoRoot = "github.com/"
TerraformCommandContextKey engineClientsKey = iota
LocksContextKey engineLocksKey = iota
LatestVersionsContextKey engineLocksKey = iota
engineVersion = 1
engineCookieKey = "engine"
engineCookieValue = "terragrunt"

defaultCacheDir = ".cache"
defaultEngineCachePath = "terragrunt/plugins/iac-engine"
prefixTrim = "terragrunt-"
fileNameFormat = "terragrunt-iac-%s_%s_%s_%s_%s"
checksumFileNameFormat = "terragrunt-iac-%s_%s_%s_SHA256SUMS"

defaultEngineRepoRoot = "github.com/"
terraformCommandContextKey engineClientsKey = iota
locksContextKey engineLocksKey = iota
latestVersionsContextKey engineLocksKey = iota
)

type engineClientsKey byte
Expand Down Expand Up @@ -129,20 +127,16 @@ func Run(

// WithEngineValues add to context default values for engine.
func WithEngineValues(ctx context.Context) context.Context {
if !IsEngineEnabled() {
return ctx
}

ctx = context.WithValue(ctx, TerraformCommandContextKey, &sync.Map{})
ctx = context.WithValue(ctx, LocksContextKey, util.NewKeyLocks())
ctx = context.WithValue(ctx, LatestVersionsContextKey, cache.NewCache[string]("engineVersions"))
ctx = context.WithValue(ctx, terraformCommandContextKey, &sync.Map{})
ctx = context.WithValue(ctx, locksContextKey, util.NewKeyLocks())
ctx = context.WithValue(ctx, latestVersionsContextKey, cache.NewCache[string]("engineVersions"))

return ctx
}

// DownloadEngine downloads the engine for the given options.
func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error {
if !IsEngineEnabled() {
if !opts.EngineEnabled {
return nil
}

Expand All @@ -165,7 +159,7 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error
}
}

path, err := engineDir(e)
path, err := engineDir(opts)
if err != nil {
return errors.WithStackTrace(err)
}
Expand Down Expand Up @@ -225,7 +219,7 @@ func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error
}
}

if !skipEngineCheck() && checksumFile != "" && checksumSigFile != "" {
if !opts.EngineSkipChecksumCheck && checksumFile != "" && checksumSigFile != "" {
opts.Logger.Infof("Verifying checksum for %s", downloadFile)

if err := verifyFile(downloadFile, checksumFile, checksumSigFile); err != nil {
Expand Down Expand Up @@ -350,25 +344,26 @@ func extractArchive(opts *options.TerragruntOptions, downloadFile string, engine
}

// engineDir returns the directory path where engine files are stored.
func engineDir(e *options.EngineOptions) (string, error) {
if util.FileExists(e.Source) {
return filepath.Dir(e.Source), nil
func engineDir(terragruntOptions *options.TerragruntOptions) (string, error) {
engine := terragruntOptions.Engine
if util.FileExists(engine.Source) {
return filepath.Dir(engine.Source), nil
}

cacheDir := os.Getenv(EngineCachePathEnv)
if cacheDir == "" {
cacheDir := terragruntOptions.EngineCachePath
if len(cacheDir) == 0 {
homeDir, err := os.UserHomeDir()
if err != nil {
return "", errors.WithStackTrace(err)
}

cacheDir = filepath.Join(homeDir, DefaultCacheDir)
cacheDir = filepath.Join(homeDir, defaultCacheDir)
}

platform := runtime.GOOS
arch := runtime.GOARCH

return filepath.Join(cacheDir, EngineCacheDir, e.Type, e.Version, platform, arch), nil
return filepath.Join(cacheDir, defaultEngineCachePath, engine.Type, engine.Version, platform, arch), nil
}

// engineFileName returns the file name for the engine.
Expand All @@ -381,18 +376,18 @@ func engineFileName(e *options.EngineOptions) string {

platform := runtime.GOOS
arch := runtime.GOARCH
engineName = strings.TrimPrefix(engineName, PrefixTrim)
engineName = strings.TrimPrefix(engineName, prefixTrim)

return fmt.Sprintf(FileNameFormat, engineName, e.Type, e.Version, platform, arch)
return fmt.Sprintf(fileNameFormat, engineName, e.Type, e.Version, platform, arch)
}

// engineChecksumName returns the file name of engine checksum file
func engineChecksumName(e *options.EngineOptions) string {
engineName := filepath.Base(e.Source)

engineName = strings.TrimPrefix(engineName, PrefixTrim)
engineName = strings.TrimPrefix(engineName, prefixTrim)

return fmt.Sprintf(ChecksumFileNameFormat, engineName, e.Type, e.Version)
return fmt.Sprintf(checksumFileNameFormat, engineName, e.Type, e.Version)
}

// engineChecksumSigName returns the file name of engine checksum file signature
Expand Down Expand Up @@ -420,7 +415,7 @@ func isArchiveByHeader(filePath string) bool {

// engineClientsFromContext returns the engine clients map from the context.
func engineClientsFromContext(ctx context.Context) (*sync.Map, error) {
val := ctx.Value(TerraformCommandContextKey)
val := ctx.Value(terraformCommandContextKey)
if val == nil {
return nil, errors.WithStackTrace(goErrors.New("failed to fetch engine clients from context"))
}
Expand All @@ -435,7 +430,7 @@ func engineClientsFromContext(ctx context.Context) (*sync.Map, error) {

// downloadLocksFromContext returns the locks map from the context.
func downloadLocksFromContext(ctx context.Context) (*util.KeyLocks, error) {
val := ctx.Value(LocksContextKey)
val := ctx.Value(locksContextKey)
if val == nil {
return nil, errors.WithStackTrace(goErrors.New("failed to fetch engine clients from context"))
}
Expand All @@ -449,7 +444,7 @@ func downloadLocksFromContext(ctx context.Context) (*util.KeyLocks, error) {
}

func engineVersionsCacheFromContext(ctx context.Context) (*cache.Cache[string], error) {
val := ctx.Value(LatestVersionsContextKey)
val := ctx.Value(latestVersionsContextKey)
if val == nil {
return nil, errors.WithStackTrace(goErrors.New("failed to fetch engine versions cache from context"))
}
Expand All @@ -462,15 +457,9 @@ func engineVersionsCacheFromContext(ctx context.Context) (*cache.Cache[string],
return result, nil
}

// IsEngineEnabled returns true if the experimental engine is enabled.
func IsEngineEnabled() bool {
ok, _ := strconv.ParseBool(os.Getenv(EnableExperimentalEngineEnvName)) //nolint:errcheck
return ok
}

// Shutdown shuts down the experimental engine.
func Shutdown(ctx context.Context) error {
if !IsEngineEnabled() {
func Shutdown(ctx context.Context, opts *options.TerragruntOptions) error {
if !opts.EngineEnabled {
return nil
}

Expand Down Expand Up @@ -498,7 +487,7 @@ func Shutdown(ctx context.Context) error {

// createEngine create engine for working directory
func createEngine(terragruntOptions *options.TerragruntOptions) (*proto.EngineClient, *plugin.Client, error) {
path, err := engineDir(terragruntOptions.Engine)
path, err := engineDir(terragruntOptions)
if err != nil {
return nil, nil, errors.WithStackTrace(err)
}
Expand All @@ -508,7 +497,9 @@ func createEngine(terragruntOptions *options.TerragruntOptions) (*proto.EngineCl
localChecksumSigFile := filepath.Join(path, engineChecksumSigName(terragruntOptions.Engine))

// validate engine before loading if verification is not disabled
if !skipEngineCheck() && util.FileExists(localEnginePath) && util.FileExists(localChecksumFile) && util.FileExists(localChecksumSigFile) {
skipCheck := terragruntOptions.EngineSkipChecksumCheck
if !skipCheck && util.FileExists(localEnginePath) && util.FileExists(localChecksumFile) &&
util.FileExists(localChecksumSigFile) {
if err := verifyFile(localEnginePath, localChecksumFile, localChecksumSigFile); err != nil {
return nil, nil, errors.WithStackTrace(err)
}
Expand All @@ -518,10 +509,21 @@ func createEngine(terragruntOptions *options.TerragruntOptions) (*proto.EngineCl

terragruntOptions.Logger.Debugf("Creating engine %s", localEnginePath)

engineLogLevel := terragruntOptions.EngineLogLevel
if len(engineLogLevel) == 0 {
engineLogLevel = terragruntOptions.LogLevel.String()
// turn off log formatting if disabled for Terragrunt
if terragruntOptions.DisableLog {
engineLogLevel = hclog.Off.String()
}
}

logger := hclog.NewInterceptLogger(&hclog.LoggerOptions{
Level: hclog.Debug,
Level: hclog.LevelFromString(engineLogLevel),
Output: terragruntOptions.Logger.Writer(),
})

cmd := exec.Command(localEnginePath)
client := plugin.NewClient(&plugin.ClientConfig{
Logger: logger,
HandshakeConfig: plugin.HandshakeConfig{
Expand All @@ -532,7 +534,7 @@ func createEngine(terragruntOptions *options.TerragruntOptions) (*proto.EngineCl
Plugins: map[string]plugin.Plugin{
"plugin": &engine.TerragruntGRPCEngine{},
},
Cmd: exec.Command(localEnginePath),
Cmd: cmd,
GRPCDialOptions: []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
},
Expand Down Expand Up @@ -799,9 +801,3 @@ func ConvertMetaToProtobuf(meta map[string]interface{}) (map[string]*anypb.Any,

return protoMeta, nil
}

// skipChecksumCheck returns true if the engine checksum check is skipped.
func skipEngineCheck() bool {
ok, _ := strconv.ParseBool(os.Getenv(EngineSkipCheckEnv)) //nolint:errcheck
return ok
}
12 changes: 0 additions & 12 deletions engine/engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,6 @@ import (
"github.com/stretchr/testify/require"
)

func TestIsEngineEnabled(t *testing.T) {
t.Setenv("TG_EXPERIMENTAL_ENGINE", "true")

assert.True(t, engine.IsEngineEnabled())

t.Setenv("TG_EXPERIMENTAL_ENGINE", "false")
assert.False(t, engine.IsEngineEnabled())

t.Setenv("TG_EXPERIMENTAL_ENGINE", "")
assert.False(t, engine.IsEngineEnabled())
}

func TestConvertMetaToProtobuf(t *testing.T) {
t.Parallel()
meta := map[string]interface{}{
Expand Down
Loading

0 comments on commit e3a1cf8

Please sign in to comment.