-
Notifications
You must be signed in to change notification settings - Fork 94
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
generate/hcl: support module calls through source
keyword
#130
Open
tormath1
wants to merge
9
commits into
cycloidio:master
Choose a base branch
from
tormath1:mt-56-support-remote-sources
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
9 commits
Select commit
Hold shift + click to select a range
aef5207
testdata: add Terraform from stack GKE
tormath1 425ed5c
mod: add go-getter and terracognita
tormath1 ccf8e6c
generate/hcl: fetch terraform config required modules
tormath1 6d58168
cmd/root: add `debug` option to display TF logs
tormath1 272e1f9
sum: go mod tidy
tormath1 974fd5a
fixup! mod: add go-getter and terracognita
tormath1 b386772
fixup! sum: go mod tidy
tormath1 8320f1e
fixup! cmd/root: add `debug` option to display TF logs
tormath1 0409a56
fixup! generate/hcl: fetch terraform config required modules
tormath1 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,23 +3,39 @@ package generate | |
import ( | ||
"errors" | ||
"fmt" | ||
"os" | ||
"path" | ||
"path/filepath" | ||
"strings" | ||
|
||
"github.com/adrg/xdg" | ||
"github.com/cycloidio/inframap/errcode" | ||
"github.com/cycloidio/inframap/graph" | ||
"github.com/cycloidio/inframap/provider" | ||
"github.com/hashicorp/go-getter" | ||
"github.com/hashicorp/go-version" | ||
"github.com/hashicorp/hcl/v2" | ||
"github.com/hashicorp/hcl/v2/hclsyntax" | ||
"github.com/hashicorp/hcl/v2/hclwrite" | ||
"github.com/hashicorp/terraform/configs" | ||
"github.com/hashicorp/terraform/configs/hcl2shim" | ||
"github.com/hashicorp/terraform/registry" | ||
"github.com/hashicorp/terraform/registry/regsrc" | ||
uuid "github.com/satori/go.uuid" | ||
"github.com/spf13/afero" | ||
) | ||
|
||
var ( | ||
localSourcePrefixes = []string{ | ||
"./", | ||
"../", | ||
} | ||
cachePath = path.Join(xdg.CacheHome, "inframap", "modules") | ||
) | ||
|
||
// FromHCL generates a new graph from the HCL on the path, | ||
// it can be a file or a Module/Dir | ||
func FromHCL(fs afero.Fs, path string, opt Options) (*graph.Graph, error) { | ||
func FromHCL(fs afero.Fs, p string, opt Options) (*graph.Graph, error) { | ||
parser := configs.NewParser(fs) | ||
|
||
g := graph.New() | ||
|
@@ -30,10 +46,10 @@ func FromHCL(fs afero.Fs, path string, opt Options) (*graph.Graph, error) { | |
err error | ||
) | ||
|
||
if parser.IsConfigDir(path) { | ||
mod, diags = parser.LoadConfigDir(path) | ||
if parser.IsConfigDir(p) { | ||
mod, diags = parser.LoadConfigDir(p) | ||
} else { | ||
f, dgs := parser.LoadConfigFile(path) | ||
f, dgs := parser.LoadConfigFile(p) | ||
if dgs.HasErrors() { | ||
return nil, errors.New(dgs.Error()) | ||
} | ||
|
@@ -44,6 +60,21 @@ func FromHCL(fs afero.Fs, path string, opt Options) (*graph.Graph, error) { | |
return nil, errors.New(diags.Error()) | ||
} | ||
|
||
managedResources := make(map[string]*configs.Resource) | ||
for rk, rv := range mod.ManagedResources { | ||
managedResources[rk] = rv | ||
} | ||
|
||
installedModules := make(map[string]struct{}) | ||
calls := make([]*configs.ModuleCall, 0) | ||
for _, call := range mod.ModuleCalls { | ||
calls = append(calls, call) | ||
} | ||
p, _ = filepath.Abs(p) | ||
if err := moduleInstall(calls, &managedResources, p, installedModules); err != nil { | ||
return nil, fmt.Errorf("unable to fetch all modules: %w", err) | ||
} | ||
|
||
// nodeCanID holds as key the `aws_alb.front` (graph.Node.Canonical) | ||
// and as value the UUID (graph.Node.ID) we give to it | ||
nodeCanID := make(map[string]string) | ||
|
@@ -64,7 +95,7 @@ func FromHCL(fs afero.Fs, path string, opt Options) (*graph.Graph, error) { | |
} | ||
} | ||
|
||
for rk, rv := range mod.ManagedResources { | ||
for rk, rv := range managedResources { | ||
pv, rs, err := getProviderAndResource(rk, opt) | ||
if err != nil { | ||
if errors.Is(err, errcode.ErrProviderNotFound) { | ||
|
@@ -277,3 +308,139 @@ func checkHCLProviders(mod *configs.Module, opt Options) (Options, error) { | |
|
||
return opt, nil | ||
} | ||
|
||
// moduleInstall will recursively walk through the module calls required by the Terraform config, it will store the downloaded module | ||
// in $XDG_CACHE directory and stop once all the required modules have been downloaded. | ||
func moduleInstall(calls []*configs.ModuleCall, mRes *map[string]*configs.Resource, pwd string, installedModules map[string]struct{}) error { | ||
// stop condition, if there is no module to | ||
// fetch we stop | ||
if len(calls) == 0 { | ||
return nil | ||
} | ||
|
||
call := calls[0] | ||
name := call.Name | ||
|
||
// we check if the module is already installed | ||
// or not | ||
if _, ok := installedModules[name]; ok { | ||
return nil | ||
} | ||
|
||
var ( | ||
src string = call.SourceAddr | ||
vers string | ||
) | ||
|
||
// we check if the module is a Terraform registry module | ||
// in order to get its source address from Terraform registry | ||
if regMod, err := regsrc.ParseModuleSource(src); err == nil { | ||
client := registry.NewClient(nil, nil) | ||
// we get the list of available module versions | ||
resp, err := client.ModuleVersions(regMod) | ||
if err != nil { | ||
return fmt.Errorf("unable to get module versions: %w", err) | ||
} | ||
|
||
if len(resp.Modules) < 1 { | ||
return fmt.Errorf("unable to find suitable versions") | ||
} | ||
meta := resp.Modules[0] | ||
|
||
var ( | ||
latest *version.Version | ||
// match holds the version matching the | ||
// source constraints set in the module call | ||
match *version.Version | ||
) | ||
for _, vers := range meta.Versions { | ||
v, err := version.NewVersion(vers.Version) | ||
if err != nil { | ||
return fmt.Errorf("unable to create version from string: %w", err) | ||
} | ||
|
||
if latest == nil || v.GreaterThan(latest) { | ||
latest = v | ||
} | ||
|
||
if call.Version.Required.Check(v) { | ||
if match == nil || v.GreaterThan(match) { | ||
match = v | ||
} | ||
} | ||
} | ||
Comment on lines
+356
to
+371
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Comment a bit this logic. |
||
|
||
vers = match.String() | ||
// we finally get the module location, it will return | ||
// a string `go-getter` compliant | ||
src, err = client.ModuleLocation(regMod, vers) | ||
if err != nil { | ||
return fmt.Errorf("unable to fetch module location: %w", err) | ||
} | ||
} | ||
|
||
// since go-getter does not support yet in-memory fs, | ||
// we need to initialize the parser using actual fs | ||
// https://github.com/hashicorp/go-getter/issues/83 | ||
pars := configs.NewParser(nil) | ||
|
||
// we check if the module is a local one by checking | ||
// its prefix "./", "../", etc. | ||
var isLocal bool | ||
for _, prefix := range localSourcePrefixes { | ||
if strings.HasPrefix(src, prefix) { | ||
isLocal = true | ||
} | ||
} | ||
xescugc marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
var ( | ||
m *configs.Module | ||
diags hcl.Diagnostics | ||
) | ||
|
||
// the module is not a local one or a Terraform registry one | ||
// it should be handle by `go-getter` | ||
if !isLocal { | ||
dst := path.Join(cachePath, fmt.Sprintf("%s-%s", name, vers)) | ||
// TODO: we should add a logic to invalidate | ||
// the cache | ||
if _, err := os.Stat(dst); os.IsNotExist(err) { | ||
client := &getter.Client{ | ||
Src: src, | ||
Dst: dst, | ||
Pwd: pwd, | ||
Mode: getter.ClientModeDir, | ||
} | ||
if err := client.Get(); err != nil { | ||
return fmt.Errorf("unable to get remote module: %w", err) | ||
} | ||
} | ||
m, diags = pars.LoadConfigDir(dst) | ||
} else { | ||
m, diags = pars.LoadConfigDir(path.Join(pwd, src)) | ||
} | ||
if diags.HasErrors() { | ||
return fmt.Errorf("unable to load config directory: %s", diags.Error()) | ||
} | ||
|
||
// fill the final map of managed resources | ||
// using the config freshly loaded | ||
for rk, rv := range m.ManagedResources { | ||
(*mRes)[rk] = rv | ||
} | ||
|
||
Comment on lines
+426
to
+431
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Potentially prefix it with them module if more than one or we'll have collision. |
||
// keep a trace of the imported / loaded module | ||
// to avoid infinite recursion | ||
installedModules[name] = struct{}{} | ||
|
||
// create the next slice of module calls to | ||
// check before merging it with the current we | ||
// still have | ||
next := make([]*configs.ModuleCall, 0) | ||
for _, call := range m.ModuleCalls { | ||
next = append(next, call) | ||
} | ||
calls = append(calls[1:len(calls)], next...) | ||
|
||
return moduleInstall(calls, mRes, pwd, installedModules) | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
One thing we are doing right now on the State is check if we have to prefix a resource with the module because the same resource could exist on another module
aws_instance.front
could be in multiple modules.We only do that when we have more than 1 module we should try to do the same as we'll find the same issue.