diff --git a/aws/client.go b/aws/client.go index 911bc11..bdcc551 100644 --- a/aws/client.go +++ b/aws/client.go @@ -5,6 +5,7 @@ package aws import ( "context" "fmt" + "time" "github.com/apex/log" "github.com/aws/aws-sdk-go-v2/aws/external" @@ -711,6 +712,7 @@ func NewClient(configs ...external.Config) (*Client, error) { log.WithFields(log.Fields{ "profile": profile, "region": cfg.Region, + "time": time.Now().Format("04:05.000"), }).Debugf("created new instance of AWS client") return client, nil diff --git a/go.mod b/go.mod index 5eec1ad..021978d 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/fatih/color v1.9.0 github.com/gobwas/glob v0.2.3 github.com/gruntwork-io/terratest v0.23.0 + github.com/jckuester/awstools-lib v0.0.0-20210215194522-14607cce3470 github.com/jckuester/terradozer v0.1.3 github.com/onsi/gomega v1.9.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 20c7a8a..acf7e9b 100644 --- a/go.sum +++ b/go.sum @@ -712,6 +712,30 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jckuester/awstools-lib v0.0.0-20210212200752-307022362457 h1:xf/avUnf5zhBDJCD5u6/IgT1hceQElx1q9rHW1pcvpI= +github.com/jckuester/awstools-lib v0.0.0-20210212200752-307022362457/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213113828-144d01dfb776 h1:ZisH10yJl3L1AuRmkPNhOZj6clFZX86dMUslcbAwvCw= +github.com/jckuester/awstools-lib v0.0.0-20210213113828-144d01dfb776/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213120546-79923f22ec17 h1:1qkVmHqW4kHb2iQ1vF7MwuoAb/VueOCh1j1YEeXt658= +github.com/jckuester/awstools-lib v0.0.0-20210213120546-79923f22ec17/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213122015-b91b58160d27 h1:6k7/gnLiMTSaCUi+6RE3TKnuG4bNa/H9Jj63xr3LeZ4= +github.com/jckuester/awstools-lib v0.0.0-20210213122015-b91b58160d27/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213184508-58fef899e0e7 h1:rjlUOTJ5d/vZ2P+vyDrKND4xToMI9X0hOz+6hd36Ipk= +github.com/jckuester/awstools-lib v0.0.0-20210213184508-58fef899e0e7/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213201536-b3d8a93d67a0 h1:lDkRtsLz8Rs9OSj9ChYAbfrGfk+0tBDtMO1SVtEkDSA= +github.com/jckuester/awstools-lib v0.0.0-20210213201536-b3d8a93d67a0/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213202606-fb656cd226fd h1:cHLGcwvlMpNTN+tsNCRNdy289iypeSzLzmYyM/kMIZc= +github.com/jckuester/awstools-lib v0.0.0-20210213202606-fb656cd226fd/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210213203113-a32463ae719a h1:6UYnZKnoFV3RUiN58F0Jliob5duwpgiR9Go5mLdnEk8= +github.com/jckuester/awstools-lib v0.0.0-20210213203113-a32463ae719a/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210214123157-3a7301153e57 h1:TkDopPYfCb6R5EIfzPJi8cW55yRmBbI/HYDfiauezsU= +github.com/jckuester/awstools-lib v0.0.0-20210214123157-3a7301153e57/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210214124804-d61556f035ab h1:WUR10Dj0XUMDtAFou/4LlBLHctLo5kR7q9BP9NvFlQk= +github.com/jckuester/awstools-lib v0.0.0-20210214124804-d61556f035ab/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210214131601-bbe567b52f2b h1:jitMIosIgzDzr1J6yq6/PUlOUK5cbJs3WQoCClpe380= +github.com/jckuester/awstools-lib v0.0.0-20210214131601-bbe567b52f2b/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= +github.com/jckuester/awstools-lib v0.0.0-20210215194522-14607cce3470 h1:d//j4SZKiysxLC/1/hLZ/VdQGJBrQG6hKHl6v61loWA= +github.com/jckuester/awstools-lib v0.0.0-20210215194522-14607cce3470/go.mod h1:71+ozVxBDbhfKZH5ybJVxWo+mO1SVfpGiLfwXU5qD/w= github.com/jckuester/terradozer v0.1.3 h1:xrRxr+L58QAVz5Kwq2fyWCNiK1NWOuKo8g5Q2664WZ4= github.com/jckuester/terradozer v0.1.3/go.mod h1:ER3EJojZmO2u6lfcdgnmC+Nrg/TV2T2bacY5FZpqgks= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= @@ -723,6 +747,7 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/joyent/triton-go v0.0.0-20180313100802-d8f9c0314926/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= diff --git a/main.go b/main.go index 6ca6dba..a77f2e9 100644 --- a/main.go +++ b/main.go @@ -1,23 +1,25 @@ package main import ( + "context" + "errors" "fmt" "io/ioutil" stdlog "log" "os" + "os/signal" "strings" - "text/tabwriter" "time" - "github.com/jckuester/awsls/util" - "github.com/apex/log" "github.com/apex/log/handlers/cli" aws_ssmhelpers "github.com/disneystreaming/go-ssmhelpers/aws" "github.com/fatih/color" - "github.com/jckuester/awsls/aws" + awsls "github.com/jckuester/awsls/aws" "github.com/jckuester/awsls/internal" - "github.com/jckuester/awsls/resource" + resource "github.com/jckuester/awsls/resource" + "github.com/jckuester/awstools-lib/aws" + "github.com/jckuester/awstools-lib/terraform" flag "github.com/spf13/pflag" ) @@ -121,7 +123,7 @@ func mainExitCode() int { profiles = profilesFromConfig } - clients, err := util.NewAWSClientPool(profiles, regions) + clients, err := aws.NewClientPool(profiles, regions) if err != nil { fmt.Fprint(os.Stderr, color.RedString("\nError: %s\n", err)) @@ -131,132 +133,85 @@ func mainExitCode() int { // suppress provider debug and info logs log.SetLevel(log.ErrorLevel) - clientKeys := make([]util.AWSClientKey, 0, len(clients)) + clientKeys := make([]aws.ClientKey, 0, len(clients)) for k := range clients { clientKeys = append(clientKeys, k) } + ctx := context.Background() + + // trap Ctrl+C and call cancel on the context + ctx, cancel := context.WithCancel(ctx) + signalCh := make(chan os.Signal, 1) + signal.Notify(signalCh, ignoreSignals...) + signal.Notify(signalCh, forwardSignals...) + defer func() { + signal.Stop(signalCh) + cancel() + }() + go func() { + select { + case <-signalCh: + fmt.Fprint(os.Stderr, color.RedString("\nAborting...\n")) + cancel() + case <-ctx.Done(): + } + }() + + if logDebug { + log.SetLevel(log.DebugLevel) + } + // initialize a Terraform AWS provider for each AWS client with a matching config - providers, err := util.NewProviderPool(clientKeys, "v3.16.0", "~/.awsls", 10*time.Second) + providers, err := terraform.NewProviderPool(ctx, clientKeys, "v3.16.0", "~/.awsls", 10*time.Second) if err != nil { - fmt.Fprint(os.Stderr, color.RedString("\nError: %s\n", err)) - + if !errors.Is(err, context.Canceled) { + fmt.Fprint(os.Stderr, color.RedString("\nError: %s\n", err)) + } return 1 } - defer func() { for _, p := range providers { _ = p.Close() } }() - if logDebug { - log.SetLevel(log.DebugLevel) - } - for _, rType := range matchedTypes { - var resources []aws.Resource - var hasAttrs map[string]bool - - for key, client := range clients { - err := client.SetAccountID() - if err != nil { - fmt.Fprint(os.Stderr, color.RedString("Error %s: %s\n", rType, err)) + // any provider here is sufficient to check if a resource type has attributes + p := providers[clientKeys[0]] - return 1 - } - - res, err := aws.ListResourcesByType(&client, rType) - if err != nil { - fmt.Fprint(os.Stderr, color.RedString("Error %s: %s\n", rType, err)) - - continue - } - - provider := providers[key] + hasAttrs, err := resource.HasAttributes(attributes, rType, &p) + if err != nil { + fmt.Fprint(os.Stderr, color.RedString("Error: failed to check if resource type has attribute: "+ + "%s\n", err)) + return 1 + } - hasAttrs, err = resource.HasAttributes(attributes, rType, &provider) - if err != nil { - fmt.Fprint(os.Stderr, color.RedString("Error: failed to check if resource type has attribute: "+ - "%s\n", err)) + var resources []awsls.Resource - continue - } + resourcesCh := make(chan resource.UpdatedResources, 1) + go func() { resourcesCh <- resource.ListInMultipleAccountsAndRegions(rType, hasAttrs, clients, providers) }() + select { + case <-ctx.Done(): + return 1 + case result := <-resourcesCh: + resources = result.Resources - if len(hasAttrs) > 0 { - // for performance reasons: - // only fetch state if some attributes need to be displayed for this resource type - res = resource.GetStates(res, providers) + for _, err := range result.Errors { + fmt.Fprint(os.Stderr, color.RedString("Error %s: %s\n", rType, err)) } - - resources = append(resources, res...) } if len(resources) == 0 { continue } - printResources(resources, hasAttrs, attributes) + resource.PrintResources(resources, hasAttrs, attributes) } return 0 } -func printResources(resources []aws.Resource, hasAttrs map[string]bool, attributes []string) { - const padding = 3 - w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.TabIndent) - - printHeader(w, attributes) - - for _, r := range resources { - profile := `N/A` - if r.Profile != "" { - profile = r.Profile - } - - fmt.Fprintf(w, "%s\t%s\t%s\t%s", r.Type, r.ID, profile, r.Region) - - if r.CreatedAt != nil { - fmt.Fprintf(w, "\t%s", r.CreatedAt.Format("2006-01-02 15:04:05")) - } else { - fmt.Fprint(w, "\tN/A") - } - - for _, attr := range attributes { - v := "N/A" - - _, ok := hasAttrs[attr] - if ok { - var err error - v, err = resource.GetAttribute(attr, &r) - if err != nil { - log.WithFields(log.Fields{ - "type": r.Type, - "id": r.ID}).WithError(err).Debug("failed to get attribute") - v = "error" - } - } - - fmt.Fprintf(w, "\t%s", v) - } - - fmt.Fprintf(w, "\t\n") - } - - w.Flush() - fmt.Println() -} - -func printHeader(w *tabwriter.Writer, attributes []string) { - fmt.Fprintf(w, "TYPE\tID\tPROFILE\tREGION\tCREATED") - - for _, attribute := range attributes { - fmt.Fprintf(w, "\t%s", strings.ToUpper(attribute)) - } - - fmt.Fprintf(w, "\t\n") -} - func printHelp(fs *flag.FlagSet) { fmt.Fprintf(os.Stderr, "\n"+strings.TrimSpace(help)+"\n") fs.PrintDefaults() diff --git a/resource/list.go b/resource/list.go new file mode 100644 index 0000000..7a6fb9d --- /dev/null +++ b/resource/list.go @@ -0,0 +1,84 @@ +package resource + +import ( + "fmt" + "os" + "sync" + "time" + + "github.com/apex/log" + "github.com/fatih/color" + awsls "github.com/jckuester/awsls/aws" + "github.com/jckuester/awsls/internal" + "github.com/jckuester/awstools-lib/aws" + "github.com/jckuester/awstools-lib/terraform" + "github.com/jckuester/terradozer/pkg/provider" +) + +type UpdatedResources struct { + Resources []awsls.Resource + Errors []error +} + +// ListInMultipleAccountsAndRegions lists resources of given resource type in parallel +// for multiple accounts and regions. +func ListInMultipleAccountsAndRegions(rType string, hasAttrs map[string]bool, + clients map[aws.ClientKey]awsls.Client, providers map[aws.ClientKey]provider.TerraformProvider) UpdatedResources { + var wg sync.WaitGroup + sem := internal.NewSemaphore(10) + + resources := terraform.ResourcesThreadSafe{ + Resources: []awsls.Resource{}, + } + + for key, client := range clients { + log.WithFields(log.Fields{ + "type": rType, + "region": key.Region, + "profile": key.Profile, + "time": time.Now().Format("04:05.000"), + }).Debugf("start listing resources") + + wg.Add(1) + + go func(client awsls.Client) { + defer wg.Done() + + // Acquire a semaphore so that we can limit concurrency + sem.Acquire() + defer sem.Release() + + err := client.SetAccountID() + if err != nil { + fmt.Fprint(os.Stderr, color.RedString("Error %s: %s\n", rType, err)) + return + } + + res, err := awsls.ListResourcesByType(&client, rType) + if err != nil { + fmt.Fprint(os.Stderr, color.RedString("Error %s: %s\n", rType, err)) + return + } + + if len(hasAttrs) > 0 { + // for performance reasons: + // only fetch state if some attributes need to be displayed for this resource type + updatesRes, errs := terraform.UpdateStates(res, providers, 10) + res = updatesRes + + resources.Lock() + resources.Errors = append(resources.Errors, errs...) + resources.Unlock() + } + + resources.Lock() + resources.Resources = append(resources.Resources, res...) + resources.Unlock() + }(client) + } + + // Wait until listing resources of this type completes for every account and region + wg.Wait() + + return UpdatedResources{resources.Resources, resources.Errors} +} diff --git a/resource/print.go b/resource/print.go new file mode 100644 index 0000000..c3262a3 --- /dev/null +++ b/resource/print.go @@ -0,0 +1,66 @@ +package resource + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + + "github.com/apex/log" + awsls "github.com/jckuester/awsls/aws" +) + +func PrintResources(resources []awsls.Resource, hasAttrs map[string]bool, attributes []string) { + const padding = 3 + w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.TabIndent) + + printHeader(w, attributes) + + for _, r := range resources { + profile := `N/A` + if r.Profile != "" { + profile = r.Profile + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s", r.Type, r.ID, profile, r.Region) + + if r.CreatedAt != nil { + fmt.Fprintf(w, "\t%s", r.CreatedAt.Format("2006-01-02 15:04:05")) + } else { + fmt.Fprint(w, "\tN/A") + } + + for _, attr := range attributes { + v := "N/A" + + _, ok := hasAttrs[attr] + if ok { + var err error + v, err = GetAttribute(attr, &r) + if err != nil { + log.WithFields(log.Fields{ + "type": r.Type, + "id": r.ID}).WithError(err).Debug("failed to get attribute") + v = "error" + } + } + + fmt.Fprintf(w, "\t%s", v) + } + + fmt.Fprintf(w, "\t\n") + } + + w.Flush() + fmt.Println() +} + +func printHeader(w *tabwriter.Writer, attributes []string) { + fmt.Fprintf(w, "TYPE\tID\tPROFILE\tREGION\tCREATED") + + for _, attribute := range attributes { + fmt.Fprintf(w, "\t%s", strings.ToUpper(attribute)) + } + + fmt.Fprintf(w, "\t\n") +} diff --git a/resource/utils.go b/resource/utils.go index 99011e1..5451e99 100644 --- a/resource/utils.go +++ b/resource/utils.go @@ -2,24 +2,17 @@ package resource import ( "fmt" - "os" "sort" "strconv" "strings" - "sync" - - "github.com/jckuester/awsls/util" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" "github.com/apex/log" - "github.com/fatih/color" "github.com/gobwas/glob" "github.com/jckuester/awsls/aws" - "github.com/jckuester/awsls/internal" "github.com/jckuester/terradozer/pkg/provider" - terradozerRes "github.com/jckuester/terradozer/pkg/resource" ) // IsType returns true if the given string is a Terraform AWS resource type. @@ -81,69 +74,6 @@ func SupportsTags(s string) bool { return false } -// resourcesThreadSafe is a list implementation to store resources concurrently. -type resourcesThreadSafe struct { - sync.Mutex - resources []aws.Resource -} - -// GetStates fetches the Terraform state for each resource via the Terraform AWS Provider. -// Returns only resources which still exist (i.e. state isn't of type cty.Nil after update). -func GetStates(resources []aws.Resource, providers map[util.AWSClientKey]provider.TerraformProvider) []aws.Resource { - var wg sync.WaitGroup - - result := &resourcesThreadSafe{ - resources: []aws.Resource{}, - } - - sem := internal.NewSemaphore(5) - for i := range resources { - wg.Add(1) - go func(i int) { - defer wg.Done() - - // Acquire a semaphore so that we can limit concurrency - sem.Acquire() - defer sem.Release() - - r := &resources[i] - - key := util.AWSClientKey{ - Profile: r.Profile, - Region: r.Region, - } - - p, ok := providers[key] - - if !ok { - panic(fmt.Sprintf("could not find Terraform AWS Provider for key: %v", key)) - } - - r.UpdatableResource = terradozerRes.New(r.Type, r.ID, nil, &p) - - err := r.UpdateState() - if err != nil { - fmt.Fprint(os.Stderr, color.RedString("Error: %s\n", err)) - } - - // filter out resources that don't exist anymore - // (e.g., ECS clusters in state INACTIVE) - if r.State() != nil && r.State().IsNull() { - return - } - - result.Lock() - result.resources = append(result.resources, *r) - result.Unlock() - }(i) - } - - // Wait for all updates to complete - wg.Wait() - - return resources -} - // HasAttributes returns only the attributes that the given Terraform resource type supports out of a given // list of attributes. func HasAttributes(attributes []string, terraformType string, provider *provider.TerraformProvider) (map[string]bool, error) { diff --git a/resource/utils_test.go b/resource/utils_test.go index 556da74..a95ef3f 100644 --- a/resource/utils_test.go +++ b/resource/utils_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/jckuester/awsls/resource" + resource "github.com/jckuester/awsls/resource" "github.com/stretchr/testify/assert" ) diff --git a/signal_unix.go b/signal_unix.go new file mode 100644 index 0000000..573964c --- /dev/null +++ b/signal_unix.go @@ -0,0 +1,11 @@ +// +build !windows + +package main + +import ( + "os" + "syscall" +) + +var ignoreSignals = []os.Signal{os.Interrupt} +var forwardSignals = []os.Signal{syscall.SIGTERM} diff --git a/signal_windows.go b/signal_windows.go new file mode 100644 index 0000000..ad1da29 --- /dev/null +++ b/signal_windows.go @@ -0,0 +1,10 @@ +// +build windows + +package main + +import ( + "os" +) + +var ignoreSignals = []os.Signal{os.Interrupt} +var forwardSignals []os.Signal diff --git a/util/clients.go b/util/clients.go deleted file mode 100644 index 6b6737b..0000000 --- a/util/clients.go +++ /dev/null @@ -1,116 +0,0 @@ -package util - -import ( - "sync" - - "github.com/aws/aws-sdk-go-v2/aws/external" - "github.com/jckuester/awsls/aws" -) - -// awsClientPoolThreadSafe is a concurrent map implementation to store multiple AWS clients. -type awsClientPoolThreadSafe struct { - sync.Mutex - clients map[AWSClientKey]aws.Client -} - -type AWSClientKey struct { - Profile, Region string -} - -// NewAWSClientPool creates an AWS client for each permutation of the given profiles and regions. -// If profiles, regions, or both are empty, credentials and regions are picked up via the usual default provider chain, -// respectively. For example, if regions are empty, the region is first looked for via the according region environment variable -// or second the default region for each profile is used from `~/.aws/config`. -func NewAWSClientPool(profiles []string, regions []string) (map[AWSClientKey]aws.Client, error) { - errors := make(chan error) - wgDone := make(chan bool) - - var wg sync.WaitGroup - - clientPool := &awsClientPoolThreadSafe{ - clients: make(map[AWSClientKey]aws.Client), - } - - if len(profiles) > 0 && len(regions) > 0 { - wg.Add(len(profiles) * len(regions)) - - for _, profile := range profiles { - for _, region := range regions { - - go func(p string, r string) { - defer wg.Done() - - client, err := aws.NewClient( - external.WithSharedConfigProfile(p), - external.WithRegion(r)) - if err != nil { - errors <- err - return - } - - clientPool.Lock() - clientPool.clients[AWSClientKey{p, client.Region}] = *client - clientPool.Unlock() - }(profile, region) - } - } - } else if len(profiles) > 0 { - wg.Add(len(profiles)) - - for _, profile := range profiles { - go func(p string) { - defer wg.Done() - - client, err := aws.NewClient(external.WithSharedConfigProfile(p)) - if err != nil { - errors <- err - return - } - - clientPool.Lock() - clientPool.clients[AWSClientKey{p, client.Region}] = *client - clientPool.Unlock() - }(profile) - } - } else if len(regions) > 0 { - wg.Add(len(regions)) - - for _, region := range regions { - go func(r string) { - defer wg.Done() - - client, err := aws.NewClient(external.WithRegion(r)) - if err != nil { - errors <- err - return - } - - clientPool.Lock() - clientPool.clients[AWSClientKey{"", client.Region}] = *client - clientPool.Unlock() - }(region) - } - } else { - client, err := aws.NewClient() - if err != nil { - return nil, err - } - - return map[AWSClientKey]aws.Client{AWSClientKey{"", client.Region}: *client}, nil - } - - go func() { - wg.Wait() - close(wgDone) - }() - - select { - case <-wgDone: - break - case err := <-errors: - close(errors) - return nil, err - } - - return clientPool.clients, nil -} diff --git a/util/clients_test.go b/util/clients_test.go deleted file mode 100644 index 5ddadf9..0000000 --- a/util/clients_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package util_test - -import ( - "testing" - - "github.com/jckuester/awsls/test" - - "github.com/jckuester/awsls/util" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewAWSClientPool(t *testing.T) { - type args struct { - profiles []string - regions []string - } - tests := []struct { - name string - args args - envs map[string]string - want []util.AWSClientKey - wantErr bool - }{ - { - name: "no profiles and regions via flag, default region via env", - args: args{}, - envs: map[string]string{ - "AWS_DEFAULT_REGION": "us-test-1", - }, - want: []util.AWSClientKey{ - {"", "us-test-1"}, - }, - }, - { - name: "profiles via flag, default region via config file", - args: args{ - profiles: []string{"profile1", "profile2"}, - }, - envs: map[string]string{ - "AWS_CONFIG_FILE": "../test/test-fixtures/aws-config", - }, - want: []util.AWSClientKey{ - {"profile1", "us-test-1"}, - {"profile2", "us-test-2"}, - }, - }, - { - name: "profiles via flag, default region via env", - args: args{ - profiles: []string{"profile1", "profile2"}, - }, - envs: map[string]string{ - "AWS_DEFAULT_REGION": "us-test-3", - "AWS_CONFIG_FILE": "../test/test-fixtures/aws-config", - }, - want: []util.AWSClientKey{ - {"profile1", "us-test-3"}, - {"profile2", "us-test-3"}, - }, - }, - { - name: "no profiles and regions via flag, profile via env", - args: args{}, - envs: map[string]string{ - "AWS_CONFIG_FILE": "../test/test-fixtures/aws-config", - "AWS_PROFILE": "profile1", - }, - // Note: unfortunately, if the profile is not explicitly added to the config - // we cannot retrieve the profile name from the config ex-post - want: []util.AWSClientKey{ - {"", "us-test-1"}, - }, - }, - { - name: "no profiles but regions via flag", - args: args{ - regions: []string{"us-test-1", "us-test-2"}, - }, - want: []util.AWSClientKey{ - {"", "us-test-1"}, - {"", "us-test-2"}, - }, - }, - { - name: "permutation of multiple profiles and regions via flag", - args: args{ - profiles: []string{"profile1", "profile2"}, - regions: []string{"us-test-1", "us-test-2"}, - }, - want: []util.AWSClientKey{ - {"profile1", "us-test-1"}, - {"profile1", "us-test-2"}, - {"profile2", "us-test-1"}, - {"profile2", "us-test-2"}, - }, - }, - { - name: "permutation of multiple, duplicate profiles and regions via flag", - args: args{ - profiles: []string{"profile1", "profile2", "profile1"}, - regions: []string{"us-test-1", "us-test-2", "us-test-2"}, - }, - want: []util.AWSClientKey{ - {"profile1", "us-test-1"}, - {"profile1", "us-test-2"}, - {"profile2", "us-test-1"}, - {"profile2", "us-test-2"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := test.UnsetAWSEnvs() - require.NoError(t, err) - - err = test.SetMultiEnvs(tt.envs) - require.NoError(t, err) - - got, err := util.NewAWSClientPool(tt.args.profiles, tt.args.regions) - if (err != nil) != tt.wantErr { - t.Errorf("NewAWSClientPool() error = %v, wantErr %v", err, tt.wantErr) - return - } - - require.Len(t, got, len(tt.want)) - - for _, clientKey := range tt.want { - actualClient, ok := got[clientKey] - if !ok { - t.Fatal("AWS client does not exist") - } - assert.Equal(t, clientKey.Profile, actualClient.Profile) - assert.Equal(t, clientKey.Region, actualClient.Region) - } - }) - } -} diff --git a/util/providers.go b/util/providers.go deleted file mode 100644 index 2b1e2c4..0000000 --- a/util/providers.go +++ /dev/null @@ -1,103 +0,0 @@ -package util - -import ( - "fmt" - "os" - "sync" - "time" - - "github.com/fatih/color" - "github.com/jckuester/terradozer/pkg/provider" - "github.com/zclconf/go-cty/cty" -) - -// providerPoolThreadSafe is a concurrent map implementation to store multiple Terraform AWS Providers. -type providerPoolThreadSafe struct { - sync.Mutex - providers map[AWSClientKey]provider.TerraformProvider -} - -// NewProviderPool launches a set of Terraform AWS Providers with the configuration of the given clientKeys -// (combination of AWS profile and region). -func NewProviderPool(clientKeys []AWSClientKey, version, installDir string, timeout time.Duration) ( - map[AWSClientKey]provider.TerraformProvider, error) { - - metaPlugin, err := provider.Install("aws", version, installDir) - if err != nil { - fmt.Fprint(os.Stderr, color.RedString("failed to install provider (%s): %s", "aws", err)) - } - - errors := make(chan error) - wgDone := make(chan bool) - - var wg sync.WaitGroup - - providerPool := &providerPoolThreadSafe{ - providers: make(map[AWSClientKey]provider.TerraformProvider), - } - - if len(clientKeys) > 0 { - wg.Add(len(clientKeys)) - - for _, clientKey := range clientKeys { - go func(p string, r string) { - defer wg.Done() - - pr, err := provider.Launch(metaPlugin.Path, timeout) - if err != nil { - errors <- fmt.Errorf("failed to launch provider (%s): %s", metaPlugin.Path, err) - return - } - - config := cty.ObjectVal(map[string]cty.Value{ - "profile": cty.StringVal(p), - "region": cty.StringVal(r), - "access_key": cty.UnknownVal(cty.DynamicPseudoType), - "allowed_account_ids": cty.UnknownVal(cty.DynamicPseudoType), - "assume_role": cty.UnknownVal(cty.DynamicPseudoType), - "endpoints": cty.UnknownVal(cty.DynamicPseudoType), - "forbidden_account_ids": cty.UnknownVal(cty.DynamicPseudoType), - "insecure": cty.UnknownVal(cty.DynamicPseudoType), - "max_retries": cty.UnknownVal(cty.DynamicPseudoType), - "s3_force_path_style": cty.UnknownVal(cty.DynamicPseudoType), - "secret_key": cty.UnknownVal(cty.DynamicPseudoType), - "shared_credentials_file": cty.UnknownVal(cty.DynamicPseudoType), - "skip_credentials_validation": cty.UnknownVal(cty.DynamicPseudoType), - "skip_get_ec2_platforms": cty.UnknownVal(cty.DynamicPseudoType), - "skip_metadata_api_check": cty.UnknownVal(cty.DynamicPseudoType), - "skip_region_validation": cty.UnknownVal(cty.DynamicPseudoType), - "skip_requesting_account_id": cty.UnknownVal(cty.DynamicPseudoType), - "token": cty.UnknownVal(cty.DynamicPseudoType), - "ignore_tag_prefixes": cty.UnknownVal(cty.DynamicPseudoType), - "ignore_tags": cty.UnknownVal(cty.DynamicPseudoType), - }) - - err = pr.Configure(config) - if err != nil { - errors <- fmt.Errorf("failed to configure provider (name=%s, version=%s): %s", - metaPlugin.Name, metaPlugin.Version, err) - return - } - - providerPool.Lock() - providerPool.providers[AWSClientKey{p, r}] = *pr - providerPool.Unlock() - }(clientKey.Profile, clientKey.Region) - } - } - - go func() { - wg.Wait() - close(wgDone) - }() - - select { - case <-wgDone: - break - case err := <-errors: - close(errors) - return nil, err - } - - return providerPool.providers, nil -}