From 5f23dd6353b41a02956f171409fde7ced26bf9a7 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 4 Sep 2024 13:24:21 +0545 Subject: [PATCH] feat(aws connection): use from duty and support default regions --- .github/dependabot.yml | 12 +- api/v1/aws.go | 5 +- api/v1/common.go | 31 +++-- api/v1/zz_generated.deepcopy.go | 4 +- ...configs.flanksource.com_scrapeconfigs.yaml | 3 +- config/schemas/config_aws.schema.json | 19 ++- config/schemas/scrape_config.schema.json | 19 ++- go.mod | 6 +- scrapers/aws/aws.go | 27 ++-- scrapers/aws/aws_session.go | 120 ------------------ scrapers/aws/cost.go | 30 +++-- scrapers/run.go | 4 +- 12 files changed, 83 insertions(+), 197 deletions(-) delete mode 100644 scrapers/aws/aws_session.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4b160e65..437aeb3d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,9 @@ version: 2 -groups: - dependabot: - patterns: - - "*" - exclude-patterns: - - "flanksource/*" updates: - - package-ecosystem: "gomod" - directory: "/" + - package-ecosystem: 'gomod' + directory: '/' schedule: - interval: "daily" + interval: 'daily' - package-ecosystem: github-actions directory: / diff --git a/api/v1/aws.go b/api/v1/aws.go index 8b945010..2d313747 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -10,8 +10,9 @@ import ( // AWS ... type AWS struct { - BaseScraper `json:",inline"` - AWSConnection `json:",inline"` + BaseScraper `yaml:",inline" json:",inline"` + AWSConnection `yaml:",inline" json:",inline"` + Compliance bool `json:"compliance,omitempty"` CloudTrail CloudTrail `json:"cloudtrail,omitempty"` Include []string `json:"include,omitempty"` diff --git a/api/v1/common.go b/api/v1/common.go index 74d640f2..564d60dc 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/flanksource/duty" + "github.com/flanksource/duty/connection" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "github.com/flanksource/gomplate/v3" @@ -302,28 +303,30 @@ func (auth Authentication) GetDomain() string { return "" } -// AWSConnection ... +// AWSConnection is a mirror or duty's AWSConnection. +// It has a slice of []region instead of duty's single Region field. type AWSConnection struct { // ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey. ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` AccessKey types.EnvVar `yaml:"accessKey,omitempty" json:"accessKey,omitempty"` SecretKey types.EnvVar `yaml:"secretKey,omitempty" json:"secretKey,omitempty"` - Region []string `yaml:"region,omitempty" json:"region"` - Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` - SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"` AssumeRole string `yaml:"assumeRole,omitempty" json:"assumeRole,omitempty"` + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // Skip TLS verify when connecting to aws + SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"` + + Regions []string `yaml:"region,omitempty" json:"region,omitempty"` } -func (aws AWSConnection) GetModel() *models.Connection { - return &models.Connection{ - URL: aws.Endpoint, - Username: aws.AccessKey.String(), - Password: aws.SecretKey.String(), - Properties: types.JSONStringMap{ - "region": strings.Join(aws.Region, ","), - "assumeRole": aws.AssumeRole, - }, - InsecureTLS: aws.SkipTLSVerify, +func (aws AWSConnection) ToDutyAWSConnection(region string) *connection.AWSConnection { + return &connection.AWSConnection{ + ConnectionName: aws.ConnectionName, + AccessKey: aws.AccessKey, + SecretKey: aws.SecretKey, + AssumeRole: aws.AssumeRole, + Endpoint: aws.Endpoint, + SkipTLSVerify: aws.SkipTLSVerify, + Region: region, } } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index a1d4308c..a4d2d34a 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -61,8 +61,8 @@ func (in *AWSConnection) DeepCopyInto(out *AWSConnection) { *out = *in in.AccessKey.DeepCopyInto(&out.AccessKey) in.SecretKey.DeepCopyInto(&out.SecretKey) - if in.Region != nil { - in, out := &in.Region, &out.Region + if in.Regions != nil { + in, out := &in.Regions, &out.Regions *out = make([]string, len(*in)) copy(*out, *in) } diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index 37141abb..da44803f 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -285,6 +285,7 @@ spec: type: object type: object skipTLSVerify: + description: Skip TLS verify when connecting to aws type: boolean status: description: A static value or JSONPath expression to use as @@ -523,8 +524,6 @@ spec: description: A static value or JSONPath expression to use as the type for the resource. type: string - required: - - region type: object type: array azure: diff --git a/config/schemas/config_aws.schema.json b/config/schemas/config_aws.schema.json index 1e53a513..3c16bf39 100644 --- a/config/schemas/config_aws.schema.json +++ b/config/schemas/config_aws.schema.json @@ -71,11 +71,8 @@ "secretKey": { "$ref": "#/$defs/EnvVar" }, - "region": { - "items": { - "type": "string" - }, - "type": "array" + "assumeRole": { + "type": "string" }, "endpoint": { "type": "string" @@ -83,8 +80,11 @@ "skipTLSVerify": { "type": "boolean" }, - "assumeRole": { - "type": "string" + "region": { + "items": { + "type": "string" + }, + "type": "array" }, "compliance": { "type": "boolean" @@ -109,10 +109,7 @@ } }, "additionalProperties": false, - "type": "object", - "required": [ - "region" - ] + "type": "object" }, "ChangeMapping": { "properties": { diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index 6edfd8c8..f8d3c03f 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -71,11 +71,8 @@ "secretKey": { "$ref": "#/$defs/EnvVar" }, - "region": { - "items": { - "type": "string" - }, - "type": "array" + "assumeRole": { + "type": "string" }, "endpoint": { "type": "string" @@ -83,8 +80,11 @@ "skipTLSVerify": { "type": "boolean" }, - "assumeRole": { - "type": "string" + "region": { + "items": { + "type": "string" + }, + "type": "array" }, "compliance": { "type": "boolean" @@ -109,10 +109,7 @@ } }, "additionalProperties": false, - "type": "object", - "required": [ - "region" - ] + "type": "object" }, "Authentication": { "properties": { diff --git a/go.mod b/go.mod index 0df75c10..b725070a 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager v1.0.0 github.com/Jeffail/gabs/v2 v2.7.0 github.com/aws/aws-sdk-go-v2 v1.30.4 - github.com/aws/aws-sdk-go-v2/config v1.27.29 - github.com/aws/aws-sdk-go-v2/credentials v1.17.29 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.3 github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.42.3 github.com/aws/aws-sdk-go-v2/service/configservice v1.48.3 @@ -60,7 +58,6 @@ require ( github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a github.com/google/uuid v1.6.0 github.com/hashicorp/go-getter v1.7.5 - github.com/henvic/httpretty v0.1.3 github.com/hexops/gotextdiff v1.0.3 github.com/labstack/echo-contrib v0.17.1 github.com/labstack/echo/v4 v4.12.0 @@ -116,6 +113,8 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/asecurityteam/rolling v2.0.4+incompatible // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.29 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.29 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect @@ -157,6 +156,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl/v2 v2.21.0 // indirect + github.com/henvic/httpretty v0.1.3 // indirect github.com/hirochachacha/go-smb2 v1.1.0 // indirect github.com/invopop/jsonschema v0.12.0 // indirect github.com/itchyny/gojq v0.12.16 // indirect diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index b3aaa208..254ce787 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -75,12 +75,17 @@ func (ctx AWSContext) String() string { } func (aws Scraper) getContext(ctx api.ScrapeContext, awsConfig v1.AWS, region string) (*AWSContext, error) { - session, err := NewSession(ctx, awsConfig.AWSConnection, region) + awsConn := awsConfig.AWSConnection.ToDutyAWSConnection(region) + if err := awsConn.Populate(ctx); err != nil { + return nil, err + } + + session, err := awsConn.Client(ctx.Context) if err != nil { return nil, fmt.Errorf("failed to create AWS session for region=%q: %w", region, err) } - STS := sts.NewFromConfig(*session) + STS := sts.NewFromConfig(session) caller, err := STS.GetCallerIdentity(ctx, nil) if err != nil { return nil, fmt.Errorf("failed to get identity for region=%q: %w", region, err) @@ -91,15 +96,15 @@ func (aws Scraper) getContext(ctx api.ScrapeContext, awsConfig v1.AWS, region st return &AWSContext{ ScrapeContext: ctx, - Session: session, + Session: &session, Caller: caller, STS: STS, Support: support.NewFromConfig(usEast1), - EC2: ec2.NewFromConfig(*session), - SSM: ssm.NewFromConfig(*session), - IAM: iam.NewFromConfig(*session), + EC2: ec2.NewFromConfig(session), + SSM: ssm.NewFromConfig(session), + IAM: iam.NewFromConfig(session), Subnets: make(map[string]Zone), - Config: configservice.NewFromConfig(*session), + Config: configservice.NewFromConfig(session), }, nil } @@ -1719,7 +1724,13 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { for _, awsConfig := range ctx.ScrapeConfig().Spec.AWS { results := &v1.ScrapeResults{} var totalResults int - for _, region := range awsConfig.Region { + + if len(awsConfig.Regions) == 0 { + // Use an empty region and the sdk picks the default region + awsConfig.Regions = []string{""} + } + + for _, region := range awsConfig.Regions { awsCtx, err := aws.getContext(ctx, awsConfig, region) if err != nil { results.Errorf(err, "failed to create AWS context") diff --git a/scrapers/aws/aws_session.go b/scrapers/aws/aws_session.go deleted file mode 100644 index 8c4f8965..00000000 --- a/scrapers/aws/aws_session.go +++ /dev/null @@ -1,120 +0,0 @@ -package aws - -import ( - "context" - "crypto/tls" - "fmt" - "net/http" - - "github.com/flanksource/config-db/api" - v1 "github.com/flanksource/config-db/api/v1" - "github.com/henvic/httpretty" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/credentials/stscreds" - "github.com/aws/aws-sdk-go-v2/service/sts" -) - -// NewSession ... -func NewSession(ctx api.ScrapeContext, conn v1.AWSConnection, region string) (*aws.Config, error) { - cfg, err := loadConfig(ctx, conn, region) - if err != nil { - return nil, err - } - if conn.AssumeRole != "" { - cfg.Credentials = aws.NewCredentialsCache(stscreds.NewAssumeRoleProvider(sts.NewFromConfig(*cfg), conn.AssumeRole)) - } - return cfg, nil -} - -// EndpointResolver ... -type EndpointResolver struct { - Endpoint string - Region string -} - -// ResolveEndpoint ... -// nolint:staticcheck -// FIXME: deprecated global endpoint resolver -func (e EndpointResolver) ResolveEndpoint(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{ - URL: e.Endpoint, - HostnameImmutable: true, - Source: aws.EndpointSourceCustom, - SigningRegion: e.Region, - }, nil -} - -func loadConfig(ctx api.ScrapeContext, conn v1.AWSConnection, region string) (*aws.Config, error) { - if conn.ConnectionName != "" { - connection, err := ctx.HydrateConnectionByURL(conn.ConnectionName) - if err != nil { - return nil, fmt.Errorf("could not hydrate connection: %w", err) - } else if connection == nil { - return nil, fmt.Errorf("connection %s not found", conn.ConnectionName) - } - - conn.AccessKey.ValueStatic = connection.Username - conn.SecretKey.ValueStatic = connection.Password - conn.Endpoint = connection.URL - } - - var tr http.RoundTripper - tr = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: conn.SkipTLSVerify}, - } - - if ctx.IsTrace() { - httplogger := &httpretty.Logger{ - Time: true, - TLS: true, - RequestHeader: true, - RequestBody: true, - ResponseHeader: true, - ResponseBody: true, - Colors: true, // erase line if you don't like colors - Formatters: []httpretty.Formatter{&httpretty.JSONFormatter{}}, - } - tr = httplogger.RoundTripper(tr) - } - - options := []func(*config.LoadOptions) error{ - config.WithRegion(region), - config.WithHTTPClient(&http.Client{Transport: tr}), - } - - if conn.Endpoint != "" { - // nolint:staticcheck - // FIXME: deprecated global endpoint resolver - options = append(options, config.WithEndpointResolverWithOptions(EndpointResolver{Endpoint: conn.Endpoint, Region: region})) - } - - if !conn.AccessKey.IsEmpty() { - accessKey, secretKey, err := getAccessAndSecretKey(ctx, conn) - if err != nil { - return nil, err - } - - options = append(options, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""))) - } - - cfg, err := config.LoadDefaultConfig(context.Background(), options...) - return &cfg, err -} - -// getAccessAndSecretKey retrieves the access and secret keys from the Kubernetes cache. -func getAccessAndSecretKey(ctx api.ScrapeContext, conn v1.AWSConnection) (string, string, error) { - accessKey, err := ctx.GetEnvValueFromCache(conn.AccessKey, ctx.Namespace()) - if err != nil { - return "", "", fmt.Errorf("error getting access key: %w", err) - } - - secretKey, err := ctx.GetEnvValueFromCache(conn.SecretKey, ctx.Namespace()) - if err != nil { - return "", "", fmt.Errorf("error getting secret key: %w", err) - } - - return accessKey, secretKey, nil -} diff --git a/scrapers/aws/cost.go b/scrapers/aws/cost.go index 6006c47c..777c07ea 100644 --- a/scrapers/aws/cost.go +++ b/scrapers/aws/cost.go @@ -47,26 +47,23 @@ const costQueryTemplate = ` ON cost_30d.line_item_product_code = items.line_item_product_code AND items.line_item_resource_id = cost_30d.line_item_resource_id ` -func getAWSAthenaConfig(ctx api.ScrapeContext, awsConfig v1.AWS) (*athena.Config, error) { +func getAWSAthenaConfig(awsConfig v1.AWS) (*athena.Config, error) { conf := athena.NewNoOpsConfig() if err := conf.SetRegion(awsConfig.CostReporting.Region); err != nil { return nil, err } - if err := conf.SetOutputBucket(awsConfig.CostReporting.S3BucketPath); err != nil { - return nil, err - } - accessKey, secretKey, err := getAccessAndSecretKey(ctx, awsConfig.AWSConnection) - if err != nil { + if err := conf.SetOutputBucket(awsConfig.CostReporting.S3BucketPath); err != nil { return nil, err } - if len(accessKey) > 0 && len(secretKey) > 0 { - if err = conf.SetAccessID(accessKey); err != nil { + if len(awsConfig.AWSConnection.AccessKey.ValueStatic) > 0 && len(awsConfig.AWSConnection.SecretKey.ValueStatic) > 0 { + if err := conf.SetAccessID(awsConfig.AWSConnection.AccessKey.ValueStatic); err != nil { return nil, err } - if err = conf.SetSecretAccessKey(secretKey); err != nil { + + if err := conf.SetSecretAccessKey(awsConfig.AWSConnection.SecretKey.ValueStatic); err != nil { return nil, err } } @@ -83,10 +80,10 @@ type LineItemRow struct { Cost30d float64 } -func fetchCosts(ctx api.ScrapeContext, config v1.AWS) ([]LineItemRow, error) { +func fetchCosts(config v1.AWS) ([]LineItemRow, error) { var lineItemRows []LineItemRow - athenaConf, err := getAWSAthenaConfig(ctx, config) + athenaConf, err := getAWSAthenaConfig(config) if err != nil { return lineItemRows, err } @@ -144,19 +141,24 @@ func (awsCost CostScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { var results v1.ScrapeResults for _, awsConfig := range ctx.ScrapeConfig().Spec.AWS { - session, err := NewSession(ctx, awsConfig.AWSConnection, awsConfig.Region[0]) + awsConn := awsConfig.AWSConnection.ToDutyAWSConnection(awsConfig.Regions[0]) + if err := awsConn.Populate(ctx); err != nil { + return results.Errorf(err, "failed to populate AWS connection") + } + + session, err := awsConn.Client(ctx.Context) if err != nil { return results.Errorf(err, "failed to create AWS session") } - stsClient := sts.NewFromConfig(*session) + stsClient := sts.NewFromConfig(session) caller, err := stsClient.GetCallerIdentity(ctx, nil) if err != nil { return results.Errorf(err, "failed to get identity") } accountID := *caller.Account - rows, err := fetchCosts(ctx, awsConfig) + rows, err := fetchCosts(awsConfig) if err != nil { return results.Errorf(err, "failed to fetch costs") } diff --git a/scrapers/run.go b/scrapers/run.go index 3b451c0b..2a6bf8e0 100644 --- a/scrapers/run.go +++ b/scrapers/run.go @@ -29,7 +29,9 @@ func RunScraper(ctx api.ScrapeContext) (*ScrapeOutput, error) { } ctx = ctx.WithValue(contextKeyScrapeStart, time.Now()) - ctx.Context = ctx.WithName(fmt.Sprintf("%s/%s", ctx.ScrapeConfig().Namespace, ctx.ScrapeConfig().Name)) + ctx.Context = ctx. + WithName(fmt.Sprintf("%s/%s", ctx.ScrapeConfig().Namespace, ctx.ScrapeConfig().Name)). + WithNamespace(ctx.ScrapeConfig().Namespace) results, scraperErr := Run(ctx) if scraperErr != nil {