feat(aws connection): use from duty and support default regions

.github/dependabot.yml

@@ -1,15 +1,9 @@
 version: 2
-groups:
-  dependabot:
-    patterns:
-      - "*"
-    exclude-patterns:
-      - "flanksource/*"
 updates:
-  - package-ecosystem: "gomod"
-    directory: "/"
+  - package-ecosystem: 'gomod'
+    directory: '/'
     schedule:
-      interval: "daily"
+      interval: 'daily'
 
   - package-ecosystem: github-actions
     directory: /

api/v1/aws.go

@@ -10,8 +10,9 @@ import (
 
 // AWS ...
 type AWS struct {
-	BaseScraper   `json:",inline"`
-	AWSConnection `json:",inline"`
+	BaseScraper   `yaml:",inline" json:",inline"`
+	AWSConnection `yaml:",inline" json:",inline"`
+
 	Compliance    bool          `json:"compliance,omitempty"`
 	CloudTrail    CloudTrail    `json:"cloudtrail,omitempty"`
 	Include       []string      `json:"include,omitempty"`

api/v1/common.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/flanksource/duty"
+	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/gomplate/v3"
@@ -302,28 +303,30 @@ func (auth Authentication) GetDomain() string {
 	return ""
 }
 
-// AWSConnection ...
+// AWSConnection is a mirror or duty's AWSConnection.
+// It has a slice of []region instead of duty's single Region field.
 type AWSConnection struct {
 	// ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey.
 	ConnectionName string       `yaml:"connection,omitempty" json:"connection,omitempty"`
 	AccessKey      types.EnvVar `yaml:"accessKey,omitempty" json:"accessKey,omitempty"`
 	SecretKey      types.EnvVar `yaml:"secretKey,omitempty" json:"secretKey,omitempty"`
-	Region         []string     `yaml:"region,omitempty" json:"region"`
-	Endpoint       string       `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
-	SkipTLSVerify  bool         `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"`
 	AssumeRole     string       `yaml:"assumeRole,omitempty" json:"assumeRole,omitempty"`
+	Endpoint       string       `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
+	// Skip TLS verify when connecting to aws
+	SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"`
+
+	Regions []string `yaml:"region,omitempty" json:"region,omitempty"`
 }
 
-func (aws AWSConnection) GetModel() *models.Connection {
-	return &models.Connection{
-		URL:      aws.Endpoint,
-		Username: aws.AccessKey.String(),
-		Password: aws.SecretKey.String(),
-		Properties: types.JSONStringMap{
-			"region":     strings.Join(aws.Region, ","),
-			"assumeRole": aws.AssumeRole,
-		},
-		InsecureTLS: aws.SkipTLSVerify,
+func (aws AWSConnection) ToDutyAWSConnection(region string) *connection.AWSConnection {
+	return &connection.AWSConnection{
+		ConnectionName: aws.ConnectionName,
+		AccessKey:      aws.AccessKey,
+		SecretKey:      aws.SecretKey,
+		AssumeRole:     aws.AssumeRole,
+		Endpoint:       aws.Endpoint,
+		SkipTLSVerify:  aws.SkipTLSVerify,
+		Region:         region,
 	}
 }
 

chart/crds/configs.flanksource.com_scrapeconfigs.yaml

@@ -285,6 +285,7 @@ spec:
                           type: object
                       type: object
                     skipTLSVerify:
+                      description: Skip TLS verify when connecting to aws
                       type: boolean
                     status:
                       description: A static value or JSONPath expression to use as
@@ -523,8 +524,6 @@ spec:
                       description: A static value or JSONPath expression to use as
                         the type for the resource.
                       type: string
-                  required:
-                  - region
                   type: object
                 type: array
               azure:

config/schemas/config_aws.schema.json

@@ -71,20 +71,20 @@
         "secretKey": {
           "$ref": "#/$defs/EnvVar"
         },
-        "region": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
+        "assumeRole": {
+          "type": "string"
         },
         "endpoint": {
           "type": "string"
         },
         "skipTLSVerify": {
           "type": "boolean"
         },
-        "assumeRole": {
-          "type": "string"
+        "region": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
         },
         "compliance": {
           "type": "boolean"
@@ -109,10 +109,7 @@
         }
       },
       "additionalProperties": false,
-      "type": "object",
-      "required": [
-        "region"
-      ]
+      "type": "object"
     },
     "ChangeMapping": {
       "properties": {

config/schemas/scrape_config.schema.json

@@ -71,20 +71,20 @@
         "secretKey": {
           "$ref": "#/$defs/EnvVar"
         },
-        "region": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
+        "assumeRole": {
+          "type": "string"
         },
         "endpoint": {
           "type": "string"
         },
         "skipTLSVerify": {
           "type": "boolean"
         },
-        "assumeRole": {
-          "type": "string"
+        "region": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
         },
         "compliance": {
           "type": "boolean"
@@ -109,10 +109,7 @@
         }
       },
       "additionalProperties": false,
-      "type": "object",
-      "required": [
-        "region"
-      ]
+      "type": "object"
     },
     "Authentication": {
       "properties": {

go.mod

@@ -25,8 +25,6 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager v1.0.0
 	github.com/Jeffail/gabs/v2 v2.7.0
 	github.com/aws/aws-sdk-go-v2 v1.30.4
-	github.com/aws/aws-sdk-go-v2/config v1.27.29
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.29
 	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.53.3
 	github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.42.3
 	github.com/aws/aws-sdk-go-v2/service/configservice v1.48.3
@@ -60,7 +58,6 @@ require (
 	github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-getter v1.7.5
-	github.com/henvic/httpretty v0.1.3
 	github.com/hexops/gotextdiff v1.0.3
 	github.com/labstack/echo-contrib v0.17.1
 	github.com/labstack/echo/v4 v4.12.0
@@ -116,6 +113,8 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/asecurityteam/rolling v2.0.4+incompatible // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.29 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.29 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
@@ -157,6 +156,7 @@ require (
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.21.0 // indirect
+	github.com/henvic/httpretty v0.1.3 // indirect
 	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
 	github.com/invopop/jsonschema v0.12.0 // indirect
 	github.com/itchyny/gojq v0.12.16 // indirect

scrapers/aws/aws.go

@@ -75,12 +75,17 @@ func (ctx AWSContext) String() string {
 }
 
 func (aws Scraper) getContext(ctx api.ScrapeContext, awsConfig v1.AWS, region string) (*AWSContext, error) {
-	session, err := NewSession(ctx, awsConfig.AWSConnection, region)
+	awsConn := awsConfig.AWSConnection.ToDutyAWSConnection(region)
+	if err := awsConn.Populate(ctx); err != nil {
+		return nil, err
+	}
+
+	session, err := awsConn.Client(ctx.Context)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create AWS session for region=%q: %w", region, err)
 	}
 
-	STS := sts.NewFromConfig(*session)
+	STS := sts.NewFromConfig(session)
 	caller, err := STS.GetCallerIdentity(ctx, nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get identity for region=%q: %w", region, err)
@@ -91,15 +96,15 @@ func (aws Scraper) getContext(ctx api.ScrapeContext, awsConfig v1.AWS, region st
 
 	return &AWSContext{
 		ScrapeContext: ctx,
-		Session:       session,
+		Session:       &session,
 		Caller:        caller,
 		STS:           STS,
 		Support:       support.NewFromConfig(usEast1),
-		EC2:           ec2.NewFromConfig(*session),
-		SSM:           ssm.NewFromConfig(*session),
-		IAM:           iam.NewFromConfig(*session),
+		EC2:           ec2.NewFromConfig(session),
+		SSM:           ssm.NewFromConfig(session),
+		IAM:           iam.NewFromConfig(session),
 		Subnets:       make(map[string]Zone),
-		Config:        configservice.NewFromConfig(*session),
+		Config:        configservice.NewFromConfig(session),
 	}, nil
 }
 
@@ -1719,7 +1724,13 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults {
 	for _, awsConfig := range ctx.ScrapeConfig().Spec.AWS {
 		results := &v1.ScrapeResults{}
 		var totalResults int
-		for _, region := range awsConfig.Region {
+
+		if len(awsConfig.Regions) == 0 {
+			// Use an empty region and the sdk picks the default region
+			awsConfig.Regions = []string{""}
+		}
+
+		for _, region := range awsConfig.Regions {
 			awsCtx, err := aws.getContext(ctx, awsConfig, region)
 			if err != nil {
 				results.Errorf(err, "failed to create AWS context")