From b6d2460f54ffcce344b1f17b72c9e1bf4c74e1d5 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 26 Jun 2024 22:04:25 +0545 Subject: [PATCH 1/8] feat(aws): scrape lambda functions --- api/v1/aws.go | 1 + go.mod | 1 + go.sum | 5 +++ scrapers/aws/aws.go | 107 +++++++++++++++++++++++++++++--------------- 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/api/v1/aws.go b/api/v1/aws.go index 2e466a06..a3ac23b7 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -48,6 +48,7 @@ const ( AWSEC2Instance = "AWS::EC2::Instance" AWSEKSCluster = "AWS::EKS::Cluster" AWSS3Bucket = "AWS::S3::Bucket" + AWSLambdaFunction = "AWS::Lambda::Function" AWSLoadBalancer = "AWS::ElasticLoadBalancing::LoadBalancer" AWSLoadBalancerV2 = "AWS::ElasticLoadBalancingV2::LoadBalancer" AWSEBSVolume = "AWS::EBS::Volume" diff --git a/go.mod b/go.mod index 36fe71c0..b62e5f3f 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.6 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.7 github.com/aws/aws-sdk-go-v2/service/iam v1.19.8 + github.com/aws/aws-sdk-go-v2/service/lambda v1.30.0 github.com/aws/aws-sdk-go-v2/service/rds v1.42.0 github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 diff --git a/go.sum b/go.sum index dfacfb0a..e88600ce 100644 --- a/go.sum +++ b/go.sum @@ -701,6 +701,7 @@ github.com/aws/aws-sdk-go v1.37.32/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.49.16 h1:KAQwhLg296hfffRdh+itA9p7Nx/3cXS/qOa3uF9ssig= github.com/aws/aws-sdk-go v1.49.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= @@ -712,9 +713,11 @@ github.com/aws/aws-sdk-go-v2/credentials v1.13.24 h1:PjiYyls3QdCrzqUN35jMWtUK1vq github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 h1:jJPgroehGvjrde3XufFIJUZVK5A2L9a3KwSFgKy9n8w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= @@ -749,6 +752,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27 h1:0iKliEXAc github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2 h1:NbWkRxEEIRSCqxhsHQuMiTH7yo+JZW1gp8v3elSVMTQ= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.2/go.mod h1:4tfW5l4IAB32VWCDEBxCRtR9T4BWy4I4kr1spr8NgZM= +github.com/aws/aws-sdk-go-v2/service/lambda v1.30.0 h1:i2AFUTfisQPZP0iZlUEJiGfOBxEN7Yy+d3zBfDYRmnQ= +github.com/aws/aws-sdk-go-v2/service/lambda v1.30.0/go.mod h1:iPDYs5hrSZ+/8Ifoq9ZpoiuHZXDEJx9Udurdoq20958= github.com/aws/aws-sdk-go-v2/service/rds v1.42.0 h1:/FdXrQQyMCi1UwkrkrTRVv+PCj3cYC8cKn30YuSIVso= github.com/aws/aws-sdk-go-v2/service/rds v1.42.0/go.mod h1:es+Xl+GSYsY3ESUW8H6zwieX0ePwycTheaC91KgrpJI= github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5 h1:m223LdVWU3SPDQ4dk1qjupdcr8j5iGO2xoVbxbpKz3g= diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 1d6a78ae..07cfc3b5 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -5,7 +5,6 @@ import ( "net/url" "slices" "strings" - "time" "github.com/Jeffail/gabs/v2" "github.com/aws/aws-sdk-go-v2/aws" @@ -16,12 +15,12 @@ import ( "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/sts" - "github.com/aws/smithy-go/ptr" "github.com/samber/lo" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" @@ -141,6 +140,42 @@ func (aws Scraper) containerImages(ctx *AWSContext, config v1.AWS, results *v1.S } } +func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if config.Excludes("lambda") { + return + } + + lambdaClient := lambda.NewFromConfig(*ctx.Session) + input := &lambda.ListFunctionsInput{} + + for { + functions, err := lambdaClient.ListFunctions(ctx, input) + if err != nil { + results.Errorf(err, "Failed to list Lambda functions") + return + } + + var resourceHealth health.HealthStatus // TODO: Lambda health check + for _, function := range functions.Functions { + *results = append(*results, v1.ScrapeResult{ + Type: v1.AWSLambdaFunction, + ID: *function.FunctionName, + Name: *function.FunctionName, + Config: function, + ConfigClass: "Lamba", + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSLambdaFunction, lo.FromPtr(function.FunctionName))}, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, + }.WithHealthStatus(resourceHealth)) + } + + if functions.NextMarker == nil { + break + } + input.Marker = functions.NextMarker + } +} + func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { if !config.Includes("EKS") { return @@ -1107,39 +1142,38 @@ func (aws Scraper) iamProfiles(ctx *AWSContext, config v1.AWS, results *v1.Scrap } } -//nolint:all -func (aws Scraper) ami(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { - if !config.Includes("Images") { - return - } - - amis, err := ctx.EC2.DescribeImages(ctx, &ec2.DescribeImagesInput{}) - if err != nil { - results.Errorf(err, "failed to get amis") - return - } - - for _, image := range amis.Images { - createdAt, err := time.Parse(time.RFC3339, *image.CreationDate) - if err != nil { - createdAt = time.Now() - } - - labels := make(map[string]string) - labels["region"] = lo.FromPtr(ctx.Caller.Account) - *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSEC2AMI, - CreatedAt: &createdAt, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2AMI, lo.FromPtr(image.ImageId))}, - Config: image, - Labels: labels, - ConfigClass: "Image", - Name: ptr.ToString(image.Name), - ID: *image.ImageId, - }) - } -} +// func (aws Scraper) ami(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { +// if !config.Includes("Images") { +// return +// } + +// amis, err := ctx.EC2.DescribeImages(ctx, &ec2.DescribeImagesInput{}) +// if err != nil { +// results.Errorf(err, "failed to get amis") +// return +// } + +// for _, image := range amis.Images { +// createdAt, err := time.Parse(time.RFC3339, *image.CreationDate) +// if err != nil { +// createdAt = time.Now() +// } + +// labels := make(map[string]string) +// labels["region"] = lo.FromPtr(ctx.Caller.Account) +// *results = append(*results, v1.ScrapeResult{ +// Type: v1.AWSEC2AMI, +// CreatedAt: &createdAt, +// BaseScraper: config.BaseScraper, +// Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2AMI, lo.FromPtr(image.ImageId))}, +// Config: image, +// Labels: labels, +// ConfigClass: "Image", +// Name: ptr.ToString(image.Name), +// ID: *image.ImageId, +// }) +// } +// } func (aws Scraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.AWS) > 0 @@ -1159,6 +1193,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { } ctx.Logger.V(1).Infof("scraping %s", awsCtx) + aws.lambdaFunctions(awsCtx, awsConfig, results) aws.subnets(awsCtx, awsConfig, results) aws.instances(awsCtx, awsConfig, results) aws.vpcs(awsCtx, awsConfig, results) @@ -1246,6 +1281,8 @@ func getConsoleLink(region, resourceType, resourceID string) *types.Property { url = fmt.Sprintf("https://%s.console.aws.amazon.com/s3/buckets/%s", region, resourceID) case v1.AWSEC2Subnet: url = fmt.Sprintf("https://%s.console.aws.amazon.com/vpcconsole/home?region=%s#SubnetDetails:subnetId=%s", region, region, resourceID) + case v1.AWSLambdaFunction: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/lambda/home?region=%s#/functions/%s", region, region, resourceID) case v1.AWSEC2Instance: url = fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/v2/home?region=%s#Instances:search=%s", region, region, resourceID) case v1.AWSEKSCluster: From 75160bbff2d6248833b6ac506054512147989c01 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 11:39:30 +0545 Subject: [PATCH 2/8] feat(aws): scrape sqs queues --- api/v1/aws.go | 1 + go.mod | 1 + go.sum | 2 ++ scrapers/aws/aws.go | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/api/v1/aws.go b/api/v1/aws.go index a3ac23b7..8609df34 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -43,6 +43,7 @@ type CostReporting struct { } const ( + AWSSQS = "AWS::SQS::Queue" AWSRegion = "AWS::Region" AWSZone = "AWS::Route53::HostedZone" AWSEC2Instance = "AWS::EC2::Instance" diff --git a/go.mod b/go.mod index b62e5f3f..0c3cef76 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/rds v1.42.0 github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 + github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6 github.com/aws/aws-sdk-go-v2/service/ssm v1.36.0 github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 github.com/aws/aws-sdk-go-v2/service/support v1.14.7 diff --git a/go.sum b/go.sum index e88600ce..dac5a48f 100644 --- a/go.sum +++ b/go.sum @@ -760,6 +760,8 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5 h1:m223LdVWU3SPDQ4dk1qjupdc github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5/go.mod h1:AE/SlJyaSHVHnpp0eYkHwtGIr3ly5TizD1w8Fni2G/o= github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1o4ZDAM9rQp5NsY= github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= +github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6 h1:4P/vyx7zCI5yBhlDZ2kwhoLjMJi0X7iR3cxqjNfbego= +github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6/go.mod h1:HQHh1eChX10zDnGmD53WLYk8nPhUKO/JkAUUzDZ530Y= github.com/aws/aws-sdk-go-v2/service/ssm v1.36.0 h1:L1gK0SF7Filotf8Jbhiq0Y+rKVs/W1av8MH0+AXPrAg= github.com/aws/aws-sdk-go-v2/service/ssm v1.36.0/go.mod h1:nCdeJmEFby1HKwKhDdKdVxPOJQUNht7Ngw+ejzbzvDU= github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 h1:UBQjaMTCKwyUYwiVnUt6toEJwGXsLBI6al083tpjJzY= diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 07cfc3b5..100e6ed9 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -19,6 +19,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sqs" + + sqsTypes "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/samber/lo" @@ -140,6 +143,41 @@ func (aws Scraper) containerImages(ctx *AWSContext, config v1.AWS, results *v1.S } } +func (aws Scraper) sqs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if !config.Includes("sqs") { + return + } + + SQS := sqs.NewFromConfig(*ctx.Session) + listQueuesOutput, err := SQS.ListQueues(ctx, &sqs.ListQueuesInput{}) + if err != nil { + results.Errorf(err, "failed to list SQS queues") + return + } + + for _, queueURL := range listQueuesOutput.QueueUrls { + getQueueAttributesOutput, err := SQS.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{ + QueueUrl: &queueURL, + AttributeNames: []sqsTypes.QueueAttributeName{sqsTypes.QueueAttributeNameAll}, + }) + if err != nil { + results.Errorf(err, "failed to get attributes for SQS queue: %s", queueURL) + continue + } + + *results = append(*results, v1.ScrapeResult{ + Type: v1.AWSSQS, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSSQS, queueURL)}, + Config: getQueueAttributesOutput, + Labels: getQueueAttributesOutput.Attributes, + ConfigClass: "Queue", + Name: queueURL, + ID: queueURL, + }) + } +} + func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { if config.Excludes("lambda") { return @@ -1194,6 +1232,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.lambdaFunctions(awsCtx, awsConfig, results) + aws.sqs(awsCtx, awsConfig, results) aws.subnets(awsCtx, awsConfig, results) aws.instances(awsCtx, awsConfig, results) aws.vpcs(awsCtx, awsConfig, results) @@ -1271,6 +1310,8 @@ func getRegionFromArn(arn, resourceType string) string { func getConsoleLink(region, resourceType, resourceID string) *types.Property { var url string switch resourceType { + case v1.AWSSQS: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/sqs/v2/home?region=%s#/queues/%s", region, region, resourceID) case "AWS::ECR::Repository": url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecr/repositories/%s", region, resourceID) case "AWS::EFS::FileSystem": From a36dc294bb24202d1fc4eeb8bf7591907b2c0562 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 11:43:51 +0545 Subject: [PATCH 3/8] feat(aws): scrape sns --- api/v1/aws.go | 1 + go.mod | 1 + go.sum | 2 ++ scrapers/aws/aws.go | 50 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/api/v1/aws.go b/api/v1/aws.go index 8609df34..a994c977 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -43,6 +43,7 @@ type CostReporting struct { } const ( + AWSSNSTopic = "AWS::SNS::Topic" AWSSQS = "AWS::SQS::Queue" AWSRegion = "AWS::Region" AWSZone = "AWS::Route53::HostedZone" diff --git a/go.mod b/go.mod index 0c3cef76..a1619f54 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/rds v1.42.0 github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 + github.com/aws/aws-sdk-go-v2/service/sns v1.20.6 github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6 github.com/aws/aws-sdk-go-v2/service/ssm v1.36.0 github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 diff --git a/go.sum b/go.sum index dac5a48f..e73a0a1e 100644 --- a/go.sum +++ b/go.sum @@ -760,6 +760,8 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5 h1:m223LdVWU3SPDQ4dk1qjupdc github.com/aws/aws-sdk-go-v2/service/route53 v1.27.5/go.mod h1:AE/SlJyaSHVHnpp0eYkHwtGIr3ly5TizD1w8Fni2G/o= github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 h1:O+9nAy9Bb6bJFTpeNFtd9UfHbgxO1o4ZDAM9rQp5NsY= github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1/go.mod h1:J9kLNzEiHSeGMyN7238EjJmBpCniVzFda75Gxl/NqB8= +github.com/aws/aws-sdk-go-v2/service/sns v1.20.6 h1:s8ukppSyVyRWktx1km5pNttWVIyFAnZjjAlgXlONO2M= +github.com/aws/aws-sdk-go-v2/service/sns v1.20.6/go.mod h1:8o/0aAt6gOxdVFubsp4L8Bry0EBss7OhM+II2p607JE= github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6 h1:4P/vyx7zCI5yBhlDZ2kwhoLjMJi0X7iR3cxqjNfbego= github.com/aws/aws-sdk-go-v2/service/sqs v1.20.6/go.mod h1:HQHh1eChX10zDnGmD53WLYk8nPhUKO/JkAUUzDZ530Y= github.com/aws/aws-sdk-go-v2/service/ssm v1.36.0 h1:L1gK0SF7Filotf8Jbhiq0Y+rKVs/W1av8MH0+AXPrAg= diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 100e6ed9..f1ad947d 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -14,28 +14,27 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" + "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" - sqsTypes "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/sts" - "github.com/samber/lo" - - "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" - "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" - "github.com/aws/aws-sdk-go-v2/service/support" "github.com/flanksource/commons/logger" + "github.com/flanksource/duty/types" + "github.com/flanksource/is-healthy/pkg/health" + "github.com/samber/lo" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/utils" - "github.com/flanksource/duty/types" - "github.com/flanksource/is-healthy/pkg/health" ) // Scraper ... @@ -178,6 +177,38 @@ func (aws Scraper) sqs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults } } +func (aws Scraper) snsTopics(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if !config.Includes("sns") { + return + } + + client := sns.NewFromConfig(*ctx.Session) + topics, err := client.ListTopics(ctx, nil) + if err != nil { + results.Errorf(err, "failed to list SNS topics") + return + } + + for _, topic := range topics.Topics { + topicArn := lo.FromPtr(topic.TopicArn) + labels := make(map[string]string) + labels["region"] = ctx.Session.Region + + *results = append(*results, v1.ScrapeResult{ + Type: v1.AWSSNSTopic, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSSNSTopic, topicArn)}, + Config: topic, + Labels: labels, + ConfigClass: "Topic", + Name: topicArn, + Aliases: []string{topicArn}, + ID: topicArn, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, + }) + } +} + func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { if config.Excludes("lambda") { return @@ -1232,6 +1263,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.lambdaFunctions(awsCtx, awsConfig, results) + aws.snsTopics(awsCtx, awsConfig, results) aws.sqs(awsCtx, awsConfig, results) aws.subnets(awsCtx, awsConfig, results) aws.instances(awsCtx, awsConfig, results) @@ -1312,6 +1344,8 @@ func getConsoleLink(region, resourceType, resourceID string) *types.Property { switch resourceType { case v1.AWSSQS: url = fmt.Sprintf("https://%s.console.aws.amazon.com/sqs/v2/home?region=%s#/queues/%s", region, region, resourceID) + case v1.AWSSNSTopic: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/sns/v3/home?region=%s#/topics/%s", region, region, resourceID) case "AWS::ECR::Repository": url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecr/repositories/%s", region, resourceID) case "AWS::EFS::FileSystem": From 5be1f0c838df6fdd40602e50431440f798f629c0 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 11:48:59 +0545 Subject: [PATCH 4/8] feat(aws): scrape ecs cluster, services & tasks --- api/v1/aws.go | 3 + go.mod | 1 + go.sum | 2 + scrapers/aws/aws.go | 135 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 140 insertions(+), 1 deletion(-) diff --git a/api/v1/aws.go b/api/v1/aws.go index a994c977..e7d53e38 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -43,6 +43,9 @@ type CostReporting struct { } const ( + AWSECSTask = "AWS::ECS:Task" + AWSECSService = "AWS::EC2::Service" + AWSECSCluster = "AWS::ECS::Cluster" AWSSNSTopic = "AWS::SNS::Topic" AWSSQS = "AWS::SQS::Queue" AWSRegion = "AWS::Region" diff --git a/go.mod b/go.mod index a1619f54..06a04018 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/configservice v1.30.1 github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1 github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7 + github.com/aws/aws-sdk-go-v2/service/ecs v1.24.2 github.com/aws/aws-sdk-go-v2/service/efs v1.19.9 github.com/aws/aws-sdk-go-v2/service/eks v1.27.8 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.6 diff --git a/go.sum b/go.sum index e73a0a1e..766e57ee 100644 --- a/go.sum +++ b/go.sum @@ -733,6 +733,8 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1 h1:xn5CI639mnWvdiweqoRx/H221Ia9 github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1/go.mod h1:ZZLfkd1Y7fjXujjMg1CFqNmaTl314eCbShlHQO7VTWo= github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7 h1:oQ1Esut3iaL2Dydt2RBd9gbuUevToXpdTI+Uh1xXryI= github.com/aws/aws-sdk-go-v2/service/ecr v1.18.7/go.mod h1:RHhgOMnMIkgB4TmxQat9obSnZ6fF1fuA27+itZKUi1o= +github.com/aws/aws-sdk-go-v2/service/ecs v1.24.2 h1:W94oEzOVUhefAqBtt33gOnsIEB0qFwK4akzhfD/eReI= +github.com/aws/aws-sdk-go-v2/service/ecs v1.24.2/go.mod h1:fMCHV5nbbpjoVHlKIcasH51tyDKha+ofZHVhQyXLRlI= github.com/aws/aws-sdk-go-v2/service/efs v1.19.9 h1:IZM3sHHZy+NoCXKEiB04LkhkyftaCXhBxV3OCCmK6vA= github.com/aws/aws-sdk-go-v2/service/efs v1.19.9/go.mod h1:WkMJD5lMsA0tugbGhOvj49IYsWKBCJxta2J9gezQJJ8= github.com/aws/aws-sdk-go-v2/service/eks v1.27.8 h1:o3cnwMi1lTjOB1N5kKMHc1yjq6al8wmMT9uyI2VLhpc= diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index f1ad947d..fa7c25aa 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -12,6 +12,7 @@ import ( ec2 "github.com/aws/aws-sdk-go-v2/service/ec2" ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/ecr" + "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" @@ -209,6 +210,130 @@ func (aws Scraper) snsTopics(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR } } +func (aws Scraper) ecsClusters(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if !config.Includes("ecs") { + return + } + + client := ecs.NewFromConfig(*ctx.Session) + clusters, err := client.ListClusters(ctx, nil) + if err != nil { + results.Errorf(err, "failed to list ECS clusters") + return + } + + for _, clusterArn := range clusters.ClusterArns { + cluster, err := client.DescribeClusters(ctx, &ecs.DescribeClustersInput{ + Clusters: []string{clusterArn}, + }) + if err != nil { + results.Errorf(err, "failed to describe ECS cluster") + continue + } + + labels := make(map[string]string) + for _, tag := range cluster.Clusters[0].Tags { + labels[*tag.Key] = *tag.Value + } + + for _, clusterInfo := range cluster.Clusters { + *results = append(*results, v1.ScrapeResult{ + Type: v1.AWSECSCluster, + Labels: labels, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSCluster, clusterArn)}, + Config: clusterInfo, + ConfigClass: "ECSCluster", + Name: *clusterInfo.ClusterName, + Aliases: []string{clusterArn}, + ID: clusterArn, + }) + + aws.ecsServices(ctx, config, client, *clusterInfo.ClusterArn, results) + } + } +} + +func (aws Scraper) ecsServices(ctx *AWSContext, config v1.AWS, client *ecs.Client, cluster string, results *v1.ScrapeResults) { + if !config.Includes("ecsservice") { + return + } + + services, err := client.ListServices(ctx, &ecs.ListServicesInput{ + Cluster: &cluster, + }) + if err != nil { + results.Errorf(err, "failed to list ECS services in cluster %s", cluster) + return + } + + describeServicesOutput, err := client.DescribeServices(ctx, &ecs.DescribeServicesInput{ + Cluster: &cluster, + Services: services.ServiceArns, + }) + if err != nil { + results.Errorf(err, "failed to describe ECS services in cluster %s", cluster) + return + } + + for _, service := range describeServicesOutput.Services { + *results = append(*results, v1.ScrapeResult{ + Type: v1.AWSECSService, + ID: *service.ServiceArn, + Name: *service.ServiceName, + Config: service, + ConfigClass: "ECSService", + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSService, *service.ServiceArn)}, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSECSCluster, ExternalID: cluster}}, + }) + } +} + +func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if !config.Includes("ecstask") { + return + } + + client := ecs.NewFromConfig(*ctx.Session) + input := &ecs.ListTasksInput{} + paginator := ecs.NewListTasksPaginator(client, input) + + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + results.Errorf(err, "failed to list ECS tasks") + return + } + + if len(output.TaskArns) == 0 { + continue + } + + tasksOutput, err := client.DescribeTasks(ctx, &ecs.DescribeTasksInput{ + Tasks: output.TaskArns, + }) + if err != nil { + results.Errorf(err, "failed to describe ECS tasks") + return + } + + for _, task := range tasksOutput.Tasks { + result := v1.ScrapeResult{ + Type: v1.AWSECSTask, + ID: *task.TaskArn, + Name: *task.TaskDefinitionArn, + Config: task, + ConfigClass: "ECSTask", + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSTask, *task.TaskArn)}, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSECSCluster, ExternalID: *task.ClusterArn}}, + } + *results = append(*results, result) + } + } +} + func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { if config.Excludes("lambda") { return @@ -220,7 +345,7 @@ func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.S for { functions, err := lambdaClient.ListFunctions(ctx, input) if err != nil { - results.Errorf(err, "Failed to list Lambda functions") + results.Errorf(err, "failed to list Lambda functions") return } @@ -1263,6 +1388,8 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.lambdaFunctions(awsCtx, awsConfig, results) + aws.ecsClusters(awsCtx, awsConfig, results) + aws.ecsTasks(awsCtx, awsConfig, results) aws.snsTopics(awsCtx, awsConfig, results) aws.sqs(awsCtx, awsConfig, results) aws.subnets(awsCtx, awsConfig, results) @@ -1342,6 +1469,12 @@ func getRegionFromArn(arn, resourceType string) string { func getConsoleLink(region, resourceType, resourceID string) *types.Property { var url string switch resourceType { + case v1.AWSECSTask: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/home?region=%s#/tasks/%s", region, region, resourceID) + case v1.AWSECSService: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/home?region=%s#/clusters/%s", region, region, resourceID) + case v1.AWSECSCluster: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/v2/clusters/%s/tasks?region=%s", region, resourceID, region) case v1.AWSSQS: url = fmt.Sprintf("https://%s.console.aws.amazon.com/sqs/v2/home?region=%s#/queues/%s", region, region, resourceID) case v1.AWSSNSTopic: From 0e6c61d232d7543076e7ea3d94183853df1ac90d Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 14:46:11 +0545 Subject: [PATCH 5/8] feat(aws): scrape fargate profiles --- api/v1/aws.go | 1 + scrapers/aws/aws.go | 49 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/api/v1/aws.go b/api/v1/aws.go index e7d53e38..c5103d03 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -43,6 +43,7 @@ type CostReporting struct { } const ( + AWSEKSFargateProfile = "AWS::ECS::FargateProfile" AWSECSTask = "AWS::ECS:Task" AWSECSService = "AWS::EC2::Service" AWSECSCluster = "AWS::ECS::Cluster" diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index fa7c25aa..d76b800b 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -334,6 +334,54 @@ func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRe } } +func (aws Scraper) eksFargateProfiles(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if !config.Includes("eksfargateprofile") { + return + } + + client := eks.NewFromConfig(*ctx.Session) + listClustersInput := &eks.ListClustersInput{} + listClustersOutput, err := client.ListClusters(ctx, listClustersInput) + if err != nil { + results.Errorf(err, "failed to list EKS clusters") + return + } + + for _, clusterName := range listClustersOutput.Clusters { + listFargateProfilesInput := &eks.ListFargateProfilesInput{ + ClusterName: &clusterName, + } + listFargateProfilesOutput, err := client.ListFargateProfiles(ctx, listFargateProfilesInput) + if err != nil { + results.Errorf(err, "failed to list Fargate profiles for cluster %s", clusterName) + continue + } + + for _, profileName := range listFargateProfilesOutput.FargateProfileNames { + describeFargateProfileInput := &eks.DescribeFargateProfileInput{ + ClusterName: &clusterName, + FargateProfileName: &profileName, + } + describeFargateProfileOutput, err := client.DescribeFargateProfile(ctx, describeFargateProfileInput) + if err != nil { + results.Errorf(err, "failed to describe Fargate profile %s for cluster %s", profileName, clusterName) + continue + } + + *results = append(*results, v1.ScrapeResult{ + ID: *describeFargateProfileOutput.FargateProfile.FargateProfileName, + Type: v1.AWSEKSFargateProfile, + BaseScraper: config.BaseScraper, + Config: describeFargateProfileOutput.FargateProfile, + ConfigClass: "FargateProfile", + Tags: []v1.Tag{{Name: "cluster", Value: clusterName}}, + Name: *describeFargateProfileOutput.FargateProfile.FargateProfileName, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEKSCluster, ExternalID: clusterName}}, + }) + } + } +} + func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { if config.Excludes("lambda") { return @@ -1399,6 +1447,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { aws.routes(awsCtx, awsConfig, results) aws.dhcp(awsCtx, awsConfig, results) aws.eksClusters(awsCtx, awsConfig, results) + aws.eksFargateProfiles(awsCtx, awsConfig, results) aws.ebs(awsCtx, awsConfig, results) aws.efs(awsCtx, awsConfig, results) aws.rds(awsCtx, awsConfig, results) From 9fa2ff41bf8eca7f81d0af137bbac91cd33aad52 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 15:00:53 +0545 Subject: [PATCH 6/8] feat(aws): scrape fargate profiles --- api/v1/aws.go | 9 +++--- scrapers/aws/aws.go | 68 +++++++++++++++++++-------------------------- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/api/v1/aws.go b/api/v1/aws.go index c5103d03..ef9c0deb 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -43,10 +43,12 @@ type CostReporting struct { } const ( - AWSEKSFargateProfile = "AWS::ECS::FargateProfile" - AWSECSTask = "AWS::ECS:Task" - AWSECSService = "AWS::EC2::Service" AWSECSCluster = "AWS::ECS::Cluster" + AWSECSService = "AWS::EC2::Service" + AWSECSTask = "AWS::ECS:Task" + AWSEKSFargateProfile = "AWS::ECS::FargateProfile" + AWSElastiCacheCluster = "AWS::ElastiCache::CacheCluster" + AWSLambdaFunction = "AWS::Lambda::Function" AWSSNSTopic = "AWS::SNS::Topic" AWSSQS = "AWS::SQS::Queue" AWSRegion = "AWS::Region" @@ -54,7 +56,6 @@ const ( AWSEC2Instance = "AWS::EC2::Instance" AWSEKSCluster = "AWS::EKS::Cluster" AWSS3Bucket = "AWS::S3::Bucket" - AWSLambdaFunction = "AWS::Lambda::Function" AWSLoadBalancer = "AWS::ElasticLoadBalancing::LoadBalancer" AWSLoadBalancerV2 = "AWS::ElasticLoadBalancingV2::LoadBalancer" AWSEBSVolume = "AWS::EBS::Volume" diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index d76b800b..72a5f9f2 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -250,6 +250,7 @@ func (aws Scraper) ecsClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap }) aws.ecsServices(ctx, config, client, *clusterInfo.ClusterArn, results) + aws.ecsTasks(ctx, config, client, *clusterInfo.ClusterArn, results) } } } @@ -290,13 +291,12 @@ func (aws Scraper) ecsServices(ctx *AWSContext, config v1.AWS, client *ecs.Clien } } -func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { +func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, client *ecs.Client, cluster string, results *v1.ScrapeResults) { if !config.Includes("ecstask") { return } - client := ecs.NewFromConfig(*ctx.Session) - input := &ecs.ListTasksInput{} + input := &ecs.ListTasksInput{Cluster: &cluster} paginator := ecs.NewListTasksPaginator(client, input) for paginator.HasMorePages() { @@ -334,51 +334,41 @@ func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRe } } -func (aws Scraper) eksFargateProfiles(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { - if !config.Includes("eksfargateprofile") { +func (aws Scraper) eksFargateProfiles(ctx *AWSContext, config v1.AWS, client *eks.Client, clusterName string, results *v1.ScrapeResults) { + if !config.Includes("FargateProfile") { return } - client := eks.NewFromConfig(*ctx.Session) - listClustersInput := &eks.ListClustersInput{} - listClustersOutput, err := client.ListClusters(ctx, listClustersInput) + listFargateProfilesInput := &eks.ListFargateProfilesInput{ + ClusterName: &clusterName, + } + listFargateProfilesOutput, err := client.ListFargateProfiles(ctx, listFargateProfilesInput) if err != nil { - results.Errorf(err, "failed to list EKS clusters") + results.Errorf(err, "failed to list Fargate profiles for cluster %s", clusterName) return } - for _, clusterName := range listClustersOutput.Clusters { - listFargateProfilesInput := &eks.ListFargateProfilesInput{ - ClusterName: &clusterName, + for _, profileName := range listFargateProfilesOutput.FargateProfileNames { + describeFargateProfileInput := &eks.DescribeFargateProfileInput{ + ClusterName: &clusterName, + FargateProfileName: &profileName, } - listFargateProfilesOutput, err := client.ListFargateProfiles(ctx, listFargateProfilesInput) + describeFargateProfileOutput, err := client.DescribeFargateProfile(ctx, describeFargateProfileInput) if err != nil { - results.Errorf(err, "failed to list Fargate profiles for cluster %s", clusterName) + results.Errorf(err, "failed to describe Fargate profile %s for cluster %s", profileName, clusterName) continue } - for _, profileName := range listFargateProfilesOutput.FargateProfileNames { - describeFargateProfileInput := &eks.DescribeFargateProfileInput{ - ClusterName: &clusterName, - FargateProfileName: &profileName, - } - describeFargateProfileOutput, err := client.DescribeFargateProfile(ctx, describeFargateProfileInput) - if err != nil { - results.Errorf(err, "failed to describe Fargate profile %s for cluster %s", profileName, clusterName) - continue - } - - *results = append(*results, v1.ScrapeResult{ - ID: *describeFargateProfileOutput.FargateProfile.FargateProfileName, - Type: v1.AWSEKSFargateProfile, - BaseScraper: config.BaseScraper, - Config: describeFargateProfileOutput.FargateProfile, - ConfigClass: "FargateProfile", - Tags: []v1.Tag{{Name: "cluster", Value: clusterName}}, - Name: *describeFargateProfileOutput.FargateProfile.FargateProfileName, - Parents: []v1.ConfigExternalKey{{Type: v1.AWSEKSCluster, ExternalID: clusterName}}, - }) - } + *results = append(*results, v1.ScrapeResult{ + ID: *describeFargateProfileOutput.FargateProfile.FargateProfileName, + Type: v1.AWSEKSFargateProfile, + BaseScraper: config.BaseScraper, + Config: describeFargateProfileOutput.FargateProfile, + ConfigClass: "FargateProfile", + Tags: []v1.Tag{{Name: "cluster", Value: clusterName}}, + Name: *describeFargateProfileOutput.FargateProfile.FargateProfileName, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEKSCluster, ExternalID: clusterName}}, + }) } } @@ -485,6 +475,8 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: *cluster.Cluster.ResourcesVpcConfig.VpcId}}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth)) + + aws.eksFargateProfiles(ctx, config, EKS, clusterName, results) } } @@ -1435,9 +1427,8 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { } ctx.Logger.V(1).Infof("scraping %s", awsCtx) - aws.lambdaFunctions(awsCtx, awsConfig, results) aws.ecsClusters(awsCtx, awsConfig, results) - aws.ecsTasks(awsCtx, awsConfig, results) + aws.lambdaFunctions(awsCtx, awsConfig, results) aws.snsTopics(awsCtx, awsConfig, results) aws.sqs(awsCtx, awsConfig, results) aws.subnets(awsCtx, awsConfig, results) @@ -1447,7 +1438,6 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { aws.routes(awsCtx, awsConfig, results) aws.dhcp(awsCtx, awsConfig, results) aws.eksClusters(awsCtx, awsConfig, results) - aws.eksFargateProfiles(awsCtx, awsConfig, results) aws.ebs(awsCtx, awsConfig, results) aws.efs(awsCtx, awsConfig, results) aws.rds(awsCtx, awsConfig, results) From c60a025bacbdc8726b4b3c7b925b905f60eaba6a Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 15:01:31 +0545 Subject: [PATCH 7/8] feat(aws): scrape elastiCache --- go.mod | 1 + go.sum | 2 ++ scrapers/aws/aws.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/go.mod b/go.mod index 06a04018..fd51ef2e 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecs v1.24.2 github.com/aws/aws-sdk-go-v2/service/efs v1.19.9 github.com/aws/aws-sdk-go-v2/service/eks v1.27.8 + github.com/aws/aws-sdk-go-v2/service/elasticache v1.26.6 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.6 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.7 github.com/aws/aws-sdk-go-v2/service/iam v1.19.8 diff --git a/go.sum b/go.sum index 766e57ee..798e17cb 100644 --- a/go.sum +++ b/go.sum @@ -739,6 +739,8 @@ github.com/aws/aws-sdk-go-v2/service/efs v1.19.9 h1:IZM3sHHZy+NoCXKEiB04Lkhkyfta github.com/aws/aws-sdk-go-v2/service/efs v1.19.9/go.mod h1:WkMJD5lMsA0tugbGhOvj49IYsWKBCJxta2J9gezQJJ8= github.com/aws/aws-sdk-go-v2/service/eks v1.27.8 h1:o3cnwMi1lTjOB1N5kKMHc1yjq6al8wmMT9uyI2VLhpc= github.com/aws/aws-sdk-go-v2/service/eks v1.27.8/go.mod h1:XNcs5UYWyXvctSc/tVZIfq6sj2QA1jTy2hMkTzSDZ3c= +github.com/aws/aws-sdk-go-v2/service/elasticache v1.26.6 h1:PRbssBtMKTVa+Ge+ARwMSFcwT3s+KGPn5A//ckodgcg= +github.com/aws/aws-sdk-go-v2/service/elasticache v1.26.6/go.mod h1:YG1Qmu7Nan+5htEwo6iVa4mcZjoKm+NjbJUE5f+Oq9k= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.6 h1:rY5DlQpQVGje6jiZQ+8xb4WbUyq4EFoPowpAnE3LrGg= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.15.6/go.mod h1:by4vXHvRiuHkKrp3h++wo776M2E80m4Wmyt+Yj6uF1M= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.7 h1:XpIms0tmerNg/t6IiGrbKU6Au25CHyXqs8Yc3zOET5o= diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 72a5f9f2..f3e2e0f4 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -15,6 +15,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/efs" "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/elasticache" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/iam" @@ -372,6 +373,32 @@ func (aws Scraper) eksFargateProfiles(ctx *AWSContext, config v1.AWS, client *ek } } +func (aws Scraper) elastiCache(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if config.Excludes("ElastiCache") { + return + } + + svc := elasticache.NewFromConfig(*ctx.Session) + + clusters, err := svc.DescribeCacheClusters(ctx, &elasticache.DescribeCacheClustersInput{}) + if err != nil { + results.Errorf(err, "failed to describe ElastiCache clusters") + return + } + + for _, cluster := range clusters.CacheClusters { + *results = append(*results, v1.ScrapeResult{ + ID: *cluster.CacheClusterId, + Type: v1.AWSElastiCacheCluster, + BaseScraper: config.BaseScraper, + Config: cluster, + ConfigClass: "Cache", + Name: *cluster.CacheClusterId, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, + }) + } +} + func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { if config.Excludes("lambda") { return @@ -1428,6 +1455,8 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.ecsClusters(awsCtx, awsConfig, results) + aws.elastiCache(awsCtx, awsConfig, results) + aws.lambdaFunctions(awsCtx, awsConfig, results) aws.lambdaFunctions(awsCtx, awsConfig, results) aws.snsTopics(awsCtx, awsConfig, results) aws.sqs(awsCtx, awsConfig, results) From f2021ff5b3424aca4291c3fd0bd7623678667bee Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 27 Jun 2024 15:39:12 +0545 Subject: [PATCH 8/8] chore(aws): cleanup --- api/v1/aws.go | 6 +- db/update.go | 5 +- scrapers/aws/aws.go | 183 ++++++++++++++++++++++++++----------- scrapers/aws/cloudtrail.go | 3 + scrapers/aws/config.go | 2 + 5 files changed, 143 insertions(+), 56 deletions(-) diff --git a/api/v1/aws.go b/api/v1/aws.go index ef9c0deb..28ccb171 100644 --- a/api/v1/aws.go +++ b/api/v1/aws.go @@ -44,9 +44,9 @@ type CostReporting struct { const ( AWSECSCluster = "AWS::ECS::Cluster" - AWSECSService = "AWS::EC2::Service" - AWSECSTask = "AWS::ECS:Task" - AWSEKSFargateProfile = "AWS::ECS::FargateProfile" + AWSECSService = "AWS::ECS::Service" + AWSECSTaskDefinition = "AWS::ECS::TaskDefinition" + AWSEKSFargateProfile = "AWS::EKS::FargateProfile" AWSElastiCacheCluster = "AWS::ElastiCache::CacheCluster" AWSLambdaFunction = "AWS::Lambda::Function" AWSSNSTopic = "AWS::SNS::Topic" diff --git a/db/update.go b/db/update.go index 9f5aba28..1819fb2f 100644 --- a/db/update.go +++ b/db/update.go @@ -12,6 +12,7 @@ import ( "github.com/dominikbraun/graph" jsonpatch "github.com/evanphx/json-patch" "github.com/flanksource/commons/logger" + cUtils "github.com/flanksource/commons/utils" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db/models" @@ -569,7 +570,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations continue } if configID == "" { - ctx.Logger.V(1).Infof("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID) + ctx.Logger.V(2).Infof("unable to form relationship. failed to find the parent config %s for config %s", relationship.ConfigExternalID, cUtils.Coalesce(relationship.RelatedConfigID, relationship.RelatedExternalID.String())) continue } } @@ -584,7 +585,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations continue } if relatedID == "" { - logger.V(6).Infof("related external config item(id=%s) not found.", relationship.RelatedExternalID) + ctx.Logger.V(2).Infof("unable to form relationship. failed to find related config %s for config %s", relationship.RelatedExternalID, configID) continue } } diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index f3e2e0f4..9ba3cda7 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -117,6 +117,8 @@ func (aws Scraper) containerImages(ctx *AWSContext, config v1.AWS, results *v1.S return } + ctx.Logger.V(2).Infof("scraping ECR") + ECR := ecr.NewFromConfig(*ctx.Session) images, err := ECR.DescribeRepositories(ctx, &ecr.DescribeRepositoriesInput{}) if err != nil { @@ -149,6 +151,8 @@ func (aws Scraper) sqs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults return } + ctx.Logger.V(2).Infof("scraping SQS queues") + SQS := sqs.NewFromConfig(*ctx.Session) listQueuesOutput, err := SQS.ListQueues(ctx, &sqs.ListQueuesInput{}) if err != nil { @@ -184,6 +188,8 @@ func (aws Scraper) snsTopics(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR return } + ctx.Logger.V(2).Infof("scraping SNS topics") + client := sns.NewFromConfig(*ctx.Session) topics, err := client.ListTopics(ctx, nil) if err != nil { @@ -216,6 +222,8 @@ func (aws Scraper) ecsClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap return } + ctx.Logger.V(2).Infof("scraping ECS clusters") + client := ecs.NewFromConfig(*ctx.Session) clusters, err := client.ListClusters(ctx, nil) if err != nil { @@ -242,7 +250,7 @@ func (aws Scraper) ecsClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap Type: v1.AWSECSCluster, Labels: labels, BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSCluster, clusterArn)}, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSCluster, *clusterInfo.ClusterName)}, Config: clusterInfo, ConfigClass: "ECSCluster", Name: *clusterInfo.ClusterName, @@ -251,7 +259,6 @@ func (aws Scraper) ecsClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap }) aws.ecsServices(ctx, config, client, *clusterInfo.ClusterArn, results) - aws.ecsTasks(ctx, config, client, *clusterInfo.ClusterArn, results) } } } @@ -261,6 +268,8 @@ func (aws Scraper) ecsServices(ctx *AWSContext, config v1.AWS, client *ecs.Clien return } + ctx.Logger.V(2).Infof("scraping ECS services") + services, err := client.ListServices(ctx, &ecs.ListServicesInput{ Cluster: &cluster, }) @@ -269,6 +278,10 @@ func (aws Scraper) ecsServices(ctx *AWSContext, config v1.AWS, client *ecs.Clien return } + if len(services.ServiceArns) == 0 { + return + } + describeServicesOutput, err := client.DescribeServices(ctx, &ecs.DescribeServicesInput{ Cluster: &cluster, Services: services.ServiceArns, @@ -279,26 +292,38 @@ func (aws Scraper) ecsServices(ctx *AWSContext, config v1.AWS, client *ecs.Clien } for _, service := range describeServicesOutput.Services { + var relationships []v1.RelationshipResult + // ECS Task Definition to ECS Service relationship + relationships = append(relationships, v1.RelationshipResult{ + RelatedExternalID: v1.ExternalID{ExternalID: []string{*service.ServiceArn}, ConfigType: v1.AWSECSService}, + ConfigExternalID: v1.ExternalID{ExternalID: []string{*service.TaskDefinition}, ConfigType: v1.AWSECSTaskDefinition}, + Relationship: "ECSTaskDefinitionECSService", + }) + *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSECSService, - ID: *service.ServiceArn, - Name: *service.ServiceName, - Config: service, - ConfigClass: "ECSService", - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSService, *service.ServiceArn)}, - Parents: []v1.ConfigExternalKey{{Type: v1.AWSECSCluster, ExternalID: cluster}}, + Type: v1.AWSECSService, + ID: *service.ServiceArn, + Name: *service.ServiceName, + Config: service, + RelationshipResults: relationships, + ConfigClass: "ECSService", + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSService, *service.ServiceArn)}, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSECSCluster, ExternalID: cluster}}, }) } } -func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, client *ecs.Client, cluster string, results *v1.ScrapeResults) { - if !config.Includes("ecstask") { +func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + if !config.Includes("ECSTaskDefinition") { return } - input := &ecs.ListTasksInput{Cluster: &cluster} - paginator := ecs.NewListTasksPaginator(client, input) + ctx.Logger.V(2).Infof("scraping ECS task definitions") + + client := ecs.NewFromConfig(*ctx.Session) + input := &ecs.ListTaskDefinitionsInput{} + paginator := ecs.NewListTaskDefinitionsPaginator(client, input) for paginator.HasMorePages() { output, err := paginator.NextPage(ctx) @@ -307,30 +332,34 @@ func (aws Scraper) ecsTasks(ctx *AWSContext, config v1.AWS, client *ecs.Client, return } - if len(output.TaskArns) == 0 { - continue - } - - tasksOutput, err := client.DescribeTasks(ctx, &ecs.DescribeTasksInput{ - Tasks: output.TaskArns, - }) - if err != nil { - results.Errorf(err, "failed to describe ECS tasks") + if len(output.TaskDefinitionArns) == 0 { return } - for _, task := range tasksOutput.Tasks { - result := v1.ScrapeResult{ - Type: v1.AWSECSTask, - ID: *task.TaskArn, - Name: *task.TaskDefinitionArn, - Config: task, - ConfigClass: "ECSTask", - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSTask, *task.TaskArn)}, - Parents: []v1.ConfigExternalKey{{Type: v1.AWSECSCluster, ExternalID: *task.ClusterArn}}, + for _, taskDefinitionArn := range output.TaskDefinitionArns { + describeTaskDefinitionOutput, err := client.DescribeTaskDefinition(ctx, &ecs.DescribeTaskDefinitionInput{ + TaskDefinition: &taskDefinitionArn, + }) + if err != nil { + results.Errorf(err, "failed to describe ECS task definition %s", taskDefinitionArn) + continue + } + + labels := make(map[string]string) + for _, tag := range describeTaskDefinitionOutput.Tags { + labels[*tag.Key] = *tag.Value } - *results = append(*results, result) + + *results = append(*results, v1.ScrapeResult{ + Type: v1.AWSECSTaskDefinition, + Labels: labels, + ID: *describeTaskDefinitionOutput.TaskDefinition.TaskDefinitionArn, + Name: *describeTaskDefinitionOutput.TaskDefinition.Family, + Config: describeTaskDefinitionOutput.TaskDefinition, + ConfigClass: "ECSTaskDefinition", + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSECSTaskDefinition, *describeTaskDefinitionOutput.TaskDefinition.Family)}, + }) } } } @@ -340,6 +369,8 @@ func (aws Scraper) eksFargateProfiles(ctx *AWSContext, config v1.AWS, client *ek return } + ctx.Logger.V(2).Infof("scraping EKS Fargate profiles") + listFargateProfilesInput := &eks.ListFargateProfilesInput{ ClusterName: &clusterName, } @@ -378,6 +409,8 @@ func (aws Scraper) elastiCache(ctx *AWSContext, config v1.AWS, results *v1.Scrap return } + ctx.Logger.V(2).Infof("scraping elastiCache") + svc := elasticache.NewFromConfig(*ctx.Session) clusters, err := svc.DescribeCacheClusters(ctx, &elasticache.DescribeCacheClustersInput{}) @@ -403,6 +436,7 @@ func (aws Scraper) lambdaFunctions(ctx *AWSContext, config v1.AWS, results *v1.S if config.Excludes("lambda") { return } + ctx.Logger.V(2).Infof("scraping lambda functions") lambdaClient := lambda.NewFromConfig(*ctx.Session) input := &lambda.ListFunctionsInput{} @@ -440,6 +474,8 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap return } + ctx.Logger.V(2).Infof("scraping EKS clusters") + EKS := eks.NewFromConfig(*ctx.Session) clusters, err := EKS.ListClusters(ctx, nil) if err != nil { @@ -512,6 +548,8 @@ func (aws Scraper) efs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults return } + ctx.Logger.V(2).Infof("scraping EFS") + describeInput := &efs.DescribeFileSystemsInput{} EFS := efs.NewFromConfig(*ctx.Session) describeOutput, err := EFS.DescribeFileSystems(ctx, describeInput) @@ -541,6 +579,8 @@ func (aws Scraper) efs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults // availabilityZones fetches all the availability zones in the region set in givne the aws session. func (aws Scraper) availabilityZones(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + ctx.Logger.V(2).Infof("scraping availability zones") + azDescribeInput := &ec2.DescribeAvailabilityZonesInput{} azDescribeOutput, err := ctx.EC2.DescribeAvailabilityZones(ctx, azDescribeInput) if err != nil { @@ -585,6 +625,8 @@ func (aws Scraper) account(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes return } + ctx.Logger.V(2).Infof("scraping AWS account") + summary, err := ctx.IAM.GetAccountSummary(ctx, nil) if err != nil { results.Errorf(err, "failed to get account summary") @@ -655,6 +697,9 @@ func (aws Scraper) users(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResul if !config.Includes("User") { return } + + ctx.Logger.V(2).Infof("scraping IAM Users") + users, err := ctx.IAM.ListUsers(ctx, nil) if err != nil { results.Errorf(err, "failed to get users") @@ -685,6 +730,8 @@ func (aws Scraper) ebs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults return } + ctx.Logger.V(2).Infof("scraping EBS") + describeInput := &ec2.DescribeVolumesInput{} describeOutput, err := ctx.EC2.DescribeVolumes(ctx, describeInput) if err != nil { @@ -722,6 +769,8 @@ func (aws Scraper) rds(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults return } + ctx.Logger.V(2).Infof("scraping RDS") + describeInput := &rds.DescribeDBInstancesInput{} RDS := rds.NewFromConfig(*ctx.Session) describeOutput, err := RDS.DescribeDBInstances(ctx, describeInput) @@ -776,6 +825,8 @@ func (aws Scraper) vpcs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult return } + ctx.Logger.V(2).Infof("scraping VPCs") + describeInput := &ec2.DescribeVpcsInput{} describeOutput, err := ctx.EC2.DescribeVpcs(ctx, describeInput) if err != nil { @@ -812,6 +863,7 @@ func (aws Scraper) vpcs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult *results = append(*results, v1.ScrapeResult{ Type: v1.AWSEC2VPC, Labels: labels, + Tags: []v1.Tag{{Name: "region", Value: ctx.Session.Region}}, BaseScraper: config.BaseScraper, Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2VPC, lo.FromPtr(vpc.VpcId))}, Config: vpc, @@ -830,6 +882,8 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR return } + ctx.Logger.V(2).Infof("scraping EC2 instances") + describeOutput, err := ctx.EC2.DescribeInstances(ctx, &ec2.DescribeInstancesInput{}) if err != nil { results.Errorf(err, "failed to describe instances") @@ -881,11 +935,12 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR }) } - relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{*i.ImageId}, ConfigType: v1.AWSEC2AMI}, - RelatedExternalID: selfExternalID, - Relationship: "AMIInstance", - }) + // NOTE: commenting out for now since we aren't currently scraping AMIs + // relationships = append(relationships, v1.RelationshipResult{ + // ConfigExternalID: v1.ExternalID{ExternalID: []string{*i.ImageId}, ConfigType: v1.AWSEC2AMI}, + // RelatedExternalID: selfExternalID, + // Relationship: "AMIInstance", + // }) relationships = append(relationships, v1.RelationshipResult{ ConfigExternalID: selfExternalID, @@ -906,7 +961,7 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR }) relationships = append(relationships, v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ExternalID: []string{ctx.Subnets[lo.FromPtr(i.SubnetId)].Zone}, ConfigType: v1.AWSZone}, + ConfigExternalID: v1.ExternalID{ExternalID: []string{ctx.Subnets[lo.FromPtr(i.SubnetId)].Zone}, ConfigType: v1.AWSAvailabilityZone}, RelatedExternalID: selfExternalID, Relationship: "ZoneInstance", }) @@ -947,6 +1002,8 @@ func (aws Scraper) securityGroups(ctx *AWSContext, config v1.AWS, results *v1.Sc return } + ctx.Logger.V(2).Infof("scraping security groups") + describeInput := &ec2.DescribeSecurityGroupsInput{} describeOutput, err := ctx.EC2.DescribeSecurityGroups(ctx, describeInput) if err != nil { @@ -975,6 +1032,9 @@ func (aws Scraper) routes(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResu if !config.Includes("Route") { return } + + ctx.Logger.V(2).Infof("scraping route tables") + describeInput := &ec2.DescribeRouteTablesInput{} describeOutput, err := ctx.EC2.DescribeRouteTables(ctx, describeInput) if err != nil { @@ -1003,6 +1063,8 @@ func (aws Scraper) dhcp(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult return } + ctx.Logger.V(2).Infof("scraping DHCP options") + describeInput := &ec2.DescribeDhcpOptionsInput{} describeOutput, err := ctx.EC2.DescribeDhcpOptions(ctx, describeInput) if err != nil { @@ -1031,6 +1093,8 @@ func (aws Scraper) s3Buckets(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR return } + ctx.Logger.V(2).Infof("scraping S3 buckets") + S3 := s3.NewFromConfig(*ctx.Session) buckets, err := S3.ListBuckets(ctx, nil) if err != nil { @@ -1061,6 +1125,9 @@ func (aws Scraper) dnsZones(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRe if !config.Includes("DNSZone") { return } + + ctx.Logger.V(2).Infof("scraping DNS zones") + Route53 := route53.NewFromConfig(*ctx.Session) zones, err := Route53.ListHostedZones(ctx, nil) if err != nil { @@ -1088,8 +1155,10 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr if !config.Includes("LoadBalancer") { return } - elb := elasticloadbalancing.NewFromConfig(*ctx.Session) + ctx.Logger.V(2).Infof("scraping load balancers") + + elb := elasticloadbalancing.NewFromConfig(*ctx.Session) loadbalancers, err := elb.DescribeLoadBalancers(ctx, nil) if err != nil { results.Errorf(err, "failed to describe load balancers") @@ -1115,7 +1184,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr clusterPrefix := "kubernetes.io/cluster/" elbTagsOutput, err := elb.DescribeTags(ctx, &elasticloadbalancing.DescribeTagsInput{LoadBalancerNames: []string{*lb.LoadBalancerName}}) if err != nil { - logger.Errorf("error while fetching elb tags: %v", err) + logger.Errorf("error while scraping elb tags: %v", err) continue } for _, tagDesc := range elbTagsOutput.TagDescriptions { @@ -1177,7 +1246,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr var relationships []v1.RelationshipResult elbv2TagsOutput, err := elbv2.DescribeTags(ctx, &elasticloadbalancingv2.DescribeTagsInput{ResourceArns: []string{*lb.LoadBalancerArn}}) if err != nil { - logger.Errorf("error while fetching elbv2 tags: %v", err) + logger.Errorf("error while scraping elbv2 tags: %v", err) continue } for _, tagDesc := range elbv2TagsOutput.TagDescriptions { @@ -1223,6 +1292,8 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr } func (aws Scraper) subnets(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { + ctx.Logger.V(2).Infof("scraping subnets") + // we always need to scrape subnets to get the zone for other resources subnets, err := ctx.EC2.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{}) if err != nil { @@ -1329,6 +1400,8 @@ func (aws Scraper) iamProfiles(ctx *AWSContext, config v1.AWS, results *v1.Scrap return } + ctx.Logger.V(2).Infof("scraping IAM profiles") + profiles, err := ctx.IAM.ListInstanceProfiles(ctx, nil) if err != nil { results.Errorf(err, "failed to get profiles") @@ -1445,6 +1518,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { for _, awsConfig := range ctx.ScrapeConfig().Spec.AWS { results := &v1.ScrapeResults{} + var totalResults int for _, region := range awsConfig.Region { awsCtx, err := aws.getContext(ctx, awsConfig, region) if err != nil { @@ -1455,9 +1529,9 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.ecsClusters(awsCtx, awsConfig, results) + aws.ecsTasks(awsCtx, awsConfig, results) aws.elastiCache(awsCtx, awsConfig, results) aws.lambdaFunctions(awsCtx, awsConfig, results) - aws.lambdaFunctions(awsCtx, awsConfig, results) aws.snsTopics(awsCtx, awsConfig, results) aws.sqs(awsCtx, awsConfig, results) aws.subnets(awsCtx, awsConfig, results) @@ -1472,11 +1546,13 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { aws.rds(awsCtx, awsConfig, results) aws.config(awsCtx, awsConfig, results) aws.loadBalancers(awsCtx, awsConfig, results) + aws.availabilityZones(awsCtx, awsConfig, results) aws.containerImages(awsCtx, awsConfig, results) aws.cloudtrail(awsCtx, awsConfig, results) - aws.availabilityZones(awsCtx, awsConfig, results) // We are querying half a million amis, need to optimize for this // aws.ami(awsCtx, awsConfig, results) + + ctx.Logger.V(2).Infof("scraped %d results from region %s", len(*results)-totalResults, region) } awsCtx, err := aws.getContext(ctx, awsConfig, "us-east-1") @@ -1535,18 +1611,23 @@ func getRegionFromArn(arn, resourceType string) string { } func getConsoleLink(region, resourceType, resourceID string) *types.Property { + // resourceID can be a URL in itself - so we encode it. + resourceID = url.QueryEscape(resourceID) + var url string switch resourceType { - case v1.AWSECSTask: - url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/home?region=%s#/tasks/%s", region, region, resourceID) + case v1.AWSEKSFargateProfile: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/home?region=%s#/clusters/%s/fargate-profiles/%s", region, region, resourceID, resourceID) + case v1.AWSECSTaskDefinition: + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/v2/task-definitions/%s?region=%s", region, resourceID, region) case v1.AWSECSService: - url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/home?region=%s#/clusters/%s", region, region, resourceID) + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/v2/services/%s?region=%s", region, resourceID, region) case v1.AWSECSCluster: - url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/v2/clusters/%s/tasks?region=%s", region, resourceID, region) + url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecs/v2/clusters/%s/services?region=%s", region, resourceID, region) case v1.AWSSQS: - url = fmt.Sprintf("https://%s.console.aws.amazon.com/sqs/v2/home?region=%s#/queues/%s", region, region, resourceID) + url = fmt.Sprintf("https://%s.console.aws.amazon.com/sqs/v3/home?region=%s#/queues/%s", region, region, resourceID) case v1.AWSSNSTopic: - url = fmt.Sprintf("https://%s.console.aws.amazon.com/sns/v3/home?region=%s#/topics/%s", region, region, resourceID) + url = fmt.Sprintf("https://%s.console.aws.amazon.com/sns/v3/home?region=%s#/topic/%s", region, region, resourceID) case "AWS::ECR::Repository": url = fmt.Sprintf("https://%s.console.aws.amazon.com/ecr/repositories/%s", region, resourceID) case "AWS::EFS::FileSystem": diff --git a/scrapers/aws/cloudtrail.go b/scrapers/aws/cloudtrail.go index 76832a9d..2d387078 100644 --- a/scrapers/aws/cloudtrail.go +++ b/scrapers/aws/cloudtrail.go @@ -67,6 +67,9 @@ func (aws Scraper) cloudtrail(ctx *AWSContext, config v1.AWS, results *v1.Scrape if config.Excludes("cloudtrail") { return } + + ctx.Logger.V(2).Infof("scraping cloudtrail") + if len(config.CloudTrail.Exclude) == 0 { config.CloudTrail.Exclude = []string{"AssumeRole"} } diff --git a/scrapers/aws/config.go b/scrapers/aws/config.go index fa816cdb..9dfd958f 100644 --- a/scrapers/aws/config.go +++ b/scrapers/aws/config.go @@ -11,6 +11,8 @@ func (aws Scraper) config(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResu return } + ctx.Logger.V(2).Infof("scraping Config Rules") + rules, err := ctx.Config.DescribeConfigRules(ctx, nil) if err != nil { results.Errorf(err, "Failed to describe config rules")