From 68e5ee73c244a67179869f712f257cb173936c58 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 16 Oct 2024 21:13:37 +0530 Subject: [PATCH] fix: config external lookup references for aws --- api/v1/common.go | 3 +++ api/v1/interface.go | 2 +- api/v1/types.go | 3 ++- db/config.go | 6 ++++-- scrapers/aws/aws.go | 6 ++---- scrapers/kubernetes/kubernetes.go | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api/v1/common.go b/api/v1/common.go index 564d60dc..1e0174ed 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -13,6 +13,9 @@ import ( "github.com/flanksource/gomplate/v3" ) +// List of types which should not have scraper_id +var ScraperLessTypes = []string{AWSRegion, AWSAvailabilityZone} + // ConfigFieldExclusion defines fields with JSONPath that needs to // be removed from the config. type ConfigFieldExclusion struct { diff --git a/api/v1/interface.go b/api/v1/interface.go index 0fbb3d56..c039627d 100644 --- a/api/v1/interface.go +++ b/api/v1/interface.go @@ -596,7 +596,7 @@ type ScrapeResult struct { LastScrapedTime *time.Time `json:"last_scraped_time"` // ScraperLess when true indicates that this config item - // do not belong to any scraper. Example: AWS region & availability zone. + // does not belong to any scraper. Example: AWS region & availability zone. ScraperLess bool `json:"scraper_less,omitempty"` // List of candidate parents in order of precision. diff --git a/api/v1/types.go b/api/v1/types.go index 598b9c9e..a6cfa811 100644 --- a/api/v1/types.go +++ b/api/v1/types.go @@ -2,6 +2,7 @@ package v1 import ( "fmt" + "slices" "strings" "github.com/flanksource/config-db/utils" @@ -118,7 +119,7 @@ func (e ExternalID) Find(db *gorm.DB) *gorm.DB { if e.ConfigType != "" { query = query.Where("type = ?", e.ConfigType) } - if e.ScraperID != "all" && e.ScraperID != "" { + if e.ScraperID != "all" && e.ScraperID != "" && !slices.Contains(ScraperLessTypes, e.ConfigType) { query = query.Where("scraper_id = ?", e.ScraperID) } return query diff --git a/db/config.go b/db/config.go index fca35854..e04fd378 100644 --- a/db/config.go +++ b/db/config.go @@ -3,6 +3,7 @@ package db import ( "encoding/json" "fmt" + "slices" "github.com/flanksource/commons/hash" "github.com/flanksource/commons/logger" @@ -145,10 +146,11 @@ func NewConfigItemFromResult(ctx api.ScrapeContext, result v1.ScrapeResult) (*mo Parents: result.Parents, Health: lo.ToPtr(dutyModels.HealthUnknown), Children: result.Children, + ScraperID: ctx.ScrapeConfig().GetPersistedID(), } - if !result.ScraperLess { - ci.ScraperID = ctx.ScrapeConfig().GetPersistedID() + if result.ScraperLess || slices.Contains(v1.ScraperLessTypes, ci.Type) { + ci.ScraperID = nil } if parsed, err := result.Tags.AsMap(); err != nil { diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index afe5799a..feeaafd6 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -803,7 +803,6 @@ func (aws Scraper) availabilityZones(ctx *AWSContext, config v1.AWS, results *v1 ConfigClass: "AvailabilityZone", Aliases: nil, Name: lo.FromPtr(az.ZoneId), - ScraperLess: true, Parents: []v1.ConfigExternalKey{{Type: v1.AWSRegion, ExternalID: lo.FromPtr(az.RegionName)}}, }) @@ -874,7 +873,6 @@ func (aws Scraper) account(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes BaseScraper: config.BaseScraper, Name: *region.RegionName, ID: *region.RegionName, - ScraperLess: true, } if *region.OptInStatus != "not-opted-in" { @@ -919,7 +917,7 @@ func (aws Scraper) users(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResul ConfigClass: "User", Labels: labels, Name: *user.UserName, - Aliases: []string{*user.UserId, *user.Arn}, + Aliases: []string{*user.UserId, *user.UserName}, Ignore: []string{"arn", "userId", "createDate", "userName"}, ID: *user.Arn, // UserId is not often referenced Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, @@ -1435,7 +1433,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr Name: *lb.LoadBalancerName, Labels: labels, Tags: tags, - Aliases: []string{"AWSELB/" + arn, lo.FromPtr(lb.CanonicalHostedZoneName)}, + Aliases: []string{"AWSELB/" + arn, lo.FromPtr(lb.CanonicalHostedZoneName), *lb.LoadBalancerName}, ID: arn, Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(lb.VPCId)}}, RelationshipResults: relationships, diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 849e0a4e..3c020e75 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -389,7 +389,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v if strings.HasSuffix(hostname, "elb.amazonaws.com") { relationships = append(relationships, v1.RelationshipResult{ ConfigID: string(obj.GetUID()), - RelatedExternalID: v1.ExternalID{ExternalID: hostname, ConfigType: v1.AWSLoadBalancer}, + RelatedExternalID: v1.ExternalID{ExternalID: hostname, ConfigType: v1.AWSLoadBalancer, ScraperID: "all"}, }) } }