Skip to content

Commit

Permalink
feat: link kubernetes LB services to AWS ELB
Browse files Browse the repository at this point in the history 
* fix source vertex not found error by namespacing all look ups to the
  scraper.
  • Loading branch information
@adityathebe
adityathebe committed Jul 8, 2024
1 parent 6c8bbc3 commit c2b5d44
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 15 deletions.
19 changes: 11 additions & 8 deletions api/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,23 @@ type TempCache struct {
aliases map[string]string
}

func (t TempCache) FindExternal(ext v1.ExternalID) (*models.ConfigItem, error) {
return t.Find(ext.ConfigType, ext.ExternalID[0])
}

func (t TempCache) FindExternalID(ext v1.ExternalID) (string, error) {
if item, err := t.Find(ext.ConfigType, ext.ExternalID[0]); err != nil {
func (t TempCache) FindExternalID(scraperID string, ext v1.ExternalID) (string, error) {
if item, err := t.Find(scraperID, ext.ConfigType, ext.ExternalID[0]); err != nil {
return "", err
} else if item != nil {
return item.ID, nil
}
return "", nil
}

func (t TempCache) Find(typ, id string) (*models.ConfigItem, error) {
func (t TempCache) Find(scraperID, typ, id string) (*models.ConfigItem, error) {
typ = strings.ToLower(typ)

if strings.HasPrefix(typ, "kubernetes::") && uuid.Validate(id) == nil {
// kubernetes external ids are stored are the same as the config ids
return t.Get(id)
}

if t.aliases == nil {
t.aliases = make(map[string]string)
}
Expand All @@ -47,13 +44,19 @@ func (t TempCache) Find(typ, id string) (*models.ConfigItem, error) {
}

result := models.ConfigItem{}
if err := t.ctx.DB().Limit(1).Find(&result, "lower(type) = ? and external_id @> ?", typ, pq.StringArray{id}).Error; err != nil {
query := t.ctx.DB().Limit(1)
if id != "" {
query = query.Where("scraper_id = ?", scraperID)
}
if err := query.Find(&result, "lower(type) = ? and external_id @> ?", typ, pq.StringArray{id}).Error; err != nil {
return nil, err
}

if result.ID != "" {
t.Insert(result)
return &result, nil
}

return nil, nil
}

Expand Down
12 changes: 6 additions & 6 deletions db/update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.C
if change.ConfigID == "" && change.GetExternalID().IsEmpty() && ci != nil {
change.ConfigID = ci.ID
} else if !change.GetExternalID().IsEmpty() {
if ci, err := ctx.TempCache().FindExternalID(change.GetExternalID()); err != nil {
if ci, err := ctx.TempCache().FindExternalID(string(ctx.ScrapeConfig().GetUID()), change.GetExternalID()); err != nil {
return nil, nil, changeSummary, fmt.Errorf("failed to get config from change (externalID=%s): %w", change.GetExternalID(), err)
} else if ci != "" {
change.ConfigID = ci
Expand Down Expand Up @@ -289,7 +289,7 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.C

func upsertAnalysis(ctx api.ScrapeContext, result *v1.ScrapeResult) error {
analysis := result.AnalysisResult.ToConfigAnalysis()
ciID, err := ctx.TempCache().Find(analysis.ConfigType, analysis.ExternalID)
ciID, err := ctx.TempCache().Find(string(ctx.ScrapeConfig().GetUID()), analysis.ConfigType, analysis.ExternalID)
if err != nil {
return err
}
Expand Down Expand Up @@ -578,7 +578,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations
if relationship.ConfigID != "" {
configID = relationship.ConfigID
} else {
configID, err = ctx.TempCache().FindExternalID(relationship.ConfigExternalID)
configID, err = ctx.TempCache().FindExternalID(string(ctx.ScrapeConfig().GetUID()), relationship.ConfigExternalID)
if err != nil {
logger.Errorf("error fetching config item(id=%s): %v", relationship.ConfigExternalID, err)
continue
Expand All @@ -593,7 +593,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations
if relationship.RelatedConfigID != "" {
relatedID = relationship.RelatedConfigID
} else {
relatedID, err = ctx.TempCache().FindExternalID(relationship.RelatedExternalID)
relatedID, err = ctx.TempCache().FindExternalID(string(ctx.ScrapeConfig().GetUID()), relationship.RelatedExternalID)
if err != nil {
logger.Errorf("error fetching external config item(id=%s): %v", relationship.RelatedExternalID, err)
continue
Expand Down Expand Up @@ -676,7 +676,7 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime
return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err)
}
} else {
if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil {
if existing, err = ctx.TempCache().Find(string(ctx.ScrapeConfig().GetUID()), *ci.Type, ci.ExternalID[0]); err != nil {
return nil, nil, nil, nil, allChangeSummary, fmt.Errorf("unable to lookup external id(%s): %w", ci, err)
}
}
Expand Down Expand Up @@ -776,7 +776,7 @@ func setConfigParents(ctx api.ScrapeContext, parentTypeToConfigMap map[configExt
break
}

if found, err := ctx.TempCache().Find(parent.Type, parent.ExternalID); err != nil {
if found, err := ctx.TempCache().Find(string(ctx.ScrapeConfig().GetUID()), parent.Type, parent.ExternalID); err != nil {
return err
} else if found != nil {
ci.ParentID = &found.ID
Expand Down
2 changes: 1 addition & 1 deletion scrapers/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1264,7 +1264,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr
Name: *lb.LoadBalancerName,
Labels: labels,
Tags: tags,
Aliases: []string{"AWSELB/" + arn, arn},
Aliases: []string{"AWSELB/" + arn, arn, lo.FromPtr(lb.CanonicalHostedZoneName)},
ID: *lb.LoadBalancerName,
Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(lb.VPCId)}},
RelationshipResults: relationships,
Expand Down
7 changes: 7 additions & 0 deletions scrapers/kubernetes/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,13 @@ func ExtractResults(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstruc
if ingress, ok := ing.(map[string]any); ok {
if hostname, ok := ingress["hostname"].(string); ok && hostname != "" {
labels["hostname"] = hostname

if strings.HasSuffix(hostname, "elb.amazonaws.com") {
relationships = append(relationships, v1.RelationshipResult{
ConfigID: string(obj.GetUID()),
RelatedExternalID: v1.ExternalID{ExternalID: []string{hostname}, ConfigType: v1.AWSLoadBalancer},
})
}
}

if ip, ok := ingress["ip"].(string); ok && ip != "" {
Expand Down

0 comments on commit c2b5d44

Please sign in to comment.