From 1b84b0f595e4b7784dd7e002c48aa0744208e7a3 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 14 May 2024 17:51:53 +0545 Subject: [PATCH] fix: don't create config items for change results --- db/update.go | 90 +++++++++++++++++++++++++-------------------- scrapers/aws/aws.go | 2 +- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/db/update.go b/db/update.go index 3925d49d..54a626aa 100644 --- a/db/update.go +++ b/db/update.go @@ -117,7 +117,7 @@ func updateCI(ctx api.ScrapeContext, result v1.ScrapeResult, ci, existing *model } else if changeResult != nil { ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0]) result.Changes = []v1.ChangeResult{*changeResult} - if newChanges, _, err := extractChanges(ctx, &result); err != nil { + if newChanges, _, err := extractChanges(ctx, &result, ci); err != nil { return nil, err } else { changes = append(changes, newChanges...) @@ -214,7 +214,7 @@ func shouldExcludeChange(result *v1.ScrapeResult, changeResult v1.ChangeResult) return false, nil } -func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.ConfigChange, []*models.ConfigChange, error) { +func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.ConfigItem) ([]*models.ConfigChange, []*models.ConfigChange, error) { var ( newOnes = []*models.ConfigChange{} updates = []*models.ConfigChange{} @@ -253,7 +253,9 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.C } } - if !change.GetExternalID().IsEmpty() { + if change.ConfigID == "" && change.GetExternalID().IsEmpty() && ci != nil { + change.ConfigID = ci.ID + } else if !change.GetExternalID().IsEmpty() { if ci, err := ctx.TempCache().FindExternalID(change.GetExternalID()); err != nil { return nil, nil, fmt.Errorf("failed to get config from change (externalID=%s): %w", change.GetExternalID(), err) } else if ci != "" { @@ -265,6 +267,8 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.C } if change.ConfigID == "" { + // Some scrapers can generate changes for config items that don't exist on our db. + // Example: Cloudtrail scraper reporting changes for a resource that has been excluded. ctx.Logger.V(1).Infof("(type=%s source=%s) change doesn't have an associated config", change.ChangeType, change.Source) continue } @@ -514,7 +518,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations continue } if configID == "" { - logger.Warnf("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID) + ctx.Logger.V(1).Infof("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID) continue } } @@ -579,52 +583,60 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime for _, result := range results { result.LastScrapedTime = &scrapeStartTime - ci, err := NewConfigItemFromResult(ctx, result) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to create config item(%s): %w", result, err) - } - - ci.ScraperID = ctx.ScrapeConfig().GetPersistedID() - if len(ci.ExternalID) == 0 { - return nil, nil, nil, nil, fmt.Errorf("config item %s has no external id", ci) - } - - if isTreeRoot(lo.FromPtr(ci.Type)) { - root = ci.ID - } - parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)} - parentTypeToConfigMap[parentExternalKey] = ci.ID + var ci *models.ConfigItem + var err error - existing := &models.ConfigItem{} - if ci.ID != "" { - if existing, err = ctx.TempCache().Get(ci.ID); err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err) + if result.ID != "" { + // A result that only contains changes (example a result created by Cloudtrail scraper) + // doesn't have any id. + ci, err = NewConfigItemFromResult(ctx, result) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to create config item(%s): %w", result, err) } - } else { - if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to lookup external id(%s): %w", ci, err) + + ci.ScraperID = ctx.ScrapeConfig().GetPersistedID() + if len(ci.ExternalID) == 0 { + return nil, nil, nil, nil, fmt.Errorf("config item %s has no external id", ci) } - } - allConfigs = append(allConfigs, ci) - if result.Config != nil { - if err := tree.AddVertex(ci.ID); err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to add vertex(%s): %w", ci, err) + if isTreeRoot(lo.FromPtr(ci.Type)) { + root = ci.ID } - if existing == nil || existing.ID == "" { - newConfigs = append(newConfigs, ci) + parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)} + parentTypeToConfigMap[parentExternalKey] = ci.ID + + existing := &models.ConfigItem{} + if ci.ID != "" { + if existing, err = ctx.TempCache().Get(ci.ID); err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err) + } } else { - configsToUpdate = append(configsToUpdate, &updateConfigArgs{ - Result: result, - Existing: existing, - New: ci, - }) + if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to lookup external id(%s): %w", ci, err) + } + } + + allConfigs = append(allConfigs, ci) + if result.Config != nil { + if err := tree.AddVertex(ci.ID); err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to add vertex(%s): %w", ci, err) + } + + if existing == nil || existing.ID == "" { + newConfigs = append(newConfigs, ci) + } else { + configsToUpdate = append(configsToUpdate, &updateConfigArgs{ + Result: result, + Existing: existing, + New: ci, + }) + } } } - if toCreate, toUpdate, err := extractChanges(ctx, &result); err != nil { + if toCreate, toUpdate, err := extractChanges(ctx, &result, ci); err != nil { return nil, nil, nil, nil, err } else { newChanges = append(newChanges, toCreate...) diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 7b176175..cd0ee7e9 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -1128,7 +1128,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { continue } - ctx.Logger.V(2).Infof("scraping %s", awsCtx) + ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.subnets(awsCtx, awsConfig, results) aws.instances(awsCtx, awsConfig, results) aws.vpcs(awsCtx, awsConfig, results)