Skip to content

Commit

Permalink
fix: don't create config items for change results
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityathebe
adityathebe committed May 14, 2024
1 parent 8bb3745 commit b1137e6
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 40 deletions.
92 changes: 53 additions & 39 deletions db/update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func updateCI(ctx api.ScrapeContext, result v1.ScrapeResult, ci, existing *model
} else if changeResult != nil {
ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0])
result.Changes = []v1.ChangeResult{*changeResult}
if newChanges, _, err := extractChanges(ctx, &result); err != nil {
if newChanges, _, err := extractChanges(ctx, &result, ci); err != nil {
return nil, err
} else {
changes = append(changes, newChanges...)
Expand Down Expand Up @@ -214,7 +214,7 @@ func shouldExcludeChange(result *v1.ScrapeResult, changeResult v1.ChangeResult)
return false, nil
}

func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.ConfigChange, []*models.ConfigChange, error) {
func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.ConfigItem) ([]*models.ConfigChange, []*models.ConfigChange, error) {
var (
newOnes = []*models.ConfigChange{}
updates = []*models.ConfigChange{}
Expand Down Expand Up @@ -253,7 +253,13 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.C
}
}

if !change.GetExternalID().IsEmpty() {
if change.ChangeType == "CreateTags" {
_ = change
}

if change.ConfigID == "" && change.GetExternalID().IsEmpty() && ci != nil {
change.ConfigID = ci.ID
} else if !change.GetExternalID().IsEmpty() {
if ci, err := ctx.TempCache().FindExternalID(change.GetExternalID()); err != nil {
return nil, nil, fmt.Errorf("failed to get config from change (externalID=%s): %w", change.GetExternalID(), err)
} else if ci != "" {
Expand Down Expand Up @@ -514,7 +520,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations
continue
}
if configID == "" {
logger.Warnf("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID)
ctx.Logger.V(1).Infof("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID)
continue
}
}
Expand Down Expand Up @@ -579,52 +585,60 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime

for _, result := range results {
result.LastScrapedTime = &scrapeStartTime
ci, err := NewConfigItemFromResult(ctx, result)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to create config item(%s): %w", result, err)
}

ci.ScraperID = ctx.ScrapeConfig().GetPersistedID()
if len(ci.ExternalID) == 0 {
return nil, nil, nil, nil, fmt.Errorf("config item %s has no external id", ci)
}

if isTreeRoot(lo.FromPtr(ci.Type)) {
root = ci.ID
}

parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)}
parentTypeToConfigMap[parentExternalKey] = ci.ID
var ci *models.ConfigItem
var err error

existing := &models.ConfigItem{}
if ci.ID != "" {
if existing, err = ctx.TempCache().Get(ci.ID); err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err)
if result.ID != "" {
// A result that only contains changes (example a result created by Cloudtrail scraper)
// doesn't have any id.
ci, err = NewConfigItemFromResult(ctx, result)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to create config item(%s): %w", result, err)
}
} else {
if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to lookup external id(%s): %w", ci, err)

ci.ScraperID = ctx.ScrapeConfig().GetPersistedID()
if len(ci.ExternalID) == 0 {
return nil, nil, nil, nil, fmt.Errorf("config item %s has no external id", ci)
}
}

allConfigs = append(allConfigs, ci)
if result.Config != nil {
if err := tree.AddVertex(ci.ID); err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to add vertex(%s): %w", ci, err)
if isTreeRoot(lo.FromPtr(ci.Type)) {
root = ci.ID
}

if existing == nil || existing.ID == "" {
newConfigs = append(newConfigs, ci)
parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)}
parentTypeToConfigMap[parentExternalKey] = ci.ID

existing := &models.ConfigItem{}
if ci.ID != "" {
if existing, err = ctx.TempCache().Get(ci.ID); err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err)
}
} else {
configsToUpdate = append(configsToUpdate, &updateConfigArgs{
Result: result,
Existing: existing,
New: ci,
})
if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to lookup external id(%s): %w", ci, err)
}
}

allConfigs = append(allConfigs, ci)
if result.Config != nil {
if err := tree.AddVertex(ci.ID); err != nil {
return nil, nil, nil, nil, fmt.Errorf("unable to add vertex(%s): %w", ci, err)
}

if existing == nil || existing.ID == "" {
newConfigs = append(newConfigs, ci)
} else {
configsToUpdate = append(configsToUpdate, &updateConfigArgs{
Result: result,
Existing: existing,
New: ci,
})
}
}
}

if toCreate, toUpdate, err := extractChanges(ctx, &result); err != nil {
if toCreate, toUpdate, err := extractChanges(ctx, &result, ci); err != nil {
return nil, nil, nil, nil, err
} else {
newChanges = append(newChanges, toCreate...)
Expand Down
2 changes: 1 addition & 1 deletion scrapers/aws/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1106,7 +1106,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults {
continue
}

ctx.Logger.V(2).Infof("scraping %s", awsCtx)
ctx.Logger.V(1).Infof("scraping %s", awsCtx)
aws.subnets(awsCtx, awsConfig, results)
aws.instances(awsCtx, awsConfig, results)
aws.vpcs(awsCtx, awsConfig, results)
Expand Down
1 change: 1 addition & 0 deletions scrapers/cron.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ func scheduleScraperJob(sc api.ScrapeContext) error {
Schedule: schedule,
Singleton: true,
JobHistory: true,
RunNow: true,
Retention: job.RetentionBalanced,
ResourceID: sc.ScrapeConfig().GetPersistedID().String(),
ResourceType: job.ResourceTypeScraper,
Expand Down

0 comments on commit b1137e6

Please sign in to comment.