From b1137e617aa08447f942e8593fa32b7358e66ec8 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 14 May 2024 17:51:53 +0545 Subject: [PATCH] fix: don't create config items for change results --- db/update.go | 92 ++++++++++++++++++++++++++------------------- scrapers/aws/aws.go | 2 +- scrapers/cron.go | 1 + 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/db/update.go b/db/update.go index 3925d49dc..82c7fe60b 100644 --- a/db/update.go +++ b/db/update.go @@ -117,7 +117,7 @@ func updateCI(ctx api.ScrapeContext, result v1.ScrapeResult, ci, existing *model } else if changeResult != nil { ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0]) result.Changes = []v1.ChangeResult{*changeResult} - if newChanges, _, err := extractChanges(ctx, &result); err != nil { + if newChanges, _, err := extractChanges(ctx, &result, ci); err != nil { return nil, err } else { changes = append(changes, newChanges...) @@ -214,7 +214,7 @@ func shouldExcludeChange(result *v1.ScrapeResult, changeResult v1.ChangeResult) return false, nil } -func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.ConfigChange, []*models.ConfigChange, error) { +func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.ConfigItem) ([]*models.ConfigChange, []*models.ConfigChange, error) { var ( newOnes = []*models.ConfigChange{} updates = []*models.ConfigChange{} @@ -253,7 +253,13 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.C } } - if !change.GetExternalID().IsEmpty() { + if change.ChangeType == "CreateTags" { + _ = change + } + + if change.ConfigID == "" && change.GetExternalID().IsEmpty() && ci != nil { + change.ConfigID = ci.ID + } else if !change.GetExternalID().IsEmpty() { if ci, err := ctx.TempCache().FindExternalID(change.GetExternalID()); err != nil { return nil, nil, fmt.Errorf("failed to get config from change (externalID=%s): %w", change.GetExternalID(), err) } else if ci != "" { @@ -514,7 +520,7 @@ func relationshipResultHandler(ctx api.ScrapeContext, relationships v1.Relations continue } if configID == "" { - logger.Warnf("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID) + ctx.Logger.V(1).Infof("unable to form relationship. failed to find the parent config %s", relationship.ConfigExternalID) continue } } @@ -579,52 +585,60 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime for _, result := range results { result.LastScrapedTime = &scrapeStartTime - ci, err := NewConfigItemFromResult(ctx, result) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to create config item(%s): %w", result, err) - } - - ci.ScraperID = ctx.ScrapeConfig().GetPersistedID() - if len(ci.ExternalID) == 0 { - return nil, nil, nil, nil, fmt.Errorf("config item %s has no external id", ci) - } - if isTreeRoot(lo.FromPtr(ci.Type)) { - root = ci.ID - } - - parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)} - parentTypeToConfigMap[parentExternalKey] = ci.ID + var ci *models.ConfigItem + var err error - existing := &models.ConfigItem{} - if ci.ID != "" { - if existing, err = ctx.TempCache().Get(ci.ID); err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err) + if result.ID != "" { + // A result that only contains changes (example a result created by Cloudtrail scraper) + // doesn't have any id. + ci, err = NewConfigItemFromResult(ctx, result) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to create config item(%s): %w", result, err) } - } else { - if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to lookup external id(%s): %w", ci, err) + + ci.ScraperID = ctx.ScrapeConfig().GetPersistedID() + if len(ci.ExternalID) == 0 { + return nil, nil, nil, nil, fmt.Errorf("config item %s has no external id", ci) } - } - allConfigs = append(allConfigs, ci) - if result.Config != nil { - if err := tree.AddVertex(ci.ID); err != nil { - return nil, nil, nil, nil, fmt.Errorf("unable to add vertex(%s): %w", ci, err) + if isTreeRoot(lo.FromPtr(ci.Type)) { + root = ci.ID } - if existing == nil || existing.ID == "" { - newConfigs = append(newConfigs, ci) + parentExternalKey := configExternalKey{externalID: ci.ExternalID[0], parentType: lo.FromPtr(ci.Type)} + parentTypeToConfigMap[parentExternalKey] = ci.ID + + existing := &models.ConfigItem{} + if ci.ID != "" { + if existing, err = ctx.TempCache().Get(ci.ID); err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to lookup existing config(%s): %w", ci, err) + } } else { - configsToUpdate = append(configsToUpdate, &updateConfigArgs{ - Result: result, - Existing: existing, - New: ci, - }) + if existing, err = ctx.TempCache().Find(*ci.Type, ci.ExternalID[0]); err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to lookup external id(%s): %w", ci, err) + } + } + + allConfigs = append(allConfigs, ci) + if result.Config != nil { + if err := tree.AddVertex(ci.ID); err != nil { + return nil, nil, nil, nil, fmt.Errorf("unable to add vertex(%s): %w", ci, err) + } + + if existing == nil || existing.ID == "" { + newConfigs = append(newConfigs, ci) + } else { + configsToUpdate = append(configsToUpdate, &updateConfigArgs{ + Result: result, + Existing: existing, + New: ci, + }) + } } } - if toCreate, toUpdate, err := extractChanges(ctx, &result); err != nil { + if toCreate, toUpdate, err := extractChanges(ctx, &result, ci); err != nil { return nil, nil, nil, nil, err } else { newChanges = append(newChanges, toCreate...) diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index e423c46d2..e68cdb236 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -1106,7 +1106,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { continue } - ctx.Logger.V(2).Infof("scraping %s", awsCtx) + ctx.Logger.V(1).Infof("scraping %s", awsCtx) aws.subnets(awsCtx, awsConfig, results) aws.instances(awsCtx, awsConfig, results) aws.vpcs(awsCtx, awsConfig, results) diff --git a/scrapers/cron.go b/scrapers/cron.go index f58d943dc..5aa9ee864 100644 --- a/scrapers/cron.go +++ b/scrapers/cron.go @@ -144,6 +144,7 @@ func scheduleScraperJob(sc api.ScrapeContext) error { Schedule: schedule, Singleton: true, JobHistory: true, + RunNow: true, Retention: job.RetentionBalanced, ResourceID: sc.ScrapeConfig().GetPersistedID().String(), ResourceType: job.ResourceTypeScraper,