fix: saving changes for non existing configs

flanksource · May 14, 2024 · 8bb3745 · 8bb3745
1 parent 81a5c2e
commit 8bb3745
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 14 deletions.
diff --git a/db/update.go b/db/update.go
@@ -51,11 +51,11 @@ func deleteChangeHandler(ctx api.ScrapeContext, change v1.ChangeResult) error {
 		return errors.Wrapf(tx.Error, "unable to delete config item %s/%s", change.ConfigType, change.ExternalID)
 	}
 	if tx.RowsAffected == 0 || len(configs) == 0 {
-		logger.Warnf("Attempt to delete non-existent config item %s/%s", change.ConfigType, change.ExternalID)
+		ctx.Logger.V(2).Infof("attempt to delete non-existent config item %s/%s", change.ConfigType, change.ExternalID)
 		return nil
 	}
 
-	ctx.Logger.V(3).Infof("Deleted %s from change %s", configs[0].ID, change)
+	ctx.Logger.V(3).Infof("deleted %s from change %s", configs[0].ID, change)
 	return nil
 }
 
@@ -117,7 +117,7 @@ func updateCI(ctx api.ScrapeContext, result v1.ScrapeResult, ci, existing *model
 	} else if changeResult != nil {
 		ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0])
 		result.Changes = []v1.ChangeResult{*changeResult}
-		if newChanges, _, err := extractChanges(ctx, &result, ci); err != nil {
+		if newChanges, _, err := extractChanges(ctx, &result); err != nil {
 			return nil, err
 		} else {
 			changes = append(changes, newChanges...)
@@ -214,7 +214,7 @@ func shouldExcludeChange(result *v1.ScrapeResult, changeResult v1.ChangeResult)
 	return false, nil
 }
 
-func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.ConfigItem) ([]*models.ConfigChange, []*models.ConfigChange, error) {
+func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult) ([]*models.ConfigChange, []*models.ConfigChange, error) {
 	var (
 		newOnes = []*models.ConfigChange{}
 		updates = []*models.ConfigChange{}
@@ -229,7 +229,7 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.C
 		if exclude, err := shouldExcludeChange(result, changeResult); err != nil {
 			ctx.JobHistory().AddError(fmt.Sprintf("error running change exclusion: %v", err))
 		} else if exclude {
-			ctx.DutyContext().Tracef("excluded change: %v", changeResult)
+			ctx.Logger.V(3).Infof("excluded change: %v", changeResult)
 			continue
 		}
 
@@ -253,19 +253,22 @@ func extractChanges(ctx api.ScrapeContext, result *v1.ScrapeResult, ci *models.C
 			}
 		}
 
-		if change.ConfigID == "" && change.GetExternalID().IsEmpty() && ci != nil {
-			change.ConfigID = ci.ID
-		} else if !change.GetExternalID().IsEmpty() {
+		if !change.GetExternalID().IsEmpty() {
 			if ci, err := ctx.TempCache().FindExternalID(change.GetExternalID()); err != nil {
 				return nil, nil, fmt.Errorf("failed to get config from change (externalID=%s): %w", change.GetExternalID(), err)
 			} else if ci != "" {
 				change.ConfigID = ci
 			} else if ci == "" {
-				logger.Warnf("[%s/%s] unable to find config item for change: %v", change.ConfigType, change.ExternalID, change.ChangeType)
+				ctx.Logger.V(1).Infof("[%s/%s] unable to find config item for change: %v", change.ConfigType, change.ExternalID, change.ChangeType)
 				continue
 			}
 		}
 
+		if change.ConfigID == "" {
+			ctx.Logger.V(1).Infof("(type=%s source=%s) change doesn't have an associated config", change.ChangeType, change.Source)
+			continue
+		}
+
 		if changeResult.UpdateExisting {
 			updates = append(updates, change)
 		} else {
@@ -621,7 +624,7 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, scrapeStartTime
 			}
 		}
 
-		if toCreate, toUpdate, err := extractChanges(ctx, &result, ci); err != nil {
+		if toCreate, toUpdate, err := extractChanges(ctx, &result); err != nil {
 			return nil, nil, nil, nil, err
 		} else {
 			newChanges = append(newChanges, toCreate...)

diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go
@@ -1106,7 +1106,7 @@ func (aws Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults {
 				continue
 			}
 
-			logger.Infof("Scraping %s", awsCtx)
+			ctx.Logger.V(2).Infof("scraping %s", awsCtx)
 			aws.subnets(awsCtx, awsConfig, results)
 			aws.instances(awsCtx, awsConfig, results)
 			aws.vpcs(awsCtx, awsConfig, results)

diff --git a/scrapers/aws/cloudtrail.go b/scrapers/aws/cloudtrail.go
@@ -124,7 +124,7 @@ func (aws Scraper) cloudtrail(ctx *AWSContext, config v1.AWS, results *v1.Scrape
 		}
 
 		LastEventTime.Store(lastEventKey, maxTime)
-		logger.Infof("Processed %d events, changes=%d ignored=%d", count, len(*results), ignored)
+		ctx.Logger.V(3).Infof("processed %d cloudtrail events, changes=%d ignored=%d", count, len(*results), ignored)
 		wg.Done()
 	}()
 

diff --git a/scrapers/cron.go b/scrapers/cron.go
@@ -40,10 +40,9 @@ func SyncScrapeConfigs(sc api.ScrapeContext) {
 		Fn: func(jr job.JobRuntime) error {
 			scraperConfigsDB, err := db.GetScrapeConfigsOfAgent(sc, uuid.Nil)
 			if err != nil {
-				logger.Fatalf("error getting configs from database: %v", err)
+				return fmt.Errorf("error getting configs from database: %v", err)
 			}
 
-			logger.Infof("Starting %d scrapers", len(scraperConfigsDB))
 			for _, scraper := range scraperConfigsDB {
 				_scraper, err := v1.ScrapeConfigFromModel(scraper)
 				if err != nil {
@@ -58,6 +57,7 @@ func SyncScrapeConfigs(sc api.ScrapeContext) {
 
 				jr.History.SuccessCount += 1
 			}
+
 			return nil
 		},
 	}