From 1a513a1fbbb6ef3e555f57d13fcdcdc1af08cfa2 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 5 Sep 2024 11:14:05 +0545 Subject: [PATCH] test: external change ID collision prevention --- changes/externalIDCache.go | 5 +-- cmd/operator.go | 4 +-- cmd/server.go | 4 +-- db/changes.go | 6 ++-- scrapers/runscrapers_test.go | 63 ++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 8 deletions(-) diff --git a/changes/externalIDCache.go b/changes/externalIDCache.go index 01275ce65..6c8ccfdec 100644 --- a/changes/externalIDCache.go +++ b/changes/externalIDCache.go @@ -3,7 +3,8 @@ package changes import ( "sync" - "github.com/flanksource/config-db/api" + "github.com/flanksource/duty/context" + "github.com/flanksource/config-db/db/models" ) @@ -17,7 +18,7 @@ func AddToExteranlChangeIDCache(changes []*models.ConfigChange) { } } -func InitExternalChangeIDCache(ctx api.ScrapeContext) error { +func InitExternalChangeIDCache(ctx context.Context) error { var externalIDs []string if err := ctx.DB().Select("external_change_id").Model(&models.ConfigChange{}).Where("external_change_id IS NOT NULL").Find(&externalIDs).Error; err != nil { return err diff --git a/cmd/operator.go b/cmd/operator.go index 783d1c5bd..1663cb315 100644 --- a/cmd/operator.go +++ b/cmd/operator.go @@ -58,11 +58,11 @@ func run(cmd *cobra.Command, args []string) error { logger.SetLogLevel(k8sLogLevel) dedupWindow := api.DefaultContext.Properties().Duration("changes.dedup.window", time.Hour) - if err := db.InitChangeFingerprintCache(api.DefaultContext, dedupWindow); err != nil { + if err := db.InitChangeFingerprintCache(api.DefaultContext.Context, dedupWindow); err != nil { return fmt.Errorf("failed to initialize change fingerprint cache: %w", err) } - if err := changes.InitExternalChangeIDCache(api.DefaultContext); err != nil { + if err := changes.InitExternalChangeIDCache(api.DefaultContext.Context); err != nil { return fmt.Errorf("failed to initialize external change ID cache: %w", err) } diff --git a/cmd/server.go b/cmd/server.go index 9f17e48bb..a126efd3d 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -48,11 +48,11 @@ var Serve = &cobra.Command{ api.DefaultContext = api.NewScrapeContext(dutyCtx) dedupWindow := api.DefaultContext.Properties().Duration("changes.dedup.window", time.Hour) - if err := db.InitChangeFingerprintCache(api.DefaultContext, dedupWindow); err != nil { + if err := db.InitChangeFingerprintCache(api.DefaultContext.Context, dedupWindow); err != nil { return fmt.Errorf("failed to initialize change fingerprint cache: %w", err) } - if err := changes.InitExternalChangeIDCache(api.DefaultContext); err != nil { + if err := changes.InitExternalChangeIDCache(api.DefaultContext.Context); err != nil { return fmt.Errorf("failed to initialize external change ID cache: %w", err) } diff --git a/db/changes.go b/db/changes.go index f95825593..195711b43 100644 --- a/db/changes.go +++ b/db/changes.go @@ -4,9 +4,11 @@ import ( "fmt" "time" + "github.com/flanksource/duty/context" + "github.com/patrickmn/go-cache" + "github.com/flanksource/config-db/api" "github.com/flanksource/config-db/db/models" - "github.com/patrickmn/go-cache" ) var changeCacheByFingerprint = cache.New(time.Hour, time.Hour) @@ -15,7 +17,7 @@ func changeFingeprintCacheKey(configID, fingerprint string) string { return fmt.Sprintf("%s:%s", configID, fingerprint) } -func InitChangeFingerprintCache(ctx api.ScrapeContext, window time.Duration) error { +func InitChangeFingerprintCache(ctx context.Context, window time.Duration) error { var changes []*models.ConfigChange if err := ctx.DB().Where("fingerprint IS NOT NULL").Where("NOW() - created_at <= ?", window).Find(&changes).Error; err != nil { return err diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index 27149f1ba..5781be6ed 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -10,11 +10,13 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" + "github.com/flanksource/config-db/changes" "github.com/flanksource/config-db/db" "github.com/flanksource/config-db/db/models" "github.com/flanksource/duty" dutymodels "github.com/flanksource/duty/models" "github.com/flanksource/duty/query" + "github.com/flanksource/duty/tests/fixtures/dummy" "github.com/flanksource/duty/types" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" @@ -76,6 +78,67 @@ var _ = Describe("CRD Sync test", Ordered, func() { }) }) +var _ = Describe("prevent config change external_change_id collision on dedup", Ordered, func() { + var createdChanges []*models.ConfigChange + var scraperCtx api.ScrapeContext + + BeforeAll(func() { + scraperCtx = api.NewScrapeContext(DefaultContext).WithScrapeConfig(&v1.ScrapeConfig{}) + }) + + AfterAll(func() { + for _, change := range createdChanges { + err := DefaultContext.DB().Delete(change).Error + Expect(err).NotTo(HaveOccurred(), "failed to delete config change") + } + }) + + It("should create a new change", func() { + firstChange := models.ConfigChange{ + ConfigID: dummy.EKSCluster.ID.String(), + CreatedAt: time.Now().Add(-time.Minute * 5), + ExternalChangeID: lo.ToPtr("first"), + Details: v1.JSON{"id": 1}, + Fingerprint: lo.ToPtr("642d77b89087c2c1b898880f2ab8321f"), + } + err := DefaultContext.DB().Create(&firstChange).Error + Expect(err).NotTo(HaveOccurred()) + + createdChanges = append(createdChanges, &firstChange) + }) + + It("should create a second change", func() { + secondChange := models.ConfigChange{ + ConfigID: dummy.EKSCluster.ID.String(), + CreatedAt: time.Now().Add(-time.Minute), + ExternalChangeID: lo.ToPtr("second"), + Details: v1.JSON{"id": 100}, + Fingerprint: lo.ToPtr("642d77b89087c2c1b898880f2ab8321f"), + } + err := DefaultContext.DB().Create(&secondChange).Error + Expect(err).NotTo(HaveOccurred()) + + createdChanges = append(createdChanges, &secondChange) + }) + + It("should save without deduping", func() { + changes.InitExternalChangeIDCache(DefaultContext) + err := db.InitChangeFingerprintCache(DefaultContext, time.Minute*2) + Expect(err).NotTo(HaveOccurred()) + + _, err = db.SaveResults(scraperCtx, []v1.ScrapeResult{ + {Changes: []v1.ChangeResult{ + { + ConfigID: dummy.EKSCluster.ID.String(), + ExternalChangeID: "first", + Details: v1.JSON{"id": 1}, + }, + }}, + }) + Expect(err).NotTo(HaveOccurred()) + }) +}) + var _ = Describe("Dedup test", Ordered, func() { configA := apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{