From 9f30224785ae42cd1a98ea864374391f377ec3b3 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 6 Dec 2023 14:55:48 +0530 Subject: [PATCH 1/2] chore: add config-id to retention partition query --- scrapers/retention.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapers/retention.go b/scrapers/retention.go index 6aa57d07..81c419bf 100644 --- a/scrapers/retention.go +++ b/scrapers/retention.go @@ -36,7 +36,7 @@ func ProcessChangeRetention(ctx context.Context, scraperID uuid.UUID, spec v1.Ch query := fmt.Sprintf(` WITH latest_config_changes AS ( - SELECT id, change_type, created_at, ROW_NUMBER() OVER(ORDER BY created_at DESC) AS seq + SELECT id, change_type, created_at, ROW_NUMBER() OVER(PARTITION BY config_id ORDER BY created_at DESC) AS seq FROM config_changes WHERE change_type = @changeType AND From 02f0553b4ec8563918c503b863be54d22bb14d78 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 6 Dec 2023 16:06:37 +0530 Subject: [PATCH 2/2] chore: add more tests for retention --- scrapers/runscrapers_test.go | 40 +++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index af28abf6..72f9bcb9 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -238,8 +238,16 @@ var _ = Describe("Scrapers test", Ordered, func() { ConfigClass: "Test", ScraperID: &dummyScraper.ID, } + configItemID2 := uuid.New().String() + dummyCI2 := models.ConfigItem{ + ID: configItemID2, + ConfigClass: "Test", + ScraperID: &dummyScraper.ID, + } err = db.DefaultDB().Create(&dummyCI).Error Expect(err).To(BeNil()) + err = db.DefaultDB().Create(&dummyCI2).Error + Expect(err).To(BeNil()) twoDaysAgo := time.Now().Add(-2 * 24 * time.Hour) fiveDaysAgo := time.Now().Add(-5 * 24 * time.Hour) @@ -262,6 +270,8 @@ var _ = Describe("Scrapers test", Ordered, func() { {ConfigID: configItemID, ChangeType: "TestDiff", CreatedAt: &fiveDaysAgo, ExternalChangeId: uuid.New().String()}, {ConfigID: configItemID, ChangeType: "TestDiff", CreatedAt: &tenDaysAgo, ExternalChangeId: uuid.New().String()}, {ConfigID: configItemID, ChangeType: "TestDiff", CreatedAt: &tenDaysAgo, ExternalChangeId: uuid.New().String()}, + {ConfigID: configItemID2, ChangeType: "TestDiff", ExternalChangeId: uuid.New().String()}, + {ConfigID: configItemID2, ChangeType: "TestDiff", ExternalChangeId: uuid.New().String()}, } err = db.DefaultDB().Table("config_changes").Create(&configChanges).Error @@ -269,7 +279,7 @@ var _ = Describe("Scrapers test", Ordered, func() { var currentCount int err = db.DefaultDB(). - Raw(`SELECT COUNT(*) FROM config_changes WHERE change_type = ? AND config_id = ?`, "TestDiff", configItemID). + Raw(`SELECT COUNT(*) FROM config_changes WHERE change_type = ?`, "TestDiff"). Scan(¤tCount). Error Expect(err).To(BeNil()) @@ -288,6 +298,15 @@ var _ = Describe("Scrapers test", Ordered, func() { Expect(err).To(BeNil()) Expect(count1).To(Equal(15)) + // The other config item should not be touched + var otherCount1 int + err = db.DefaultDB(). + Raw(`SELECT COUNT(*) FROM config_changes WHERE change_type = ? AND config_id = ?`, "TestDiff", configItemID2). + Scan(&otherCount1). + Error + Expect(err).To(BeNil()) + Expect(otherCount1).To(Equal(2)) + // Only keep latest 12 config changes err = ProcessChangeRetention(ctx, dummyScraper.ID, v1.ChangeRetentionSpec{Name: "TestDiff", Count: 12}) Expect(err).To(BeNil()) @@ -299,6 +318,15 @@ var _ = Describe("Scrapers test", Ordered, func() { Expect(err).To(BeNil()) Expect(count2).To(Equal(12)) + // The other config item should not be touched + var otherCount2 int + err = db.DefaultDB(). + Raw(`SELECT COUNT(*) FROM config_changes WHERE change_type = ? AND config_id = ?`, "TestDiff", configItemID2). + Scan(&otherCount2). + Error + Expect(err).To(BeNil()) + Expect(otherCount2).To(Equal(2)) + // Keep config changes which are newer than 3 days and max count can be 10 err = ProcessChangeRetention(ctx, dummyScraper.ID, v1.ChangeRetentionSpec{Name: "TestDiff", Age: "3d", Count: 10}) Expect(err).To(BeNil()) @@ -313,6 +341,16 @@ var _ = Describe("Scrapers test", Ordered, func() { // No params in ChangeRetentionSpec should fail err = ProcessChangeRetention(ctx, dummyScraper.ID, v1.ChangeRetentionSpec{Name: "TestDiff"}) Expect(err).ToNot(BeNil()) + + // The other config item should not be touched + var otherCount3 int + err = db.DefaultDB(). + Raw(`SELECT COUNT(*) FROM config_changes WHERE change_type = ? AND config_id = ?`, "TestDiff", configItemID2). + Scan(&otherCount3). + Error + Expect(err).To(BeNil()) + Expect(otherCount3).To(Equal(2)) + }) }) })