chore: add job for scrape cleanup

flanksource · Nov 2, 2024 · 8749042 · 8749042
1 parent 12d785c
commit 8749042
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 3 deletions.
diff --git a/controllers/scrapeconfig_controller.go b/controllers/scrapeconfig_controller.go
@@ -76,7 +76,7 @@ func (r *ScrapeConfigReconciler) Reconcile(c context.Context, req ctrl.Request)
 	// Check if it is deleted, remove scrape config
 	if !scrapeConfig.DeletionTimestamp.IsZero() {
 		logger.Info("Deleting scrape config", "id", scrapeConfig.GetUID())
-		if err := db.DeleteScrapeConfig(ctx, string(scrapeConfig.GetUID())); err != nil {
+		if err := db.DeleteScrapeConfig(ctx.Context, string(scrapeConfig.GetUID())); err != nil {
 			logger.Error(err, "failed to delete scrape config")
 			return ctrl.Result{Requeue: true, RequeueAfter: 2 * time.Minute}, err
 		}

diff --git a/db/config_scraper.go b/db/config_scraper.go
@@ -6,7 +6,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/flanksource/config-db/api"
 	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/config-db/utils"
 	"github.com/flanksource/duty/context"
@@ -28,7 +27,7 @@ func FindScraper(ctx context.Context, id string) (*models.ConfigScraper, error)
 	return &configScraper, nil
 }
 
-func DeleteScrapeConfig(ctx api.ScrapeContext, id string) error {
+func DeleteScrapeConfig(ctx context.Context, id string) error {
 	if err := ctx.DB().Table("config_scrapers").
 		Where("id = ?", id).
 		Update("deleted_at", time.Now()).

diff --git a/jobs/cleanup.go b/jobs/cleanup.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/flanksource/commons/properties"
 	v1 "github.com/flanksource/config-db/api/v1"
+	cdb "github.com/flanksource/config-db/db"
 	"github.com/flanksource/config-db/scrapers"
 	"github.com/flanksource/duty/db"
 	"github.com/flanksource/duty/job"
@@ -32,6 +33,7 @@ var cleanupJobs = []*job.Job{
 	CleanupConfigChanges,
 	CleanupConfigItems,
 	SoftDeleteAgentStaleItems,
+	CleanupConfigScrapers,
 }
 
 var CleanupConfigAnalysis = &job.Job{
@@ -158,3 +160,29 @@ var SoftDeleteAgentStaleItems = &job.Job{
 		return nil
 	},
 }
+
+var CleanupConfigScrapers = &job.Job{
+	Name:       "CleanupConfigScrapers",
+	Schedule:   "15 2 * * *", // Everynight at 2:15 AM
+	Singleton:  true,
+	JobHistory: true,
+	Retention:  job.RetentionFew,
+	Fn: func(ctx job.JobRuntime) error {
+		ctx.History.ResourceType = JobResourceType
+
+		var deletedIDs []string
+		if err := ctx.DB().Model(&models.ConfigScraper{}).Select("id").Where("deleted_at IS NOT NULL").Find(&deletedIDs).Error; err != nil {
+			return fmt.Errorf("error fetching deleted config_scraper ids: %w", db.ErrorDetails(err))
+		}
+
+		for _, id := range deletedIDs {
+			if err := cdb.DeleteScrapeConfig(ctx.Context, id); err != nil {
+				ctx.History.AddErrorf("error deleting scrape config[%s]: %v", id, err)
+			} else {
+				ctx.History.SuccessCount += 1
+			}
+		}
+
+		return nil
+	},
+}