chore: cleanup config_items using deferred constraints

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/fergusstrange/embedded-postgres v1.25.0
 	github.com/flanksource/commons v1.22.1
-	github.com/flanksource/duty v1.0.500
+	github.com/flanksource/duty v1.0.517
 	github.com/flanksource/is-healthy v1.0.10
 	github.com/flanksource/ketall v1.1.7
 	github.com/flanksource/mapstructure v1.6.0

go.sum

@@ -854,8 +854,8 @@ github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7Ttk
 github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw=
 github.com/flanksource/commons v1.22.1 h1:Ycg8r26bx537UTdAEFgngDW1r2j5bX6Lr3NGxLICpiw=
 github.com/flanksource/commons v1.22.1/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
-github.com/flanksource/duty v1.0.500 h1:lHMB7Q/cnyOtePJTldtFFAKyXCMONzn3/6VZkouYaFs=
-github.com/flanksource/duty v1.0.500/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg=
+github.com/flanksource/duty v1.0.517 h1:Xwespbps+jLlQHs5vrr1mkgwAH4M4yH8DdYgrQoWmzo=
+github.com/flanksource/duty v1.0.517/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk=
 github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo=

hack/generate-schemas/go.mod

@@ -5,7 +5,7 @@ go 1.22.2
 require (
 	github.com/flanksource/commons v1.22.1
 	github.com/flanksource/config-db v0.0.65
-	github.com/flanksource/duty v1.0.500
+	github.com/flanksource/duty v1.0.517
 	github.com/spf13/cobra v1.7.0
 )
 

hack/generate-schemas/go.sum

@@ -243,8 +243,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/flanksource/commons v1.22.1 h1:Ycg8r26bx537UTdAEFgngDW1r2j5bX6Lr3NGxLICpiw=
 github.com/flanksource/commons v1.22.1/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY=
-github.com/flanksource/duty v1.0.500 h1:lHMB7Q/cnyOtePJTldtFFAKyXCMONzn3/6VZkouYaFs=
-github.com/flanksource/duty v1.0.500/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg=
+github.com/flanksource/duty v1.0.517 h1:Xwespbps+jLlQHs5vrr1mkgwAH4M4yH8DdYgrQoWmzo=
+github.com/flanksource/duty v1.0.517/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg=
 github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk=
 github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo=
 github.com/flanksource/is-healthy v1.0.10 h1:g7cyLTK/a5PqTJmRG35+4l7I6vzQqGAQAxpwrw0efkE=

jobs/cleanup.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/flanksource/duty"
 	"github.com/flanksource/duty/job"
 )
 
@@ -74,82 +75,29 @@ var CleanupConfigChanges = &job.Job{
 
 var CleanupConfigItems = &job.Job{
 	Name:       "CleanupConfigItems",
-	Schedule:   "@every 24h",
+	Schedule:   "0 2 * * *", // Everynight at 2 AM
 	Singleton:  true,
 	JobHistory: true,
 	Retention:  job.RetentionBalanced,
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = JobResourceType
 		retention := ctx.Properties().Duration("config.retention.period", (time.Hour * 24 * time.Duration(ConfigItemRetentionDays)))
-		seconds := int64(retention.Seconds())
-
-		ctx.Tracef("cleaning up config items older than %v", retention)
-
-		linkedConfigsQuery := `
-		SELECT config_id FROM evidences WHERE config_id IS NOT NULL
-		UNION
-		SELECT config_id FROM config_changes WHERE id IN (SELECT config_change_id FROM evidences)
-		UNION
-		SELECT config_id FROM config_analysis WHERE id IN (SELECT config_analysis_id FROM evidences)
-		UNION
-		SELECT config_id FROM playbook_RUNS WHERE config_id IS NOT NULL
-		`
+		days := int64(retention.Hours() / 24)
 
-		relationshipDeleteQuery := fmt.Sprintf(`
-		DELETE FROM config_relationships
-		WHERE deleted_at < NOW() - interval '1 SECONDS' * ?
-		OR config_id in (SELECT id FROM config_items WHERE id NOT IN (%s) AND deleted_at < NOW() - interval '1 SECONDS' * ?)
-		OR related_id in (SELECT id FROM config_items WHERE id NOT IN (%s) AND deleted_at < NOW() - interval '1 SECONDS' * ?)`, linkedConfigsQuery, linkedConfigsQuery)
-		if tx := ctx.Context.DB().Exec(relationshipDeleteQuery, seconds, seconds, seconds); tx.Error != nil {
-			return fmt.Errorf("failed to delete config relationships: %w", tx.Error)
-		} else {
-			ctx.Tracef("deleted %d config relationships", tx.RowsAffected)
+		var prevCount int64
+		if err := ctx.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&prevCount).Error; err != nil {
+			return fmt.Errorf("failed to count config items: %w", duty.DBErrorDetails(err))
 		}
 
-		// break the parent relationship of deleted configs
-		breakParentRelationshipQuery := fmt.Sprintf(`
-		UPDATE config_items
-		SET parent_id = NULL
-		WHERE
-            id NOT IN (%s) AND
-            parent_id IS NOT NULL AND
-            deleted_at < NOW() - interval '1 SECONDS' * ?`,
-			linkedConfigsQuery)
-		if tx := ctx.Context.DB().Exec(breakParentRelationshipQuery, seconds); tx.Error != nil {
-			return fmt.Errorf("failed to remove config parent relationships: %w", tx.Error)
-		} else {
-			ctx.Tracef("removed %d config parent relationships", tx.RowsAffected)
+		if err := ctx.DB().Exec("SELECT delete_old_config_items(?)", days).Error; err != nil {
+			return fmt.Errorf("failed to delete config items: %w", duty.DBErrorDetails(err))
 		}
 
-		var iter int
-		deleteBatchSize := ctx.Properties().Int("config.retention.delete_batch_size", 500)
-		configDeleteQuery := fmt.Sprintf(`
-		WITH ordered_rows AS (
-			SELECT id
-			FROM config_items
-			WHERE
-				deleted_at < NOW() - interval '1 SECONDS' * ? AND
-				id NOT IN (%s)
-			ORDER BY length(path) DESC
-			LIMIT ?
-		)
-		DELETE FROM config_items
-		WHERE id IN (SELECT id FROM ordered_rows)`, linkedConfigsQuery)
-		for {
-			iter++
-			tx := ctx.Context.DB().Exec(configDeleteQuery, seconds, deleteBatchSize)
-			if tx.Error != nil {
-				return fmt.Errorf("failed to delete config items: %w", tx.Error)
-			}
-
-			if tx.RowsAffected == 0 {
-				break
-			}
-
-			ctx.Logger.V(2).Infof("hard deleted %d config items [iter=%d, batchsize=%d]", iter, deleteBatchSize, tx.RowsAffected)
-			ctx.History.SuccessCount += int(tx.RowsAffected)
+		var newCount int64
+		if err := ctx.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&newCount).Error; err != nil {
+			return fmt.Errorf("failed to count config items: %w", duty.DBErrorDetails(err))
 		}
-
+		ctx.History.SuccessCount = int(newCount - prevCount)
 		return nil
 	},
 }

jobs/sync_upstream.go

@@ -36,12 +36,12 @@ var ReconcileConfigs = &job.Job{
 	Fn: func(ctx job.JobRuntime) error {
 		ctx.History.ResourceType = job.ResourceTypeUpstream
 		ctx.History.ResourceID = api.UpstreamConfig.Host
-		if count, err := upstream.ReconcileSome(ctx.Context, api.UpstreamConfig, ReconcilePageSize, tablesToReconcile...); err != nil {
+		count, fkFailed, err := upstream.ReconcileSome(ctx.Context, api.UpstreamConfig, ReconcilePageSize, tablesToReconcile...)
+		if err != nil {
 			ctx.History.AddError(err.Error())
-		} else {
-			ctx.History.SuccessCount += count
 		}
-
+		ctx.History.ErrorCount += fkFailed
+		ctx.History.SuccessCount += count
 		return nil
 	},
 }