diff --git a/go.mod b/go.mod index 36fe71c0..4c826e56 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible github.com/fergusstrange/embedded-postgres v1.25.0 github.com/flanksource/commons v1.22.1 - github.com/flanksource/duty v1.0.500 + github.com/flanksource/duty v1.0.517 github.com/flanksource/is-healthy v1.0.10 github.com/flanksource/ketall v1.1.7 github.com/flanksource/mapstructure v1.6.0 diff --git a/go.sum b/go.sum index dfacfb0a..6a28267b 100644 --- a/go.sum +++ b/go.sum @@ -854,8 +854,8 @@ github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7Ttk github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/flanksource/commons v1.22.1 h1:Ycg8r26bx537UTdAEFgngDW1r2j5bX6Lr3NGxLICpiw= github.com/flanksource/commons v1.22.1/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY= -github.com/flanksource/duty v1.0.500 h1:lHMB7Q/cnyOtePJTldtFFAKyXCMONzn3/6VZkouYaFs= -github.com/flanksource/duty v1.0.500/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg= +github.com/flanksource/duty v1.0.517 h1:Xwespbps+jLlQHs5vrr1mkgwAH4M4yH8DdYgrQoWmzo= +github.com/flanksource/duty v1.0.517/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk= github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo= diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod index 3243eae9..b55e0878 100644 --- a/hack/generate-schemas/go.mod +++ b/hack/generate-schemas/go.mod @@ -5,7 +5,7 @@ go 1.22.2 require ( github.com/flanksource/commons v1.22.1 github.com/flanksource/config-db v0.0.65 - github.com/flanksource/duty v1.0.500 + github.com/flanksource/duty v1.0.517 github.com/spf13/cobra v1.7.0 ) diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum index e51f3371..4cf37597 100644 --- a/hack/generate-schemas/go.sum +++ b/hack/generate-schemas/go.sum @@ -243,8 +243,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flanksource/commons v1.22.1 h1:Ycg8r26bx537UTdAEFgngDW1r2j5bX6Lr3NGxLICpiw= github.com/flanksource/commons v1.22.1/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY= -github.com/flanksource/duty v1.0.500 h1:lHMB7Q/cnyOtePJTldtFFAKyXCMONzn3/6VZkouYaFs= -github.com/flanksource/duty v1.0.500/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg= +github.com/flanksource/duty v1.0.517 h1:Xwespbps+jLlQHs5vrr1mkgwAH4M4yH8DdYgrQoWmzo= +github.com/flanksource/duty v1.0.517/go.mod h1:4renLmQZsi9QMFT9xBO+bLxjn5KzJ9PI/iq3Wm2+5Zg= github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk= github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo= github.com/flanksource/is-healthy v1.0.10 h1:g7cyLTK/a5PqTJmRG35+4l7I6vzQqGAQAxpwrw0efkE= diff --git a/jobs/cleanup.go b/jobs/cleanup.go index e3737217..d159fcac 100644 --- a/jobs/cleanup.go +++ b/jobs/cleanup.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/flanksource/duty" "github.com/flanksource/duty/job" ) @@ -74,82 +75,29 @@ var CleanupConfigChanges = &job.Job{ var CleanupConfigItems = &job.Job{ Name: "CleanupConfigItems", - Schedule: "@every 24h", + Schedule: "0 2 * * *", // Everynight at 2 AM Singleton: true, JobHistory: true, Retention: job.RetentionBalanced, Fn: func(ctx job.JobRuntime) error { ctx.History.ResourceType = JobResourceType retention := ctx.Properties().Duration("config.retention.period", (time.Hour * 24 * time.Duration(ConfigItemRetentionDays))) - seconds := int64(retention.Seconds()) - - ctx.Tracef("cleaning up config items older than %v", retention) - - linkedConfigsQuery := ` - SELECT config_id FROM evidences WHERE config_id IS NOT NULL - UNION - SELECT config_id FROM config_changes WHERE id IN (SELECT config_change_id FROM evidences) - UNION - SELECT config_id FROM config_analysis WHERE id IN (SELECT config_analysis_id FROM evidences) - UNION - SELECT config_id FROM playbook_RUNS WHERE config_id IS NOT NULL - ` + days := int64(retention.Hours() / 24) - relationshipDeleteQuery := fmt.Sprintf(` - DELETE FROM config_relationships - WHERE deleted_at < NOW() - interval '1 SECONDS' * ? - OR config_id in (SELECT id FROM config_items WHERE id NOT IN (%s) AND deleted_at < NOW() - interval '1 SECONDS' * ?) - OR related_id in (SELECT id FROM config_items WHERE id NOT IN (%s) AND deleted_at < NOW() - interval '1 SECONDS' * ?)`, linkedConfigsQuery, linkedConfigsQuery) - if tx := ctx.Context.DB().Exec(relationshipDeleteQuery, seconds, seconds, seconds); tx.Error != nil { - return fmt.Errorf("failed to delete config relationships: %w", tx.Error) - } else { - ctx.Tracef("deleted %d config relationships", tx.RowsAffected) + var prevCount int64 + if err := ctx.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&prevCount).Error; err != nil { + return fmt.Errorf("failed to count config items: %w", duty.DBErrorDetails(err)) } - // break the parent relationship of deleted configs - breakParentRelationshipQuery := fmt.Sprintf(` - UPDATE config_items - SET parent_id = NULL - WHERE - id NOT IN (%s) AND - parent_id IS NOT NULL AND - deleted_at < NOW() - interval '1 SECONDS' * ?`, - linkedConfigsQuery) - if tx := ctx.Context.DB().Exec(breakParentRelationshipQuery, seconds); tx.Error != nil { - return fmt.Errorf("failed to remove config parent relationships: %w", tx.Error) - } else { - ctx.Tracef("removed %d config parent relationships", tx.RowsAffected) + if err := ctx.DB().Exec("SELECT delete_old_config_items(?)", days).Error; err != nil { + return fmt.Errorf("failed to delete config items: %w", duty.DBErrorDetails(err)) } - var iter int - deleteBatchSize := ctx.Properties().Int("config.retention.delete_batch_size", 500) - configDeleteQuery := fmt.Sprintf(` - WITH ordered_rows AS ( - SELECT id - FROM config_items - WHERE - deleted_at < NOW() - interval '1 SECONDS' * ? AND - id NOT IN (%s) - ORDER BY length(path) DESC - LIMIT ? - ) - DELETE FROM config_items - WHERE id IN (SELECT id FROM ordered_rows)`, linkedConfigsQuery) - for { - iter++ - tx := ctx.Context.DB().Exec(configDeleteQuery, seconds, deleteBatchSize) - if tx.Error != nil { - return fmt.Errorf("failed to delete config items: %w", tx.Error) - } - - if tx.RowsAffected == 0 { - break - } - - ctx.Logger.V(2).Infof("hard deleted %d config items [iter=%d, batchsize=%d]", iter, deleteBatchSize, tx.RowsAffected) - ctx.History.SuccessCount += int(tx.RowsAffected) + var newCount int64 + if err := ctx.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&newCount).Error; err != nil { + return fmt.Errorf("failed to count config items: %w", duty.DBErrorDetails(err)) } - + ctx.History.SuccessCount = int(newCount - prevCount) return nil }, } diff --git a/jobs/sync_upstream.go b/jobs/sync_upstream.go index 91c7eea1..ef5ca86b 100644 --- a/jobs/sync_upstream.go +++ b/jobs/sync_upstream.go @@ -36,12 +36,12 @@ var ReconcileConfigs = &job.Job{ Fn: func(ctx job.JobRuntime) error { ctx.History.ResourceType = job.ResourceTypeUpstream ctx.History.ResourceID = api.UpstreamConfig.Host - if count, err := upstream.ReconcileSome(ctx.Context, api.UpstreamConfig, ReconcilePageSize, tablesToReconcile...); err != nil { + count, fkFailed, err := upstream.ReconcileSome(ctx.Context, api.UpstreamConfig, ReconcilePageSize, tablesToReconcile...) + if err != nil { ctx.History.AddError(err.Error()) - } else { - ctx.History.SuccessCount += count } - + ctx.History.ErrorCount += fkFailed + ctx.History.SuccessCount += count return nil }, }