-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: change retention * chore: clean up * fix: config retention tests and flow * chore: change config retention to a job
- Loading branch information
1 parent
72e711a
commit 7fd0755
Showing
9 changed files
with
277 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package jobs | ||
|
|
||
| import ( | ||
| gocontext "context" | ||
| "encoding/json" | ||
|
|
||
| "github.com/flanksource/commons/logger" | ||
| "github.com/flanksource/config-db/api/v1" | ||
| "github.com/flanksource/config-db/db" | ||
| "github.com/flanksource/config-db/scrapers" | ||
| "github.com/flanksource/duty/context" | ||
| "github.com/flanksource/duty/models" | ||
| ) | ||
|
|
||
| func ProcessChangeRetentionRules() { | ||
| ctx := context.NewContext(gocontext.Background()).WithDB(db.DefaultDB(), db.Pool) | ||
| jobHistory := models.NewJobHistory("ProcessChangeRetentionRules", "", "").Start() | ||
| _ = db.PersistJobHistory(jobHistory) | ||
| defer func() { | ||
| _ = db.PersistJobHistory(jobHistory.End()) | ||
| }() | ||
|
|
||
| var activeScrapers []models.ConfigScraper | ||
| if err := ctx.DB().Where("deleted_at IS NULL").Find(&activeScrapers).Error; err != nil { | ||
| logger.Errorf("Error querying config scrapers from db: %v", err) | ||
| jobHistory.AddError(err.Error()) | ||
| return | ||
| } | ||
|
|
||
| for _, s := range activeScrapers { | ||
| var spec v1.ScraperSpec | ||
| if err := json.Unmarshal([]byte(s.Spec), &spec); err != nil { | ||
| logger.Errorf("Error unmarshaling config scraper[%s] into json: %v", s.ID, err) | ||
| jobHistory.AddError(err.Error()) | ||
| continue | ||
| } | ||
|
|
||
| for _, changeSpec := range spec.Retention.Changes { | ||
| err := scrapers.ProcessChangeRetention(ctx, s.ID, changeSpec) | ||
| if err != nil { | ||
| logger.Errorf("Error processing change retention for scraper[%s] config analysis: %v", s.ID, err) | ||
| jobHistory.AddError(err.Error()) | ||
| } else { | ||
| jobHistory.IncrSuccess() | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package scrapers | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/flanksource/commons/duration" | ||
| "github.com/flanksource/commons/logger" | ||
| v1 "github.com/flanksource/config-db/api/v1" | ||
| "github.com/flanksource/duty/context" | ||
| "github.com/google/uuid" | ||
| ) | ||
|
|
||
| func ProcessChangeRetention(ctx context.Context, scraperID uuid.UUID, spec v1.ChangeRetentionSpec) error { | ||
| var whereClauses []string | ||
|
|
||
| var ageMinutes int | ||
| if spec.Age != "" { | ||
| age, err := duration.ParseDuration(spec.Age) | ||
| if err != nil { | ||
| return fmt.Errorf("error parsing age %s as duration: %w", spec.Age, err) | ||
| } | ||
| ageMinutes = int(age.Minutes()) | ||
|
|
||
| whereClauses = append(whereClauses, `((now()- created_at) > interval '1 minute' * @ageMinutes)`) | ||
| } | ||
|
|
||
| if spec.Count > 0 { | ||
| whereClauses = append(whereClauses, `seq > @count`) | ||
| } | ||
|
|
||
| if len(whereClauses) == 0 { | ||
| return fmt.Errorf("both age and count cannot be empty") | ||
| } | ||
|
|
||
| query := fmt.Sprintf(` | ||
| WITH latest_config_changes AS ( | ||
| SELECT id, change_type, created_at, ROW_NUMBER() OVER(ORDER BY created_at DESC) AS seq | ||
| FROM config_changes | ||
| WHERE | ||
| change_type = @changeType AND | ||
| config_id IN (SELECT id FROM config_items WHERE scraper_id = @scraperID) | ||
| ) | ||
| DELETE FROM config_changes | ||
| WHERE id IN ( | ||
| SELECT id from latest_config_changes | ||
| WHERE %s | ||
| ) | ||
| `, strings.Join(whereClauses, " OR ")) | ||
|
|
||
| result := ctx.DB().Exec(query, | ||
| sql.Named("changeType", spec.Name), | ||
| sql.Named("scraperID", scraperID), | ||
| sql.Named("ageMinutes", ageMinutes), | ||
| sql.Named("count", spec.Count), | ||
| ) | ||
| if err := result.Error; err != nil { | ||
| return fmt.Errorf("error retaining config changes: %w", err) | ||
| } | ||
|
|
||
| if result.RowsAffected > 0 { | ||
| logger.Infof("Deleted %d config_changes as per ChangeRetentionSpec[%s]", result.RowsAffected, spec.Name) | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters