From 81160fdb1dda4168bdd9fe5aafa42bf84895d08a Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Tue, 21 Nov 2023 19:04:21 +0530 Subject: [PATCH] feat: improve deletion logic with delete_reason --- api/v1/interface.go | 1 + api/v1/types.go | 8 ++++++ db/config.go | 11 ++++++-- db/models/config_item.go | 47 ++++++++++++++++--------------- scrapers/kubernetes/kubernetes.go | 3 ++ scrapers/stale.go | 31 ++++++++++++++++++-- 6 files changed, 73 insertions(+), 28 deletions(-) diff --git a/api/v1/interface.go b/api/v1/interface.go index dd7f915dd..c45e576d1 100644 --- a/api/v1/interface.go +++ b/api/v1/interface.go @@ -192,6 +192,7 @@ func (s *ScrapeResults) Errorf(e error, msg string, args ...interface{}) ScrapeR type ScrapeResult struct { CreatedAt *time.Time `json:"created_at,omitempty"` DeletedAt *time.Time `json:"deleted_at,omitempty"` + DeleteReason ConfigDeleteReason `json:"delete_reason,omitempty"` LastModified time.Time `json:"last_modified,omitempty"` ConfigClass string `json:"config_class,omitempty"` Type string `json:"config_type,omitempty"` diff --git a/api/v1/types.go b/api/v1/types.go index d5503c628..00ca61369 100644 --- a/api/v1/types.go +++ b/api/v1/types.go @@ -77,3 +77,11 @@ func (e ExternalID) CacheKey() string { func (e ExternalID) WhereClause(db *gorm.DB) *gorm.DB { return db.Where("type = ? AND external_id @> ?", e.ConfigType, pq.StringArray(e.ExternalID)) } + +type ConfigDeleteReason string + +var ( + DeletedReasonMissingScrape ConfigDeleteReason = "MISSING_SCRAPE" + DeletedReasonFromAttribute ConfigDeleteReason = "FROM_ATTRIBUTE" + DeletedReasonFromEvent ConfigDeleteReason = "FROM_EVENT" +) diff --git a/db/config.go b/db/config.go index 4cc9b7b57..d7aff344f 100644 --- a/db/config.go +++ b/db/config.go @@ -78,8 +78,14 @@ func UpdateConfigItem(ci *models.ConfigItem) error { } // Since gorm ignores nil fields, we are setting deleted_at explicitly - if ci.TouchDeletedAt { - if err := db.Table("config_items").Where("id = ?", ci.ID).UpdateColumn("deleted_at", nil).Error; err != nil { + // TODO Add deleted reason check + if ci.TouchDeletedAt && ci.DeleteReason != "FROM_EVENT" { + if err := db.Table("config_items"). + Where("id = ?", ci.ID). + Updates(map[string]any{ + "deleted_at": nil, + "delete_reason": nil, + }).Error; err != nil { return err } } @@ -167,6 +173,7 @@ func NewConfigItemFromResult(result v1.ScrapeResult) (*models.ConfigItem, error) if result.DeletedAt != nil { ci.DeletedAt = result.DeletedAt + ci.DeleteReason = result.DeleteReason } if result.ParentExternalID != "" && result.ParentType != "" { diff --git a/db/models/config_item.go b/db/models/config_item.go index d91087274..2be305e9b 100644 --- a/db/models/config_item.go +++ b/db/models/config_item.go @@ -12,29 +12,30 @@ import ( // ConfigItem represents the config item database table type ConfigItem struct { - ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" ` - ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"` - ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" ` - ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" ` - Type *string `gorm:"column:type;default:null" json:"type,omitempty" ` - Status *string `gorm:"column:status;default:null" json:"status,omitempty" ` - Name *string `gorm:"column:name;default:null" json:"name,omitempty" ` - Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" ` - Description *string `gorm:"column:description;default:null" json:"description,omitempty" ` - Account *string `gorm:"column:account;default:null" json:"account,omitempty" ` - Config *string `gorm:"column:config;default:null" json:"config,omitempty" ` - Source *string `gorm:"column:source;default:null" json:"source,omitempty" ` - ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"` - Path string `gorm:"column:path;default:null" json:"path,omitempty"` - CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"` - CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"` - CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"` - CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"` - Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" ` - CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` - UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"` - DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"` - TouchDeletedAt bool `gorm:"-" json:"-"` + ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" ` + ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"` + ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" ` + ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" ` + Type *string `gorm:"column:type;default:null" json:"type,omitempty" ` + Status *string `gorm:"column:status;default:null" json:"status,omitempty" ` + Name *string `gorm:"column:name;default:null" json:"name,omitempty" ` + Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" ` + Description *string `gorm:"column:description;default:null" json:"description,omitempty" ` + Account *string `gorm:"column:account;default:null" json:"account,omitempty" ` + Config *string `gorm:"column:config;default:null" json:"config,omitempty" ` + Source *string `gorm:"column:source;default:null" json:"source,omitempty" ` + ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"` + Path string `gorm:"column:path;default:null" json:"path,omitempty"` + CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"` + CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"` + CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"` + CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"` + Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" ` + CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"` + DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"` + DeleteReason v1.ConfigDeleteReason `gorm:"column:delete_reason" json:"delete_reason"` + TouchDeletedAt bool `gorm:"-" json:"-"` } func (ci ConfigItem) String() string { diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 569a367f1..023f93750 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -142,8 +142,10 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul createdAt := obj.GetCreationTimestamp().Time var deletedAt *time.Time + var deleteReason v1.ConfigDeleteReason if !obj.GetDeletionTimestamp().IsZero() { deletedAt = &obj.GetDeletionTimestamp().Time + deleteReason = v1.DeletedReasonFromAttribute } // Evicted Pods must be considered deleted @@ -170,6 +172,7 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul Description: description, CreatedAt: &createdAt, DeletedAt: deletedAt, + DeleteReason: deleteReason, Config: cleanKubernetesObject(obj.Object), ID: string(obj.GetUID()), Tags: stripLabels(convertStringInterfaceMapToStringMap(tags), "-hash"), diff --git a/scrapers/stale.go b/scrapers/stale.go index ff5375358..69968952e 100644 --- a/scrapers/stale.go +++ b/scrapers/stale.go @@ -3,6 +3,7 @@ package scrapers import ( "github.com/flanksource/commons/duration" "github.com/flanksource/commons/logger" + v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" "github.com/google/uuid" ) @@ -19,15 +20,19 @@ func DeleteStaleConfigItems(scraperID uuid.UUID) error { } staleMinutes := int(staleDuration.Minutes()) - query := ` + // TODO Check deleted_at against updated_at and if updated_at is greater + // and reason is missing scrape, remove deleted_at + deleteQuery := ` UPDATE config_items - SET deleted_at = NOW() + SET + deleted_at = NOW(), + delete_reason = ? WHERE ((NOW() - updated_at) > INTERVAL '1 minute' * ?) AND deleted_at IS NULL AND scraper_id = ?` - result := db.DefaultDB().Exec(query, staleMinutes, scraperID) + result := db.DefaultDB().Exec(deleteQuery, v1.DeletedReasonMissingScrape, staleMinutes, scraperID) if err := result.Error; err != nil { return err } @@ -35,5 +40,25 @@ func DeleteStaleConfigItems(scraperID uuid.UUID) error { if result.RowsAffected > 0 { logger.Infof("Marked %d items as deleted", result.RowsAffected) } + + undeleteQuery := ` + UPDATE config_items + SET + deleted_at = NULL, + delete_reason = NULL + WHERE + deleted_at IS NOT NULL AND + delete_reason = ? AND + updated_at > deleted_at AND + scraper_id = ?` + + result = db.DefaultDB().Exec(undeleteQuery, v1.DeletedReasonMissingScrape, scraperID) + if err := result.Error; err != nil { + return err + } + + if result.RowsAffected > 0 { + logger.Infof("Marked %d items as not deleted", result.RowsAffected) + } return nil }