Skip to content

Commit

Permalink
feat: improve deletion logic with delete_reason
Browse files Browse the repository at this point in the history
  • Loading branch information
@yashmehrotra
yashmehrotra committed Nov 21, 2023
1 parent 65a73f9 commit 81160fd
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 28 deletions.
1 change: 1 addition & 0 deletions api/v1/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ func (s *ScrapeResults) Errorf(e error, msg string, args ...interface{}) ScrapeR
type ScrapeResult struct {
CreatedAt *time.Time `json:"created_at,omitempty"`
DeletedAt *time.Time `json:"deleted_at,omitempty"`
DeleteReason ConfigDeleteReason `json:"delete_reason,omitempty"`
LastModified time.Time `json:"last_modified,omitempty"`
ConfigClass string `json:"config_class,omitempty"`
Type string `json:"config_type,omitempty"`
Expand Down
8 changes: 8 additions & 0 deletions api/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,11 @@ func (e ExternalID) CacheKey() string {
func (e ExternalID) WhereClause(db *gorm.DB) *gorm.DB {
return db.Where("type = ? AND external_id @> ?", e.ConfigType, pq.StringArray(e.ExternalID))
}

type ConfigDeleteReason string

var (
DeletedReasonMissingScrape ConfigDeleteReason = "MISSING_SCRAPE"
DeletedReasonFromAttribute ConfigDeleteReason = "FROM_ATTRIBUTE"
DeletedReasonFromEvent ConfigDeleteReason = "FROM_EVENT"
)
11 changes: 9 additions & 2 deletions db/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,14 @@ func UpdateConfigItem(ci *models.ConfigItem) error {
}

// Since gorm ignores nil fields, we are setting deleted_at explicitly
if ci.TouchDeletedAt {
if err := db.Table("config_items").Where("id = ?", ci.ID).UpdateColumn("deleted_at", nil).Error; err != nil {
// TODO Add deleted reason check
if ci.TouchDeletedAt && ci.DeleteReason != "FROM_EVENT" {
if err := db.Table("config_items").
Where("id = ?", ci.ID).
Updates(map[string]any{
"deleted_at": nil,
"delete_reason": nil,
}).Error; err != nil {
return err
}
}
Expand Down Expand Up @@ -167,6 +173,7 @@ func NewConfigItemFromResult(result v1.ScrapeResult) (*models.ConfigItem, error)

if result.DeletedAt != nil {
ci.DeletedAt = result.DeletedAt
ci.DeleteReason = result.DeleteReason
}

if result.ParentExternalID != "" && result.ParentType != "" {
Expand Down
47 changes: 24 additions & 23 deletions db/models/config_item.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,30 @@ import (

// ConfigItem represents the config item database table
type ConfigItem struct {
ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type *string `gorm:"column:type;default:null" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Account *string `gorm:"column:account;default:null" json:"account,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" `
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
TouchDeletedAt bool `gorm:"-" json:"-"`
ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type *string `gorm:"column:type;default:null" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Account *string `gorm:"column:account;default:null" json:"account,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" `
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
DeleteReason v1.ConfigDeleteReason `gorm:"column:delete_reason" json:"delete_reason"`
TouchDeletedAt bool `gorm:"-" json:"-"`
}

func (ci ConfigItem) String() string {
Expand Down
3 changes: 3 additions & 0 deletions scrapers/kubernetes/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,10 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul

createdAt := obj.GetCreationTimestamp().Time
var deletedAt *time.Time
var deleteReason v1.ConfigDeleteReason
if !obj.GetDeletionTimestamp().IsZero() {
deletedAt = &obj.GetDeletionTimestamp().Time
deleteReason = v1.DeletedReasonFromAttribute
}

// Evicted Pods must be considered deleted
Expand All @@ -170,6 +172,7 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul
Description: description,
CreatedAt: &createdAt,
DeletedAt: deletedAt,
DeleteReason: deleteReason,
Config: cleanKubernetesObject(obj.Object),
ID: string(obj.GetUID()),
Tags: stripLabels(convertStringInterfaceMapToStringMap(tags), "-hash"),
Expand Down
31 changes: 28 additions & 3 deletions scrapers/stale.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package scrapers
import (
"github.com/flanksource/commons/duration"
"github.com/flanksource/commons/logger"
v1 "github.com/flanksource/config-db/api/v1"
"github.com/flanksource/config-db/db"
"github.com/google/uuid"
)
Expand All @@ -19,21 +20,45 @@ func DeleteStaleConfigItems(scraperID uuid.UUID) error {
}
staleMinutes := int(staleDuration.Minutes())

query := `
// TODO Check deleted_at against updated_at and if updated_at is greater
// and reason is missing scrape, remove deleted_at
deleteQuery := `
UPDATE config_items
SET deleted_at = NOW()
SET
deleted_at = NOW(),
delete_reason = ?
WHERE
((NOW() - updated_at) > INTERVAL '1 minute' * ?) AND
deleted_at IS NULL AND
scraper_id = ?`

result := db.DefaultDB().Exec(query, staleMinutes, scraperID)
result := db.DefaultDB().Exec(deleteQuery, v1.DeletedReasonMissingScrape, staleMinutes, scraperID)
if err := result.Error; err != nil {
return err
}

if result.RowsAffected > 0 {
logger.Infof("Marked %d items as deleted", result.RowsAffected)
}

undeleteQuery := `
UPDATE config_items
SET
deleted_at = NULL,
delete_reason = NULL
WHERE
deleted_at IS NOT NULL AND
delete_reason = ? AND
updated_at > deleted_at AND
scraper_id = ?`

result = db.DefaultDB().Exec(undeleteQuery, v1.DeletedReasonMissingScrape, scraperID)
if err := result.Error; err != nil {
return err
}

if result.RowsAffected > 0 {
logger.Infof("Marked %d items as not deleted", result.RowsAffected)
}
return nil
}

0 comments on commit 81160fd

Please sign in to comment.