Skip to content

Commit

Permalink
Merge pull request #298 from flanksource/handle-evicted-pods
Browse files Browse the repository at this point in the history 
fix: mark evicted pods as deleted
  • Loading branch information
@moshloop
moshloop authored Sep 22, 2023
2 parents 2827540 + efce409 commit d8f7b94
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 29 deletions.
5 changes: 2 additions & 3 deletions db/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
// GetConfigItem returns a single config item result
func GetConfigItem(extType, extID string) (*models.ConfigItem, error) {
ci := models.ConfigItem{}
tx := db.Limit(1).Find(&ci, "type = ? and external_id @> ?", extType, pq.StringArray{extID})
tx := db.Limit(1).Select("id", "config_class", "type", "config").Find(&ci, "type = ? and external_id @> ?", extType, pq.StringArray{extID})
if tx.RowsAffected == 0 {
return nil, nil
}
Expand Down Expand Up @@ -73,13 +73,12 @@ func CreateConfigItem(ci *models.ConfigItem) error {

// UpdateConfigItem updates all the fields of a given config item row
func UpdateConfigItem(ci *models.ConfigItem) error {

if err := db.Updates(ci).Error; err != nil {
return err
}

// Since gorm ignores nil fields, we are setting deleted_at explicitly
if ci.DeletedAt != nil {
if ci.TouchDeletedAt {
if err := db.Table("config_items").Where("id = ?", ci.ID).UpdateColumn("deleted_at", nil).Error; err != nil {
return err
}
Expand Down
45 changes: 23 additions & 22 deletions db/models/config_item.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,28 +12,29 @@ import (

// ConfigItem represents the config item database table
type ConfigItem struct {
ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type *string `gorm:"column:type;default:null" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Account *string `gorm:"column:account;default:null" json:"account,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" `
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type *string `gorm:"column:type;default:null" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Account *string `gorm:"column:account;default:null" json:"account,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" `
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
TouchDeletedAt bool `gorm:"-" json:"-"`
}

func (ci ConfigItem) String() string {
Expand Down
20 changes: 16 additions & 4 deletions db/update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,13 @@ func updateCI(ctx *v1.ScrapeContext, ci models.ConfigItem) error {
}

ci.ID = existing.ID
ci.DeletedAt = existing.DeletedAt

// In case a resource was marked as deleted but is un-deleted now
// we set an update flag as gorm ignores nil pointers
if ci.DeletedAt != existing.DeletedAt {
ci.TouchDeletedAt = true
}

if err := UpdateConfigItem(&ci); err != nil {
if err := CreateConfigItem(&ci); err != nil {
return fmt.Errorf("[%s] failed to update item %v", ci, err)
Expand Down Expand Up @@ -177,16 +183,22 @@ func updateChange(ctx *v1.ScrapeContext, result *v1.ScrapeResult) error {

func upsertAnalysis(ctx *v1.ScrapeContext, result *v1.ScrapeResult) error {
analysis := result.AnalysisResult.ToConfigAnalysis()
ci, err := GetConfigItem(analysis.ConfigType, analysis.ExternalID)
if ci == nil {
ciID, err := FindConfigItemID(v1.ExternalID{
ConfigType: analysis.ConfigType,
ExternalID: []string{analysis.ExternalID},
})
if ciID == nil {
logger.Warnf("[Source=%s] [%s/%s] unable to find config item for analysis: %+v", analysis.Source, analysis.ConfigType, analysis.ExternalID, analysis)
return nil
} else if err != nil {
return err
}

logger.Tracef("[%s/%s] ==> %s", analysis.ConfigType, analysis.ExternalID, analysis)
analysis.ConfigID = uuid.MustParse(ci.ID)
analysis.ConfigID, err = uuid.Parse(*ciID)
if err != nil {
return err
}
analysis.ID = uuid.MustParse(ulid.MustNew().AsUUID())
analysis.ScraperID = ctx.ScrapeConfig.GetPersistedID()
if analysis.Status == "" {
Expand Down
20 changes: 20 additions & 0 deletions scrapers/kubernetes/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
"time"

"github.com/Jeffail/gabs/v2"
"github.com/flanksource/commons/collections"
Expand Down Expand Up @@ -124,6 +125,24 @@ func (kubernetes KubernetesScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResul
}

createdAt := obj.GetCreationTimestamp().Time
var deletedAt *time.Time
if !obj.GetDeletionTimestamp().IsZero() {
deletedAt = &obj.GetDeletionTimestamp().Time
}

// Evicted Pods must be considered deleted
if obj.GetKind() == "Pod" && status == string(health.HealthStatusDegraded) {
objStatus := obj.Object["status"].(map[string]any)
if val, ok := objStatus["reason"].(string); ok && val == "Evicted" {
// Use time.Now() as default and try to parse the evict time
timeNow := time.Now()
deletedAt = &timeNow
if evictTime, err := time.Parse(time.RFC3339, objStatus["startTime"].(string)); err != nil {
deletedAt = &evictTime
}
}
}

parentType, parentExternalID := getKubernetesParent(obj, resourceIDMap)
results = append(results, v1.ScrapeResult{
BaseScraper: config.BaseScraper,
Expand All @@ -134,6 +153,7 @@ func (kubernetes KubernetesScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResul
Status: status,
Description: description,
CreatedAt: &createdAt,
DeletedAt: deletedAt,
Config: cleanKubernetesObject(obj.Object),
ID: string(obj.GetUID()),
Tags: stripLabels(convertStringInterfaceMapToStringMap(tags), "-hash"),
Expand Down

0 comments on commit d8f7b94

Please sign in to comment.