Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: improve deletion logic with delete_reason #314

Merged
merged 3 commits into from
Nov 24, 2023
Merged

feat: improve deletion logic with delete_reason #314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
26b2fa3
feat: improve deletion logic with delete_reason
yashmehrotra Nov 21, 2023
901d41f
chore: bump duty
yashmehrotra Nov 23, 2023
2018b96
chore: add deletefield delete reason
yashmehrotra Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/v1/interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ func (s *ScrapeResults) Errorf(e error, msg string, args ...interface{}) ScrapeR
type ScrapeResult struct {
CreatedAt *time.Time `json:"created_at,omitempty"`
DeletedAt *time.Time `json:"deleted_at,omitempty"`
DeleteReason ConfigDeleteReason `json:"delete_reason,omitempty"`
LastModified time.Time `json:"last_modified,omitempty"`
ConfigClass string `json:"config_class,omitempty"`
Type string `json:"config_type,omitempty"`
Expand Down
9 changes: 9 additions & 0 deletions api/v1/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,12 @@ func (e ExternalID) CacheKey() string {
func (e ExternalID) WhereClause(db *gorm.DB) *gorm.DB {
return db.Where("type = ? AND external_id @> ?", e.ConfigType, pq.StringArray(e.ExternalID))
}

type ConfigDeleteReason string

var (
DeletedReasonMissingScrape ConfigDeleteReason = "MISSING_SCRAPE"
DeletedReasonFromAttribute ConfigDeleteReason = "FROM_ATTRIBUTE"
DeletedReasonFromEvent ConfigDeleteReason = "FROM_EVENT"
DeletedReasonFromDeleteField ConfigDeleteReason = "FROM_DELETE_FIELD"
)
11 changes: 9 additions & 2 deletions db/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,14 @@ func UpdateConfigItem(ci *models.ConfigItem) error {
}

// Since gorm ignores nil fields, we are setting deleted_at explicitly
if ci.TouchDeletedAt {
if err := db.Table("config_items").Where("id = ?", ci.ID).UpdateColumn("deleted_at", nil).Error; err != nil {
// TODO Add deleted reason check
if ci.TouchDeletedAt && ci.DeleteReason != v1.DeletedReasonFromEvent {
if err := db.Table("config_items").
Where("id = ?", ci.ID).
Updates(map[string]any{
"deleted_at": nil,
"delete_reason": nil,
}).Error; err != nil {
return err
}
}
Expand Down Expand Up @@ -167,6 +173,7 @@ func NewConfigItemFromResult(result v1.ScrapeResult) (*models.ConfigItem, error)

if result.DeletedAt != nil {
ci.DeletedAt = result.DeletedAt
ci.DeleteReason = result.DeleteReason
}

if result.ParentExternalID != "" && result.ParentType != "" {
Expand Down
47 changes: 24 additions & 23 deletions db/models/config_item.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,30 @@ import (

// ConfigItem represents the config item database table
type ConfigItem struct {
ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type *string `gorm:"column:type;default:null" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Account *string `gorm:"column:account;default:null" json:"account,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" `
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
TouchDeletedAt bool `gorm:"-" json:"-"`
ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type *string `gorm:"column:type;default:null" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Namespace *string `gorm:"column:namespace;default:null" json:"namespace,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Account *string `gorm:"column:account;default:null" json:"account,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Tags *v1.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty" `
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
DeleteReason v1.ConfigDeleteReason `gorm:"column:delete_reason" json:"delete_reason"`
TouchDeletedAt bool `gorm:"-" json:"-"`
}

func (ci ConfigItem) String() string {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ require (
github.com/evanphx/json-patch v5.6.0+incompatible
github.com/fergusstrange/embedded-postgres v1.21.0
github.com/flanksource/commons v1.17.1
github.com/flanksource/duty v1.0.222
github.com/flanksource/duty v1.0.224
github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7
github.com/flanksource/ketall v1.1.1
github.com/flanksource/postq v0.1.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,6 +868,8 @@ github.com/flanksource/commons v1.17.1 h1:jd114sxRwe2VWcbG/PVVEAWsEkialL6eltbqFG
github.com/flanksource/commons v1.17.1/go.mod h1:RDdQI0/QYC4GzicbDaXIvBPjWuQWKLzX8/rFBbFjG5U=
github.com/flanksource/duty v1.0.222 h1:X2MYmg2YP8228JrSHF5H13Ba0lGQZ+JQWGo5zf+sGEc=
github.com/flanksource/duty v1.0.222/go.mod h1:IUXGDXwYdChv9hKB7X7C9ZH79Fh7tHZ9bF7nde2ZBTo=
github.com/flanksource/duty v1.0.224 h1:5EfDcCTaemGsZmsYtygB0aNcmrPTAHnCyLOKkudV4TA=
github.com/flanksource/duty v1.0.224/go.mod h1:IUXGDXwYdChv9hKB7X7C9ZH79Fh7tHZ9bF7nde2ZBTo=
github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
github.com/flanksource/gomplate/v3 v3.20.16 h1:Bfn+nbD0iK0iGQcu6alV8Nr7O5+KpeDo8OD9WOu831Q=
github.com/flanksource/gomplate/v3 v3.20.16/go.mod h1:2GgHZ2vWmtDspJMBfUIryOuzJSwc8jU7Kw9fDLr0TMA=
Expand Down
92 changes: 91 additions & 1 deletion hack/generate-schemas/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,64 +10,154 @@ require (
)

require (
ariga.io/atlas v0.14.2 // indirect
cloud.google.com/go v0.110.8 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.2 // indirect
cloud.google.com/go/storage v1.33.0 // indirect
github.com/AlekSi/pointer v1.1.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/TomOnTime/utfutil v0.0.0-20210710122150-437f72b26edf // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
github.com/flanksource/duty v1.0.222 // indirect
github.com/antonmedv/expr v1.15.3 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/aws/aws-sdk-go v1.45.24 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/flanksource/duty v1.0.224 // indirect
github.com/flanksource/gomplate/v3 v3.20.16 // indirect
github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7 // indirect
github.com/flanksource/kommons v0.31.4 // indirect
github.com/flanksource/mapstructure v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/inflect v0.19.0 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/cel-go v0.18.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gosimple/slug v1.13.1 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf // indirect
github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-getter v1.7.3 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/hcl/v2 v2.18.1 // indirect
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/itchyny/gojq v0.12.13 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.4.3 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/liamylian/jsontime/v2 v2.0.0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/robertkrimen/otto v0.2.1 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/tidwall/gjson v1.17.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 // indirect
github.com/yuin/gopher-lua v1.1.0 // indirect
github.com/zclconf/go-cty v1.14.1 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/api v0.146.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/flanksource/yaml.v3 v3.2.3 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/sourcemap.v1 v1.0.5 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/postgres v1.5.3 // indirect
gorm.io/gorm v1.25.5 // indirect
k8s.io/api v0.28.2 // indirect
k8s.io/apiextensions-apiserver v0.28.0 // indirect
k8s.io/apimachinery v0.28.2 // indirect
k8s.io/cli-runtime v0.28.0 // indirect
k8s.io/client-go v0.28.2 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf // indirect
sigs.k8s.io/controller-runtime v0.15.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize v2.0.3+incompatible // indirect
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
Expand Down
Loading
Loading