Skip to content

Commit

Permalink
feat: add lag metrics & use informer for events (#1203)
Browse files Browse the repository at this point in the history 
* chore: remove updateLag db queries

* chore: wrap events & objects from shared informers into a QueueItem

* feat: informer and event receive lag metrics

* feat: informer & event queued to consume lag

* chore: update histogram buckets

* feat: informer update lag

* feat: per item consumer lag

[skip ci]

* counters for config changes

[skip ci]

* feat: remove event watcher and use informers for events

* chore: add labels to config changes metrics

* fix: add initial delay to metrics registration for informers
  • Loading branch information
@adityathebe
adityathebe authored Dec 2, 2024
1 parent f7c692c commit 03f96d7
Show file tree
Hide file tree
Showing 11 changed files with 316 additions and 461 deletions.
13 changes: 0 additions & 13 deletions api/v1/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,6 @@ func (summary ScrapeSummary) HasUpdates() bool {

}

func (summary *ScrapeSummary) UpdateLag(m map[string]float64) {
for k, v := range m {
summary.AddLag(k, v)
}
}

func (summary ConfigTypeScrapeSummary) String() string {
s := []string{}

Expand Down Expand Up @@ -255,12 +249,6 @@ func (t *ScrapeSummary) AddWarning(configType, warning string) {
(*t)[configType] = v
}

func (t *ScrapeSummary) AddLag(key string, lag float64) {
v := (*t)[key]
v.Lag = lag
(*t)[key] = v
}

type ChangeSummary struct {
Orphaned map[string]int `json:"orphaned,omitempty"`
Ignored map[string]int `json:"ignored,omitempty"`
Expand Down Expand Up @@ -332,7 +320,6 @@ type ConfigTypeScrapeSummary struct {
Unchanged int `json:"unchanged,omitempty"`
Change *ChangeSummary `json:"change,omitempty"`
Warnings []string `json:"warnings,omitempty"`
Lag float64 `json:"lag,omitempty"`
}

// +kubebuilder:object:generate=false
Expand Down
16 changes: 16 additions & 0 deletions api/v1/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ var DefaultWatchKinds = []KubernetesResourceToWatch{
{ApiVersion: "batch/v1", Kind: "CronJob"},
{ApiVersion: "batch/v1", Kind: "Job"},
{ApiVersion: "v1", Kind: "Node"},
{ApiVersion: "v1", Kind: "Event"},
{ApiVersion: "v1", Kind: "Pod"},
}

Expand All @@ -194,6 +195,21 @@ type KubernetesResourceToWatch struct {
Kind string `json:"kind"`
}

func AddEventResourceToWatch(watches []KubernetesResourceToWatch) []KubernetesResourceToWatch {
for _, w := range watches {
if w.ApiVersion == "v1" && w.Kind == "Event" {
return watches
}
}

watches = append(watches, KubernetesResourceToWatch{
ApiVersion: "v1",
Kind: "Event",
})

return watches
}

type Kubernetes struct {
BaseScraper `json:",inline"`
ClusterName string `json:"clusterName"`
Expand Down
50 changes: 8 additions & 42 deletions db/update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,14 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum
dedupWindow := ctx.Properties().Duration("changes.dedup.window", time.Hour)
newChanges, deduped := dedupChanges(dedupWindow, extractResult.newChanges)

for _, c := range newChanges {
ctx.Counter("config_changes", "scraper_id", ctx.ScraperID(), "change_type", c.ChangeType, "config_type", c.ConfigType).Add(1)
}

for _, c := range deduped {
ctx.Counter("config_changes_deduped", "scraper_id", ctx.ScraperID(), "change_type", c.Change.ChangeType, "config_type", c.Change.ConfigType).Add(1)
}

if err := ctx.DB().CreateInBatches(&newChanges, configItemsBulkInsertSize).Error; err != nil {
if !dutydb.IsForeignKeyError(err) {
return summary, fmt.Errorf("failed to create config changes: %w", dutydb.ErrorDetails(err))
Expand Down Expand Up @@ -719,51 +727,9 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum
ctx.Logger.V(4).Infof("No Update: %s", summary)
}

// Add lag time in summary
if lagMap, err := calculateScrapeLag(ctx); err != nil {
ctx.Errorf("error calculating lag: %v", err)
} else {
summary.UpdateLag(lagMap)
}
return summary, nil
}

func calculateScrapeLag(ctx api.ScrapeContext) (map[string]float64, error) {
if ctx.ScraperID() == "" {
return nil, nil
}

q := `
SELECT COALESCE(ROUND(EXTRACT(EPOCH FROM (MAX(inserted_at - created_at))), 2), 0) AS lag
FROM config_items
WHERE
scraper_id = ? AND
inserted_at >= last_scraped_time
`
var configItemLag float64
if err := ctx.DB().Raw(q, ctx.ScraperID()).Scan(&configItemLag).Error; err != nil {
return nil, fmt.Errorf("error querying config_item scrape lag: %w", dutydb.ErrorDetails(err))
}

q = `
SELECT COALESCE(ROUND(EXTRACT(EPOCH FROM (MAX(cc.inserted_at - cc.created_at))), 2), 0) AS lag
FROM config_changes cc
INNER JOIN config_items ci ON cc.config_id = ci.id
WHERE
ci.scraper_id = ? AND
cc.inserted_at >= ci.last_scraped_time
`
var configChangeLag float64
if err := ctx.DB().Raw(q, ctx.ScraperID()).Scan(&configChangeLag).Error; err != nil {
return nil, fmt.Errorf("error querying config_change scrape lag: %w", dutydb.ErrorDetails(err))
}

return map[string]float64{
"config_item_lag_seconds": configItemLag,
"config_change_lag_seconds": configChangeLag,
}, nil
}

var lastScrapedTimeMutex sync.Map

func updateLastScrapedTime(ctx api.ScrapeContext, ids []string) error {
Expand Down
9 changes: 1 addition & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/samber/lo v1.47.0
github.com/samber/oops v1.13.1
github.com/sethvargo/go-retry v0.3.0
github.com/shirou/gopsutil/v3 v3.24.5
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
Expand Down Expand Up @@ -116,10 +115,8 @@ require (
github.com/asecurityteam/rolling v2.0.4+incompatible // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.29 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.29 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.7.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cert-manager/cert-manager v1.9.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
Expand Down Expand Up @@ -159,7 +156,6 @@ require (
github.com/hashicorp/hcl/v2 v2.21.0 // indirect
github.com/henvic/httpretty v0.1.3 // indirect
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
github.com/invopop/jsonschema v0.12.0 // indirect
github.com/itchyny/gojq v0.12.16 // indirect
github.com/itchyny/timefmt-go v0.1.6 // indirect
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect
Expand Down Expand Up @@ -190,6 +186,7 @@ require (
github.com/robertkrimen/otto v0.3.0 // indirect
github.com/rodaine/table v1.3.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
Expand All @@ -204,11 +201,7 @@ require (
github.com/vadimi/go-http-ntlm v1.0.3 // indirect
github.com/vadimi/go-http-ntlm/v2 v2.4.1 // indirect
github.com/vadimi/go-ntlm v1.2.1 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
Expand Down
14 changes: 0 additions & 14 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -833,8 +833,6 @@ github.com/aws/aws-sdk-go-v2/service/support v1.24.3 h1:Bbesu6YZvEYACyydELMwUTYY
github.com/aws/aws-sdk-go-v2/service/support v1.24.3/go.mod h1:NvXUhACskXZ2tiXzECpC/97xKzyY7/Wcc1ug5rla7kY=
github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
Expand All @@ -851,8 +849,6 @@ github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0
github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
Expand Down Expand Up @@ -1309,8 +1305,6 @@ github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
Expand Down Expand Up @@ -1754,16 +1748,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
Expand Down
Loading

0 comments on commit 03f96d7

Please sign in to comment.