From eab25a22dff8838da5a408d1a3a615e947b36e38 Mon Sep 17 00:00:00 2001 From: Moshe Immermam Date: Wed, 28 Aug 2024 01:45:10 +0300 Subject: [PATCH] chore: bump duty --- cmd/root.go | 2 + cmd/run.go | 49 +++++++++++++++++------- cmd/server.go | 10 ++++- db/update.go | 7 +++- go.mod | 11 +++++- go.sum | 20 +++++++++- scrapers/kubernetes/informers.go | 2 +- scrapers/run.go | 4 +- utils/memory.go | 65 -------------------------------- 9 files changed, 83 insertions(+), 87 deletions(-) delete mode 100644 utils/memory.go diff --git a/cmd/root.go b/cmd/root.go index 7b2a96bd..4c654af5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -95,6 +95,8 @@ func ServerFlags(flags *pflag.FlagSet) { } func init() { + properties.BindFlags(Root.PersistentFlags()) + if err := properties.LoadFile(propertiesFile); err != nil { logger.Warnf("Failed to load properties file %v", err) } diff --git a/cmd/run.go b/cmd/run.go index 86efeb6e..e80ec336 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,21 +1,25 @@ package cmd import ( + gocontext "context" "fmt" + "net/http" "os" - "os/signal" "path" "time" "github.com/flanksource/commons/logger" + "github.com/flanksource/commons/timer" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" "github.com/flanksource/config-db/scrapers" - "github.com/flanksource/config-db/utils" "github.com/flanksource/duty" dutyapi "github.com/flanksource/duty/api" "github.com/flanksource/duty/context" + dutyEcho "github.com/flanksource/duty/echo" + "github.com/flanksource/duty/shutdown" + "github.com/labstack/echo/v4" "github.com/spf13/cobra" ) @@ -45,24 +49,43 @@ var Run = &cobra.Command{ api.DefaultContext = api.NewScrapeContext(dutyCtx) - if dutyapi.DefaultConfig.ConnectionString == "" && outputDir == "" { - logger.Fatalf("skipping export: neither --output-dir nor --db is specified") - } + e := echo.New() + + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.SetRequest(c.Request().WithContext(api.DefaultContext.Wrap(c.Request().Context()))) + return next(c) + } + }) + + dutyEcho.AddDebugHandlers(dutyCtx, e, func(next echo.HandlerFunc) echo.HandlerFunc { return next }) + + shutdown.AddHook(func() { + ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 1*time.Minute) + defer cancel() + + if err := e.Shutdown(ctx); err != nil { + e.Logger.Fatal(err) + } + }) + + shutdown.WaitForSignal() go func() { - quit := make(chan os.Signal, 1) - signal.Notify(quit, os.Interrupt) - <-quit - logger.Infof("Caught Ctrl+C") - // call shutdown hooks explicitly, post-run cleanup hooks will be a no-op - Shutdown() + if err := e.Start(fmt.Sprintf(":%d", httpPort)); err != nil && err != http.ErrServerClosed { + e.Logger.Fatal(err) + } }() + if dutyapi.DefaultConfig.ConnectionString == "" && outputDir == "" { + logger.Fatalf("skipping export: neither --output-dir nor --db is specified") + } + for i := range scraperConfigs { ctx, cancel, cancelTimeout := api.DefaultContext.WithScrapeConfig(&scraperConfigs[i]). WithTimeout(api.DefaultContext.Properties().Duration("scraper.timeout", 4*time.Hour)) defer cancelTimeout() - shutdownHooks = append(shutdownHooks, cancel) + shutdown.AddHook(cancel) if err := scrapeAndStore(ctx); err != nil { logger.Errorf("error scraping config: (name=%s) %v", scraperConfigs[i].Name, err) } @@ -76,7 +99,7 @@ func scrapeAndStore(ctx api.ScrapeContext) error { return err } - timer := utils.NewMemoryTimer() + timer := timer.NewMemoryTimer() results, err := scrapers.Run(ctx) if err != nil { return err diff --git a/cmd/server.go b/cmd/server.go index c727a115..6879048b 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -59,7 +59,15 @@ var Serve = &cobra.Command{ func serve(configFiles []string) { e := echo.New() - dutyEcho.AddDebugHandlers(e, func(next echo.HandlerFunc) echo.HandlerFunc { return next }) + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + ctx := api.DefaultContext.Wrap(c.Request().Context()) + c.SetRequest(c.Request().WithContext(ctx)) + return next(c) + } + }) + + dutyEcho.AddDebugHandlers(api.DefaultContext.DutyContext(), e, func(next echo.HandlerFunc) echo.HandlerFunc { return next }) e.Use(otelecho.Middleware("config-db", otelecho.WithSkipper(telemetryURLSkipper))) if logger.IsTraceEnabled() { diff --git a/db/update.go b/db/update.go index 34b8756a..e40513bc 100644 --- a/db/update.go +++ b/db/update.go @@ -124,7 +124,12 @@ func updateCI(ctx api.ScrapeContext, result v1.ScrapeResult, ci, existing *model if err != nil { ctx.Errorf("[%s] failed to check for changes: %v", ci, err) } else if changeResult != nil { - ctx.Logger.V(3).Infof("[%s/%s] detected changes %v", *ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff)) + if ctx.Logger.V(5).Enabled() { + ctx.Logger.V(5).Infof("[%s/%s] detected changes %v", *ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff)) + } else { + ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0]) + + } result.Changes = []v1.ChangeResult{*changeResult} if newChanges, _, _, err := extractChanges(ctx, &result, ci); err != nil { return false, nil, err diff --git a/go.mod b/go.mod index 06c3d2b1..f03f7b7d 100644 --- a/go.mod +++ b/go.mod @@ -51,8 +51,8 @@ require ( github.com/aws/smithy-go v1.20.4 github.com/evanphx/json-patch v5.7.0+incompatible github.com/flanksource/artifacts v1.0.8 - github.com/flanksource/commons v1.28.0 - github.com/flanksource/duty v1.0.606 + github.com/flanksource/commons v1.29.1 + github.com/flanksource/duty v1.0.607 github.com/flanksource/is-healthy v1.0.28 github.com/flanksource/ketall v1.1.7 github.com/flanksource/mapstructure v1.6.0 @@ -113,8 +113,10 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/asecurityteam/rolling v2.0.4+incompatible // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/distribution/reference v0.5.0 // indirect @@ -148,6 +150,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl/v2 v2.21.0 // indirect github.com/hirochachacha/go-smb2 v1.1.0 // indirect + github.com/invopop/jsonschema v0.12.0 // indirect github.com/itchyny/gojq v0.12.16 // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect @@ -189,6 +192,10 @@ require ( github.com/vadimi/go-http-ntlm v1.0.3 // indirect github.com/vadimi/go-http-ntlm/v2 v2.4.1 // indirect github.com/vadimi/go-ntlm v1.2.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/go.sum b/go.sum index 3b64ce63..7b3d221e 100644 --- a/go.sum +++ b/go.sum @@ -806,6 +806,8 @@ github.com/aws/aws-sdk-go-v2/service/support v1.24.3 h1:Bbesu6YZvEYACyydELMwUTYY github.com/aws/aws-sdk-go-v2/service/support v1.24.3/go.mod h1:NvXUhACskXZ2tiXzECpC/97xKzyY7/Wcc1ug5rla7kY= github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -818,6 +820,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -896,10 +900,12 @@ github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7Ttk github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/flanksource/artifacts v1.0.8 h1:Aucp8rYC0gI1+5joh3vv6+Ic8gb2VVe1rYLD6qzCFwc= github.com/flanksource/artifacts v1.0.8/go.mod h1:WUIzg0+qeHeFapPprdUVzSvZYFPLNNcqIbH1HKHZz3Q= -github.com/flanksource/commons v1.28.0 h1:7wyTIU0Q1/wgckecRmIGmCS+zcFeAkId5qNzffR9Sho= -github.com/flanksource/commons v1.28.0/go.mod h1:Q/9SkqJW5QwsVlynOYMVuVQzQtHSBGt1TJkkeX3GRjU= +github.com/flanksource/commons v1.29.1 h1:82vkqo0JFFcOwRXkXrKtMPfUlywOb3YQ+mPSSW9gkpo= +github.com/flanksource/commons v1.29.1/go.mod h1:Kcw3+JI04cVw2zYlv/XTMM1dkNkAwqLYqTxVb19Y/JA= github.com/flanksource/duty v1.0.606 h1:7sTMS6hXlEOpYybck5vCIz4bjVBAZy0TKv/9LyucxvU= github.com/flanksource/duty v1.0.606/go.mod h1:oCqo7xohZHNTnzuo6AsnUnFnzXHN+PaD1HFdo+7nKJk= +github.com/flanksource/duty v1.0.607 h1:Zvi7aSA943+GfgeIH0D2FVLf3n7Ku58pCx44MKz7TK8= +github.com/flanksource/duty v1.0.607/go.mod h1:oa3E43xRghJ31fsfMOfHPJBKy2vEDlyDEzStDcbHyfo= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.24.24 h1:u/30b75VNcoC7X+Q0M9Sk1PtvYUcnLe1nJjNpATHhdY= github.com/flanksource/gomplate/v3 v3.24.24/go.mod h1:Ezcg/bzLwOHYtycHljeOf46RX2YlvLj+xxVxt/nCo30= @@ -1182,6 +1188,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4= github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g= github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM= @@ -1550,6 +1558,14 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= diff --git a/scrapers/kubernetes/informers.go b/scrapers/kubernetes/informers.go index 0670cbcc..4f12f3c7 100644 --- a/scrapers/kubernetes/informers.go +++ b/scrapers/kubernetes/informers.go @@ -50,7 +50,7 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1 return nil } - ctx.Context = ctx.WithName(fmt.Sprintf("Watch[%s]", watchResource.Kind)) + ctx.Context = ctx.WithName("watch." + ctx.ScrapeConfig().Name) ctx.Logger.V(1).Infof("registering shared informer for: %v", watchResource) _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ diff --git a/scrapers/run.go b/scrapers/run.go index aa579e45..3b451c0b 100644 --- a/scrapers/run.go +++ b/scrapers/run.go @@ -4,10 +4,10 @@ import ( "fmt" "time" + "github.com/flanksource/commons/timer" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" - "github.com/flanksource/config-db/utils" ) type contextKey string @@ -22,7 +22,7 @@ type ScrapeOutput struct { } func RunScraper(ctx api.ScrapeContext) (*ScrapeOutput, error) { - var timer = utils.NewMemoryTimer() + var timer = timer.NewMemoryTimer() ctx, err := ctx.InitTempCache() if err != nil { return nil, err diff --git a/utils/memory.go b/utils/memory.go deleted file mode 100644 index 8558d8bd..00000000 --- a/utils/memory.go +++ /dev/null @@ -1,65 +0,0 @@ -package utils - -import ( - "fmt" - "os" - "runtime" - "time" - - "github.com/flanksource/commons/logger" - "github.com/shirou/gopsutil/v3/process" - k8sDuration "k8s.io/apimachinery/pkg/util/duration" -) - -type MemoryTimer struct { - startTime time.Time - start *runtime.MemStats -} - -func age(d time.Duration) string { - if d.Milliseconds() == 0 { - return "0ms" - } - if d.Milliseconds() < 1000 { - return fmt.Sprintf("%0.dms", d.Milliseconds()) - } - - return k8sDuration.HumanDuration(d) -} - -func NewMemoryTimer() MemoryTimer { - m := MemoryTimer{startTime: time.Now()} - if logger.IsTraceEnabled() { - s := runtime.MemStats{} - runtime.ReadMemStats(&s) - m.start = &s - } - return m -} - -func (m *MemoryTimer) End() string { - d := age(time.Since(m.startTime)) - if m.start == nil { - return d - } - - p, _ := process.NewProcess(int32(os.Getpid())) - mem, _ := p.MemoryInfo() - - end := runtime.MemStats{} - runtime.ReadMemStats(&end) - allocs := end.Mallocs - m.start.Mallocs - heap := end.HeapAlloc - m.start.HeapAlloc - totalheap := end.TotalAlloc - m.start.TotalAlloc - gc := end.NumGC - m.start.NumGC - - return fmt.Sprintf( - "%s (allocs=%dk, heap_allocs=%dmb heap_increase=%dmb, gc_count=%d rss=%dmb)", - d, - allocs/1000, - totalheap/1024/1024, - heap/1024/1024, - gc, - mem.RSS/1024/1024, - ) -}