chore: bump duty

cmd/root.go

@@ -95,6 +95,8 @@ func ServerFlags(flags *pflag.FlagSet) {
 }
 
 func init() {
+	properties.BindFlags(Root.PersistentFlags())
+
 	if err := properties.LoadFile(propertiesFile); err != nil {
 		logger.Warnf("Failed to load properties file %v", err)
 	}

cmd/run.go

@@ -1,21 +1,25 @@
 package cmd
 
 import (
+	gocontext "context"
 	"fmt"
+	"net/http"
 	"os"
-	"os/signal"
 	"path"
 	"time"
 
 	"github.com/flanksource/commons/logger"
+	"github.com/flanksource/commons/timer"
 	"github.com/flanksource/config-db/api"
 	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/config-db/db"
 	"github.com/flanksource/config-db/scrapers"
-	"github.com/flanksource/config-db/utils"
 	"github.com/flanksource/duty"
 	dutyapi "github.com/flanksource/duty/api"
 	"github.com/flanksource/duty/context"
+	dutyEcho "github.com/flanksource/duty/echo"
+	"github.com/flanksource/duty/shutdown"
+	"github.com/labstack/echo/v4"
 	"github.com/spf13/cobra"
 )
 
@@ -45,24 +49,43 @@ var Run = &cobra.Command{
 
 		api.DefaultContext = api.NewScrapeContext(dutyCtx)
 
-		if dutyapi.DefaultConfig.ConnectionString == "" && outputDir == "" {
-			logger.Fatalf("skipping export: neither --output-dir nor --db is specified")
-		}
+		e := echo.New()
+
+		e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+			return func(c echo.Context) error {
+				c.SetRequest(c.Request().WithContext(api.DefaultContext.Wrap(c.Request().Context())))
+				return next(c)
+			}
+		})
+
+		dutyEcho.AddDebugHandlers(dutyCtx, e, func(next echo.HandlerFunc) echo.HandlerFunc { return next })
+
+		shutdown.AddHook(func() {
+			ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 1*time.Minute)
+			defer cancel()
+
+			if err := e.Shutdown(ctx); err != nil {
+				e.Logger.Fatal(err)
+			}
+		})
+
+		shutdown.WaitForSignal()
 
 		go func() {
-			quit := make(chan os.Signal, 1)
-			signal.Notify(quit, os.Interrupt)
-			<-quit
-			logger.Infof("Caught Ctrl+C")
-			// call shutdown hooks explicitly, post-run cleanup hooks will be a no-op
-			Shutdown()
+			if err := e.Start(fmt.Sprintf(":%d", httpPort)); err != nil && err != http.ErrServerClosed {
+				e.Logger.Fatal(err)
+			}
 		}()
 
+		if dutyapi.DefaultConfig.ConnectionString == "" && outputDir == "" {
+			logger.Fatalf("skipping export: neither --output-dir nor --db is specified")
+		}
+
 		for i := range scraperConfigs {
 			ctx, cancel, cancelTimeout := api.DefaultContext.WithScrapeConfig(&scraperConfigs[i]).
 				WithTimeout(api.DefaultContext.Properties().Duration("scraper.timeout", 4*time.Hour))
 			defer cancelTimeout()
-			shutdownHooks = append(shutdownHooks, cancel)
+			shutdown.AddHook(cancel)
 			if err := scrapeAndStore(ctx); err != nil {
 				logger.Errorf("error scraping config: (name=%s) %v", scraperConfigs[i].Name, err)
 			}
@@ -76,7 +99,7 @@ func scrapeAndStore(ctx api.ScrapeContext) error {
 		return err
 	}
 
-	timer := utils.NewMemoryTimer()
+	timer := timer.NewMemoryTimer()
 	results, err := scrapers.Run(ctx)
 	if err != nil {
 		return err

cmd/server.go

@@ -59,7 +59,15 @@ var Serve = &cobra.Command{
 func serve(configFiles []string) {
 	e := echo.New()
 
-	dutyEcho.AddDebugHandlers(e, func(next echo.HandlerFunc) echo.HandlerFunc { return next })
+	e.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			ctx := api.DefaultContext.Wrap(c.Request().Context())
+			c.SetRequest(c.Request().WithContext(ctx))
+			return next(c)
+		}
+	})
+
+	dutyEcho.AddDebugHandlers(api.DefaultContext.DutyContext(), e, func(next echo.HandlerFunc) echo.HandlerFunc { return next })
 	e.Use(otelecho.Middleware("config-db", otelecho.WithSkipper(telemetryURLSkipper)))
 
 	if logger.IsTraceEnabled() {

db/update.go

@@ -124,7 +124,12 @@ func updateCI(ctx api.ScrapeContext, result v1.ScrapeResult, ci, existing *model
 	if err != nil {
 		ctx.Errorf("[%s] failed to check for changes: %v", ci, err)
 	} else if changeResult != nil {
-		ctx.Logger.V(3).Infof("[%s/%s] detected changes %v", *ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff))
+		if ctx.Logger.V(5).Enabled() {
+			ctx.Logger.V(5).Infof("[%s/%s] detected changes %v", *ci.Type, ci.ExternalID[0], lo.FromPtr(changeResult.Diff))
+		} else {
+			ctx.Logger.V(3).Infof("[%s/%s] detected changes", *ci.Type, ci.ExternalID[0])
+
+		}
 		result.Changes = []v1.ChangeResult{*changeResult}
 		if newChanges, _, _, err := extractChanges(ctx, &result, ci); err != nil {
 			return false, nil, err

go.mod

@@ -51,8 +51,8 @@ require (
 	github.com/aws/smithy-go v1.20.4
 	github.com/evanphx/json-patch v5.7.0+incompatible
 	github.com/flanksource/artifacts v1.0.8
-	github.com/flanksource/commons v1.28.0
-	github.com/flanksource/duty v1.0.606
+	github.com/flanksource/commons v1.29.1
+	github.com/flanksource/duty v1.0.607
 	github.com/flanksource/is-healthy v1.0.28
 	github.com/flanksource/ketall v1.1.7
 	github.com/flanksource/mapstructure v1.6.0
@@ -113,8 +113,10 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/asecurityteam/rolling v2.0.4+incompatible // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
@@ -148,6 +150,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.21.0 // indirect
 	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
+	github.com/invopop/jsonschema v0.12.0 // indirect
 	github.com/itchyny/gojq v0.12.16 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect
@@ -189,6 +192,10 @@ require (
 	github.com/vadimi/go-http-ntlm v1.0.3 // indirect
 	github.com/vadimi/go-http-ntlm/v2 v2.4.1 // indirect
 	github.com/vadimi/go-ntlm v1.2.1 // indirect
+	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect

go.sum

@@ -806,6 +806,8 @@ github.com/aws/aws-sdk-go-v2/service/support v1.24.3 h1:Bbesu6YZvEYACyydELMwUTYY
 github.com/aws/aws-sdk-go-v2/service/support v1.24.3/go.mod h1:NvXUhACskXZ2tiXzECpC/97xKzyY7/Wcc1ug5rla7kY=
 github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
 github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -818,6 +820,8 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN
 github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -896,10 +900,12 @@ github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7Ttk
 github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw=
 github.com/flanksource/artifacts v1.0.8 h1:Aucp8rYC0gI1+5joh3vv6+Ic8gb2VVe1rYLD6qzCFwc=
 github.com/flanksource/artifacts v1.0.8/go.mod h1:WUIzg0+qeHeFapPprdUVzSvZYFPLNNcqIbH1HKHZz3Q=
-github.com/flanksource/commons v1.28.0 h1:7wyTIU0Q1/wgckecRmIGmCS+zcFeAkId5qNzffR9Sho=
-github.com/flanksource/commons v1.28.0/go.mod h1:Q/9SkqJW5QwsVlynOYMVuVQzQtHSBGt1TJkkeX3GRjU=
+github.com/flanksource/commons v1.29.1 h1:82vkqo0JFFcOwRXkXrKtMPfUlywOb3YQ+mPSSW9gkpo=
+github.com/flanksource/commons v1.29.1/go.mod h1:Kcw3+JI04cVw2zYlv/XTMM1dkNkAwqLYqTxVb19Y/JA=
 github.com/flanksource/duty v1.0.606 h1:7sTMS6hXlEOpYybck5vCIz4bjVBAZy0TKv/9LyucxvU=
 github.com/flanksource/duty v1.0.606/go.mod h1:oCqo7xohZHNTnzuo6AsnUnFnzXHN+PaD1HFdo+7nKJk=
+github.com/flanksource/duty v1.0.607 h1:Zvi7aSA943+GfgeIH0D2FVLf3n7Ku58pCx44MKz7TK8=
+github.com/flanksource/duty v1.0.607/go.mod h1:oa3E43xRghJ31fsfMOfHPJBKy2vEDlyDEzStDcbHyfo=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.24.24 h1:u/30b75VNcoC7X+Q0M9Sk1PtvYUcnLe1nJjNpATHhdY=
 github.com/flanksource/gomplate/v3 v3.24.24/go.mod h1:Ezcg/bzLwOHYtycHljeOf46RX2YlvLj+xxVxt/nCo30=
@@ -1182,6 +1188,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
+github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
 github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
 github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
 github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
@@ -1550,6 +1558,14 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
+github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=

scrapers/kubernetes/informers.go

@@ -50,7 +50,7 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1
 		return nil
 	}
 
-	ctx.Context = ctx.WithName(fmt.Sprintf("Watch[%s]", watchResource.Kind))
+	ctx.Context = ctx.WithName("watch." + ctx.ScrapeConfig().Name)
 
 	ctx.Logger.V(1).Infof("registering shared informer for: %v", watchResource)
 	_, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{

scrapers/run.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/flanksource/commons/timer"
 	"github.com/flanksource/config-db/api"
 	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/config-db/db"
-	"github.com/flanksource/config-db/utils"
 )
 
 type contextKey string
@@ -22,7 +22,7 @@ type ScrapeOutput struct {
 }
 
 func RunScraper(ctx api.ScrapeContext) (*ScrapeOutput, error) {
-	var timer = utils.NewMemoryTimer()
+	var timer = timer.NewMemoryTimer()
 	ctx, err := ctx.InitTempCache()
 	if err != nil {
 		return nil, err