Merge pull request #345 from flanksource/feat/new-upstream-jobs

feat: push config changes & analyses using the is_pushed col

cmd/root.go

@@ -11,6 +11,7 @@ import (
 	"github.com/flanksource/config-db/jobs"
 	"github.com/flanksource/config-db/scrapers"
 	"github.com/flanksource/config-db/utils/kube"
+	"github.com/flanksource/duty"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -53,7 +54,7 @@ var Root = &cobra.Command{
 			db.ConnectionString = ""
 		}
 		db.Schema = readFromEnv(db.Schema)
-		db.LogLevel = readFromEnv(db.LogLevel)
+		db.PGRSTLogLevel = readFromEnv(db.PGRSTLogLevel)
 
 	},
 }
@@ -93,6 +94,7 @@ func ServerFlags(flags *pflag.FlagSet) {
 
 func init() {
 	logger.BindFlags(Root.PersistentFlags())
+	duty.BindFlags(Root.PersistentFlags())
 
 	if len(commit) > 8 {
 		version = fmt.Sprintf("%v, commit %v, built at %v", version, commit[0:8], date)

db/init.go

@@ -21,7 +21,7 @@ import (
 var (
 	ConnectionString string
 	Schema           = "public"
-	LogLevel         = "info"
+	PGRSTLogLevel    = "info"
 	HTTPEndpoint     = "http://localhost:8080/db"
 	db               *gorm.DB
 	runMigrations    = false
@@ -35,7 +35,7 @@ var (
 func Flags(flags *pflag.FlagSet) {
 	flags.StringVar(&ConnectionString, "db", "DB_URL", "Connection string for the postgres database. Use embedded://<path to dir> to use the embedded database")
 	flags.StringVar(&Schema, "db-schema", "public", "")
-	flags.StringVar(&LogLevel, "db-log-level", "warn", "")
+	flags.StringVar(&PGRSTLogLevel, "postgrest-log-level", "warn", "")
 	flags.BoolVar(&runMigrations, "db-migrations", false, "Run database migrations")
 }
 

db/postgrest.go

@@ -34,7 +34,7 @@ func getBinary() deps.BinaryFunc {
 		"PGRST_DB_ANON_ROLE":             "postgrest_api",
 		"PGRST_OPENAPI_SERVER_PROXY_URI": HTTPEndpoint,
 		"PGRST_DB_PORT":                  strconv.Itoa(PostgRESTServerPort),
-		"PGRST_LOG_LEVEL":                LogLevel,
+		"PGRST_LOG_LEVEL":                PGRSTLogLevel,
 		"PGRST_ADMIN_SERVER_PORT":        strconv.Itoa(PostgRESTAdminServerPort),
 	})
 }

go.mod

@@ -39,27 +39,27 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/support v1.14.7
 	github.com/aws/smithy-go v1.13.5
 	github.com/evanphx/json-patch v5.6.0+incompatible
-	github.com/fergusstrange/embedded-postgres v1.21.0
-	github.com/flanksource/commons v1.17.1
-	github.com/flanksource/duty v1.0.248
+	github.com/fergusstrange/embedded-postgres v1.25.0
+	github.com/flanksource/commons v1.19.3
+	github.com/flanksource/duty v1.0.274
 	github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7
 	github.com/flanksource/ketall v1.1.1
-	github.com/flanksource/postq v0.1.1
+	github.com/flanksource/postq v0.1.3
 	github.com/go-logr/zapr v1.2.4
 	github.com/gobwas/glob v0.2.3
 	github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a
 	github.com/google/cel-go v0.18.1
-	github.com/google/uuid v1.4.0
+	github.com/google/uuid v1.5.0
 	github.com/hashicorp/go-getter v1.7.3
 	github.com/henvic/httpretty v0.1.2
 	github.com/hexops/gotextdiff v1.0.3
-	github.com/jackc/pgx/v5 v5.5.0
-	github.com/labstack/echo/v4 v4.10.2
+	github.com/jackc/pgx/v5 v5.5.1
+	github.com/labstack/echo/v4 v4.11.4
 	github.com/lib/pq v1.10.9
 	github.com/ohler55/ojg v1.20.3
 	github.com/oklog/ulid/v2 v2.1.0
-	github.com/onsi/ginkgo/v2 v2.9.5
-	github.com/onsi/gomega v1.27.7
+	github.com/onsi/ginkgo/v2 v2.13.2
+	github.com/onsi/gomega v1.29.0
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/cobra v1.7.0
@@ -90,9 +90,11 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/antonmedv/expr v1.15.3 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/asecurityteam/rolling v2.0.4+incompatible // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
+	github.com/exaring/otelpgx v0.5.2 // indirect
 	github.com/flanksource/kommons v0.31.4 // indirect
 	github.com/flanksource/mapstructure v1.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -103,7 +105,7 @@ require (
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
+	github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/hashicorp/hcl/v2 v2.18.1 // indirect
 	github.com/itchyny/gojq v0.12.13 // indirect
@@ -121,6 +123,7 @@ require (
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/robertkrimen/otto v0.2.1 // indirect
+	github.com/samber/lo v1.39.0 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/sethvargo/go-retry v0.2.4 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
@@ -138,12 +141,13 @@ require (
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/tools v0.14.0 // indirect
+	golang.org/x/tools v0.16.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gorm.io/driver/postgres v1.5.3 // indirect
+	gorm.io/plugin/opentelemetry v0.1.4 // indirect
 	k8s.io/component-base v0.28.0 // indirect
 	layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf // indirect
 	sigs.k8s.io/kustomize v2.0.3+incompatible // indirect
@@ -176,7 +180,7 @@ require (
 	github.com/flanksource/gomplate/v3 v3.20.16
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
-	github.com/go-logr/logr v1.2.4
+	github.com/go-logr/logr v1.3.0
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
@@ -203,7 +207,7 @@ require (
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
 	github.com/jedib0t/go-pretty/v6 v6.4.6 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
@@ -213,12 +217,12 @@ require (
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/labstack/gommon v0.4.0 // indirect
+	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/magiconair/properties v1.8.7
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/microsoft/go-mssqldb v1.6.0
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -246,14 +250,14 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.4.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/api v0.150.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
@@ -273,3 +277,6 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.14.3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 )
+
+// replace github.com/flanksource/duty => ../duty
+// replace github.com/flanksource/postq => ../postq