wip: setup a way to scrape selected configs by ids

api/v1/interface.go

@@ -25,6 +25,13 @@ type Scraper interface {
 	CanScrape(config ScraperSpec) bool
 }
 
+// TargettedScraper can additionally scrape a predefined set of configs.
+// +kubebuilder:object:generate=false
+type TargettedScraper interface {
+	Scraper
+	ScrapeSome(ctx *ScrapeContext, configIndex int, ids []string) ScrapeResults
+}
+
 // Analyzer ...
 // +kubebuilder:object:generate=false
 type Analyzer func(configs []ScrapeResult) AnalysisResult

api/v1/kubernetes.go

@@ -3,6 +3,7 @@ package v1
 import (
 	"strings"
 
+	"github.com/flanksource/commons/collections/set"
 	"github.com/flanksource/duty/types"
 )
 
@@ -34,6 +35,21 @@ type Kubernetes struct {
 	Exclusions      []string        `json:"exclusions,omitempty"`
 	Kubeconfig      *types.EnvVar   `json:"kubeconfig,omitempty"`
 	Event           KubernetesEvent `json:"event,omitempty"`
+
+	exclusionSet set.Set[string] `json:"-"`
+}
+
+func (t *Kubernetes) IsExcluded(item string) bool {
+	if len(t.Exclusions) == 0 {
+		return false
+	}
+
+	if len(t.exclusionSet) == 0 {
+		t.exclusionSet = set.New[string]()
+		t.exclusionSet.Add(t.Exclusions...)
+	}
+
+	return t.exclusionSet.Contains(item)
 }
 
 type KubernetesFile struct {

cmd/server.go

@@ -4,16 +4,15 @@ import (
 	"context"
 	"fmt"
 	"net/url"
-	"strings"
 
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/config-db/api"
 	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/config-db/db"
+
 	"github.com/flanksource/config-db/jobs"
 	"github.com/flanksource/config-db/query"
 	"github.com/flanksource/config-db/scrapers/kubernetes"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/flanksource/config-db/scrapers"
 	"github.com/labstack/echo/v4"
@@ -108,13 +107,13 @@ func startScraperCron(configFiles []string) {
 			go exitOnError(kubernetes.WatchEvents(ctx, k, kubernetesChangeEventConsumer), "error watching events")
 		}
 
-		fn := func() {
-			ctx := api.NewScrapeContext(context.Background(), _scraper)
-			if _, err := scrapers.RunScraper(ctx); err != nil {
-				logger.Errorf("Error running scraper(id=%s): %v", scraper.ID, err)
-			}
-		}
-		go scrapers.AtomicRunner(scraper.ID.String(), fn)()
+		// fn := func() {
+		// 	ctx := api.NewScrapeContext(context.Background(), _scraper)
+		// 	if _, err := scrapers.RunScraper(ctx); err != nil {
+		// 		logger.Errorf("Error running scraper(id=%s): %v", scraper.ID, err)
+		// 	}
+		// }
+		// go scrapers.AtomicRunner(scraper.ID.String(), fn)()
 	}
 }
 
@@ -145,34 +144,15 @@ func exitOnError(err error, description string) {
 	}
 }
 
-func kubernetesChangeEventConsumer(ctx *v1.ScrapeContext, changes []*kubernetes.InvolvedObject) {
-	resourcesPerKind := make(map[string]map[string]*kubernetes.InvolvedObject)
-	for _, c := range changes {
-		if resourcesPerKind[c.Kind] == nil {
-			resourcesPerKind[c.Kind] = make(map[string]*kubernetes.InvolvedObject)
-		}
-
-		resourcesPerKind[c.Kind][c.UID] = c
-	}
-
+func kubernetesChangeEventConsumer(ctx *v1.ScrapeContext, resourcesPerKind map[string]map[string]*kubernetes.InvolvedObject) {
+	var resourceIDs []string
 	for kind, resources := range resourcesPerKind {
-		if kind != "Pod" {
-			continue
+		for _, r := range resources {
+			resourceIDs = append(resourceIDs, kubernetes.ItemID{Kind: kind, Name: r.Name, Namespace: r.Namespace}.Encode())
 		}
+	}
 
-		for _, resource := range resources {
-			if strings.Contains(resource.Name, "junit") { // Temporary
-				continue
-			}
-
-			logger.Infof("Getting pod (name=%s, namespace=%s)", resource.Name, resource.Namespace)
-			pod, err := ctx.Kubernetes.CoreV1().Pods(resource.Namespace).Get(ctx, resource.Name, metav1.GetOptions{})
-			if err != nil {
-				logger.Errorf("failed to get pod (name=%s): %v", resource.Name, err)
-				continue
-			}
-
-			logger.Infof("Got pod %s", pod.Name)
-		}
+	if _, err := scrapers.RunSome(ctx, kubernetes.KubernetesScraper{}, 0, resourceIDs); err != nil {
+		logger.Errorf("Error running scraper(id=%s): %v", ctx.ScrapeConfig.GetUID(), err)
 	}
 }

go.mod

@@ -42,12 +42,12 @@ require (
 	github.com/flanksource/commons v1.10.2
 	github.com/flanksource/duty v1.0.129
 	github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37
-	github.com/flanksource/ketall v1.1.1
+	github.com/flanksource/ketall v1.1.2
 	github.com/go-logr/zapr v1.2.3
 	github.com/gobwas/glob v0.2.3
 	github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a
 	github.com/google/cel-go v0.17.1
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.3.1
 	github.com/hashicorp/go-getter v1.7.2
 	github.com/henvic/httpretty v0.1.0
 	github.com/hexops/gotextdiff v1.0.3
@@ -56,20 +56,20 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/ohler55/ojg v1.18.3
 	github.com/oklog/ulid/v2 v2.1.0
-	github.com/onsi/ginkgo/v2 v2.9.2
+	github.com/onsi/ginkgo/v2 v2.9.4
 	github.com/onsi/gomega v1.27.6
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/spf13/cobra v1.6.1
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
 	github.com/uber/athenadriver v1.1.14
 	github.com/xo/dburl v0.13.1
 	gopkg.in/flanksource/yaml.v3 v3.2.3
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/gorm v1.25.0
-	k8s.io/apimachinery v0.26.4
-	k8s.io/client-go v0.26.4
+	k8s.io/apimachinery v0.28.0
+	k8s.io/client-go v0.28.0
 	sigs.k8s.io/controller-runtime v0.14.6
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -86,7 +86,6 @@ require (
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/flanksource/mapstructure v1.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -95,6 +94,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/hashicorp/hcl/v2 v2.16.2 // indirect
@@ -155,22 +155,21 @@ require (
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/flanksource/gomplate/v3 v3.20.9
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
+	github.com/go-errors/errors v1.5.0 // indirect
 	github.com/go-logr/logr v1.2.4
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/go-sql-driver/mysql v1.7.0
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.9
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
@@ -226,34 +225,34 @@ require (
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 // indirect
+	go.starlark.net v0.0.0-20230831151029-c9e9adf3fde2 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.134.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e // indirect
 	google.golang.org/grpc v1.57.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/api v0.26.4
+	k8s.io/api v0.28.0
 	k8s.io/apiextensions-apiserver v0.26.3 // indirect
-	k8s.io/cli-runtime v0.26.3 // indirect
+	k8s.io/cli-runtime v0.28.0 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515 // indirect
+	k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/kustomize/api v0.12.1 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
+	sigs.k8s.io/kustomize/api v0.14.0 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 )