From faeea56ce75715eab29edf175d44288870f1bc8c Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Tue, 28 Nov 2023 21:34:54 +0545 Subject: [PATCH] test: added test for kubernetes scraper to test relationship --- db/config.go | 5 +- fixtures/kubernetes.yaml | 6 +- scrapers/kubernetes/kubernetes.go | 34 ++++++++--- scrapers/run.go | 7 ++- scrapers/runscrapers_suite_test.go | 88 +++++++++++++++++++++++++++- scrapers/runscrapers_test.go | 93 +++++++++++++++++++++++++++--- scrapers/stale.go | 5 +- 7 files changed, 209 insertions(+), 29 deletions(-) diff --git a/db/config.go b/db/config.go index 0e0255c9..fd6b623a 100644 --- a/db/config.go +++ b/db/config.go @@ -95,13 +95,12 @@ func UpdateConfigItem(ci *models.ConfigItem) error { return nil } -// ConfigItemsIDs returns the uuid of config items which matches the given type, name & namespace -func ConfigItemsIDs(ctx context.Context, configType, name, namespace string) ([]uuid.UUID, error) { +// FindConfigIDsByNamespaceName returns the uuid of config items which matches the given type, name & namespace +func FindConfigIDsByNamespaceName(ctx context.Context, namespace, name string) ([]uuid.UUID, error) { var ids []uuid.UUID err := ctx.DB(). Model(&models.ConfigItem{}). Select("id"). - Where("type = ?", configType). Where("name = ?", name). Where("namespace = ?", namespace). Find(&ids).Error diff --git a/fixtures/kubernetes.yaml b/fixtures/kubernetes.yaml index b65dcec9..dedd7392 100644 --- a/fixtures/kubernetes.yaml +++ b/fixtures/kubernetes.yaml @@ -22,11 +22,11 @@ spec: - orders.acme.cert-manager.io relationships: - kind: - expr: has(spec.claimRef) && spec.claimRef.kind + expr: "has(spec.claimRef) ? spec.claimRef.kind : ''" name: - expr: has(spec.claimRef) && spec.claimRef.name + expr: "has(spec.claimRef) ? spec.claimRef.name : ''" namespace: - expr: has(spec.claimRef) && spec.claimRef.namespace + expr: "has(spec.claimRef) ? spec.claimRef.namespace : ''" - kind: value: Kustomization name: diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index ae873f19..0c3a4d0f 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -9,6 +9,8 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" + "github.com/flanksource/duty/context" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" @@ -32,6 +34,8 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul var ( results v1.ScrapeResults changeResults v1.ScrapeResults + + err error ) for _, config := range ctx.ScrapeConfig().Spec.Kubernetes { @@ -51,7 +55,10 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul }) opts := options.NewDefaultCmdOptions() - opts = updateOptions(opts, config) + opts, err = updateOptions(ctx.DutyContext(), opts, config) + if err != nil { + return results.Errorf(err, "error setting up kube config") + } objs := ketall.KetAll(opts) resourceIDMap := getResourceIDsFromObjs(objs) @@ -131,6 +138,10 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul return results.Errorf(err, "failed to evaluate kind: %v for config relationship", f.Kind) } + if kind != obj.GetKind() { + continue // Try matching another relationship + } + name, err := f.Name.Eval(obj.GetLabels(), env) if err != nil { return results.Errorf(err, "failed to evaluate name: %v for config relationship", f.Name) @@ -141,9 +152,9 @@ func (kubernetes KubernetesScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResul return results.Errorf(err, "failed to evaluate namespace: %v for config relationship", f.Namespace) } - linkedConfigItemIDs, err := db.ConfigItemsIDs(ctx.DutyContext(), fmt.Sprintf("%s%s", ConfigTypePrefix, kind), name, namespace) + linkedConfigItemIDs, err := db.FindConfigIDsByNamespaceName(ctx.DutyContext(), namespace, name) if err != nil { - return results.Errorf(err, "failed to get linked config items: kind=%s, name=%s, namespace=%s", kind, name, namespace) + return results.Errorf(err, "failed to get linked config items: name=%s, namespace=%s", name, namespace) } for _, id := range linkedConfigItemIDs { @@ -274,7 +285,7 @@ func getKubernetesAlias(obj *unstructured.Unstructured) []string { return []string{strings.Join([]string{"Kubernetes", obj.GetKind(), obj.GetNamespace(), obj.GetName()}, "/")} } -func updateOptions(opts *options.KetallOptions, config v1.Kubernetes) *options.KetallOptions { +func updateOptions(ctx context.Context, opts *options.KetallOptions, config v1.Kubernetes) (*options.KetallOptions, error) { opts.AllowIncomplete = config.AllowIncomplete opts.Namespace = config.Namespace opts.Scope = config.Scope @@ -284,11 +295,16 @@ func updateOptions(opts *options.KetallOptions, config v1.Kubernetes) *options.K opts.MaxInflight = config.MaxInflight opts.Exclusions = config.Exclusions opts.Since = config.Since - //TODO: update kubeconfig reference if provided by user - // if config.Kubeconfig != nil { - // opts.Kubeconfig = config.Kubeconfig.GetValue() - // } - return opts + if config.Kubeconfig != nil { + val, err := ctx.GetEnvValueFromCache(*config.Kubeconfig) + if err != nil { + return nil, err + } + + opts.GenericCliFlags.KubeConfig = &val + } + + return opts, nil } func extractDeployNameFromReplicaSet(rs string) string { diff --git a/scrapers/run.go b/scrapers/run.go index ad0010eb..96086365 100644 --- a/scrapers/run.go +++ b/scrapers/run.go @@ -22,8 +22,11 @@ func RunScraper(ctx api.ScrapeContext) (v1.ScrapeResults, error) { // If error in any of the scrape results, don't delete old items if len(results) > 0 && !v1.ScrapeResults(results).HasErr() { - if err := DeleteStaleConfigItems(*ctx.ScrapeConfig().GetPersistedID()); err != nil { - return nil, fmt.Errorf("error deleting stale config items: %w", err) + persistedID := ctx.ScrapeConfig().GetPersistedID() + if persistedID != nil { + if err := DeleteStaleConfigItems(ctx.DutyContext(), *persistedID); err != nil { + return nil, fmt.Errorf("error deleting stale config items: %w", err) + } } } diff --git a/scrapers/runscrapers_suite_test.go b/scrapers/runscrapers_suite_test.go index c2e51824..d4c724f7 100644 --- a/scrapers/runscrapers_suite_test.go +++ b/scrapers/runscrapers_suite_test.go @@ -2,14 +2,24 @@ package scrapers import ( "os" + "path/filepath" "testing" epg "github.com/fergusstrange/embedded-postgres" "github.com/flanksource/commons/logger" + v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" "github.com/flanksource/duty" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gorm.io/gorm" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + // +kubebuilder:scaffold:imports ) func TestRunScrapers(t *testing.T) { @@ -17,7 +27,10 @@ func TestRunScrapers(t *testing.T) { RunSpecs(t, "Scrapers Suite") } -var postgres *epg.EmbeddedPostgres +var ( + postgres *epg.EmbeddedPostgres + gormDB *gorm.DB +) const ( pgUrl = "postgres://postgres:postgres@localhost:9876/test?sslmode=disable" @@ -25,6 +38,8 @@ const ( ) var _ = BeforeSuite(func() { + var err error + postgres = epg.NewDatabase(epg.DefaultConfig().Database("test").Port(pgPort)) if err := postgres.Start(); err != nil { Fail(err.Error()) @@ -38,14 +53,85 @@ var _ = BeforeSuite(func() { Fail(err.Error()) } + gormDB, err = duty.NewGorm(pgUrl, duty.DefaultGormConfig()) + Expect(err).ToNot(HaveOccurred()) + if err := os.Chdir(".."); err != nil { Fail(err.Error()) } + + setupTestK8s() }) var _ = AfterSuite(func() { + if err := testEnv.Stop(); err != nil { + logger.Errorf("Error stopping test environment: %v", err) + } + logger.Infof("Stopping postgres") if err := postgres.Stop(); err != nil { Fail(err.Error()) } }) + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + kubeConfigPath string +) + +func setupTestK8s() { + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("chart", "crds")}, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(cfg).ToNot(BeNil()) + + err = v1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(k8sClient).ToNot(BeNil()) + + kubeConfigPath, err = createKubeconfigFileForRestConfig(*cfg) + Expect(err).ToNot(HaveOccurred()) +} + +func createKubeconfigFileForRestConfig(restConfig rest.Config) (string, error) { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: restConfig.Host, + CertificateAuthorityData: restConfig.CAData, + } + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + AuthInfo: "default-user", + } + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default-user"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: restConfig.CertData, + ClientKeyData: restConfig.KeyData, + } + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + kubeConfigFile, err := os.CreateTemp("", "kubeconfig-*") + if err != nil { + return "", err + } + _ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name()) + return kubeConfigFile.Name(), nil +} diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index cbec797d..36a74d01 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -1,7 +1,7 @@ package scrapers import ( - "context" + gocontext "context" "encoding/json" "fmt" "os" @@ -12,11 +12,15 @@ import ( "github.com/flanksource/config-db/db" "github.com/flanksource/config-db/db/models" "github.com/flanksource/duty" + "github.com/flanksource/duty/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var _ = Describe("Scrapers test", func() { +var _ = Describe("Scrapers test", Ordered, func() { Describe("DB initialization", func() { It("should be able to run migrations", func() { logger.Infof("Running migrations against %s", pgUrl) @@ -26,14 +30,85 @@ var _ = Describe("Scrapers test", func() { }) It("Gorm can connect", func() { - gorm, err := duty.NewGorm(pgUrl, duty.DefaultGormConfig()) - Expect(err).ToNot(HaveOccurred()) var people int64 - Expect(gorm.Table("people").Count(&people).Error).ToNot(HaveOccurred()) + Expect(gormDB.Table("people").Count(&people).Error).ToNot(HaveOccurred()) Expect(people).To(Equal(int64(1))) }) }) + Describe("Test kubernetes relationship", func() { + var scrapeConfig v1.ScrapeConfig + + It("should prepare scrape config", func() { + scrapeConfig = getConfigSpec("kubernetes") + scrapeConfig.Spec.Kubernetes[0].Kubeconfig = &types.EnvVar{ + ValueStatic: kubeConfigPath, + } + scrapeConfig.Spec.Kubernetes[0].Relationships = append(scrapeConfig.Spec.Kubernetes[0].Relationships, v1.KubernetesRelationship{ + Kind: v1.KubernetesRelationshipLookup{Value: "ConfigMap"}, + Name: v1.KubernetesRelationshipLookup{Label: "flanksource/name"}, + Namespace: v1.KubernetesRelationshipLookup{Label: "flanksource/namespace"}, + }) + }) + + It("should save a configMap", func() { + first := &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first-config", + Namespace: "default", + Labels: map[string]string{ + "flanksource/name": "second-config", + "flanksource/namespace": "default", + }, + }, + Data: map[string]string{"key": "value"}, + } + + err := k8sClient.Create(gocontext.TODO(), first) + Expect(err).NotTo(HaveOccurred(), "failed to create test MyKind resource") + }) + + It("should save second configMap", func() { + first := &apiv1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second-config", + Namespace: "default", + }, + Data: map[string]string{"key": "value"}, + } + + err := k8sClient.Create(gocontext.TODO(), first) + Expect(err).NotTo(HaveOccurred(), "failed to create test MyKind resource") + }) + + It("should successfully complete first scrape run", func() { + scraperCtx := api.NewScrapeContext(gocontext.TODO(), gormDB, nil).WithScrapeConfig(&scrapeConfig) + _, err := RunScraper(scraperCtx) + Expect(err).To(BeNil()) + }) + + It("should have saved the two config items to database", func() { + var configItems []models.ConfigItem + err := gormDB.Where("name IN (?, ?)", "first-config", "second-config").Find(&configItems).Error + Expect(err).To(BeNil()) + + Expect(len(configItems)).To(Equal(2)) + }) + + It("should correctly setup kubernetes relationship", func() { + scraperCtx := api.NewScrapeContext(gocontext.TODO(), gormDB, nil).WithScrapeConfig(&scrapeConfig) + _, err := RunScraper(scraperCtx) + Expect(err).To(BeNil()) + + var configRelationships []models.ConfigRelationship + err = gormDB.Find(&configRelationships).Error + Expect(err).To(BeNil()) + + Expect(len(configRelationships)).To(Equal(1)) + Expect(configRelationships[0].Relation).To(Equal("ConfigMapConfigMap")) + }) + }) + Describe("Testing file fixtures", func() { fixtures := []string{ "file-git", @@ -48,7 +123,7 @@ var _ = Describe("Scrapers test", func() { It(fixture, func() { config := getConfigSpec(fixture) expected := getFixtureResult(fixture) - ctx := api.NewScrapeContext(context.Background(), nil, nil).WithScrapeConfig(&config) + ctx := api.NewScrapeContext(gocontext.Background(), nil, nil).WithScrapeConfig(&config) results, err := Run(ctx) Expect(err).To(BeNil()) @@ -83,7 +158,7 @@ var _ = Describe("Scrapers test", func() { configScraper, err := db.PersistScrapeConfigFromFile(config) Expect(err).To(BeNil()) - ctx := api.NewScrapeContext(context.Background(), nil, nil).WithScrapeConfig(&config) + ctx := api.NewScrapeContext(gocontext.Background(), nil, nil).WithScrapeConfig(&config) results, err := Run(ctx) Expect(err).To(BeNil()) @@ -107,7 +182,7 @@ var _ = Describe("Scrapers test", func() { It("should store the changes from the config", func() { config := getConfigSpec("file-car-change") - ctx := api.NewScrapeContext(context.Background(), nil, nil).WithScrapeConfig(&config) + ctx := api.NewScrapeContext(gocontext.Background(), nil, nil).WithScrapeConfig(&config) results, err := Run(ctx) Expect(err).To(BeNil()) @@ -123,7 +198,7 @@ var _ = Describe("Scrapers test", func() { Expect(configItemID).ToNot(BeNil()) // Expect the config_changes to be stored - items, err := db.FindConfigChangesByItemID(context.Background(), *configItemID) + items, err := db.FindConfigChangesByItemID(gocontext.Background(), *configItemID) Expect(err).To(BeNil()) Expect(len(items)).To(Equal(1)) Expect(items[0].ConfigID).To(Equal(storedConfigItem.ID)) diff --git a/scrapers/stale.go b/scrapers/stale.go index 69968952..cbfd509e 100644 --- a/scrapers/stale.go +++ b/scrapers/stale.go @@ -5,6 +5,7 @@ import ( "github.com/flanksource/commons/logger" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" + "github.com/flanksource/duty/context" "github.com/google/uuid" ) @@ -12,7 +13,7 @@ var ( StaleTimeout string ) -func DeleteStaleConfigItems(scraperID uuid.UUID) error { +func DeleteStaleConfigItems(ctx context.Context, scraperID uuid.UUID) error { // Get stale timeout in relative terms staleDuration, err := duration.ParseDuration(StaleTimeout) if err != nil { @@ -32,7 +33,7 @@ func DeleteStaleConfigItems(scraperID uuid.UUID) error { deleted_at IS NULL AND scraper_id = ?` - result := db.DefaultDB().Exec(deleteQuery, v1.DeletedReasonMissingScrape, staleMinutes, scraperID) + result := ctx.DB().Exec(deleteQuery, v1.DeletedReasonMissingScrape, staleMinutes, scraperID) if err := result.Error; err != nil { return err }