From 6a34e5e089a4b8d017b8114b833b0bf0180f076b Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 31 Jul 2023 15:22:32 +0545 Subject: [PATCH] feat: use of CRD to run scrapers (#288) * feat: use of CRD to run scrapers * chore: rename kind to ScrapeConfig in all fixtures * chore: rename v1.ConfigScraper to ScraperSpec and modified ScraperCRD structure * refactor: modify Run() function to only accept a single scrape config. Eventually the scrape config will come from the context and the context must be tied to a single configuration. * refactor: use scrapeConfig instead of scraperSpec * refactor: the Scrape() method only accepts the context. The second param spec is removed. * chore: rename ctx.Scraper to ctx.ScrapeConfig * refactor: do not store id in the scrape context. It's already there in ScrapeConfig * NewScrapeContext accepts a context instead of scrape config id * chore: removed db ID field from scraperSpec * chore: final cleanup --- api/global.go | 10 +- api/v1/config.go | 29 ++-- api/v1/interface.go | 17 +- api/v1/scrapeconfig_types.go | 59 ++++++- api/v1/types.go | 13 +- api/v1/zz_generated.deepcopy.go | 158 ++++++++---------- ...configs.flanksource.com_scrapeconfigs.yaml | 2 +- cmd/run.go | 49 ++++-- cmd/server.go | 18 +- config/schemas/scrape_config.schema.json | 2 +- controllers/scrapeconfig_controller.go | 7 +- db/config_scraper.go | 19 +-- db/models/config_scraper.go | 19 --- db/update.go | 9 +- fixtures/aws.yaml | 77 +++++---- fixtures/azure-devops.yaml | 19 ++- fixtures/azure.yaml | 15 +- fixtures/expected/file-git.json | 92 +++++----- fixtures/file-car-change.yaml | 15 +- fixtures/file-car.yaml | 15 +- fixtures/file-git.yaml | 17 +- fixtures/file-local-creation-date.yaml | 27 +-- fixtures/file-local.yaml | 15 +- fixtures/file-mask.yaml | 37 ++-- fixtures/file-postgres-properties.yaml | 17 +- fixtures/file-script-gotemplate.yaml | 39 +++-- fixtures/file-script.yaml | 31 ++-- fixtures/file.yaml | 17 +- fixtures/github-actions.yaml | 15 +- fixtures/kubernetes.yaml | 61 +++---- fixtures/kubernetes_file.yaml | 23 ++- fixtures/scrapper.yaml | 17 +- fixtures/sql.yaml | 67 ++++---- fixtures/trivy.yaml | 33 ++-- scrapers/aws/aws.go | 6 +- scrapers/aws/cost.go | 6 +- scrapers/azure/azure.go | 6 +- scrapers/azure/devops/pipelines.go | 6 +- scrapers/cron.go | 11 +- scrapers/file/file.go | 6 +- scrapers/github/workflows.go | 8 +- scrapers/kubernetes/kubernetes.go | 6 +- scrapers/kubernetes/kubernetes_file.go | 8 +- scrapers/run.go | 24 +-- scrapers/run_now.go | 7 +- scrapers/runscrapers.go | 68 ++++---- scrapers/runscrapers_test.go | 22 +-- scrapers/sql/sql.go | 6 +- scrapers/trivy/trivy.go | 6 +- 49 files changed, 677 insertions(+), 579 deletions(-) delete mode 100644 db/models/config_scraper.go diff --git a/api/global.go b/api/global.go index f12b10b2..1f21c7bd 100644 --- a/api/global.go +++ b/api/global.go @@ -1,11 +1,10 @@ package api import ( - goctx "context" + "context" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" - "github.com/google/uuid" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -14,11 +13,10 @@ var KubernetesClient *kubernetes.Clientset var KubernetesRestConfig *rest.Config var Namespace string -func NewScrapeContext(scraper *v1.ConfigScraper, id *uuid.UUID) *v1.ScrapeContext { +func NewScrapeContext(ctx context.Context, scraper v1.ScrapeConfig) *v1.ScrapeContext { return &v1.ScrapeContext{ - Context: goctx.Background(), - Scraper: scraper, - ScraperID: id, + Context: ctx, + ScrapeConfig: scraper, Namespace: Namespace, Kubernetes: KubernetesClient, KubernetesRestConfig: KubernetesRestConfig, diff --git a/api/v1/config.go b/api/v1/config.go index 71823d07..301f88f2 100644 --- a/api/v1/config.go +++ b/api/v1/config.go @@ -1,16 +1,17 @@ package v1 import ( + "fmt" "io" "os" "regexp" "strings" - "github.com/pkg/errors" - yamlutil "k8s.io/apimachinery/pkg/util/yaml" ) +var yamlDividerRegexp = regexp.MustCompile(`(?m)^---\n`) + func readFile(path string) (string, error) { var data []byte var err error @@ -26,38 +27,38 @@ func readFile(path string) (string, error) { return string(data), nil } -func ParseConfigs(files ...string) ([]ConfigScraper, error) { - scrapers := []ConfigScraper{} +func ParseConfigs(files ...string) ([]ScrapeConfig, error) { + scrapers := make([]ScrapeConfig, 0, len(files)) for _, f := range files { _scrapers, err := parseConfig(f) if err != nil { - return nil, errors.Wrapf(err, "error parsing %s", f) + return nil, err } + scrapers = append(scrapers, _scrapers...) } + return scrapers, nil } // ParseConfig : Read config file -func parseConfig(configfile string) ([]ConfigScraper, error) { +func parseConfig(configfile string) ([]ScrapeConfig, error) { configs, err := readFile(configfile) if err != nil { - return nil, err + return nil, fmt.Errorf("error reading config file=%s: %w", configfile, err) } - var scrapers []ConfigScraper - - re := regexp.MustCompile(`(?m)^---\n`) - for _, chunk := range re.Split(configs, -1) { + var scrapers []ScrapeConfig + for _, chunk := range yamlDividerRegexp.Split(configs, -1) { if strings.TrimSpace(chunk) == "" { continue } - config := ConfigScraper{} - decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(chunk), 1024) + var config ScrapeConfig + decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(chunk), 1024) if err := decoder.Decode(&config); err != nil { - return nil, err + return nil, fmt.Errorf("error decoding yaml. file=%s: %w", configfile, err) } scrapers = append(scrapers, config) diff --git a/api/v1/interface.go b/api/v1/interface.go index 3f09a7b4..ac8e844d 100644 --- a/api/v1/interface.go +++ b/api/v1/interface.go @@ -13,7 +13,6 @@ import ( "github.com/flanksource/commons/logger" "github.com/flanksource/duty" "github.com/flanksource/duty/models" - "github.com/google/uuid" "gorm.io/gorm" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -22,8 +21,8 @@ import ( // Scraper ... // +kubebuilder:object:generate=false type Scraper interface { - Scrape(ctx *ScrapeContext, config ConfigScraper) ScrapeResults - CanScrape(config ConfigScraper) bool + Scrape(ctx *ScrapeContext) ScrapeResults + CanScrape(config ScraperSpec) bool } // Analyzer ... @@ -351,8 +350,7 @@ type ScrapeContext struct { Namespace string Kubernetes *kubernetes.Clientset KubernetesRestConfig *rest.Config - Scraper *ConfigScraper - ScraperID *uuid.UUID + ScrapeConfig ScrapeConfig } func (ctx ScrapeContext) Find(path string) ([]string, error) { @@ -366,13 +364,6 @@ func (ctx ScrapeContext) Read(path string) ([]byte, string, error) { return content, filename, err } -// WithScraper ... -func (ctx ScrapeContext) WithScraper(config *ConfigScraper) ScrapeContext { - ctx.Scraper = config - return ctx - -} - // GetNamespace ... func (ctx ScrapeContext) GetNamespace() string { return ctx.Namespace @@ -380,7 +371,7 @@ func (ctx ScrapeContext) GetNamespace() string { // IsTrace ... func (ctx ScrapeContext) IsTrace() bool { - return ctx.Scraper != nil && ctx.Scraper.IsTrace() + return ctx.ScrapeConfig.Spec.IsTrace() } // HydrateConnectionByURL ... diff --git a/api/v1/scrapeconfig_types.go b/api/v1/scrapeconfig_types.go index 531f8642..6729f673 100644 --- a/api/v1/scrapeconfig_types.go +++ b/api/v1/scrapeconfig_types.go @@ -17,14 +17,14 @@ limitations under the License. package v1 import ( + "encoding/json" + + "github.com/flanksource/duty/models" + "github.com/google/uuid" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" ) -// ScrapeConfigSpec defines the desired state of ScrapeConfig -type ScrapeConfigSpec struct { - ConfigScraper `json:",inline"` -} - // ScrapeConfigStatus defines the observed state of ScrapeConfig type ScrapeConfigStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty" protobuf:"varint,3,opt,name=observedGeneration"` @@ -38,10 +38,57 @@ type ScrapeConfig struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec ScrapeConfigSpec `json:"spec,omitempty"` + Spec ScraperSpec `json:"spec,omitempty"` Status ScrapeConfigStatus `json:"status,omitempty"` } +func (t *ScrapeConfig) ToModel() (models.ConfigScraper, error) { + spec, err := json.Marshal(t.Spec) + if err != nil { + return models.ConfigScraper{}, err + } + + return models.ConfigScraper{ + Name: t.Name, + Spec: string(spec), + Source: t.Annotations["source"], + }, nil +} + +func ScrapeConfigFromModel(m models.ConfigScraper) (ScrapeConfig, error) { + var spec ScraperSpec + if err := json.Unmarshal([]byte(m.Spec), &spec); err != nil { + return ScrapeConfig{}, err + } + + sc := ScrapeConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: m.Name, + Annotations: map[string]string{ + "source": m.Source, + }, + UID: k8stypes.UID(m.ID.String()), + CreationTimestamp: metav1.Time{Time: m.CreatedAt}, + }, + Spec: spec, + } + + if m.DeletedAt != nil { + sc.ObjectMeta.DeletionTimestamp = &metav1.Time{Time: *m.DeletedAt} + } + + return sc, nil +} + +func (t *ScrapeConfig) GetPersistedID() *uuid.UUID { + if t.GetUID() == "" { + return nil + } + + u, _ := uuid.Parse(string(t.GetUID())) + return &u +} + //+kubebuilder:object:root=true // ScrapeConfigList contains a list of ScrapeConfig diff --git a/api/v1/types.go b/api/v1/types.go index 2b047114..d5503c62 100644 --- a/api/v1/types.go +++ b/api/v1/types.go @@ -22,9 +22,8 @@ var AllScraperConfigs = map[string]any{ "trivy": Trivy{}, } -// ConfigScraper ... -type ConfigScraper struct { - ID string `json:"-"` +// ScraperSpec defines the desired state of Config scraper +type ScraperSpec struct { LogLevel string `json:"logLevel,omitempty"` Schedule string `json:"schedule,omitempty"` AWS []AWS `json:"aws,omitempty" yaml:"aws,omitempty"` @@ -41,20 +40,20 @@ type ConfigScraper struct { Full bool `json:"full,omitempty"` } -func (c ConfigScraper) GenerateName() (string, error) { +func (c ScraperSpec) GenerateName() (string, error) { return utils.Hash(c) } // IsEmpty ... -func (c ConfigScraper) IsEmpty() bool { +func (c ScraperSpec) IsEmpty() bool { return len(c.AWS) == 0 && len(c.File) == 0 } -func (c ConfigScraper) IsTrace() bool { +func (c ScraperSpec) IsTrace() bool { return c.LogLevel == "trace" } -func (c ConfigScraper) IsDebug() bool { +func (c ScraperSpec) IsDebug() bool { return c.LogLevel == "debug" } diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 6f29d002..c025d541 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -196,84 +196,6 @@ func (in *CloudTrail) DeepCopy() *CloudTrail { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConfigScraper) DeepCopyInto(out *ConfigScraper) { - *out = *in - if in.AWS != nil { - in, out := &in.AWS, &out.AWS - *out = make([]AWS, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.File != nil { - in, out := &in.File, &out.File - *out = make([]File, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Kubernetes != nil { - in, out := &in.Kubernetes, &out.Kubernetes - *out = make([]Kubernetes, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.KubernetesFile != nil { - in, out := &in.KubernetesFile, &out.KubernetesFile - *out = make([]KubernetesFile, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.AzureDevops != nil { - in, out := &in.AzureDevops, &out.AzureDevops - *out = make([]AzureDevops, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.GithubActions != nil { - in, out := &in.GithubActions, &out.GithubActions - *out = make([]GitHubActions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Azure != nil { - in, out := &in.Azure, &out.Azure - *out = make([]Azure, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.SQL != nil { - in, out := &in.SQL, &out.SQL - *out = make([]SQL, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Trivy != nil { - in, out := &in.Trivy, &out.Trivy - *out = make([]Trivy, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigScraper. -func (in *ConfigScraper) DeepCopy() *ConfigScraper { - if in == nil { - return nil - } - out := new(ConfigScraper) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Connection) DeepCopyInto(out *Connection) { *out = *in @@ -847,32 +769,94 @@ func (in *ScrapeConfigList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ScrapeConfigSpec) DeepCopyInto(out *ScrapeConfigSpec) { +func (in *ScrapeConfigStatus) DeepCopyInto(out *ScrapeConfigStatus) { *out = *in - in.ConfigScraper.DeepCopyInto(&out.ConfigScraper) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScrapeConfigSpec. -func (in *ScrapeConfigSpec) DeepCopy() *ScrapeConfigSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScrapeConfigStatus. +func (in *ScrapeConfigStatus) DeepCopy() *ScrapeConfigStatus { if in == nil { return nil } - out := new(ScrapeConfigSpec) + out := new(ScrapeConfigStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ScrapeConfigStatus) DeepCopyInto(out *ScrapeConfigStatus) { +func (in *ScraperSpec) DeepCopyInto(out *ScraperSpec) { *out = *in + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = make([]AWS, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.File != nil { + in, out := &in.File, &out.File + *out = make([]File, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Kubernetes != nil { + in, out := &in.Kubernetes, &out.Kubernetes + *out = make([]Kubernetes, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.KubernetesFile != nil { + in, out := &in.KubernetesFile, &out.KubernetesFile + *out = make([]KubernetesFile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AzureDevops != nil { + in, out := &in.AzureDevops, &out.AzureDevops + *out = make([]AzureDevops, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.GithubActions != nil { + in, out := &in.GithubActions, &out.GithubActions + *out = make([]GitHubActions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Azure != nil { + in, out := &in.Azure, &out.Azure + *out = make([]Azure, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SQL != nil { + in, out := &in.SQL, &out.SQL + *out = make([]SQL, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Trivy != nil { + in, out := &in.Trivy, &out.Trivy + *out = make([]Trivy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScrapeConfigStatus. -func (in *ScrapeConfigStatus) DeepCopy() *ScrapeConfigStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScraperSpec. +func (in *ScraperSpec) DeepCopy() *ScraperSpec { if in == nil { return nil } - out := new(ScrapeConfigStatus) + out := new(ScraperSpec) in.DeepCopyInto(out) return out } diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index 4eafdf98..d54fb5e6 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -33,7 +33,7 @@ spec: metadata: type: object spec: - description: ScrapeConfigSpec defines the desired state of ScrapeConfig + description: ScraperSpec defines the desired state of Config scraper properties: aws: items: diff --git a/cmd/run.go b/cmd/run.go index 8b627230..40d7f0eb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,7 +1,9 @@ package cmd import ( + "context" "encoding/json" + "fmt" "os" "path" @@ -31,30 +33,41 @@ var Run = &cobra.Command{ db.MustInit() } - ctx := api.NewScrapeContext(nil, nil) - results, err := scrapers.Run(ctx, scraperConfigs...) - if err != nil { - logger.Fatalf(err.Error()) + if db.ConnectionString == "" && outputDir == "" { + logger.Fatalf("skipping export: neither --output-dir nor --db is specified") } - if db.ConnectionString != "" { - logger.Infof("Exporting %d resources to DB", len(results)) - if err = db.SaveResults(ctx, results); err != nil { - logger.Errorf("Failed to update db: %+v", err) + for _, scraperConfig := range scraperConfigs { + ctx := api.NewScrapeContext(context.Background(), scraperConfig) + if err := scrapeAndStore(ctx); err != nil { + logger.Errorf("error scraping config: (name=%s) %v", scraperConfig.Name, err) } - } else if outputDir != "" { - logger.Infof("Exporting %d resources to %s", len(results), outputDir) + } + }, +} - for _, result := range results { - if err := exportResource(result, filename, outputDir); err != nil { - logger.Fatalf("failed to export results %v", err) - } - } +func scrapeAndStore(ctx *v1.ScrapeContext) error { + results, err := scrapers.Run(ctx) + if err != nil { + return err + } + + if db.ConnectionString != "" { + logger.Infof("Exporting %d resources to DB", len(results)) + return db.SaveResults(ctx, results) + } + + if outputDir != "" { + logger.Infof("Exporting %d resources to %s", len(results), outputDir) - } else { - logger.Fatalf("skipping export: neither --output-dir or --db is specified") + for _, result := range results { + if err := exportResource(result, filename, outputDir); err != nil { + return fmt.Errorf("failed to export results %v", err) + } } - }, + } + + return nil } func exportResource(resource v1.ScrapeResult, filename, outputDir string) error { diff --git a/cmd/server.go b/cmd/server.go index 8d740760..699610eb 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -6,9 +6,9 @@ import ( "net/url" "github.com/flanksource/commons/logger" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" - "github.com/flanksource/config-db/db/models" "github.com/flanksource/config-db/jobs" "github.com/flanksource/config-db/query" @@ -64,7 +64,6 @@ func serve(configFiles []string) { } } - // Run this in a goroutine to make it non-blocking for server start go startScraperCron(configFiles) go jobs.ScheduleJobs() @@ -81,8 +80,8 @@ func startScraperCron(configFiles []string) { } logger.Infof("Persisting %d config files", len(scraperConfigsFiles)) - for _, scraper := range scraperConfigsFiles { - _, err := db.PersistScrapeConfigFromFile(scraper) + for _, scrapeConfig := range scraperConfigsFiles { + _, err := db.PersistScrapeConfigFromFile(scrapeConfig) if err != nil { logger.Fatalf("Error persisting scrape config to db: %v", err) } @@ -95,18 +94,19 @@ func startScraperCron(configFiles []string) { logger.Infof("Starting %d scrapers", len(scraperConfigsDB)) for _, scraper := range scraperConfigsDB { - _scraper, err := models.V1ConfigScraper(scraper) + _scraper, err := v1.ScrapeConfigFromModel(scraper) if err != nil { logger.Fatalf("Error parsing config scraper: %v", err) } - scrapers.AddToCron(_scraper, scraper.ID.String()) + scrapers.AddToCron(_scraper) fn := func() { - if _, err := scrapers.RunScraper(_scraper); err != nil { - logger.Errorf("Error running scraper(id=%s): %v", _scraper.ID, err) + ctx := api.NewScrapeContext(context.Background(), _scraper) + if _, err := scrapers.RunScraper(ctx); err != nil { + logger.Errorf("Error running scraper(id=%s): %v", scraper.ID, err) } } - go scrapers.AtomicRunner(_scraper.ID, fn)() + go scrapers.AtomicRunner(scraper.ID.String(), fn)() } } diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index 80d61783..ffa9be8f 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigScraper":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"items":{"type":"string"},"type":"array"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEvent"}},"additionalProperties":false,"type":"object"},"KubernetesEvent":{"properties":{"exclusions":{"items":{"type":"string"},"type":"array"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/MaskSelector"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"MaskSelector":{"properties":{"type":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigSpec":{"required":["ConfigScraper"],"properties":{"ConfigScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigScraper"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"items":{"type":"string"},"type":"array"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEvent"}},"additionalProperties":false,"type":"object"},"KubernetesEvent":{"properties":{"exclusions":{"items":{"type":"string"},"type":"array"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/MaskSelector"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"MaskSelector":{"properties":{"type":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/controllers/scrapeconfig_controller.go b/controllers/scrapeconfig_controller.go index e3474233..ebfb15fa 100644 --- a/controllers/scrapeconfig_controller.go +++ b/controllers/scrapeconfig_controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/flanksource/config-db/api" configsv1 "github.com/flanksource/config-db/api/v1" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" @@ -97,12 +98,12 @@ func (r *ScrapeConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request // Sync jobs if new scrape config is created if changed || scrapeConfig.Generation == 1 { - scrapeConfig.Spec.ConfigScraper.ID = string(scrapeConfig.GetUID()) - if _, err := scrapers.RunScraper(scrapeConfig.Spec.ConfigScraper); err != nil { + ctx := api.NewScrapeContext(ctx, *scrapeConfig) + if _, err := scrapers.RunScraper(ctx); err != nil { logger.Error(err, "failed to run scraper") return ctrl.Result{Requeue: true, RequeueAfter: 2 * time.Minute}, err } - scrapers.AddToCron(scrapeConfig.Spec.ConfigScraper, string(scrapeConfig.GetUID())) + scrapers.AddToCron(*scrapeConfig) } return ctrl.Result{}, nil diff --git a/db/config_scraper.go b/db/config_scraper.go index 11d9e6fc..0b18bbd5 100644 --- a/db/config_scraper.go +++ b/db/config_scraper.go @@ -71,7 +71,7 @@ func PersistScrapeConfigFromCRD(scrapeConfig *v1.ScrapeConfig) (bool, error) { Name: fmt.Sprintf("%s/%s", scrapeConfig.Namespace, scrapeConfig.Name), Source: models.SourceCRD, } - configScraper.Spec, _ = utils.StructToJSON(scrapeConfig.Spec.ConfigScraper) + configScraper.Spec, _ = utils.StructToJSON(scrapeConfig.Spec) tx := db.Table("config_scrapers").Save(&configScraper) return tx.RowsAffected > 0, tx.Error @@ -83,15 +83,13 @@ func GetScrapeConfigsOfAgent(agentID uuid.UUID) ([]models.ConfigScraper, error) return configScrapers, err } -func PersistScrapeConfigFromFile(configScraperSpec v1.ConfigScraper) (models.ConfigScraper, error) { - var configScraper models.ConfigScraper - spec, err := utils.StructToJSON(configScraperSpec) +func PersistScrapeConfigFromFile(scrapeConfig v1.ScrapeConfig) (models.ConfigScraper, error) { + configScraper, err := scrapeConfig.ToModel() if err != nil { - return configScraper, fmt.Errorf("error converting scraper spec to JSON: %w", err) + return configScraper, err } - // Check if exists - tx := db.Table("config_scrapers").Where("spec = ?", spec).Find(&configScraper) + tx := db.Table("config_scrapers").Where("spec = ?", configScraper.Spec).Find(&configScraper) if tx.Error != nil { return configScraper, tx.Error } @@ -99,13 +97,10 @@ func PersistScrapeConfigFromFile(configScraperSpec v1.ConfigScraper) (models.Con return configScraper, nil } - // Create if not exists - configScraper.Spec = spec - configScraper.Name, err = configScraperSpec.GenerateName() + configScraper.Name, err = scrapeConfig.Spec.GenerateName() configScraper.Source = models.SourceConfigFile if err != nil { return configScraper, err } - tx = db.Table("config_scrapers").Create(&configScraper) - return configScraper, tx.Error + return configScraper, db.Create(&configScraper).Error } diff --git a/db/models/config_scraper.go b/db/models/config_scraper.go deleted file mode 100644 index 98bfe4fd..00000000 --- a/db/models/config_scraper.go +++ /dev/null @@ -1,19 +0,0 @@ -package models - -import ( - "encoding/json" - - v1 "github.com/flanksource/config-db/api/v1" - "github.com/flanksource/duty/models" -) - -// V1ConfigScraper generates a v1.ConfigScraper from a models.ConfigScraper -func V1ConfigScraper(cs models.ConfigScraper) (v1.ConfigScraper, error) { - var spec v1.ConfigScraper - if err := json.Unmarshal([]byte(cs.Spec), &spec); err != nil { - return spec, err - } - - spec.ID = cs.ID.String() - return spec, nil -} diff --git a/db/update.go b/db/update.go index 07abae1e..ff3f6fca 100644 --- a/db/update.go +++ b/db/update.go @@ -187,7 +187,7 @@ func upsertAnalysis(ctx *v1.ScrapeContext, result *v1.ScrapeResult) error { logger.Tracef("[%s/%s] ==> %s", analysis.ConfigType, analysis.ExternalID, analysis) analysis.ConfigID = uuid.MustParse(ci.ID) analysis.ID = uuid.MustParse(ulid.MustNew().AsUUID()) - analysis.ScraperID = ctx.ScraperID + analysis.ScraperID = ctx.ScrapeConfig.GetPersistedID() if analysis.Status == "" { analysis.Status = dutyModels.AnalysisStatusOpen } @@ -225,7 +225,8 @@ func SaveResults(ctx *v1.ScrapeContext, results []v1.ScrapeResult) error { return errors.Wrapf(err, "unable to create config item: %s", result) } - ci.ScraperID = ctx.ScraperID + ci.ScraperID = ctx.ScrapeConfig.GetPersistedID() + if err := updateCI(ctx, *ci); err != nil { return err } @@ -248,9 +249,9 @@ func SaveResults(ctx *v1.ScrapeContext, results []v1.ScrapeResult) error { } } - if !startTime.IsZero() && ctx.ScraperID != nil { + if !startTime.IsZero() && ctx.ScrapeConfig.GetPersistedID() != nil { // Any analysis that weren't observed again will be marked as resolved - if err := UpdateAnalysisStatusBefore(ctx, startTime, ctx.ScraperID.String(), dutyModels.AnalysisStatusResolved); err != nil { + if err := UpdateAnalysisStatusBefore(ctx, startTime, string(ctx.ScrapeConfig.GetUID()), dutyModels.AnalysisStatusResolved); err != nil { logger.Errorf("failed to mark analysis before %v as healthy: %v", startTime, err) } } diff --git a/fixtures/aws.yaml b/fixtures/aws.yaml index 1dfce9d6..56ade7bd 100644 --- a/fixtures/aws.yaml +++ b/fixtures/aws.yaml @@ -1,37 +1,42 @@ -aws: - - region: - - eu-west-2 - - us-east-1 - - af-south-1 - - ap-south-1 - - eu-central-1 - compliance: true - patch_states: false - trusted_advisor_check: false - patch_details: false - cost_reporting: - s3_bucket_path: s3://flanksource-cost-reports/query-results - database: athenacurcfn_flanksource_report - table: flanksource_report - region: af-south-1 - inventory: true - exclude: - - Amazon EC2 Reserved Instances Optimization - - Savings Plan - # - trusted_advisor - # - cloudtrail - # include: - # - vpc - # # - subnet - # - vpc - # - SecurityGroup - transform: +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: aws-scraper +spec: + aws: + - region: + - eu-west-2 + - us-east-1 + - af-south-1 + - ap-south-1 + - eu-central-1 + compliance: true + patch_states: false + trusted_advisor_check: false + patch_details: false + cost_reporting: + s3_bucket_path: s3://flanksource-cost-reports/query-results + database: athenacurcfn_flanksource_report + table: flanksource_report + region: af-south-1 + inventory: true exclude: - - jsonpath: $.tags - - jsonpath: $.privateDnsNameOptionsOnLaunch - # - jsonpath: availableIpAddressCount - - jsonpath: outpostArn - - jsonpath: mapCustomerOwnedIpOnLaunch - - jsonpath: subnetArn - # - jsonpath: usageOperationUpdateTime - # - jsonpath: $..privateIPAddresses + - Amazon EC2 Reserved Instances Optimization + - Savings Plan + # - trusted_advisor + # - cloudtrail + # include: + # - vpc + # # - subnet + # - vpc + # - SecurityGroup + transform: + exclude: + - jsonpath: $.tags + - jsonpath: $.privateDnsNameOptionsOnLaunch + # - jsonpath: availableIpAddressCount + - jsonpath: outpostArn + - jsonpath: mapCustomerOwnedIpOnLaunch + - jsonpath: subnetArn + # - jsonpath: usageOperationUpdateTime + # - jsonpath: $..privateIPAddresses diff --git a/fixtures/azure-devops.yaml b/fixtures/azure-devops.yaml index 1bea2d06..e81a6afa 100644 --- a/fixtures/azure-devops.yaml +++ b/fixtures/azure-devops.yaml @@ -1,7 +1,12 @@ -azureDevops: - - connection: connection://Azure Devops/Flanksource - projects: - - Demo1 - pipelines: - - "adhoc-release" - - "git automation" \ No newline at end of file +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: azure-devops-scraper +spec: + azureDevops: + - connection: connection://Azure Devops/Flanksource + projects: + - Demo1 + pipelines: + - "adhoc-release" + - "git automation" \ No newline at end of file diff --git a/fixtures/azure.yaml b/fixtures/azure.yaml index a1da8b49..e4fadcc8 100644 --- a/fixtures/azure.yaml +++ b/fixtures/azure.yaml @@ -1,5 +1,10 @@ -azure: - - organisation: flanksource - subscriptionId: e3911016-5810-415f-b075-682db169988f - region: - - eu-west-2 \ No newline at end of file +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: azure-scraper +spec: + azure: + - organisation: flanksource + subscriptionId: e3911016-5810-415f-b075-682db169988f + region: + - eu-west-2 \ No newline at end of file diff --git a/fixtures/expected/file-git.json b/fixtures/expected/file-git.json index 2395f6b1..ef24ac9a 100644 --- a/fixtures/expected/file-git.json +++ b/fixtures/expected/file-git.json @@ -1,50 +1,48 @@ -[{ - "last_modified": "0001-01-01T00:00:00Z", - "source": "github.com/flanksource/canary-checker/fixtures/minimal/http_pass_single.yaml", - "id": "http-pass-single", - "config_class": "Canary", - "config":{ - "apiVersion": "canaries.flanksource.com/v1", - "kind": "Canary", - "metadata": { - "labels": { - "canary": "http" - }, - "name": "http-pass-single" +[ + { + "last_modified": "0001-01-01T00:00:00Z", + "source": "github.com/flanksource/canary-checker/fixtures/minimal/http_pass_single.yaml", + "id": "http-pass-single", + "config_class": "Canary", + "config": { + "apiVersion": "canaries.flanksource.com/v1", + "kind": "Canary", + "metadata": { + "labels": { + "canary": "http" }, - "spec": { - "http": [ - { - "endpoint": "https://httpbin.demo.aws.flanksource.com/status/200", - "name": "http-deprecated-endpoint" - }, - { - "name": "http-minimal-check", - "url": "https://httpbin.demo.aws.flanksource.com/status/200" + "name": "http-pass-single" + }, + "spec": { + "http": [ + { + "endpoint": "https://httpbin.demo.aws.flanksource.com/status/200", + "name": "http-deprecated-endpoint" + }, + { + "name": "http-minimal-check", + "url": "https://httpbin.demo.aws.flanksource.com/status/200" + }, + { + "maxSSLExpiry": 7, + "name": "http-param-tests", + "responseCodes": [201, 200, 301], + "responseContent": "", + "url": "https://httpbin.demo.aws.flanksource.com/status/200" + }, + { + "display": { + "template": "code={{.code}}, age={{.sslAge}}" }, - { - "maxSSLExpiry": 7, - "name": "http-param-tests", - "responseCodes": [ - 201, - 200, - 301 - ], - "responseContent": "", - "url": "https://httpbin.demo.aws.flanksource.com/status/200" + "name": "http-expr-tests", + "test": { + "expr": "code in [200,201,301] && sslAge > Duration('7d')" }, - { - "display": { - "template": "code={{.code}}, age={{.sslAge}}" - }, - "name": "http-expr-tests", - "test": { - "expr": "code in [200,201,301] and sslAge \u003e Duration('7d')" - }, - "url": "https://httpbin.demo.aws.flanksource.com/status/200" - } - ], - "interval": 30 - } - } -}] + "url": "https://httpbin.demo.aws.flanksource.com/status/200" + } + ], + "interval": 30 + } + } + } +] diff --git a/fixtures/file-car-change.yaml b/fixtures/file-car-change.yaml index c4abfc29..1b0903e1 100644 --- a/fixtures/file-car-change.yaml +++ b/fixtures/file-car-change.yaml @@ -1,6 +1,11 @@ -full: true -file: - - type: Car - id: $.reg_no - paths: +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-car-change-scraper +spec: + full: true + file: + - type: Car + id: $.reg_no + paths: - fixtures/data/car_changes.json \ No newline at end of file diff --git a/fixtures/file-car.yaml b/fixtures/file-car.yaml index d6ca5b3b..b09d6e44 100644 --- a/fixtures/file-car.yaml +++ b/fixtures/file-car.yaml @@ -1,5 +1,10 @@ -file: - - type: Car - id: $.reg_no - paths: - - fixtures/data/car.json +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-car-scraper +spec: + file: + - type: Car + id: $.reg_no + paths: + - fixtures/data/car.json diff --git a/fixtures/file-git.yaml b/fixtures/file-git.yaml index eba1a7dd..3123b378 100644 --- a/fixtures/file-git.yaml +++ b/fixtures/file-git.yaml @@ -1,6 +1,11 @@ -file: - - type: $.kind - id: $.metadata.name - url: github.com/flanksource/canary-checker - paths: - - fixtures/minimal/http_pass_single.yaml +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-git-scraper +spec: + file: + - type: $.kind + id: $.metadata.name + url: github.com/flanksource/canary-checker + paths: + - fixtures/minimal/http_pass_single.yaml diff --git a/fixtures/file-local-creation-date.yaml b/fixtures/file-local-creation-date.yaml index e53b042d..c12be9e2 100644 --- a/fixtures/file-local-creation-date.yaml +++ b/fixtures/file-local-creation-date.yaml @@ -1,11 +1,16 @@ -file: - - type: $.aws[0].region - id: $.aws[0].region - createFields: - - $.aws[0].made_at - - $.aws[0].created_at - deleteFields: - - "$.aws[0].removed_at" - - "$.aws[0].deleted_at" - paths: - - fixtures/data/test.yaml +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-local-creation-date.yaml +spec: + file: + - type: $.aws[0].region + id: $.aws[0].region + createFields: + - $.aws[0].made_at + - $.aws[0].created_at + deleteFields: + - "$.aws[0].removed_at" + - "$.aws[0].deleted_at" + paths: + - fixtures/data/test.yaml diff --git a/fixtures/file-local.yaml b/fixtures/file-local.yaml index f4a1e647..76da59b0 100644 --- a/fixtures/file-local.yaml +++ b/fixtures/file-local.yaml @@ -1,5 +1,10 @@ -file: - - type: $.aws[0].region - id: $.aws[0].region - paths: - - fixtures/data/test.yaml +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-local-scraper +spec: + file: + - type: $.aws[0].region + id: $.aws[0].region + paths: + - fixtures/data/test.yaml diff --git a/fixtures/file-mask.yaml b/fixtures/file-mask.yaml index e35554f5..f75c0b6a 100644 --- a/fixtures/file-mask.yaml +++ b/fixtures/file-mask.yaml @@ -1,16 +1,21 @@ -file: - - type: Config - id: $.id - name: $.name - transform: - mask: - - selector: - type: Config - jsonpath: $.password - value: md5sum - - selector: - type: Config - jsonpath: $.secret - value: '***' - paths: - - fixtures/data/single-config.json +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-mask-scraper +spec: + file: + - type: Config + id: $.id + name: $.name + transform: + mask: + - selector: + type: Config + jsonpath: $.password + value: md5sum + - selector: + type: Config + jsonpath: $.secret + value: '***' + paths: + - fixtures/data/single-config.json diff --git a/fixtures/file-postgres-properties.yaml b/fixtures/file-postgres-properties.yaml index b0cc9cdc..67dc343d 100644 --- a/fixtures/file-postgres-properties.yaml +++ b/fixtures/file-postgres-properties.yaml @@ -1,6 +1,11 @@ -file: - - format: properties - type: PostgreSQLProperties - id: postgresql-properties - paths: - - ./fixtures/file-postgres-properties.conf +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-postgres-properties-scraper +spec: + file: + - format: properties + type: PostgreSQLProperties + id: postgresql-properties + paths: + - ./fixtures/file-postgres-properties.conf diff --git a/fixtures/file-script-gotemplate.yaml b/fixtures/file-script-gotemplate.yaml index 1c2bce93..fa438597 100644 --- a/fixtures/file-script-gotemplate.yaml +++ b/fixtures/file-script-gotemplate.yaml @@ -1,17 +1,22 @@ -file: - - type: MyConfig - id: "$.id" - name: "scraped" - transform: - script: - gotemplate: |+ - [ - { - {{range $i := .config}} - "name-{{.id}}": "hi {{.name}}", - {{end}} - "id": "1" - } - ] - paths: - - fixtures/data/multiple-configs.json +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-script-gotemplate-scraper +spec: + file: + - type: MyConfig + id: "$.id" + name: "scraped" + transform: + script: + gotemplate: |+ + [ + { + {{range $i := .config}} + "name-{{.id}}": "hi {{.name}}", + {{end}} + "id": "1" + } + ] + paths: + - fixtures/data/multiple-configs.json diff --git a/fixtures/file-script.yaml b/fixtures/file-script.yaml index fa2b12c7..e390c638 100644 --- a/fixtures/file-script.yaml +++ b/fixtures/file-script.yaml @@ -1,13 +1,18 @@ -file: - - type: Config - id: $.id - name: $.name - transform: - script: - javascript: |+ - for (var i = 0; i < config.length; i++) { - config[i].added ="a" - } - JSON.stringify(config) - paths: - - fixtures/data/multiple-configs.json \ No newline at end of file +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-script-scraper +spec: + file: + - type: Config + id: $.id + name: $.name + transform: + script: + javascript: |+ + for (var i = 0; i < config.length; i++) { + config[i].added ="a" + } + JSON.stringify(config) + paths: + - fixtures/data/multiple-configs.json \ No newline at end of file diff --git a/fixtures/file.yaml b/fixtures/file.yaml index 586ed595..5de77d2c 100644 --- a/fixtures/file.yaml +++ b/fixtures/file.yaml @@ -1,6 +1,11 @@ -file: - - type: $.Config.InstanceType - id: $.Config.InstanceId - path: - - config*.json - - test*.json +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-scraper +spec: + file: + - type: $.Config.InstanceType + id: $.Config.InstanceId + path: + - config*.json + - test*.json diff --git a/fixtures/github-actions.yaml b/fixtures/github-actions.yaml index 694eebbd..f3257723 100644 --- a/fixtures/github-actions.yaml +++ b/fixtures/github-actions.yaml @@ -1,5 +1,10 @@ -githubActions: - - owner: flanksource - repository: config-db - personalAccessToken: - value: +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: github-actions-scraper +spec: + githubActions: + - owner: flanksource + repository: config-db + personalAccessToken: + value: diff --git a/fixtures/kubernetes.yaml b/fixtures/kubernetes.yaml index a58d5c4a..4c3efc9d 100644 --- a/fixtures/kubernetes.yaml +++ b/fixtures/kubernetes.yaml @@ -1,29 +1,34 @@ -kubernetes: - - clusterName: local-kind-cluster - exclusions: - - Secret - - ReplicaSet - - APIService - - PodMetrics - - NodeMetrics - - endpoints.discovery.k8s.io - - endpointslices.discovery.k8s.io - - leases.coordination.k8s.io - - podmetrics.metrics.k8s.io - - nodemetrics.metrics.k8s.io - - customresourcedefinition - - controllerrevision - - certificaterequest - - orders.acme.cert-manager.io - event: +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: kubernetes-scraper +spec: + kubernetes: + - clusterName: local-kind-cluster exclusions: - - SuccessfulCreate - - Created - - DNSConfigForming - severityKeywords: - error: - - failed - - error - warn: - - backoff - - nodeoutofmemory + - Secret + - ReplicaSet + - APIService + - PodMetrics + - NodeMetrics + - endpoints.discovery.k8s.io + - endpointslices.discovery.k8s.io + - leases.coordination.k8s.io + - podmetrics.metrics.k8s.io + - nodemetrics.metrics.k8s.io + - customresourcedefinition + - controllerrevision + - certificaterequest + - orders.acme.cert-manager.io + event: + exclusions: + - SuccessfulCreate + - Created + - DNSConfigForming + severityKeywords: + error: + - failed + - error + warn: + - backoff + - nodeoutofmemory diff --git a/fixtures/kubernetes_file.yaml b/fixtures/kubernetes_file.yaml index 999a6816..3f82164b 100644 --- a/fixtures/kubernetes_file.yaml +++ b/fixtures/kubernetes_file.yaml @@ -1,9 +1,14 @@ -kubernetesFile: - - selector: - namespace: default - kind: Statefulset - name: postgresql - files: - - path: - - /etc/postgresql/postgresql.conf - format: properties +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: kubernetes-file-scraper +spec: + kubernetesFile: + - selector: + namespace: default + kind: Statefulset + name: postgresql + files: + - path: + - /etc/postgresql/postgresql.conf + format: properties diff --git a/fixtures/scrapper.yaml b/fixtures/scrapper.yaml index 35cdd392..e53ae1bf 100644 --- a/fixtures/scrapper.yaml +++ b/fixtures/scrapper.yaml @@ -1,6 +1,11 @@ -aws: - - region: ap-south-1 - compliance: true - patch_states: true - patch_details: true - inventory: true +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: aws-general-scraper +spec: + aws: + - region: ap-south-1 + compliance: true + patch_states: true + patch_details: true + inventory: true diff --git a/fixtures/sql.yaml b/fixtures/sql.yaml index 61c8ce41..b1b5a6b0 100644 --- a/fixtures/sql.yaml +++ b/fixtures/sql.yaml @@ -1,32 +1,37 @@ -sql: - - connection: postgresql://postgres:postgres@localhost:5432/incident_commander?sslmode=disable - type: Postgres::Database - id: "incident_commander" - items: .database - query: | - WITH settings AS ( - select json_object_agg(name, concat(setting,unit)) as setting from pg_settings where source != 'default' - ), - roles as ( - SELECT json_object_agg(usename, - CASE - WHEN usesuper AND usecreatedb THEN - CAST('superuser, create database' AS pg_catalog.text) - WHEN usesuper THEN - CAST('superuser' AS pg_catalog.text) - WHEN usecreatedb THEN - CAST('create database' AS pg_catalog.text) - ELSE - CAST('' AS pg_catalog.text) - END) as role - FROM pg_catalog.pg_user - ) +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: incident-commander-postgres-scraper +spec: + sql: + - connection: postgresql://postgres:postgres@localhost:5432/incident_commander?sslmode=disable + type: Postgres::Database + id: "incident_commander" + items: .database + query: | + WITH settings AS ( + select json_object_agg(name, concat(setting,unit)) as setting from pg_settings where source != 'default' + ), + roles as ( + SELECT json_object_agg(usename, + CASE + WHEN usesuper AND usecreatedb THEN + CAST('superuser, create database' AS pg_catalog.text) + WHEN usesuper THEN + CAST('superuser' AS pg_catalog.text) + WHEN usecreatedb THEN + CAST('create database' AS pg_catalog.text) + ELSE + CAST('' AS pg_catalog.text) + END) as role + FROM pg_catalog.pg_user + ) - select json_build_object('version', version(), 'settings', s.setting, 'roles', r.role ) as database FROM (SELECT * from settings) as s, (Select * from roles) as r - # - connection: connection://Postgres/incident-commander (Alternatively, you can use a connection) - # - connection: postgresql://$(username):$(password)@localhost:5432/incident_commander?sslmode=disable (connection string can also be templatized) - # auth: - # username: - # value: postgres - # password: - # value: postgres + select json_build_object('version', version(), 'settings', s.setting, 'roles', r.role ) as database FROM (SELECT * from settings) as s, (Select * from roles) as r + # - connection: connection://Postgres/incident-commander (Alternatively, you can use a connection) + # - connection: postgresql://$(username):$(password)@localhost:5432/incident_commander?sslmode=disable (connection string can also be templatized) + # auth: + # username: + # value: postgres + # password: + # value: postgres diff --git a/fixtures/trivy.yaml b/fixtures/trivy.yaml index f3643453..575fc61b 100644 --- a/fixtures/trivy.yaml +++ b/fixtures/trivy.yaml @@ -1,14 +1,19 @@ -trivy: - - version: "0.40.0" - ignoreUnfixed: true - severity: - - critical - - high - scanners: - - config - - license - - rbac - - secret - - vuln - kubernetes: {} - timeout: "20m" # Increased from the default 5m timeout +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: trivy-scraper +spec: + trivy: + - version: "0.40.0" + ignoreUnfixed: true + severity: + - critical + - high + scanners: + - config + - license + - rbac + - secret + - vuln + kubernetes: {} + timeout: "20m" # Increased from the default 5m timeout diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 38860df4..29a65160 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -960,14 +960,14 @@ func (aws Scraper) ami(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults } } -func (aws Scraper) CanScrape(configs v1.ConfigScraper) bool { +func (aws Scraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.AWS) > 0 } -func (aws Scraper) Scrape(ctx *v1.ScrapeContext, config v1.ConfigScraper) v1.ScrapeResults { +func (aws Scraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { results := &v1.ScrapeResults{} - for _, awsConfig := range config.AWS { + for _, awsConfig := range ctx.ScrapeConfig.Spec.AWS { for _, region := range awsConfig.Region { awsCtx, err := aws.getContext(ctx, awsConfig, region) if err != nil { diff --git a/scrapers/aws/cost.go b/scrapers/aws/cost.go index db20d2c6..f159cb3c 100644 --- a/scrapers/aws/cost.go +++ b/scrapers/aws/cost.go @@ -131,7 +131,7 @@ func fetchCosts(ctx *v1.ScrapeContext, config v1.AWS) ([]LineItemRow, error) { type CostScraper struct{} -func (awsCost CostScraper) CanScrape(config v1.ConfigScraper) bool { +func (awsCost CostScraper) CanScrape(config v1.ScraperSpec) bool { for _, awsConfig := range config.AWS { if awsConfig.CostReporting.S3BucketPath != "" || awsConfig.CostReporting.Table != "" { return true @@ -140,10 +140,10 @@ func (awsCost CostScraper) CanScrape(config v1.ConfigScraper) bool { return false } -func (awsCost CostScraper) Scrape(ctx *v1.ScrapeContext, config v1.ConfigScraper) v1.ScrapeResults { +func (awsCost CostScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { var results v1.ScrapeResults - for _, awsConfig := range config.AWS { + for _, awsConfig := range ctx.ScrapeConfig.Spec.AWS { session, err := NewSession(ctx, *awsConfig.AWSConnection, awsConfig.Region[0]) if err != nil { return results.Errorf(err, "failed to create AWS session") diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index 36a52571..f26d6cb4 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -28,13 +28,13 @@ type Scraper struct { config *v1.Azure } -func (azure Scraper) CanScrape(configs v1.ConfigScraper) bool { +func (azure Scraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.Azure) > 0 } -func (azure Scraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (azure Scraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { var results v1.ScrapeResults - for _, config := range configs.Azure { + for _, config := range ctx.ScrapeConfig.Spec.Azure { if err := config.HydrateConnection(ctx); err != nil { results.Errorf(err, "failed to populate config") continue diff --git a/scrapers/azure/devops/pipelines.go b/scrapers/azure/devops/pipelines.go index daa232a4..8d1d25ce 100644 --- a/scrapers/azure/devops/pipelines.go +++ b/scrapers/azure/devops/pipelines.go @@ -12,15 +12,15 @@ const PipelineRun = "AzureDevops::PipelineRun" type AzureDevopsScraper struct { } -func (ado AzureDevopsScraper) CanScrape(configs v1.ConfigScraper) bool { +func (ado AzureDevopsScraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.AzureDevops) > 0 } // Scrape ... -func (ado AzureDevopsScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (ado AzureDevopsScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { results := v1.ScrapeResults{} - for _, config := range configs.AzureDevops { + for _, config := range ctx.ScrapeConfig.Spec.AzureDevops { client, err := NewAzureDevopsClient(ctx, config) if err != nil { results.Errorf(err, "failed to create azure devops client for %s", config.Organization) diff --git a/scrapers/cron.go b/scrapers/cron.go index e6796091..a42555ca 100644 --- a/scrapers/cron.go +++ b/scrapers/cron.go @@ -1,9 +1,11 @@ package scrapers import ( + "context" "sync" "github.com/flanksource/commons/logger" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/robfig/cron/v3" ) @@ -38,14 +40,15 @@ func AtomicRunner(id string, fn func()) func() { } } -func AddToCron(scraper v1.ConfigScraper, id string) { +func AddToCron(scrapeConfig v1.ScrapeConfig) { fn := func() { - if _, err := RunScraper(scraper); err != nil { - logger.Errorf("failed to run scraper %s: %v", id, err) + ctx := api.NewScrapeContext(context.Background(), scrapeConfig) + if _, err := RunScraper(ctx); err != nil { + logger.Errorf("failed to run scraper %s: %v", ctx.ScrapeConfig.GetUID(), err) } } - AddFuncToCron(id, scraper.Schedule, fn) + AddFuncToCron(string(scrapeConfig.GetUID()), scrapeConfig.Spec.Schedule, fn) } func AddFuncToCron(id, schedule string, fn func()) { diff --git a/scrapers/file/file.go b/scrapers/file/file.go index 44009fd3..fc972655 100644 --- a/scrapers/file/file.go +++ b/scrapers/file/file.go @@ -63,16 +63,16 @@ func convertToLocalPath(uri string) string { return p + path.Base(_uri.Path) + "-" + hex.EncodeToString(hash[:])[0:8] } -func (file FileScraper) CanScrape(configs v1.ConfigScraper) bool { +func (file FileScraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.File) > 0 } -func (file FileScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (file FileScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { pwd, _ := os.Getwd() cacheDir := path.Join(pwd, ".config-db", "cache", "files") results := v1.ScrapeResults{} - for _, config := range configs.File { + for _, config := range ctx.ScrapeConfig.Spec.File { url := config.URL if connection, err := ctx.HydrateConnectionByURL(config.ConnectionName); err != nil { results.Errorf(err, "failed to find connection") diff --git a/scrapers/github/workflows.go b/scrapers/github/workflows.go index e6bcff87..fd80e37e 100644 --- a/scrapers/github/workflows.go +++ b/scrapers/github/workflows.go @@ -14,14 +14,14 @@ const WorkflowRun = "GitHubActions::WorkflowRun" type GithubActionsScraper struct { } -func (gh GithubActionsScraper) CanScrape(configs v1.ConfigScraper) bool { - return len(configs.GithubActions) > 0 +func (gh GithubActionsScraper) CanScrape(spec v1.ScraperSpec) bool { + return len(spec.GithubActions) > 0 } // Scrape fetches github workflows and workflow runs from github API and converts the action executions (workflow runs) to change events. -func (gh GithubActionsScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (gh GithubActionsScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { results := v1.ScrapeResults{} - for _, config := range configs.GithubActions { + for _, config := range ctx.ScrapeConfig.Spec.GithubActions { client, err := NewGitHubActionsClient(ctx, config) if err != nil { results.Errorf(err, "failed to create github actions client for owner %s with repository %v", config.Owner, config.Repository) diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 8367ee1c..666de3e2 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -21,17 +21,17 @@ type KubernetesScraper struct { const ConfigTypePrefix = "Kubernetes::" -func (kubernetes KubernetesScraper) CanScrape(configs v1.ConfigScraper) bool { +func (kubernetes KubernetesScraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.Kubernetes) > 0 } -func (kubernetes KubernetesScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (kubernetes KubernetesScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { var ( results v1.ScrapeResults changeResults v1.ScrapeResults ) - for _, config := range configs.Kubernetes { + for _, config := range ctx.ScrapeConfig.Spec.Kubernetes { if config.ClusterName == "" { logger.Fatalf("clusterName missing from kubernetes configuration") } diff --git a/scrapers/kubernetes/kubernetes_file.go b/scrapers/kubernetes/kubernetes_file.go index dd7eecf7..1ae3c677 100644 --- a/scrapers/kubernetes/kubernetes_file.go +++ b/scrapers/kubernetes/kubernetes_file.go @@ -117,20 +117,20 @@ func findBySelector(ctx *v1.ScrapeContext, client *kubernetes.Clientset, config return pods, nil } -func (kubernetes KubernetesFileScraper) CanScrape(configs v1.ConfigScraper) bool { +func (kubernetes KubernetesFileScraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.KubernetesFile) > 0 } // Scrape ... -func (kubernetes KubernetesFileScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (kubernetes KubernetesFileScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { results := v1.ScrapeResults{} - if len(configs.KubernetesFile) == 0 { + if len(ctx.ScrapeConfig.Spec.KubernetesFile) == 0 { return results } var pods []pod - for _, config := range configs.KubernetesFile { + for _, config := range ctx.ScrapeConfig.Spec.KubernetesFile { if config.Selector.Kind == "" { config.Selector.Kind = "Pod" } diff --git a/scrapers/run.go b/scrapers/run.go index 3d6f2d46..955e9ece 100644 --- a/scrapers/run.go +++ b/scrapers/run.go @@ -3,33 +3,25 @@ package scrapers import ( "fmt" - "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" - "github.com/google/uuid" ) -func RunScraper(scraper v1.ConfigScraper) (v1.ScrapeResults, error) { - id, err := uuid.Parse(scraper.ID) - if err != nil { - return nil, fmt.Errorf("failed to parse uuid[%s]: %w", scraper.ID, err) +func RunScraper(ctx *v1.ScrapeContext) (v1.ScrapeResults, error) { + results, scraperErr := Run(ctx) + if scraperErr != nil { + return nil, fmt.Errorf("failed to run scraper %v: %w", ctx.ScrapeConfig.Name, scraperErr) } - ctx := api.NewScrapeContext(&scraper, &id) - var results v1.ScrapeResults - var scraperErr, dbErr error - if results, scraperErr = Run(ctx, scraper); scraperErr != nil { - return nil, fmt.Errorf("failed to run scraper %v: %w", scraper, scraperErr) - } - - if dbErr = db.SaveResults(ctx, results); dbErr != nil { + dbErr := db.SaveResults(ctx, results) + if dbErr != nil { //FIXME cache results to save to db later return nil, fmt.Errorf("failed to update db: %w", dbErr) } // If error in any of the scrape results, don't delete old items - if scraperErr == nil && dbErr == nil && len(results) > 0 && !results.HasErr() { - if err = DeleteStaleConfigItems(id); err != nil { + if len(results) > 0 && !v1.ScrapeResults(results).HasErr() { + if err := DeleteStaleConfigItems(*ctx.ScrapeConfig.GetPersistedID()); err != nil { return nil, fmt.Errorf("error deleting stale config items: %w", err) } } diff --git a/scrapers/run_now.go b/scrapers/run_now.go index de2e330a..5b41bded 100644 --- a/scrapers/run_now.go +++ b/scrapers/run_now.go @@ -4,9 +4,9 @@ import ( "fmt" "net/http" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" - "github.com/flanksource/config-db/db/models" "github.com/labstack/echo/v4" ) @@ -22,12 +22,13 @@ func RunNowHandler(c echo.Context) error { return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("scraper with id=%s was not found", id)) } - configScraper, err := models.V1ConfigScraper(*scraper) + configScraper, err := v1.ScrapeConfigFromModel(*scraper) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "failed to transform config scraper model", err) } - results, err := RunScraper(configScraper) + ctx := api.NewScrapeContext(c.Request().Context(), configScraper) + results, err := RunScraper(ctx) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, "failed to run scraper", err) } diff --git a/scrapers/runscrapers.go b/scrapers/runscrapers.go index f3bef3b2..9033b18b 100644 --- a/scrapers/runscrapers.go +++ b/scrapers/runscrapers.go @@ -18,56 +18,52 @@ import ( ) // Run ... -func Run(ctx *v1.ScrapeContext, configs ...v1.ConfigScraper) ([]v1.ScrapeResult, error) { +func Run(ctx *v1.ScrapeContext) ([]v1.ScrapeResult, error) { cwd, _ := os.Getwd() logger.Infof("Scraping configs from (PWD: %s)", cwd) var results v1.ScrapeResults - for _, config := range configs { - for _, scraper := range All { - if !scraper.CanScrape(config) { - continue - } - - jobHistory := models.JobHistory{ - Name: fmt.Sprintf("scraper:%T", scraper), - ResourceType: "config_scraper", - } - if ctx.ScraperID != nil { - jobHistory.ResourceID = ctx.ScraperID.String() - } + for _, scraper := range All { + if !scraper.CanScrape(ctx.ScrapeConfig.Spec) { + continue + } - jobHistory.Start() - if err := db.PersistJobHistory(&jobHistory); err != nil { - logger.Errorf("Error persisting job history: %v", err) - } + jobHistory := models.JobHistory{ + Name: fmt.Sprintf("scraper:%T", scraper), + ResourceType: "config_scraper", + ResourceID: string(ctx.ScrapeConfig.GetUID()), + } - logger.Debugf("Starting to scrape [%s]", jobHistory.Name) - for _, result := range scraper.Scrape(ctx, config) { - scraped := processScrapeResult(config, result) + jobHistory.Start() + if err := db.PersistJobHistory(&jobHistory); err != nil { + logger.Errorf("Error persisting job history: %v", err) + } - for i := range scraped { - if scraped[i].Error != nil { - logger.Errorf("Error scraping %s: %v", scraped[i].ID, scraped[i].Error) - jobHistory.AddError(scraped[i].Error.Error()) - } - } + logger.Debugf("Starting to scrape [%s]", jobHistory.Name) + for _, result := range scraper.Scrape(ctx) { + scraped := processScrapeResult(ctx.ScrapeConfig.Spec, result) - if !scraped.HasErr() { - jobHistory.IncrSuccess() + for i := range scraped { + if scraped[i].Error != nil { + logger.Errorf("Error scraping %s: %v", scraped[i].ID, scraped[i].Error) + jobHistory.AddError(scraped[i].Error.Error()) } - - results = append(results, scraped...) } - jobHistory.End() - if err := db.PersistJobHistory(&jobHistory); err != nil { - logger.Errorf("Error persisting job history: %v", err) + if !scraped.HasErr() { + jobHistory.IncrSuccess() } + + results = append(results, scraped...) + } + + jobHistory.End() + if err := db.PersistJobHistory(&jobHistory); err != nil { + logger.Errorf("Error persisting job history: %v", err) } } - logger.Infof("Completed scraping %d configs with %d results.", len(configs), len(results)) + logger.Infof("Completed scraping with %d results.", len(results)) return results, nil } @@ -90,7 +86,7 @@ func summarizeChanges(changes []v1.ChangeResult) []v1.ChangeResult { } // processScrapeResult extracts possibly more configs from the result -func processScrapeResult(config v1.ConfigScraper, result v1.ScrapeResult) v1.ScrapeResults { +func processScrapeResult(config v1.ScraperSpec, result v1.ScrapeResult) v1.ScrapeResults { if result.AnalysisResult != nil { if rule, ok := analysis.Rules[result.AnalysisResult.Analyzer]; ok { result.AnalysisResult.AnalysisType = models.AnalysisType(rule.Category) diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index 6991b9c0..73af9a2e 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -7,6 +7,7 @@ import ( "os" "github.com/flanksource/commons/logger" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" "github.com/flanksource/config-db/db/models" @@ -45,11 +46,11 @@ var _ = Describe("Scrapers test", func() { for _, fixtureName := range fixtures { fixture := fixtureName It(fixture, func() { - config := getConfig(fixture) + config := getConfigSpec(fixture) expected := getFixtureResult(fixture) - ctx := &v1.ScrapeContext{Context: context.Background()} + ctx := api.NewScrapeContext(context.Background(), config) - results, err := Run(ctx, config) + results, err := Run(ctx) Expect(err).To(BeNil()) err = db.SaveResults(ctx, results) @@ -78,13 +79,13 @@ var _ = Describe("Scrapers test", func() { var storedConfigItem *models.ConfigItem It("should create a new config item", func() { - config := getConfig("file-car") + config := getConfigSpec("file-car") configScraper, err := db.PersistScrapeConfigFromFile(config) Expect(err).To(BeNil()) - ctx := &v1.ScrapeContext{ScraperID: &configScraper.ID, Context: context.Background()} + ctx := api.NewScrapeContext(context.Background(), config) - results, err := Run(ctx, config) + results, err := Run(ctx) Expect(err).To(BeNil()) logger.Infof("SCRAPER ID: %s", configScraper.ID) @@ -104,10 +105,11 @@ var _ = Describe("Scrapers test", func() { }) It("should store the changes from the config", func() { - config := getConfig("file-car-change") - ctx := &v1.ScrapeContext{Context: context.Background()} + config := getConfigSpec("file-car-change") - results, err := Run(ctx, config) + ctx := api.NewScrapeContext(context.Background(), config) + + results, err := Run(ctx) Expect(err).To(BeNil()) err = db.SaveResults(ctx, results) @@ -144,7 +146,7 @@ var _ = Describe("Scrapers test", func() { }) }) -func getConfig(name string) v1.ConfigScraper { +func getConfigSpec(name string) v1.ScrapeConfig { configs, err := v1.ParseConfigs("fixtures/" + name + ".yaml") if err != nil { Fail(fmt.Sprintf("Failed to parse config: %v", err)) diff --git a/scrapers/sql/sql.go b/scrapers/sql/sql.go index c6af9047..ae25195f 100644 --- a/scrapers/sql/sql.go +++ b/scrapers/sql/sql.go @@ -19,13 +19,13 @@ import ( type SqlScraper struct { } -func (s SqlScraper) CanScrape(configs v1.ConfigScraper) bool { +func (s SqlScraper) CanScrape(configs v1.ScraperSpec) bool { return len(configs.SQL) > 0 } -func (s SqlScraper) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (s SqlScraper) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { var results v1.ScrapeResults - for _, _config := range configs.SQL { + for _, _config := range ctx.ScrapeConfig.Spec.SQL { var ( config = _config err error diff --git a/scrapers/trivy/trivy.go b/scrapers/trivy/trivy.go index d420a268..91908ea8 100644 --- a/scrapers/trivy/trivy.go +++ b/scrapers/trivy/trivy.go @@ -25,14 +25,14 @@ const ( type Scanner struct { } -func (t Scanner) CanScrape(config v1.ConfigScraper) bool { +func (t Scanner) CanScrape(config v1.ScraperSpec) bool { return len(config.Trivy) > 0 } -func (t Scanner) Scrape(ctx *v1.ScrapeContext, configs v1.ConfigScraper) v1.ScrapeResults { +func (t Scanner) Scrape(ctx *v1.ScrapeContext) v1.ScrapeResults { var results v1.ScrapeResults - for i, config := range configs.Trivy { + for i, config := range ctx.ScrapeConfig.Spec.Trivy { if config.IsEmpty() { logger.Debugf("Trivy config [%d] is empty. Skipping ...", i+1) continue