feat: use of CRD to run scrapers (#288)

* feat: use of CRD to run scrapers

* chore: rename kind to ScrapeConfig in all fixtures

* chore: rename v1.ConfigScraper to ScraperSpec and modified ScraperCRD
structure

* refactor: modify Run() function to only accept a single scrape config.

Eventually the scrape config will come from the context and the context
must be tied to a single configuration.

* refactor: use scrapeConfig instead of scraperSpec

* refactor: the Scrape() method only accepts the context. The second param
spec is removed.

* chore: rename ctx.Scraper to ctx.ScrapeConfig

* refactor: do not store id in the scrape context. It's already there in
ScrapeConfig

* NewScrapeContext accepts a context instead of scrape config id

* chore: removed db ID field from scraperSpec

* chore: final cleanup

api/global.go

@@ -1,11 +1,10 @@
 package api
 
 import (
-	goctx "context"
+	"context"
 
 	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/config-db/db"
-	"github.com/google/uuid"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
@@ -14,11 +13,10 @@ var KubernetesClient *kubernetes.Clientset
 var KubernetesRestConfig *rest.Config
 var Namespace string
 
-func NewScrapeContext(scraper *v1.ConfigScraper, id *uuid.UUID) *v1.ScrapeContext {
+func NewScrapeContext(ctx context.Context, scraper v1.ScrapeConfig) *v1.ScrapeContext {
 	return &v1.ScrapeContext{
-		Context:              goctx.Background(),
-		Scraper:              scraper,
-		ScraperID:            id,
+		Context:              ctx,
+		ScrapeConfig:         scraper,
 		Namespace:            Namespace,
 		Kubernetes:           KubernetesClient,
 		KubernetesRestConfig: KubernetesRestConfig,

api/v1/config.go

@@ -1,16 +1,17 @@
 package v1
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"regexp"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	yamlutil "k8s.io/apimachinery/pkg/util/yaml"
 )
 
+var yamlDividerRegexp = regexp.MustCompile(`(?m)^---\n`)
+
 func readFile(path string) (string, error) {
 	var data []byte
 	var err error
@@ -26,38 +27,38 @@ func readFile(path string) (string, error) {
 	return string(data), nil
 }
 
-func ParseConfigs(files ...string) ([]ConfigScraper, error) {
-	scrapers := []ConfigScraper{}
+func ParseConfigs(files ...string) ([]ScrapeConfig, error) {
+	scrapers := make([]ScrapeConfig, 0, len(files))
 
 	for _, f := range files {
 		_scrapers, err := parseConfig(f)
 		if err != nil {
-			return nil, errors.Wrapf(err, "error parsing %s", f)
+			return nil, err
 		}
+
 		scrapers = append(scrapers, _scrapers...)
 	}
+
 	return scrapers, nil
 }
 
 // ParseConfig : Read config file
-func parseConfig(configfile string) ([]ConfigScraper, error) {
+func parseConfig(configfile string) ([]ScrapeConfig, error) {
 	configs, err := readFile(configfile)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error reading config file=%s: %w", configfile, err)
 	}
 
-	var scrapers []ConfigScraper
-
-	re := regexp.MustCompile(`(?m)^---\n`)
-	for _, chunk := range re.Split(configs, -1) {
+	var scrapers []ScrapeConfig
+	for _, chunk := range yamlDividerRegexp.Split(configs, -1) {
 		if strings.TrimSpace(chunk) == "" {
 			continue
 		}
-		config := ConfigScraper{}
-		decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(chunk), 1024)
 
+		var config ScrapeConfig
+		decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(chunk), 1024)
 		if err := decoder.Decode(&config); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error decoding yaml. file=%s: %w", configfile, err)
 		}
 
 		scrapers = append(scrapers, config)

api/v1/interface.go

@@ -13,7 +13,6 @@ import (
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty"
 	"github.com/flanksource/duty/models"
-	"github.com/google/uuid"
 	"gorm.io/gorm"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -22,8 +21,8 @@ import (
 // Scraper ...
 // +kubebuilder:object:generate=false
 type Scraper interface {
-	Scrape(ctx *ScrapeContext, config ConfigScraper) ScrapeResults
-	CanScrape(config ConfigScraper) bool
+	Scrape(ctx *ScrapeContext) ScrapeResults
+	CanScrape(config ScraperSpec) bool
 }
 
 // Analyzer ...
@@ -351,8 +350,7 @@ type ScrapeContext struct {
 	Namespace            string
 	Kubernetes           *kubernetes.Clientset
 	KubernetesRestConfig *rest.Config
-	Scraper              *ConfigScraper
-	ScraperID            *uuid.UUID
+	ScrapeConfig         ScrapeConfig
 }
 
 func (ctx ScrapeContext) Find(path string) ([]string, error) {
@@ -366,21 +364,14 @@ func (ctx ScrapeContext) Read(path string) ([]byte, string, error) {
 	return content, filename, err
 }
 
-// WithScraper ...
-func (ctx ScrapeContext) WithScraper(config *ConfigScraper) ScrapeContext {
-	ctx.Scraper = config
-	return ctx
-
-}
-
 // GetNamespace ...
 func (ctx ScrapeContext) GetNamespace() string {
 	return ctx.Namespace
 }
 
 // IsTrace ...
 func (ctx ScrapeContext) IsTrace() bool {
-	return ctx.Scraper != nil && ctx.Scraper.IsTrace()
+	return ctx.ScrapeConfig.Spec.IsTrace()
 }
 
 // HydrateConnectionByURL ...

api/v1/scrapeconfig_types.go

@@ -17,14 +17,14 @@ limitations under the License.
 package v1
 
 import (
+	"encoding/json"
+
+	"github.com/flanksource/duty/models"
+	"github.com/google/uuid"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
 )
 
-// ScrapeConfigSpec defines the desired state of ScrapeConfig
-type ScrapeConfigSpec struct {
-	ConfigScraper `json:",inline"`
-}
-
 // ScrapeConfigStatus defines the observed state of ScrapeConfig
 type ScrapeConfigStatus struct {
 	ObservedGeneration int64 `json:"observedGeneration,omitempty" protobuf:"varint,3,opt,name=observedGeneration"`
@@ -38,10 +38,57 @@ type ScrapeConfig struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec   ScrapeConfigSpec   `json:"spec,omitempty"`
+	Spec   ScraperSpec        `json:"spec,omitempty"`
 	Status ScrapeConfigStatus `json:"status,omitempty"`
 }
 
+func (t *ScrapeConfig) ToModel() (models.ConfigScraper, error) {
+	spec, err := json.Marshal(t.Spec)
+	if err != nil {
+		return models.ConfigScraper{}, err
+	}
+
+	return models.ConfigScraper{
+		Name:   t.Name,
+		Spec:   string(spec),
+		Source: t.Annotations["source"],
+	}, nil
+}
+
+func ScrapeConfigFromModel(m models.ConfigScraper) (ScrapeConfig, error) {
+	var spec ScraperSpec
+	if err := json.Unmarshal([]byte(m.Spec), &spec); err != nil {
+		return ScrapeConfig{}, err
+	}
+
+	sc := ScrapeConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: m.Name,
+			Annotations: map[string]string{
+				"source": m.Source,
+			},
+			UID:               k8stypes.UID(m.ID.String()),
+			CreationTimestamp: metav1.Time{Time: m.CreatedAt},
+		},
+		Spec: spec,
+	}
+
+	if m.DeletedAt != nil {
+		sc.ObjectMeta.DeletionTimestamp = &metav1.Time{Time: *m.DeletedAt}
+	}
+
+	return sc, nil
+}
+
+func (t *ScrapeConfig) GetPersistedID() *uuid.UUID {
+	if t.GetUID() == "" {
+		return nil
+	}
+
+	u, _ := uuid.Parse(string(t.GetUID()))
+	return &u
+}
+
 //+kubebuilder:object:root=true
 
 // ScrapeConfigList contains a list of ScrapeConfig

api/v1/types.go

@@ -22,9 +22,8 @@ var AllScraperConfigs = map[string]any{
 	"trivy":          Trivy{},
 }
 
-// ConfigScraper ...
-type ConfigScraper struct {
-	ID             string           `json:"-"`
+// ScraperSpec defines the desired state of Config scraper
+type ScraperSpec struct {
 	LogLevel       string           `json:"logLevel,omitempty"`
 	Schedule       string           `json:"schedule,omitempty"`
 	AWS            []AWS            `json:"aws,omitempty" yaml:"aws,omitempty"`
@@ -41,20 +40,20 @@ type ConfigScraper struct {
 	Full bool `json:"full,omitempty"`
 }
 
-func (c ConfigScraper) GenerateName() (string, error) {
+func (c ScraperSpec) GenerateName() (string, error) {
 	return utils.Hash(c)
 }
 
 // IsEmpty ...
-func (c ConfigScraper) IsEmpty() bool {
+func (c ScraperSpec) IsEmpty() bool {
 	return len(c.AWS) == 0 && len(c.File) == 0
 }
 
-func (c ConfigScraper) IsTrace() bool {
+func (c ScraperSpec) IsTrace() bool {
 	return c.LogLevel == "trace"
 }
 
-func (c ConfigScraper) IsDebug() bool {
+func (c ScraperSpec) IsDebug() bool {
 	return c.LogLevel == "debug"
 }