From 6cdadbf2ca609f37b3feae6ff3a57ce9c753a8bc Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 1 Feb 2024 10:50:03 +0545 Subject: [PATCH] feat: improve kubernetes relationship selector and use the db methods from duty --- api/v1/common.go | 85 +++++++++--------- api/v1/kubernetes.go | 52 ++++++++++- api/v1/zz_generated.deepcopy.go | 45 +++++----- cmd/operator.go | 7 +- cmd/root.go | 2 +- config/schemas/config_kubernetes.schema.json | 2 +- config/schemas/scrape_config.schema.json | 2 +- db/config.go | 90 +++----------------- go.mod | 8 +- go.sum | 17 ++-- hack/generate-schemas/go.mod | 4 +- hack/generate-schemas/go.sum | 8 +- jobs/cleanup.go | 12 ++- jobs/retention.go | 4 +- scrapers/kubernetes/kubernetes.go | 41 +++------ scrapers/processors/json.go | 3 +- scrapers/runscrapers_test.go | 4 +- 17 files changed, 177 insertions(+), 209 deletions(-) diff --git a/api/v1/common.go b/api/v1/common.go index 13419ccd..04099b33 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -4,14 +4,11 @@ import ( "fmt" "net/url" "regexp" - "sort" "strings" - "github.com/flanksource/commons/hash" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "github.com/flanksource/gomplate/v3" - "github.com/samber/lo" ) // ConfigFieldExclusion defines fields with JSONPath that needs to @@ -132,44 +129,36 @@ func (t *RelationshipLookup) Eval(labels map[string]string, envVar map[string]an } func (t RelationshipLookup) IsEmpty() bool { - if t.Value == "" && t.Label == "" && t.Expr == "" { - return true - } - - return false + return t.Value == "" && t.Label == "" && t.Expr == "" } // RelationshipSelector is the evaluated output of RelationshipSelector. -// -// NOTE: The fields are pointers because we need to differentiate between -// empty filter (null value) and empty result (output of the filter). type RelationshipSelector struct { - ID *string `json:"id,omitempty"` - Name *string `json:"name,omitempty"` - Type *string `json:"type,omitempty"` - Agent *string `json:"agent,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Agent string `json:"agent,omitempty"` Labels map[string]string `json:"labels,omitempty"` } -func (t *RelationshipSelector) Hash() string { - items := []string{ - lo.FromPtr(t.ID), - lo.FromPtr(t.Name), - lo.FromPtr(t.Type), - lo.FromPtr(t.Agent), - } - - labelkeys := lo.Keys(t.Labels) - sort.Slice(labelkeys, func(i, j int) bool { return labelkeys[i] < labelkeys[j] }) - for _, k := range labelkeys { - items = append(items, fmt.Sprintf("%s=%s", k, t.Labels[k])) - } - - return hash.Sha256Hex(strings.Join(items, "|")) +func (t *RelationshipSelector) IsEmpty() bool { + return t.ID == "" && t.Name == "" && t.Type == "" && t.Agent == "" && len(t.Labels) == 0 } -func (t *RelationshipSelector) IsEmpty() bool { - return t.ID == nil && t.Name == nil && t.Type == nil && t.Agent == nil && len(t.Labels) == 0 +func (t *RelationshipSelector) ToResourceSelector() types.ResourceSelector { + var labelSelector string + for k, v := range t.Labels { + labelSelector += fmt.Sprintf("%s=%s,", k, v) + } + labelSelector = strings.TrimSuffix(labelSelector, ",") + + return types.ResourceSelector{ + ID: t.ID, + Name: t.Name, + Types: []string{t.Type}, + Agent: t.Agent, + LabelSelector: labelSelector, + } } type RelationshipSelectorTemplate struct { @@ -188,38 +177,46 @@ func (t *RelationshipSelectorTemplate) IsEmpty() bool { return t.ID.IsEmpty() && t.Name.IsEmpty() && t.Type.IsEmpty() && t.Agent.IsEmpty() && len(t.Labels) == 0 } +// Eval evaluates the template and returns a RelationshipSelector. +// If any of the filter returns an empty value, the evaluation results to a nil selector. +// i.e. if a lookup is non-empty, it must return a non-empty value. func (t *RelationshipSelectorTemplate) Eval(labels map[string]string, env map[string]any) (*RelationshipSelector, error) { + if t.IsEmpty() { + return nil, nil + } + var output RelationshipSelector + var err error if !t.ID.IsEmpty() { - if res, err := t.ID.Eval(labels, env); err != nil { + if output.ID, err = t.ID.Eval(labels, env); err != nil { return nil, fmt.Errorf("failed to evaluate id: %v for config relationship: %w", t.ID, err) - } else { - output.ID = &res + } else if output.ID == "" { + return nil, nil } } if !t.Name.IsEmpty() { - if res, err := t.Name.Eval(labels, env); err != nil { + if output.Name, err = t.Name.Eval(labels, env); err != nil { return nil, fmt.Errorf("failed to evaluate name: %v for config relationship: %w", t.Name, err) - } else { - output.Name = &res + } else if output.Name == "" { + return nil, nil } } if !t.Type.IsEmpty() { - if res, err := t.Type.Eval(labels, env); err != nil { + if output.Type, err = t.Type.Eval(labels, env); err != nil { return nil, fmt.Errorf("failed to evaluate type: %v for config relationship: %w", t.Type, err) - } else { - output.Type = &res + } else if output.Type == "" { + return nil, nil } } if !t.Agent.IsEmpty() { - if res, err := t.Agent.Eval(labels, env); err != nil { + if output.Agent, err = t.Agent.Eval(labels, env); err != nil { return nil, fmt.Errorf("failed to evaluate agent_id: %v for config relationship: %w", t.Agent, err) - } else { - output.Agent = &res + } else if output.Agent == "" { + return nil, nil } } diff --git a/api/v1/kubernetes.go b/api/v1/kubernetes.go index 119f0440..e0960fd5 100644 --- a/api/v1/kubernetes.go +++ b/api/v1/kubernetes.go @@ -103,7 +103,13 @@ func (t *KubernetesExclusionConfig) Filter(name, namespace, kind string, labels return false } -type KubernetesRelationship struct { +type KubernetesRelationshipSelector struct { + Kind string `json:"kind" yaml:"kind"` + Name string `json:"name" yaml:"name"` + Namespace string `json:"namespace" yaml:"namespace"` +} + +type KubernetesRelationshipSelectorTemplate struct { // Kind defines which field to use for the kind lookup Kind RelationshipLookup `json:"kind" yaml:"kind"` // Name defines which field to use for the name lookup @@ -112,6 +118,48 @@ type KubernetesRelationship struct { Namespace RelationshipLookup `json:"namespace" yaml:"namespace"` } +func (t *KubernetesRelationshipSelectorTemplate) IsEmpty() bool { + return t.Kind.IsEmpty() && t.Name.IsEmpty() && t.Namespace.IsEmpty() +} + +// Eval evaluates the template and returns a KubernetesRelationshipSelector. +// If any of the filter returns an empty value, the evaluation results to a nil selector. +// i.e. if a lookup is non-empty, it must return a non-empty value. +func (t *KubernetesRelationshipSelectorTemplate) Eval(labels map[string]string, env map[string]any) (*KubernetesRelationshipSelector, error) { + if t.IsEmpty() { + return nil, nil + } + + var output KubernetesRelationshipSelector + var err error + + if !t.Name.IsEmpty() { + if output.Name, err = t.Name.Eval(labels, env); err != nil { + return nil, fmt.Errorf("failed to evaluate Name: %s: %w", t.Name, err) + } else if output.Name == "" { + return nil, nil + } + } + + if !t.Kind.IsEmpty() { + if output.Kind, err = t.Kind.Eval(labels, env); err != nil { + return nil, fmt.Errorf("failed to evaluate kind: %s: %w", t.Kind, err) + } else if output.Kind == "" { + return nil, nil + } + } + + if !t.Namespace.IsEmpty() { + if output.Namespace, err = t.Namespace.Eval(labels, env); err != nil { + return nil, fmt.Errorf("failed to evaluate namespace: %s: %w", t.Namespace, err) + } else if output.Namespace == "" { + return nil, nil + } + } + + return &output, nil +} + type Kubernetes struct { BaseScraper `json:",inline"` ClusterName string `json:"clusterName,omitempty"` @@ -128,7 +176,7 @@ type Kubernetes struct { Event KubernetesEventConfig `json:"event,omitempty"` // Relationships specify the fields to use to relate Kubernetes objects. - Relationships []KubernetesRelationship `json:"relationships,omitempty"` + Relationships []KubernetesRelationshipSelectorTemplate `json:"relationships,omitempty"` } type KubernetesFile struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 4f15cffc..2dc0ce9f 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -478,7 +478,7 @@ func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { in.Event.DeepCopyInto(&out.Event) if in.Relationships != nil { in, out := &in.Relationships, &out.Relationships - *out = make([]KubernetesRelationship, len(*in)) + *out = make([]KubernetesRelationshipSelectorTemplate, len(*in)) copy(*out, *in) } } @@ -634,19 +634,34 @@ func (in *KubernetesFile) DeepCopy() *KubernetesFile { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KubernetesRelationship) DeepCopyInto(out *KubernetesRelationship) { +func (in *KubernetesRelationshipSelector) DeepCopyInto(out *KubernetesRelationshipSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesRelationshipSelector. +func (in *KubernetesRelationshipSelector) DeepCopy() *KubernetesRelationshipSelector { + if in == nil { + return nil + } + out := new(KubernetesRelationshipSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesRelationshipSelectorTemplate) DeepCopyInto(out *KubernetesRelationshipSelectorTemplate) { *out = *in out.Kind = in.Kind out.Name = in.Name out.Namespace = in.Namespace } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesRelationship. -func (in *KubernetesRelationship) DeepCopy() *KubernetesRelationship { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesRelationshipSelectorTemplate. +func (in *KubernetesRelationshipSelectorTemplate) DeepCopy() *KubernetesRelationshipSelectorTemplate { if in == nil { return nil } - out := new(KubernetesRelationship) + out := new(KubernetesRelationshipSelectorTemplate) in.DeepCopyInto(out) return out } @@ -873,26 +888,6 @@ func (in RelationshipResults) DeepCopy() RelationshipResults { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RelationshipSelector) DeepCopyInto(out *RelationshipSelector) { *out = *in - if in.ID != nil { - in, out := &in.ID, &out.ID - *out = new(string) - **out = **in - } - if in.Name != nil { - in, out := &in.Name, &out.Name - *out = new(string) - **out = **in - } - if in.Type != nil { - in, out := &in.Type, &out.Type - *out = new(string) - **out = **in - } - if in.Agent != nil { - in, out := &in.Agent, &out.Agent - *out = new(string) - **out = **in - } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) diff --git a/cmd/operator.go b/cmd/operator.go index d5b38118..d7e359b6 100644 --- a/cmd/operator.go +++ b/cmd/operator.go @@ -52,9 +52,10 @@ func run(cmd *cobra.Command, args []string) { loggr := ctrlzap.NewRaw( ctrlzap.UseDevMode(true), ctrlzap.WriteTo(os.Stderr), - ctrlzap.Level(zapLogger.Level), - ctrlzap.StacktraceLevel(zapLogger.StackTraceLevel), - ctrlzap.Encoder(zapLogger.GetEncoder()), + // FIXME: Breaking after duty bump. + // ctrlzap.Level(zapLogger.Level), + // ctrlzap.StacktraceLevel(zapLogger.StackTraceLevel), + // ctrlzap.Encoder(zapLogger.GetEncoder()), ) ctrl.SetLogger(zapr.NewLogger(loggr)) diff --git a/cmd/root.go b/cmd/root.go index 41e296b7..9a8eb57d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -42,7 +42,7 @@ var Root = &cobra.Command{ count, _ := cmd.Flags().GetCount("loglevel") // logger.StandardLogger().(logsrusapi.Logger).Out = os.Stderr logger.StandardLogger().SetLogLevel(count) - logger.UseZap(cmd.Flags()) + logger.UseZap() var err error if api.KubernetesClient, api.KubernetesRestConfig, err = kube.NewK8sClient(); err != nil { diff --git a/config/schemas/config_kubernetes.schema.json b/config/schemas/config_kubernetes.schema.json index 00201690..2c2921b0 100644 --- a/config/schemas/config_kubernetes.schema.json +++ b/config/schemas/config_kubernetes.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind","name","namespace"],"properties":{"kind":{"$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"namespace":{"$ref":"#/definitions/RelationshipLookup"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipConfig":{"required":["RelationshipSelectorTemplate"],"properties":{"RelationshipSelectorTemplate":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipSelectorTemplate"},"expr":{"type":"string"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipSelectorTemplate":{"properties":{"id":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"type":{"$ref":"#/definitions/RelationshipLookup"},"agent":{"$ref":"#/definitions/RelationshipLookup"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"relationship":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipConfig"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipSelectorTemplate"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipSelectorTemplate":{"required":["kind","name","namespace"],"properties":{"kind":{"$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"namespace":{"$ref":"#/definitions/RelationshipLookup"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipConfig":{"required":["RelationshipSelectorTemplate"],"properties":{"RelationshipSelectorTemplate":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipSelectorTemplate"},"expr":{"type":"string"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipSelectorTemplate":{"properties":{"id":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"type":{"$ref":"#/definitions/RelationshipLookup"},"agent":{"$ref":"#/definitions/RelationshipLookup"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"relationship":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipConfig"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index 3bfacc51..cd7c404f 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ChangeRetentionSpec":{"properties":{"name":{"type":"string"},"age":{"type":"string"},"count":{"type":"integer"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind","name","namespace"],"properties":{"kind":{"$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"namespace":{"$ref":"#/definitions/RelationshipLookup"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipConfig":{"required":["RelationshipSelectorTemplate"],"properties":{"RelationshipSelectorTemplate":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipSelectorTemplate"},"expr":{"type":"string"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipSelectorTemplate":{"properties":{"id":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"type":{"$ref":"#/definitions/RelationshipLookup"},"agent":{"$ref":"#/definitions/RelationshipLookup"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"RetentionSpec":{"properties":{"changes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ChangeRetentionSpec"},"type":"array"},"types":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeRetentionSpec"},"type":"array"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"retention":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RetentionSpec"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"relationship":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipConfig"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeRetentionSpec":{"properties":{"name":{"type":"string"},"createdAge":{"type":"string"},"updatedAge":{"type":"string"},"deletedAge":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ChangeRetentionSpec":{"properties":{"name":{"type":"string"},"age":{"type":"string"},"count":{"type":"integer"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipSelectorTemplate"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipSelectorTemplate":{"required":["kind","name","namespace"],"properties":{"kind":{"$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"namespace":{"$ref":"#/definitions/RelationshipLookup"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipConfig":{"required":["RelationshipSelectorTemplate"],"properties":{"RelationshipSelectorTemplate":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipSelectorTemplate"},"expr":{"type":"string"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"RelationshipSelectorTemplate":{"properties":{"id":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipLookup"},"name":{"$ref":"#/definitions/RelationshipLookup"},"type":{"$ref":"#/definitions/RelationshipLookup"},"agent":{"$ref":"#/definitions/RelationshipLookup"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"RetentionSpec":{"properties":{"changes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ChangeRetentionSpec"},"type":"array"},"types":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeRetentionSpec"},"type":"array"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"retention":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RetentionSpec"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"relationship":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RelationshipConfig"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeRetentionSpec":{"properties":{"name":{"type":"string"},"createdAge":{"type":"string"},"updatedAge":{"type":"string"},"deletedAge":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/db/config.go b/db/config.go index ecd62814..40045b9d 100644 --- a/db/config.go +++ b/db/config.go @@ -5,29 +5,23 @@ import ( "database/sql" "encoding/json" "fmt" - "time" "github.com/flanksource/commons/logger" "github.com/flanksource/commons/utils" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db/models" + "github.com/flanksource/duty" "github.com/flanksource/duty/context" dutyModels "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" "github.com/google/uuid" "github.com/lib/pq" "github.com/ohler55/ojg/oj" "github.com/patrickmn/go-cache" + "github.com/samber/lo" "gorm.io/gorm/clause" ) -var ( - configIDCache = cache.New(cache.NoExpiration, cache.NoExpiration) - - configRelationshipSelectorCache = cache.New(cache.NoExpiration, cache.NoExpiration) - - configRelationshipSelectorMutableCache = cache.New(time.Minute*5, time.Minute*5) -) - // GetConfigItem returns a single config item result func GetConfigItem(extType, extID string) (*models.ConfigItem, error) { ci := models.ConfigItem{} @@ -108,89 +102,27 @@ func UpdateConfigItem(ci *models.ConfigItem) error { return nil } -// ConfigRelationshipSelectorResult represents the a subset of columns in the config item database table. -type ConfigRelationshipSelectorResult struct { - ID string `json:"id"` - Type string `json:"type"` -} - -func (t *ConfigRelationshipSelectorResult) TableName() string { - return "config_items" -} - -func FindConfigsByRelationshipSelector(ctx context.Context, selector v1.RelationshipSelector) ([]ConfigRelationshipSelectorResult, error) { +func FindConfigsByRelationshipSelector(ctx context.Context, selector v1.RelationshipSelector) ([]dutyModels.ConfigItem, error) { if selector.IsEmpty() { return nil, nil } - var cacheToUse = configRelationshipSelectorCache - if len(selector.Labels) != 0 { - cacheToUse = configRelationshipSelectorMutableCache - } - - if val, ok := cacheToUse.Get(selector.Hash()); ok { - return val.([]ConfigRelationshipSelectorResult), nil - } - - var items []ConfigRelationshipSelectorResult - query := ctx.DB().Select("config_items.id, config_items.type") - if selector.Name != nil { - query = query.Where("config_items.name = ?", *selector.Name) - } - - if selector.Type != nil { - query = query.Where("config_items.type = ?", *selector.Type) - } - - if selector.Agent != nil { - if *selector.Agent == "self" { - query = query.Where("config_items.agent_id = ?", uuid.Nil) - } else if uid, err := uuid.Parse(*selector.Agent); err == nil { - query = query.Where("config_items.agent_id = ?", uid) - } else { // assume it's an agent name - query = query.Joins("LEFT JOIN agents ON config_items.agent_id = agents.id").Where("agents.name = ?", *selector.Agent).Where("config_items.agent_id = ?", uid) - } - } - - if len(selector.Labels) > 0 { - query = query.Where("config_items.labels @> ?", selector.Labels) - } - - if err := query.Find(&items).Error; err != nil { - return nil, err - } - - if len(items) > 0 { - cacheToUse.SetDefault(selector.Hash(), items) - } else { - cacheToUse.Set(selector.Hash(), items, time.Minute*5) - } - - return items, nil + return duty.FindConfigs(ctx, []types.ResourceSelector{selector.ToResourceSelector()}, duty.PickColumns("id")) } // FindConfigIDsByNamespaceNameClass returns the uuid of config items which matches the given type, name & namespace func FindConfigIDsByNamespaceNameClass(ctx context.Context, namespace, name, configClass string) ([]uuid.UUID, error) { - cacheKey := fmt.Sprintf("%s|%s|%s", namespace, name, configClass) - if val, ok := configIDCache.Get(cacheKey); ok { - return val.([]uuid.UUID), nil + rs := types.ResourceSelector{ + Name: name, + Namespace: namespace, + FieldSelector: fmt.Sprintf("config_class=%s", configClass), } - - var ids []uuid.UUID - err := ctx.DB().Model(&models.ConfigItem{}).Select("id"). - Where("name = ?", name). - Where("namespace = ?", namespace). - Where("config_class = ?", configClass). - Find(&ids).Error + items, err := duty.FindConfigs(ctx, []types.ResourceSelector{rs}, duty.PickColumns("id")) if err != nil { return nil, err } - if len(ids) > 0 { - configIDCache.SetDefault(cacheKey, ids) - } - - return ids, err + return lo.Map(items, func(c dutyModels.ConfigItem, _ int) uuid.UUID { return c.ID }), nil } // QueryConfigItems ... diff --git a/go.mod b/go.mod index fe628149..10075c1a 100644 --- a/go.mod +++ b/go.mod @@ -40,8 +40,8 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/evanphx/json-patch v5.6.0+incompatible github.com/fergusstrange/embedded-postgres v1.25.0 - github.com/flanksource/commons v1.20.0 - github.com/flanksource/duty v1.0.282 + github.com/flanksource/commons v1.22.0 + github.com/flanksource/duty v1.0.321 github.com/flanksource/is-healthy v1.0.1 github.com/flanksource/ketall v1.1.3 github.com/flanksource/mapstructure v1.6.0 @@ -95,6 +95,8 @@ require ( github.com/asecurityteam/rolling v2.0.4+incompatible // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/eko/gocache/lib/v4 v4.1.5 // indirect + github.com/eko/gocache/store/go_cache/v4 v4.2.1 // indirect github.com/evanphx/json-patch/v5 v5.7.0 // indirect github.com/exaring/otelpgx v0.5.2 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -106,6 +108,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/pprof v0.0.0-20231212022811-ec68065c825e // indirect github.com/google/s2a-go v0.1.7 // indirect @@ -147,7 +150,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gorm.io/driver/postgres v1.5.3 // indirect - gorm.io/plugin/opentelemetry v0.1.4 // indirect k8s.io/component-base v0.28.0 // indirect layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf // indirect sigs.k8s.io/kustomize v2.0.3+incompatible // indirect diff --git a/go.sum b/go.sum index dff77e13..0096fbfb 100644 --- a/go.sum +++ b/go.sum @@ -804,6 +804,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eko/gocache/lib/v4 v4.1.5 h1:CeMQmdIzwBKKLRjk3FCDXzNFsQTyqJ01JLI7Ib0C9r8= +github.com/eko/gocache/lib/v4 v4.1.5/go.mod h1:XaNfCwW8KYW1bRZ/KoHA1TugnnkMz0/gT51NDIu7LSY= +github.com/eko/gocache/store/go_cache/v4 v4.2.1 h1:3xSksOamzCf+YZXz9l67gr6jOj3AA4hnk0mV4z3Jwbs= +github.com/eko/gocache/store/go_cache/v4 v4.2.1/go.mod h1:rd6tUbaBOdqgi5xT3+OGeU7lQuhVJGTapNWFlZou524= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -835,10 +839,10 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7TtkWXXJ+YRsxpKMDhxK0= github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= -github.com/flanksource/commons v1.20.0 h1:1z7FHGsCY7F7zMx4m0OlzYCUvCc1caREPp7vf1xyeCQ= -github.com/flanksource/commons v1.20.0/go.mod h1:bs2nMwaTCpTQGZNtJv8KNqEVyfGjFZTGUDaAbPHB1cw= -github.com/flanksource/duty v1.0.282 h1:iddsHVcOs6IQ4Wm8U7pxeazrkRn8bO6T387As4zdDaM= -github.com/flanksource/duty v1.0.282/go.mod h1:zDVxlWxpSZjbpuBv0i9ke4vlG2elgyKjYY+656XhfPs= +github.com/flanksource/commons v1.22.0 h1:LI839ZOVJ6qrNuGKqb6Z4JyIxFHXGhfDxJbXUj5VimQ= +github.com/flanksource/commons v1.22.0/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY= +github.com/flanksource/duty v1.0.321 h1:ioXzfDZWSWe3rNpj5dadOvItYwwZrJE7aIe8pwUb3qY= +github.com/flanksource/duty v1.0.321/go.mod h1:h8ertbIhbzv2hsDYOraOPWTm3koP/Twwz9RfSXHI5SQ= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.20.30 h1:zRB1DVFIp64QiL6vkJJzVJK4Hj/PazwTNQ6+mDlHaxc= github.com/flanksource/gomplate/v3 v3.20.30/go.mod h1:m2WVc04GMVBOcZhtDaz/LTtrVWKejeJhFM1Jy/h9VZQ= @@ -938,6 +942,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -1185,7 +1190,6 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67NNNpc= @@ -2143,11 +2147,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.5.3 h1:qKGY5CPHOuj47K/VxbCXJfFvIUeqMSXXadqdCY+MbBU= gorm.io/driver/postgres v1.5.3/go.mod h1:F+LtvlFhZT7UBiA81mC9W6Su3D4WUhSboc/36QZU0gk= -gorm.io/driver/sqlite v1.5.0 h1:zKYbzRCpBrT1bNijRnxLDJWPjVfImGEn0lSnUY5gZ+c= gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/plugin/opentelemetry v0.1.4 h1:7p0ocWELjSSRI7NCKPW2mVe6h43YPini99sNJcbsTuc= -gorm.io/plugin/opentelemetry v0.1.4/go.mod h1:tndJHOdvPT0pyGhOb8E2209eXJCUxhC5UpKw7bGVWeI= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod index a396a469..126d32b3 100644 --- a/hack/generate-schemas/go.mod +++ b/hack/generate-schemas/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b - github.com/flanksource/commons v1.20.0 + github.com/flanksource/commons v1.22.0 github.com/flanksource/config-db v0.0.65 github.com/spf13/cobra v1.7.0 ) @@ -21,7 +21,7 @@ require ( github.com/aws/aws-sdk-go v1.49.16 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/flanksource/duty v1.0.282 // indirect + github.com/flanksource/duty v1.0.321 // indirect github.com/flanksource/gomplate/v3 v3.20.30 // indirect github.com/flanksource/is-healthy v1.0.1 // indirect github.com/flanksource/mapstructure v1.6.0 // indirect diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum index de9f25c4..25c220c5 100644 --- a/hack/generate-schemas/go.sum +++ b/hack/generate-schemas/go.sum @@ -239,10 +239,10 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBF github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flanksource/commons v1.20.0 h1:1z7FHGsCY7F7zMx4m0OlzYCUvCc1caREPp7vf1xyeCQ= -github.com/flanksource/commons v1.20.0/go.mod h1:bs2nMwaTCpTQGZNtJv8KNqEVyfGjFZTGUDaAbPHB1cw= -github.com/flanksource/duty v1.0.282 h1:iddsHVcOs6IQ4Wm8U7pxeazrkRn8bO6T387As4zdDaM= -github.com/flanksource/duty v1.0.282/go.mod h1:zDVxlWxpSZjbpuBv0i9ke4vlG2elgyKjYY+656XhfPs= +github.com/flanksource/commons v1.22.0 h1:LI839ZOVJ6qrNuGKqb6Z4JyIxFHXGhfDxJbXUj5VimQ= +github.com/flanksource/commons v1.22.0/go.mod h1:GD5+yGvmYFPIW3WMNN+y1JkeDMJY74e05pQAsRbrvwY= +github.com/flanksource/duty v1.0.321 h1:ioXzfDZWSWe3rNpj5dadOvItYwwZrJE7aIe8pwUb3qY= +github.com/flanksource/duty v1.0.321/go.mod h1:h8ertbIhbzv2hsDYOraOPWTm3koP/Twwz9RfSXHI5SQ= github.com/flanksource/gomplate/v3 v3.20.30 h1:zRB1DVFIp64QiL6vkJJzVJK4Hj/PazwTNQ6+mDlHaxc= github.com/flanksource/gomplate/v3 v3.20.30/go.mod h1:m2WVc04GMVBOcZhtDaz/LTtrVWKejeJhFM1Jy/h9VZQ= github.com/flanksource/is-healthy v1.0.1 h1:cr36GHndzge4rb6GDw+kSh5M6bUpglcZNbqqaTDAZeE= diff --git a/jobs/cleanup.go b/jobs/cleanup.go index 7598efe1..f56e17c6 100644 --- a/jobs/cleanup.go +++ b/jobs/cleanup.go @@ -1,8 +1,11 @@ package jobs import ( + gocontext "context" + "github.com/flanksource/commons/logger" "github.com/flanksource/config-db/db" + "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" ) @@ -19,7 +22,8 @@ var ( ) func DeleteOldConfigAnalysis() { - jobHistory := models.NewJobHistory("DeleteOldConfigAnalysis", "", "").Start() + ctx := context.NewContext(gocontext.Background()).WithDB(db.DefaultDB(), db.Pool) + jobHistory := models.NewJobHistory(ctx.Logger, "DeleteOldConfigAnalysis", "", "").Start() _ = db.PersistJobHistory(jobHistory) if ConfigAnalysisRetentionDays <= 0 { @@ -43,7 +47,8 @@ func DeleteOldConfigAnalysis() { } func DeleteOldConfigChanges() { - jobHistory := models.NewJobHistory("DeleteOldConfigChanges", "", "").Start() + ctx := context.NewContext(gocontext.Background()).WithDB(db.DefaultDB(), db.Pool) + jobHistory := models.NewJobHistory(ctx.Logger, "DeleteOldConfigChanges", "", "").Start() _ = db.PersistJobHistory(jobHistory) if ConfigChangeRetentionDays <= 0 { @@ -67,7 +72,8 @@ func DeleteOldConfigChanges() { } func CleanupConfigItems() { - jobHistory := models.NewJobHistory("CleanupConfigItems", "", "").Start() + ctx := context.NewContext(gocontext.Background()).WithDB(db.DefaultDB(), db.Pool) + jobHistory := models.NewJobHistory(ctx.Logger, "CleanupConfigItems", "", "").Start() _ = db.PersistJobHistory(jobHistory) defer func() { _ = db.PersistJobHistory(jobHistory.End()) diff --git a/jobs/retention.go b/jobs/retention.go index 8593a3db..c3216b66 100644 --- a/jobs/retention.go +++ b/jobs/retention.go @@ -5,7 +5,7 @@ import ( "encoding/json" "github.com/flanksource/commons/logger" - "github.com/flanksource/config-db/api/v1" + v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" "github.com/flanksource/config-db/scrapers" "github.com/flanksource/duty/context" @@ -14,7 +14,7 @@ import ( func ProcessChangeRetentionRules() { ctx := context.NewContext(gocontext.Background()).WithDB(db.DefaultDB(), db.Pool) - jobHistory := models.NewJobHistory("ProcessChangeRetentionRules", "", "").Start() + jobHistory := models.NewJobHistory(ctx.Logger, "ProcessChangeRetentionRules", "", "").Start() _ = db.PersistJobHistory(jobHistory) defer func() { _ = db.PersistJobHistory(jobHistory.End()) diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 6533448c..5df0e2c5 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -284,36 +284,21 @@ func extractResults(ctx context.Context, config v1.Kubernetes, objs []*unstructu env["spec"] = map[string]any{} } - kind, err := f.Kind.Eval(obj.GetLabels(), env) - if err != nil { - return results.Errorf(err, "failed to evaluate kind: %v for config relationship", f.Kind) - } - - name, err := f.Name.Eval(obj.GetLabels(), env) - if err != nil { - return results.Errorf(err, "failed to evaluate name: %v for config relationship", f.Name) - } - - namespace, err := f.Namespace.Eval(obj.GetLabels(), env) - if err != nil { - return results.Errorf(err, "failed to evaluate namespace: %v for config relationship", f.Namespace) - } - - linkedConfigItemIDs, err := db.FindConfigIDsByNamespaceNameClass(ctx, namespace, name, kind) - if err != nil { - return results.Errorf(err, "failed to get linked config items: name=%s, namespace=%s", name, namespace) - } + if selector, err := f.Eval(obj.GetLabels(), env); err != nil { + return results.Errorf(err, "failed to evaluate selector: %v for config relationship", f) + } else if selector != nil { + linkedConfigItemIDs, err := db.FindConfigIDsByNamespaceNameClass(ctx, selector.Namespace, selector.Name, selector.Kind) + if err != nil { + return results.Errorf(err, "failed to get linked config items by kubernetes selector(%v)", selector) + } - for _, id := range linkedConfigItemIDs { - rel := v1.RelationshipResult{ - ConfigExternalID: v1.ExternalID{ - ExternalID: []string{string(obj.GetUID())}, - ConfigType: ConfigTypePrefix + obj.GetKind(), - }, - RelatedConfigID: id.String(), - Relationship: kind + obj.GetKind(), + for _, id := range linkedConfigItemIDs { + rel := v1.RelationshipResult{ + ConfigExternalID: v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: ConfigTypePrefix + obj.GetKind()}, + RelatedConfigID: id.String(), + } + relationships = append(relationships, rel) } - relationships = append(relationships, rel) } } diff --git a/scrapers/processors/json.go b/scrapers/processors/json.go index 15c0ed73..113e6eea 100644 --- a/scrapers/processors/json.go +++ b/scrapers/processors/json.go @@ -242,8 +242,7 @@ func getRelationshipsFromRelationshipConfigs(ctx context.Context, input v1.Scrap for _, itemToLink := range linkedConfigItems { rel := v1.RelationshipResult{ ConfigExternalID: v1.ExternalID{ExternalID: []string{input.ID}, ConfigType: input.Type}, - RelatedConfigID: itemToLink.ID, - Relationship: itemToLink.Type + input.Type, + RelatedConfigID: itemToLink.ID.String(), } relationships = append(relationships, rel) diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index 231a57dc..b107bb96 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -50,7 +50,7 @@ var _ = Describe("Scrapers test", Ordered, func() { scrapeConfig.Spec.Kubernetes[0].Kubeconfig = &types.EnvVar{ ValueStatic: kubeConfigPath, } - scrapeConfig.Spec.Kubernetes[0].Relationships = append(scrapeConfig.Spec.Kubernetes[0].Relationships, v1.KubernetesRelationship{ + scrapeConfig.Spec.Kubernetes[0].Relationships = append(scrapeConfig.Spec.Kubernetes[0].Relationships, v1.KubernetesRelationshipSelectorTemplate{ Kind: v1.RelationshipLookup{Value: "ConfigMap"}, Name: v1.RelationshipLookup{Label: "flanksource/name"}, Namespace: v1.RelationshipLookup{Label: "flanksource/namespace"}, @@ -117,6 +117,8 @@ var _ = Describe("Scrapers test", Ordered, func() { }) It("should correctly setup kubernetes relationship", func() { + duty.CleanCache() + scraperCtx := api.NewScrapeContext(gocontext.TODO(), gormDB, nil).WithScrapeConfig(&scrapeConfig) _, err := RunScraper(scraperCtx) Expect(err).To(BeNil())