From c85e9775d6db3c19bc268c0795dc20ac2e5db092 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Wed, 31 Jan 2024 16:52:32 +0545 Subject: [PATCH] feat: config field exclusion by types * fix: config class and type mapping and added tests * feat: if config class is empty, use the type --- api/v1/common.go | 38 ++-- api/v1/zz_generated.deepcopy.go | 46 ++-- ...configs.flanksource.com_scrapeconfigs.yaml | 198 ++++++++++++------ config/schemas/config_aws.schema.json | 2 +- config/schemas/config_azure.schema.json | 2 +- config/schemas/config_azuredevops.schema.json | 2 +- config/schemas/config_file.schema.json | 2 +- .../schemas/config_githubactions.schema.json | 2 +- config/schemas/config_kubernetes.schema.json | 2 +- .../schemas/config_kubernetesfile.schema.json | 2 +- config/schemas/config_sql.schema.json | 2 +- config/schemas/config_trivy.schema.json | 2 +- config/schemas/scrape_config.schema.json | 2 +- fixtures/expected/file-exclusion.json | 25 +++ fixtures/expected/file-git.json | 1 + fixtures/expected/file-mask.json | 1 + .../expected/file-postgres-properties.json | 125 +++++------ fixtures/expected/file-script-gotemplate.json | 1 + fixtures/expected/file-script.json | 2 + fixtures/file-car.yaml | 1 + fixtures/file-exclusion.yaml | 24 +++ fixtures/file-local-creation-date.yaml | 1 + fixtures/file-local.yaml | 1 + fixtures/file-mask.yaml | 1 + fixtures/file.yaml | 1 + fixtures/kubernetes.yaml | 7 + scrapers/azure/azure.go | 2 +- scrapers/azure/devops/pipelines.go | 8 +- scrapers/github/workflows.go | 4 +- scrapers/kubernetes/events.go | 6 +- scrapers/processors/json.go | 59 ++++-- scrapers/runscrapers_suite_test.go | 11 +- scrapers/runscrapers_test.go | 10 +- utils/match.go | 30 --- 34 files changed, 395 insertions(+), 228 deletions(-) create mode 100644 fixtures/expected/file-exclusion.json create mode 100644 fixtures/file-exclusion.yaml delete mode 100644 utils/match.go diff --git a/api/v1/common.go b/api/v1/common.go index 55a4ae7d..4a3dc209 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -11,8 +11,15 @@ import ( "github.com/flanksource/gomplate/v3" ) -type Filter struct { - JSONPath string `json:"jsonpath,omitempty"` +// ConfigFieldExclusion defines fields with JSONPath that needs to +// be removed from the config. +type ConfigFieldExclusion struct { + // Optionally specify the config types + // from which the JSONPath fields need to be removed. + // If left empty, all config types are considered. + Types []string `json:"types,omitempty"` + + JSONPath string `json:"jsonpath"` } type Script struct { @@ -89,12 +96,15 @@ type TransformChange struct { Exclude []string `json:"exclude,omitempty"` } +func (t *TransformChange) IsEmpty() bool { + return len(t.Exclude) == 0 +} + type Transform struct { - Script Script `yaml:",inline" json:",inline"` - Include []Filter `json:"include,omitempty"` + Script Script `yaml:",inline" json:",inline"` // Fields to remove from the config, useful for removing sensitive data and fields // that change often without a material impact i.e. Last Scraped Time - Exclude []Filter `json:"exclude,omitempty"` + Exclude []ConfigFieldExclusion `json:"exclude,omitempty"` // Masks consist of configurations to replace sensitive fields // with hash functions or static string. Masks MaskList `json:"mask,omitempty"` @@ -102,7 +112,7 @@ type Transform struct { } func (t Transform) IsEmpty() bool { - return t.Script.IsEmpty() && len(t.Include) == 0 && len(t.Exclude) == 0 && t.Masks.IsEmpty() + return t.Script.IsEmpty() && t.Change.IsEmpty() && len(t.Exclude) == 0 && t.Masks.IsEmpty() } func (t Transform) String() string { @@ -112,16 +122,17 @@ func (t Transform) String() string { } if !t.Masks.IsEmpty() { - s += fmt.Sprintf("masks=%s", t.Masks) - } - - if len(t.Include) > 0 { - s += fmt.Sprintf(" include=%s", t.Include) + s += fmt.Sprintf(" masks=%s", t.Masks) } if len(t.Exclude) > 0 { s += fmt.Sprintf(" exclude=%s", t.Exclude) } + + if !t.Change.IsEmpty() { + s += fmt.Sprintf(" change=%s", t.Change) + } + return s } @@ -140,7 +151,10 @@ type BaseScraper struct { // items are extracted first and then the ID,Name,Type and transformations are applied for each item. Items string `json:"items,omitempty"` // A static value or JSONPath expression to use as the type for the resource. - Type string `json:"type,omitempty"` + Type string `json:"type,omitempty"` + // A static value or JSONPath expression to use as the class for the resource. + Class string `json:"class,omitempty"` + Transform Transform `json:"transform,omitempty"` // Format of config item, defaults to JSON, available options are JSON, properties Format string `json:"format,omitempty"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index c30671cd..ad9f88ad 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -244,6 +244,26 @@ func (in *CloudTrail) DeepCopy() *CloudTrail { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigFieldExclusion) DeepCopyInto(out *ConfigFieldExclusion) { + *out = *in + if in.Types != nil { + in, out := &in.Types, &out.Types + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigFieldExclusion. +func (in *ConfigFieldExclusion) DeepCopy() *ConfigFieldExclusion { + if in == nil { + return nil + } + out := new(ConfigFieldExclusion) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigProperties) DeepCopyInto(out *ConfigProperties) { *out = *in @@ -352,21 +372,6 @@ func (in *FileLocation) DeepCopy() *FileLocation { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Filter) DeepCopyInto(out *Filter) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Filter. -func (in *Filter) DeepCopy() *Filter { - if in == nil { - return nil - } - out := new(Filter) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCPConnection) DeepCopyInto(out *GCPConnection) { *out = *in @@ -1138,15 +1143,12 @@ func (in *Template) DeepCopy() *Template { func (in *Transform) DeepCopyInto(out *Transform) { *out = *in out.Script = in.Script - if in.Include != nil { - in, out := &in.Include, &out.Include - *out = make([]Filter, len(*in)) - copy(*out, *in) - } if in.Exclude != nil { in, out := &in.Exclude, &out.Exclude - *out = make([]Filter, len(*in)) - copy(*out, *in) + *out = make([]ConfigFieldExclusion, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Masks != nil { in, out := &in.Masks, &out.Masks diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index d82f434c..5abda241 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -84,6 +84,10 @@ spec: type: object assumeRole: type: string + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string cloudtrail: properties: exclude: @@ -298,22 +302,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -351,6 +359,10 @@ spec: azure: items: properties: + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string clientID: properties: name: @@ -578,22 +590,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -631,6 +647,10 @@ spec: azureDevops: items: properties: + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string connection: type: string createFields: @@ -809,22 +829,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -862,6 +886,10 @@ spec: items: description: File ... properties: + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string connection: description: ConnectionName is used to populate the URL type: string @@ -998,22 +1026,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -1053,6 +1085,10 @@ spec: githubActions: items: properties: + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string connection: description: ConnectionName, if provided, will be used to populate personalAccessToken @@ -1227,22 +1263,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -1287,6 +1327,10 @@ spec: properties: allowIncomplete: type: boolean + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string clusterName: type: string createFields: @@ -1566,22 +1610,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -1617,6 +1665,10 @@ spec: kubernetesFile: items: properties: + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string container: type: string createFields: @@ -1766,22 +1818,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -1941,6 +1997,10 @@ spec: - password - username type: object + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string connection: description: Connection is either the name of the connection to lookup or the connection string itself. @@ -2072,22 +2132,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: @@ -2124,6 +2188,10 @@ spec: trivy: items: properties: + class: + description: A static value or JSONPath expression to use as + the class for the resource. + type: string compliance: items: type: string @@ -2284,22 +2352,26 @@ spec: removing sensitive data and fields that change often without a material impact i.e. Last Scraped Time items: + description: ConfigFieldExclusion defines fields with + JSONPath that needs to be removed from the config. properties: jsonpath: type: string + types: + description: Optionally specify the config types from + which the JSONPath fields need to be removed. If + left empty, all config types are considered. + items: + type: string + type: array + required: + - jsonpath type: object type: array expr: type: string gotemplate: type: string - include: - items: - properties: - jsonpath: - type: string - type: object - type: array javascript: type: string jsonpath: diff --git a/config/schemas/config_aws.schema.json b/config/schemas/config_aws.schema.json index 02e9f53c..cb3707a6 100644 --- a/config/schemas/config_aws.schema.json +++ b/config/schemas/config_aws.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_azure.schema.json b/config/schemas/config_azure.schema.json index b691eeb4..f1d19b42 100644 --- a/config/schemas/config_azure.schema.json +++ b/config/schemas/config_azure.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure","definitions":{"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure","definitions":{"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_azuredevops.schema.json b/config/schemas/config_azuredevops.schema.json index c6d25f48..895a3edd 100644 --- a/config/schemas/config_azuredevops.schema.json +++ b/config/schemas/config_azuredevops.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops","definitions":{"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops","definitions":{"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_file.schema.json b/config/schemas/config_file.schema.json index f5e480ca..536a195c 100644 --- a/config/schemas/config_file.schema.json +++ b/config/schemas/config_file.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_githubactions.schema.json b/config/schemas/config_githubactions.schema.json index bd67847d..351698c2 100644 --- a/config/schemas/config_githubactions.schema.json +++ b/config/schemas/config_githubactions.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_kubernetes.schema.json b/config/schemas/config_kubernetes.schema.json index 38eee80a..dc99e0e1 100644 --- a/config/schemas/config_kubernetes.schema.json +++ b/config/schemas/config_kubernetes.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind","name","namespace"],"properties":{"kind":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipLookup"},"name":{"$ref":"#/definitions/KubernetesRelationshipLookup"},"namespace":{"$ref":"#/definitions/KubernetesRelationshipLookup"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind","name","namespace"],"properties":{"kind":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipLookup"},"name":{"$ref":"#/definitions/KubernetesRelationshipLookup"},"namespace":{"$ref":"#/definitions/KubernetesRelationshipLookup"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_kubernetesfile.schema.json b/config/schemas/config_kubernetesfile.schema.json index e8a3c0b1..24b96fe4 100644 --- a/config/schemas/config_kubernetesfile.schema.json +++ b/config/schemas/config_kubernetesfile.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_sql.schema.json b/config/schemas/config_sql.schema.json index 9e3fd710..db6d7faf 100644 --- a/config/schemas/config_sql.schema.json +++ b/config/schemas/config_sql.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL","definitions":{"Authentication":{"required":["username","password"],"properties":{"username":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL","definitions":{"Authentication":{"required":["username","password"],"properties":{"username":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/config_trivy.schema.json b/config/schemas/config_trivy.schema.json index b55caa63..cab31979 100644 --- a/config/schemas/config_trivy.schema.json +++ b/config/schemas/config_trivy.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy","definitions":{"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index b4daa735..5790aacf 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ChangeRetentionSpec":{"properties":{"name":{"type":"string"},"age":{"type":"string"},"count":{"type":"integer"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"Filter":{"properties":{"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind","name","namespace"],"properties":{"kind":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipLookup"},"name":{"$ref":"#/definitions/KubernetesRelationshipLookup"},"namespace":{"$ref":"#/definitions/KubernetesRelationshipLookup"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"RetentionSpec":{"properties":{"changes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ChangeRetentionSpec"},"type":"array"},"types":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeRetentionSpec"},"type":"array"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"retention":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RetentionSpec"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"include":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Filter"},"type":"array"},"exclude":{"items":{"$ref":"#/definitions/Filter"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeRetentionSpec":{"properties":{"name":{"type":"string"},"createdAge":{"type":"string"},"updatedAge":{"type":"string"},"deletedAge":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfig","definitions":{"AWS":{"required":["BaseScraper","AWSConnection"],"properties":{"BaseScraper":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/BaseScraper"},"AWSConnection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWSConnection"},"patch_states":{"type":"boolean"},"patch_details":{"type":"boolean"},"inventory":{"type":"boolean"},"compliance":{"type":"boolean"},"cloudtrail":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CloudTrail"},"trusted_advisor_check":{"type":"boolean"},"include":{"items":{"type":"string"},"type":"array"},"exclude":{"items":{"type":"string"},"type":"array"},"cost_reporting":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/CostReporting"}},"additionalProperties":false,"type":"object"},"AWSConnection":{"required":["region"],"properties":{"connection":{"type":"string"},"accessKey":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVar"},"secretKey":{"$ref":"#/definitions/EnvVar"},"region":{"items":{"type":"string"},"type":"array"},"endpoint":{"type":"string"},"skipTLSVerify":{"type":"boolean"},"assumeRole":{"type":"string"}},"additionalProperties":false,"type":"object"},"Authentication":{"required":["username","password"],"properties":{"username":{"$ref":"#/definitions/EnvVar"},"password":{"$ref":"#/definitions/EnvVar"}},"additionalProperties":false,"type":"object"},"Azure":{"required":["BaseScraper","subscriptionID","organisation","tenantID"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"subscriptionID":{"type":"string"},"organisation":{"type":"string"},"clientID":{"$ref":"#/definitions/EnvVar"},"clientSecret":{"$ref":"#/definitions/EnvVar"},"tenantID":{"type":"string"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureExclusions"}},"additionalProperties":false,"type":"object"},"AzureDevops":{"required":["BaseScraper","projects","pipelines"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"connection":{"type":"string"},"organization":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"projects":{"items":{"type":"string"},"type":"array"},"pipelines":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"AzureExclusions":{"properties":{"activityLogs":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"BaseScraper":{"properties":{"id":{"type":"string"},"name":{"type":"string"},"items":{"type":"string"},"type":{"type":"string"},"class":{"type":"string"},"transform":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Transform"},"format":{"type":"string"},"timestampFormat":{"type":"string"},"createFields":{"items":{"type":"string"},"type":"array"},"deleteFields":{"items":{"type":"string"},"type":"array"},"tags":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"properties":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigProperties"},"type":"array"}},"additionalProperties":false,"type":"object"},"ChangeRetentionSpec":{"properties":{"name":{"type":"string"},"age":{"type":"string"},"count":{"type":"integer"}},"additionalProperties":false,"type":"object"},"CloudTrail":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"},"max_age":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigFieldExclusion":{"required":["jsonpath"],"properties":{"types":{"items":{"type":"string"},"type":"array"},"jsonpath":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigMapKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"ConfigProperties":{"properties":{"label":{"type":"string"},"name":{"type":"string"},"tooltip":{"type":"string"},"icon":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"order":{"type":"integer"},"headline":{"type":"boolean"},"text":{"type":"string"},"value":{"type":"integer"},"unit":{"type":"string"},"max":{"type":"integer"},"min":{"type":"integer"},"status":{"type":"string"},"lastTransition":{"type":"string"},"links":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Link"},"type":"array"},"filter":{"type":"string"}},"additionalProperties":false,"type":"object"},"Connection":{"required":["connection"],"properties":{"connection":{"type":"string"},"auth":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Authentication"}},"additionalProperties":false,"type":"object"},"CostReporting":{"properties":{"s3_bucket_path":{"type":"string"},"table":{"type":"string"},"database":{"type":"string"},"region":{"type":"string"}},"additionalProperties":false,"type":"object"},"EnvVar":{"properties":{"name":{"type":"string"},"value":{"type":"string"},"valueFrom":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/EnvVarSource"}},"additionalProperties":false,"type":"object"},"EnvVarSource":{"properties":{"serviceAccount":{"type":"string"},"helmRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/HelmRefKeySelector"},"configMapKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigMapKeySelector"},"secretKeyRef":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SecretKeySelector"}},"additionalProperties":false,"type":"object"},"FieldsV1":{"properties":{},"additionalProperties":false,"type":"object"},"File":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"url":{"type":"string"},"paths":{"items":{"type":"string"},"type":"array"},"ignore":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"},"icon":{"type":"string"},"connection":{"type":"string"}},"additionalProperties":false,"type":"object"},"GitHubActions":{"required":["BaseScraper","owner","repository","personalAccessToken","workflows"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"owner":{"type":"string"},"repository":{"type":"string"},"personalAccessToken":{"$ref":"#/definitions/EnvVar"},"connection":{"type":"string"},"workflows":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"HelmRefKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"Kubernetes":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"clusterName":{"type":"string"},"namespace":{"type":"string"},"useCache":{"type":"boolean"},"allowIncomplete":{"type":"boolean"},"scope":{"type":"string"},"since":{"type":"string"},"selector":{"type":"string"},"fieldSelector":{"type":"string"},"maxInflight":{"type":"integer"},"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesExclusionConfig"},"kubeconfig":{"$ref":"#/definitions/EnvVar"},"event":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventConfig"},"relationships":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationship"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesEventConfig":{"properties":{"exclusions":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesEventExclusions"},"severityKeywords":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SeverityKeywords"}},"additionalProperties":false,"type":"object"},"KubernetesEventExclusions":{"properties":{"name":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"reason":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesExclusionConfig":{"required":["name","kind","namespace"],"properties":{"name":{"items":{"type":"string"},"type":"array"},"kind":{"items":{"type":"string"},"type":"array"},"namespace":{"items":{"type":"string"},"type":"array"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"}},"additionalProperties":false,"type":"object"},"KubernetesFile":{"required":["BaseScraper","selector"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"selector":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ResourceSelector"},"container":{"type":"string"},"files":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/PodFile"},"type":"array"}},"additionalProperties":false,"type":"object"},"KubernetesRelationship":{"required":["kind","name","namespace"],"properties":{"kind":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesRelationshipLookup"},"name":{"$ref":"#/definitions/KubernetesRelationshipLookup"},"namespace":{"$ref":"#/definitions/KubernetesRelationshipLookup"}},"additionalProperties":false,"type":"object"},"KubernetesRelationshipLookup":{"properties":{"expr":{"type":"string"},"value":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Link":{"required":["Text"],"properties":{"type":{"type":"string"},"url":{"type":"string"},"Text":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Text"}},"additionalProperties":false,"type":"object"},"ManagedFieldsEntry":{"properties":{"manager":{"type":"string"},"operation":{"type":"string"},"apiVersion":{"type":"string"},"time":{"$ref":"#/definitions/Time"},"fieldsType":{"type":"string"},"fieldsV1":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/FieldsV1"},"subresource":{"type":"string"}},"additionalProperties":false,"type":"object"},"Mask":{"properties":{"selector":{"type":"string"},"jsonpath":{"type":"string"},"value":{"type":"string"}},"additionalProperties":false,"type":"object"},"ObjectMeta":{"properties":{"name":{"type":"string"},"generateName":{"type":"string"},"namespace":{"type":"string"},"selfLink":{"type":"string"},"uid":{"type":"string"},"resourceVersion":{"type":"string"},"generation":{"type":"integer"},"creationTimestamp":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Time"},"deletionTimestamp":{"$ref":"#/definitions/Time"},"deletionGracePeriodSeconds":{"type":"integer"},"labels":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"annotations":{"patternProperties":{".*":{"type":"string"}},"type":"object"},"ownerReferences":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/OwnerReference"},"type":"array"},"finalizers":{"items":{"type":"string"},"type":"array"},"managedFields":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ManagedFieldsEntry"},"type":"array"}},"additionalProperties":false,"type":"object"},"OwnerReference":{"required":["apiVersion","kind","name","uid"],"properties":{"apiVersion":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"uid":{"type":"string"},"controller":{"type":"boolean"},"blockOwnerDeletion":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"PodFile":{"properties":{"path":{"items":{"type":"string"},"type":"array"},"format":{"type":"string"}},"additionalProperties":false,"type":"object"},"ResourceSelector":{"properties":{"namespace":{"type":"string"},"kind":{"type":"string"},"name":{"type":"string"},"labelSelector":{"type":"string"},"fieldSelector":{"type":"string"}},"additionalProperties":false,"type":"object"},"RetentionSpec":{"properties":{"changes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ChangeRetentionSpec"},"type":"array"},"types":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeRetentionSpec"},"type":"array"}},"additionalProperties":false,"type":"object"},"SQL":{"required":["BaseScraper","Connection","query"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"Connection":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Connection"},"driver":{"type":"string"},"query":{"type":"string"}},"additionalProperties":false,"type":"object"},"ScrapeConfig":{"required":["TypeMeta"],"properties":{"TypeMeta":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TypeMeta"},"metadata":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ObjectMeta"},"spec":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScraperSpec"},"status":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ScrapeConfigStatus"}},"additionalProperties":false,"type":"object"},"ScrapeConfigStatus":{"properties":{"observedGeneration":{"type":"integer"}},"additionalProperties":false,"type":"object"},"ScraperSpec":{"properties":{"logLevel":{"type":"string"},"schedule":{"type":"string"},"aws":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AWS"},"type":"array"},"file":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/File"},"type":"array"},"kubernetes":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Kubernetes"},"type":"array"},"kubernetesFile":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/KubernetesFile"},"type":"array"},"azureDevops":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/AzureDevops"},"type":"array"},"githubActions":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/GitHubActions"},"type":"array"},"azure":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Azure"},"type":"array"},"sql":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/SQL"},"type":"array"},"trivy":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Trivy"},"type":"array"},"retention":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/RetentionSpec"},"full":{"type":"boolean"}},"additionalProperties":false,"type":"object"},"SecretKeySelector":{"required":["key"],"properties":{"name":{"type":"string"},"key":{"type":"string"}},"additionalProperties":false,"type":"object"},"SeverityKeywords":{"properties":{"warn":{"items":{"type":"string"},"type":"array"},"error":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Text":{"properties":{"tooltip":{"type":"string"},"icon":{"type":"string"},"text":{"type":"string"},"label":{"type":"string"}},"additionalProperties":false,"type":"object"},"Time":{"properties":{},"additionalProperties":false,"type":"object"},"Transform":{"properties":{"gotemplate":{"type":"string"},"jsonpath":{"type":"string"},"expr":{"type":"string"},"javascript":{"type":"string"},"exclude":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/ConfigFieldExclusion"},"type":"array"},"mask":{"items":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/Mask"},"type":"array"},"changes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TransformChange"}},"additionalProperties":false,"type":"object"},"TransformChange":{"properties":{"exclude":{"items":{"type":"string"},"type":"array"}},"additionalProperties":false,"type":"object"},"Trivy":{"required":["BaseScraper"],"properties":{"BaseScraper":{"$ref":"#/definitions/BaseScraper"},"version":{"type":"string"},"compliance":{"items":{"type":"string"},"type":"array"},"ignoredLicenses":{"items":{"type":"string"},"type":"array"},"ignoreUnfixed":{"type":"boolean"},"licenseFull":{"type":"boolean"},"severity":{"items":{"type":"string"},"type":"array"},"vulnType":{"items":{"type":"string"},"type":"array"},"scanners":{"items":{"type":"string"},"type":"array"},"timeout":{"type":"string"},"kubernetes":{"$schema":"http://json-schema.org/draft-04/schema#","$ref":"#/definitions/TrivyK8sOptions"}},"additionalProperties":false,"type":"object"},"TrivyK8sOptions":{"properties":{"components":{"items":{"type":"string"},"type":"array"},"context":{"type":"string"},"kubeconfig":{"type":"string"},"namespace":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeMeta":{"properties":{"kind":{"type":"string"},"apiVersion":{"type":"string"}},"additionalProperties":false,"type":"object"},"TypeRetentionSpec":{"properties":{"name":{"type":"string"},"createdAge":{"type":"string"},"updatedAge":{"type":"string"},"deletedAge":{"type":"string"}},"additionalProperties":false,"type":"object"}}} \ No newline at end of file diff --git a/fixtures/expected/file-exclusion.json b/fixtures/expected/file-exclusion.json new file mode 100644 index 00000000..4e64b525 --- /dev/null +++ b/fixtures/expected/file-exclusion.json @@ -0,0 +1,25 @@ +[ + { + "last_modified": "0001-01-01T00:00:00Z", + "config_type": "Config1", + "config_class": "MySecrets", + "name": "Config1", + "id": "1", + "config": { + "id": 1, + "name": "Config1", + "secret": "secret_1" + } + }, + { + "last_modified": "0001-01-01T00:00:00Z", + "config_type": "Config2", + "config_class": "MySecrets", + "name": "Config2", + "id": "2", + "config": { + "id": 2, + "name": "Config2" + } + } +] diff --git a/fixtures/expected/file-git.json b/fixtures/expected/file-git.json index 259fea05..11bc2928 100644 --- a/fixtures/expected/file-git.json +++ b/fixtures/expected/file-git.json @@ -3,6 +3,7 @@ "last_modified": "0001-01-01T00:00:00Z", "source": "github.com/flanksource/canary-checker/fixtures/minimal/http_pass_single.yaml", "id": "http-pass-single", + "config_type": "Canary", "config_class": "Canary", "config": { "apiVersion": "canaries.flanksource.com/v1", diff --git a/fixtures/expected/file-mask.json b/fixtures/expected/file-mask.json index 4dc970d5..06e86697 100644 --- a/fixtures/expected/file-mask.json +++ b/fixtures/expected/file-mask.json @@ -1,6 +1,7 @@ [{ "last_modified": "0001-01-01T00:00:00Z", "config_class": "Config", + "config_type": "Config", "name": "Config1", "id": "1", "config": { diff --git a/fixtures/expected/file-postgres-properties.json b/fixtures/expected/file-postgres-properties.json index 55e2e5f9..b7b50a98 100644 --- a/fixtures/expected/file-postgres-properties.json +++ b/fixtures/expected/file-postgres-properties.json @@ -1,61 +1,64 @@ -[{ - "last_modified": "0001-01-01T00:00:00Z", - "id": "postgresql-properties", - "config_class": "PostgreSQLProperties", - "config":{ - "authentication_timeout": "1min", - "checkpoint_completion_target": "0.5", - "checkpoint_flush_after": "256kB", - "cluster_name": "'main'", - "cron.database_name": "'postgres'", - "data_directory": "'/var/lib/postgresql/data'", - "db_user_namespace": "off", - "default_text_search_config": "'pg_catalog.english'", - "effective_cache_size": "128MB", - "extra_float_digits": "0", - "hba_file": "'/etc/postgresql/pg_hba.conf'", - "ident_file": "'/etc/postgresql/pg_ident.conf'", - "jit_provider": "'llvmjit'", - "lc_messages": "'en_US.UTF-8'", - "lc_monetary": "'en_US.UTF-8'", - "lc_numeric": "'en_US.UTF-8'", - "lc_time": "'en_US.UTF-8'", - "listen_addresses": "'*'", - "log_destination": "'csvlog'", - "log_directory": "'/var/log/postgresql'", - "log_file_mode": "0640", - "log_filename": "'postgresql.log'", - "log_line_prefix": "'%h %m [%p] %q%u@%d '", - "log_rotation_age": "0", - "log_rotation_size": "0", - "log_statement": "'all'", - "log_timezone": "'UTC'", - "logging_collector": "on", - "max_replication_slots": "5", - "max_slot_wal_keep_size": "1024", - "max_wal_senders": "10", - "password_encryption": "scram-sha-256", - "pgsodium.getkey_script": "'/usr/lib/postgresql/14/bin/pgsodium_getkey_urandom.sh'", - "pljava.libjvm_location": "'/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'", - "row_security": "on", - "shared_buffers": "128MB", - "shared_preload_libraries": "'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium'", - "ssl": "off", - "ssl_ca_file": "''", - "ssl_cert_file": "''", - "ssl_ciphers": "'HIGH:MEDIUM:+3DES:!aNULL'", - "ssl_crl_dir": "''", - "ssl_crl_file": "''", - "ssl_dh_params_file": "''", - "ssl_ecdh_curve": "'prime256v1'", - "ssl_key_file": "''", - "ssl_max_protocol_version": "''", - "ssl_min_protocol_version": "'TLSv1.2'", - "ssl_passphrase_command": "''", - "ssl_passphrase_command_supports_reload": "off", - "ssl_prefer_server_ciphers": "on", - "timezone": "'UTC'", - "unix_socket_directories": "'/var/run/postgresql'", - "wal_level": "logical" - } -}] +[ + { + "last_modified": "0001-01-01T00:00:00Z", + "id": "postgresql-properties", + "config_class": "PostgreSQLProperties", + "config_type": "PostgreSQLProperties", + "config": { + "authentication_timeout": "1min", + "checkpoint_completion_target": "0.5", + "checkpoint_flush_after": "256kB", + "cluster_name": "'main'", + "cron.database_name": "'postgres'", + "data_directory": "'/var/lib/postgresql/data'", + "db_user_namespace": "off", + "default_text_search_config": "'pg_catalog.english'", + "effective_cache_size": "128MB", + "extra_float_digits": "0", + "hba_file": "'/etc/postgresql/pg_hba.conf'", + "ident_file": "'/etc/postgresql/pg_ident.conf'", + "jit_provider": "'llvmjit'", + "lc_messages": "'en_US.UTF-8'", + "lc_monetary": "'en_US.UTF-8'", + "lc_numeric": "'en_US.UTF-8'", + "lc_time": "'en_US.UTF-8'", + "listen_addresses": "'*'", + "log_destination": "'csvlog'", + "log_directory": "'/var/log/postgresql'", + "log_file_mode": "0640", + "log_filename": "'postgresql.log'", + "log_line_prefix": "'%h %m [%p] %q%u@%d '", + "log_rotation_age": "0", + "log_rotation_size": "0", + "log_statement": "'all'", + "log_timezone": "'UTC'", + "logging_collector": "on", + "max_replication_slots": "5", + "max_slot_wal_keep_size": "1024", + "max_wal_senders": "10", + "password_encryption": "scram-sha-256", + "pgsodium.getkey_script": "'/usr/lib/postgresql/14/bin/pgsodium_getkey_urandom.sh'", + "pljava.libjvm_location": "'/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'", + "row_security": "on", + "shared_buffers": "128MB", + "shared_preload_libraries": "'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium'", + "ssl": "off", + "ssl_ca_file": "''", + "ssl_cert_file": "''", + "ssl_ciphers": "'HIGH:MEDIUM:+3DES:!aNULL'", + "ssl_crl_dir": "''", + "ssl_crl_file": "''", + "ssl_dh_params_file": "''", + "ssl_ecdh_curve": "'prime256v1'", + "ssl_key_file": "''", + "ssl_max_protocol_version": "''", + "ssl_min_protocol_version": "'TLSv1.2'", + "ssl_passphrase_command": "''", + "ssl_passphrase_command_supports_reload": "off", + "ssl_prefer_server_ciphers": "on", + "timezone": "'UTC'", + "unix_socket_directories": "'/var/run/postgresql'", + "wal_level": "logical" + } + } +] diff --git a/fixtures/expected/file-script-gotemplate.json b/fixtures/expected/file-script-gotemplate.json index ac4443ce..cfb345de 100644 --- a/fixtures/expected/file-script-gotemplate.json +++ b/fixtures/expected/file-script-gotemplate.json @@ -1,6 +1,7 @@ [ { "config_class": "MyConfig", + "config_type": "MyConfig", "name": "scraped", "id": "1", "config": { diff --git a/fixtures/expected/file-script.json b/fixtures/expected/file-script.json index deec07d0..045c73a2 100644 --- a/fixtures/expected/file-script.json +++ b/fixtures/expected/file-script.json @@ -2,6 +2,7 @@ { "last_modified": "0001-01-01T00:00:00Z", "config_class": "Config", + "config_type": "Config", "name": "Config1", "id": "1", "config": { @@ -15,6 +16,7 @@ { "last_modified": "0001-01-01T00:00:00Z", "config_class": "Config", + "config_type": "Config", "name": "Config2", "id": "2", "config": { diff --git a/fixtures/file-car.yaml b/fixtures/file-car.yaml index b09d6e44..020e2c1a 100644 --- a/fixtures/file-car.yaml +++ b/fixtures/file-car.yaml @@ -5,6 +5,7 @@ metadata: spec: file: - type: Car + class: Car id: $.reg_no paths: - fixtures/data/car.json diff --git a/fixtures/file-exclusion.yaml b/fixtures/file-exclusion.yaml new file mode 100644 index 00000000..f32a450b --- /dev/null +++ b/fixtures/file-exclusion.yaml @@ -0,0 +1,24 @@ +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: exclude-file-secrets +spec: + file: + - type: $.name + id: $.id + name: $.name + class: MySecrets + transform: + script: + javascript: |+ + for (var i = 0; i < config.length; i++) { + config[i].id = i + 1 + } + JSON.stringify(config) + exclude: + - jsonpath: '.password' + - types: + - Config2 + jsonpath: '.secret' + paths: + - fixtures/data/multiple-configs.json diff --git a/fixtures/file-local-creation-date.yaml b/fixtures/file-local-creation-date.yaml index c12be9e2..8757cd94 100644 --- a/fixtures/file-local-creation-date.yaml +++ b/fixtures/file-local-creation-date.yaml @@ -5,6 +5,7 @@ metadata: spec: file: - type: $.aws[0].region + class: AWS id: $.aws[0].region createFields: - $.aws[0].made_at diff --git a/fixtures/file-local.yaml b/fixtures/file-local.yaml index 76da59b0..2b53951e 100644 --- a/fixtures/file-local.yaml +++ b/fixtures/file-local.yaml @@ -5,6 +5,7 @@ metadata: spec: file: - type: $.aws[0].region + class: $.aws[0].region id: $.aws[0].region paths: - fixtures/data/test.yaml diff --git a/fixtures/file-mask.yaml b/fixtures/file-mask.yaml index eb9b9da2..2b3da629 100644 --- a/fixtures/file-mask.yaml +++ b/fixtures/file-mask.yaml @@ -17,3 +17,4 @@ spec: value: '***' paths: - fixtures/data/single-config.json + diff --git a/fixtures/file.yaml b/fixtures/file.yaml index 5de77d2c..83faa0a4 100644 --- a/fixtures/file.yaml +++ b/fixtures/file.yaml @@ -5,6 +5,7 @@ metadata: spec: file: - type: $.Config.InstanceType + class: $.Config.InstanceType id: $.Config.InstanceId path: - config*.json diff --git a/fixtures/kubernetes.yaml b/fixtures/kubernetes.yaml index 2549505e..5ee3c1ed 100644 --- a/fixtures/kubernetes.yaml +++ b/fixtures/kubernetes.yaml @@ -14,6 +14,13 @@ spec: - selector: 'config_type == "Kubernetes::Certificate"' jsonpath: .spec.commonName value: md5sum + exclude: + - types: + - Kubernetes::* + jsonpath: '.metadata.ownerReferences' + - types: + - Kubernetes::Pod + jsonpath: '.metadata.generateName' changes: exclude: - 'details.source.component == "canary-checker" && details.reason == "Failed"' diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index f81f310c..b874a216 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -61,7 +61,7 @@ var activityLogFilter = strings.Join([]string{ "subStatus", }, ",") -var defaultExcludes = []v1.Filter{ +var defaultExcludes = []v1.ConfigFieldExclusion{ {JSONPath: "$..etag"}, // Remove etags from the config json as they produce unecessary changes. } diff --git a/scrapers/azure/devops/pipelines.go b/scrapers/azure/devops/pipelines.go index 387befdf..650f7778 100644 --- a/scrapers/azure/devops/pipelines.go +++ b/scrapers/azure/devops/pipelines.go @@ -3,9 +3,9 @@ package devops import ( "fmt" + "github.com/flanksource/commons/collections" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" - "github.com/flanksource/config-db/utils" ) const PipelineRun = "AzureDevops::PipelineRun" @@ -33,9 +33,9 @@ func (ado AzureDevopsScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { results.Errorf(err, "failed to get projects for %s", config.Organization) continue } - for _, project := range projects { - if !utils.MatchItems(project.Name, config.Projects...) { + for _, project := range projects { + if !collections.MatchItems(project.Name, config.Projects...) { continue } @@ -48,7 +48,7 @@ func (ado AzureDevopsScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { for _, _pipeline := range pipelines { var pipeline = _pipeline - if !utils.MatchItems(pipeline.Name, config.Pipelines...) { + if !collections.MatchItems(pipeline.Name, config.Pipelines...) { continue } diff --git a/scrapers/github/workflows.go b/scrapers/github/workflows.go index 92462ac4..0cc33982 100644 --- a/scrapers/github/workflows.go +++ b/scrapers/github/workflows.go @@ -4,10 +4,10 @@ import ( "fmt" "math" + "github.com/flanksource/commons/collections" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/db" - "github.com/flanksource/config-db/utils" ) const WorkflowRun = "GitHubActions::WorkflowRun" @@ -36,7 +36,7 @@ func (gh GithubActionsScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { } for _, workflow := range workflows { - if !utils.MatchItems(workflow.Name, config.Workflows...) { + if !collections.MatchItems(workflow.Name, config.Workflows...) { continue } runs, err := getNewWorkflowRuns(client, workflow) diff --git a/scrapers/kubernetes/events.go b/scrapers/kubernetes/events.go index 8c172a04..851bb372 100644 --- a/scrapers/kubernetes/events.go +++ b/scrapers/kubernetes/events.go @@ -5,19 +5,19 @@ import ( "sort" "strings" + "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" v1 "github.com/flanksource/config-db/api/v1" - "github.com/flanksource/config-db/utils" "github.com/google/uuid" "k8s.io/apimachinery/pkg/types" ) func getSeverityFromReason(reason string, errKeywords, warnKeywords []string) string { - if utils.MatchItems(reason, errKeywords...) { + if collections.MatchItems(reason, errKeywords...) { return "error" } - if utils.MatchItems(reason, warnKeywords...) { + if collections.MatchItems(reason, warnKeywords...) { return "warn" } diff --git a/scrapers/processors/json.go b/scrapers/processors/json.go index d57299f1..9589a550 100644 --- a/scrapers/processors/json.go +++ b/scrapers/processors/json.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/duty/types" @@ -44,13 +45,19 @@ type Transform struct { Masks []Mask } +// ConfigFieldExclusion instructs what fields from the given config types should be removed. +type ConfigFieldExclusion struct { + jp jp.Expr + configTypes []string +} + type Extract struct { - ID, Type, Name jp.Expr - CreatedAt, DeletedAt []jp.Expr - Items *jp.Expr - Config v1.BaseScraper - Excludes []jp.Expr - Transform Transform + ID, Type, Class, Name jp.Expr + CreatedAt, DeletedAt []jp.Expr + Items *jp.Expr + Config v1.BaseScraper + Excludes []ConfigFieldExclusion + Transform Transform } func (e Extract) WithoutItems() Extract { @@ -92,6 +99,13 @@ func NewExtractor(config v1.BaseScraper) (Extract, error) { extract.Type = x } } + if isJSONPath(config.Class) { + if x, err := jp.ParseString(config.Class); err != nil { + return extract, fmt.Errorf("failed to parse class: %s: %v", config.Class, err) + } else { + extract.Class = x + } + } if config.Items != "" { if x, err := jp.ParseString(config.Items); err != nil { return extract, fmt.Errorf("failed to parse items: %s: %v", config.Items, err) @@ -125,10 +139,10 @@ func NewExtractor(config v1.BaseScraper) (Extract, error) { } for _, exclude := range config.Transform.Exclude { - if x, err := jp.ParseString(exclude.JSONPath); err != nil { + if expr, err := jp.ParseString(exclude.JSONPath); err != nil { return extract, fmt.Errorf("failed to parse exclude: %s: %v", exclude.JSONPath, err) } else { - extract.Excludes = append(extract.Excludes, x) + extract.Excludes = append(extract.Excludes, ConfigFieldExclusion{jp: expr, configTypes: exclude.Types}) } } @@ -162,6 +176,9 @@ func (e Extract) String() string { if e.Type != nil { s += fmt.Sprintf(" Type: %s", e.Type) } + if e.Class != nil { + s += fmt.Sprintf(" Class: %s", e.Class) + } if e.Name != nil { s += fmt.Sprintf(" Name: %s", e.Name) } @@ -375,20 +392,38 @@ func (e Extract) extractAttributes(input v1.ScrapeResult) (v1.ScrapeResult, erro input.Name = input.ID } + if input.Type == "" { + input.Type, err = getString(e.Type, input.Config, e.Config.Type) + if err != nil { + return input, err + } + } + + if input.Type == "" { + return input, fmt.Errorf("no config type defined for: %s", input) + } + if input.ConfigClass == "" { - input.ConfigClass, err = getString(e.Type, input.Config, e.Config.Type) + defaultClass := e.Config.Class + if defaultClass == "" { + defaultClass = input.Type + } + + input.ConfigClass, err = getString(e.Class, input.Config, defaultClass) if err != nil { return input, err } } if input.ConfigClass == "" { - return input, fmt.Errorf("no type defined for: %s", input) + return input, fmt.Errorf("no class defined for: %s", input) } for _, exclude := range e.Excludes { - if err := exclude.Del(input.Config); err != nil { - return input, err + if len(exclude.configTypes) == 0 || collections.MatchItems(input.Type, exclude.configTypes...) { + if err := exclude.jp.Del(input.Config); err != nil { + return input, err + } } } diff --git a/scrapers/runscrapers_suite_test.go b/scrapers/runscrapers_suite_test.go index cb6b91d5..fa24aba2 100644 --- a/scrapers/runscrapers_suite_test.go +++ b/scrapers/runscrapers_suite_test.go @@ -31,6 +31,8 @@ func TestRunScrapers(t *testing.T) { var ( postgres *epg.EmbeddedPostgres gormDB *gorm.DB + + DefaultContext *context.Context ) const ( @@ -47,15 +49,14 @@ var _ = BeforeSuite(func() { } logger.Infof("Started postgres on port %d", pgPort) - if _, err := duty.NewDB(pgUrl); err != nil { - Fail(err.Error()) - } + DefaultContext, err := duty.InitDB(pgUrl, nil) + Expect(err).ToNot(HaveOccurred()) + if err := db.Init(context.Background(), pgUrl); err != nil { Fail(err.Error()) } - gormDB, err = duty.NewGorm(pgUrl, duty.DefaultGormConfig()) - Expect(err).ToNot(HaveOccurred()) + gormDB = DefaultContext.DB() if err := os.Chdir(".."); err != nil { Fail(err.Error()) diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index fcc3fe14..24c475db 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -19,6 +19,7 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -137,6 +138,7 @@ var _ = Describe("Scrapers test", Ordered, func() { "file-script", "file-script-gotemplate", "file-mask", + "file-exclusion", "file-postgres-properties", } @@ -163,6 +165,7 @@ var _ = Describe("Scrapers test", Ordered, func() { got := results[i] Expect(want.ID).To(Equal(got.ID)) + Expect(want.Type).To(Equal(got.Type)) Expect(want.ConfigClass).To(Equal(got.ConfigClass)) wantJSON, _ := json.Marshal(want.Config) gotJSON, _ := json.Marshal(got.Config) @@ -190,7 +193,7 @@ var _ = Describe("Scrapers test", Ordered, func() { Expect(err).To(BeNil()) configItemID, err := db.FindConfigItemID(v1.ExternalID{ - ConfigType: "", // Comes from file-car.yaml + ConfigType: "Car", // Comes from file-car.yaml ExternalID: []string{"A123"}, // Comes from the config mentioned in file-car.yaml }) Expect(err).To(BeNil()) @@ -213,7 +216,7 @@ var _ = Describe("Scrapers test", Ordered, func() { Expect(err).To(BeNil()) configItemID, err := db.FindConfigItemID(v1.ExternalID{ - ConfigType: "", // Comes from file-car.yaml + ConfigType: "Car", // Comes from file-car.yaml ExternalID: []string{"A123"}, // Comes from the config mentioned in file-car.yaml }) Expect(err).To(BeNil()) @@ -228,7 +231,7 @@ var _ = Describe("Scrapers test", Ordered, func() { It("should not change the original config", func() { configItemID, err := db.FindConfigItemID(v1.ExternalID{ - ConfigType: "", // Comes from file-car.yaml + ConfigType: "Car", // Comes from file-car.yaml ExternalID: []string{"A123"}, // Comes from the config mentioned in file-car.yaml }) Expect(err).To(BeNil()) @@ -254,6 +257,7 @@ var _ = Describe("Scrapers test", Ordered, func() { dummyCI := models.ConfigItem{ ID: configItemID, ConfigClass: "Test", + Type: lo.ToPtr("Test"), ScraperID: &dummyScraper.ID, } configItemID2 := uuid.New().String() diff --git a/utils/match.go b/utils/match.go deleted file mode 100644 index 7bf873f2..00000000 --- a/utils/match.go +++ /dev/null @@ -1,30 +0,0 @@ -package utils - -import "strings" - -// matchItems returns true if any of the items in the list match the item -// negative matches are supported by prefixing the item with a ! -// * matches everything -func MatchItems(item string, items ...string) bool { - if len(items) == 0 { - return true - } - - for _, i := range items { - if strings.HasPrefix(i, "!") { - if item == strings.TrimPrefix(i, "!") { - return false - } - } - } - - for _, i := range items { - if strings.HasPrefix(i, "!") { - continue - } - if i == "*" || item == i { - return true - } - } - return false -}