From 0dc84a78785a5b49cea114e5c19f6b40539598f8 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 11 Jan 2024 09:37:44 +0545 Subject: [PATCH] fix: perform deep clone of config properties --- api/v1/common.go | 3 ++- .../configs.flanksource.com_scrapeconfigs.yaml | 18 +++++++++--------- fixtures/aws.yaml | 7 +++++++ scrapers/processors/json.go | 11 ++++++++--- utils/struct.go | 13 +++++++++++++ 5 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 utils/struct.go diff --git a/api/v1/common.go b/api/v1/common.go index ee8f9807..c71156fc 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -166,7 +166,8 @@ type BaseScraper struct { // Tags allow you to set custom tags on the scraped config items. Tags JSONStringMap `json:"tags,omitempty"` - // Properties are custom templatable properties for the scraped config items. + // Properties are custom templatable properties for the scraped config items + // grouped by the config type. Properties map[string]types.Properties `json:"properties,omitempty" template:"true"` } diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index cae8c728..2f39bbd1 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -222,7 +222,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object region: items: @@ -550,7 +550,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object subscriptionID: type: string @@ -788,7 +788,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object tags: additionalProperties: @@ -980,7 +980,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object tags: additionalProperties: @@ -1210,7 +1210,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object repository: type: string @@ -1476,7 +1476,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object relationships: description: Relationships specify the fields to use to relate @@ -1716,7 +1716,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object selector: properties: @@ -2023,7 +2023,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object query: type: string @@ -2230,7 +2230,7 @@ spec: type: object type: array description: Properties are custom templatable properties for - the scraped config items. + the scraped config items grouped by the config type. type: object scanners: items: diff --git a/fixtures/aws.yaml b/fixtures/aws.yaml index 56ade7bd..a843a854 100644 --- a/fixtures/aws.yaml +++ b/fixtures/aws.yaml @@ -10,6 +10,13 @@ spec: - af-south-1 - ap-south-1 - eu-central-1 + properties: + 'AWS::IAM::Role': + - name: AWS Link + icon: aws-iam + links: + - text: AWS Link + url: https://us-east-1.console.aws.amazon.com/iamv2/home#/roles/details/{{.name}}?section=permissions compliance: true patch_states: false trusted_advisor_check: false diff --git a/scrapers/processors/json.go b/scrapers/processors/json.go index 9a9aae62..f6751055 100644 --- a/scrapers/processors/json.go +++ b/scrapers/processors/json.go @@ -9,7 +9,7 @@ import ( "github.com/flanksource/commons/logger" v1 "github.com/flanksource/config-db/api/v1" - "github.com/flanksource/duty/types" + "github.com/flanksource/config-db/utils" "github.com/flanksource/gomplate/v3" "github.com/magiconair/properties" "github.com/ohler55/ojg/jp" @@ -183,8 +183,13 @@ func (e Extract) Extract(inputs ...v1.ScrapeResult) ([]v1.ScrapeResult, error) { }, } - input.Properties = make([]*types.Property, len(properties)) - copy(input.Properties, properties) + // Need to perform a deep clone otherwise this will affect the base scraper + // and hence all the inputs that come after. + input.Properties, err = utils.CloneWithJSON(properties) + if err != nil { + return results, fmt.Errorf("failed to clone config properties: %w", err) + } + if err := templater.Walk(input.Properties); err != nil { return results, fmt.Errorf("failed to template scraper properties: %w", err) } diff --git a/utils/struct.go b/utils/struct.go new file mode 100644 index 00000000..7cfdea7a --- /dev/null +++ b/utils/struct.go @@ -0,0 +1,13 @@ +package utils + +import "encoding/json" + +func CloneWithJSON[T any](v T) (T, error) { + b, err := json.Marshal(&v) + if err != nil { + return v, err + } + + var v2 T + return v2, json.Unmarshal(b, &v2) +}