fix: perform deep clone of config properties

api/v1/common.go

@@ -166,7 +166,8 @@ type BaseScraper struct {
 	// Tags allow you to set custom tags on the scraped config items.
 	Tags JSONStringMap `json:"tags,omitempty"`
 
-	// Properties are custom templatable properties for the scraped config items.
+	// Properties are custom templatable properties for the scraped config items
+	// grouped by the config type.
 	Properties map[string]types.Properties `json:"properties,omitempty" template:"true"`
 }
 

chart/crds/configs.flanksource.com_scrapeconfigs.yaml

@@ -222,7 +222,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     region:
                       items:
@@ -550,7 +550,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     subscriptionID:
                       type: string
@@ -788,7 +788,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     tags:
                       additionalProperties:
@@ -980,7 +980,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     tags:
                       additionalProperties:
@@ -1210,7 +1210,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     repository:
                       type: string
@@ -1476,7 +1476,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     relationships:
                       description: Relationships specify the fields to use to relate
@@ -1716,7 +1716,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     selector:
                       properties:
@@ -2023,7 +2023,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     query:
                       type: string
@@ -2230,7 +2230,7 @@ spec:
                           type: object
                         type: array
                       description: Properties are custom templatable properties for
-                        the scraped config items.
+                        the scraped config items grouped by the config type.
                       type: object
                     scanners:
                       items:

fixtures/aws.yaml

@@ -10,6 +10,13 @@ spec:
         - af-south-1
         - ap-south-1
         - eu-central-1
+      properties:
+        'AWS::IAM::Role':
+          - name: AWS Link
+            icon: aws-iam
+            links:
+              - text: AWS Link
+                url: https://us-east-1.console.aws.amazon.com/iamv2/home#/roles/details/{{.name}}?section=permissions
       compliance: true
       patch_states: false
       trusted_advisor_check: false

scrapers/processors/json.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/flanksource/commons/logger"
 	v1 "github.com/flanksource/config-db/api/v1"
-	"github.com/flanksource/duty/types"
+	"github.com/flanksource/config-db/utils"
 	"github.com/flanksource/gomplate/v3"
 	"github.com/magiconair/properties"
 	"github.com/ohler55/ojg/jp"
@@ -183,8 +183,13 @@ func (e Extract) Extract(inputs ...v1.ScrapeResult) ([]v1.ScrapeResult, error) {
 				},
 			}
 
-			input.Properties = make([]*types.Property, len(properties))
-			copy(input.Properties, properties)
+			// Need to perform a deep clone otherwise this will affect the base scraper
+			// and hence all the inputs that come after.
+			input.Properties, err = utils.CloneWithJSON(properties)
+			if err != nil {
+				return results, fmt.Errorf("failed to clone config properties: %w", err)
+			}
+
 			if err := templater.Walk(input.Properties); err != nil {
 				return results, fmt.Errorf("failed to template scraper properties: %w", err)
 			}

utils/struct.go

@@ -0,0 +1,13 @@
+package utils
+
+import "encoding/json"
+
+func CloneWithJSON[T any](v T) (T, error) {
+	b, err := json.Marshal(&v)
+	if err != nil {
+		return v, err
+	}
+
+	var v2 T
+	return v2, json.Unmarshal(b, &v2)
+}