From 2d3889c26699843c959b2f757478b8ceacc0f9d3 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 11 Jul 2024 20:22:41 +0545 Subject: [PATCH] feat: allow scope in relationship selector --- ...configs.flanksource.com_scrapeconfigs.yaml | 144 ++++++++++++++++++ config/schemas/config_aws.schema.json | 46 +++--- config/schemas/config_azure.schema.json | 46 +++--- config/schemas/config_azuredevops.schema.json | 46 +++--- config/schemas/config_file.schema.json | 46 +++--- .../schemas/config_githubactions.schema.json | 46 +++--- config/schemas/config_kubernetes.schema.json | 52 ++++--- .../schemas/config_kubernetesfile.schema.json | 46 +++--- config/schemas/config_sql.schema.json | 46 +++--- config/schemas/config_trivy.schema.json | 46 +++--- config/schemas/scrape_config.schema.json | 52 ++++--- go.mod | 2 +- go.sum | 4 +- hack/generate-schemas/go.mod | 2 +- hack/generate-schemas/go.sum | 4 +- scrapers/processors/json.go | 25 ++- scrapers/runscrapers.go | 13 +- 17 files changed, 444 insertions(+), 222 deletions(-) diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index 9a94f3ce..eef37805 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -469,6 +469,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -896,6 +912,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -1273,6 +1305,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -1609,6 +1657,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -1985,6 +2049,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -2474,6 +2554,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -2880,6 +2976,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -3336,6 +3448,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value @@ -3695,6 +3823,22 @@ spec: value: type: string type: object + scope: + description: |- + Scope is the id of the parent of the resource to select. + Example: For config items, the scope is the scraper id + - for checks, it's canaries and + - for components, it's topology. + If left empty, the scope is the requester's scope. + Use `all` to disregard scope. + properties: + expr: + type: string + label: + type: string + value: + type: string + type: object type: description: Lookup offers different ways to specify a lookup value diff --git a/config/schemas/config_aws.schema.json b/config/schemas/config_aws.schema.json index afaa6c77..074e7ad8 100644 --- a/config/schemas/config_aws.schema.json +++ b/config/schemas/config_aws.schema.json @@ -327,6 +327,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -351,19 +366,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -381,21 +402,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_azure.schema.json b/config/schemas/config_azure.schema.json index a3c0ea4f..43e54b05 100644 --- a/config/schemas/config_azure.schema.json +++ b/config/schemas/config_azure.schema.json @@ -283,6 +283,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -307,19 +322,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -337,21 +358,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_azuredevops.schema.json b/config/schemas/config_azuredevops.schema.json index 77a2d56e..0bc4da0d 100644 --- a/config/schemas/config_azuredevops.schema.json +++ b/config/schemas/config_azuredevops.schema.json @@ -271,6 +271,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -295,19 +310,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -325,21 +346,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_file.schema.json b/config/schemas/config_file.schema.json index 82a2cc70..c76604fd 100644 --- a/config/schemas/config_file.schema.json +++ b/config/schemas/config_file.schema.json @@ -204,6 +204,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -228,19 +243,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -258,21 +279,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "Tag": { "properties": { "name": { diff --git a/config/schemas/config_githubactions.schema.json b/config/schemas/config_githubactions.schema.json index a2c3c1a6..cfd58b6b 100644 --- a/config/schemas/config_githubactions.schema.json +++ b/config/schemas/config_githubactions.schema.json @@ -270,6 +270,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -294,19 +309,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -324,21 +345,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_kubernetes.schema.json b/config/schemas/config_kubernetes.schema.json index 48b5183f..b4229ca3 100644 --- a/config/schemas/config_kubernetes.schema.json +++ b/config/schemas/config_kubernetes.schema.json @@ -339,13 +339,13 @@ "KubernetesRelationshipSelectorTemplate": { "properties": { "kind": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "namespace": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" } }, "additionalProperties": false, @@ -396,6 +396,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -420,19 +435,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -450,21 +471,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_kubernetesfile.schema.json b/config/schemas/config_kubernetesfile.schema.json index e46ed357..5f1a9cf3 100644 --- a/config/schemas/config_kubernetesfile.schema.json +++ b/config/schemas/config_kubernetesfile.schema.json @@ -264,6 +264,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -303,19 +318,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -333,21 +354,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "ResourceSelector": { "properties": { "namespace": { diff --git a/config/schemas/config_sql.schema.json b/config/schemas/config_sql.schema.json index 1670c95a..1aafe858 100644 --- a/config/schemas/config_sql.schema.json +++ b/config/schemas/config_sql.schema.json @@ -208,6 +208,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -232,19 +247,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -262,21 +283,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "SQL": { "properties": { "id": { diff --git a/config/schemas/config_trivy.schema.json b/config/schemas/config_trivy.schema.json index 2d469fed..9c1cc946 100644 --- a/config/schemas/config_trivy.schema.json +++ b/config/schemas/config_trivy.schema.json @@ -129,6 +129,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "Mask": { "properties": { "selector": { @@ -153,19 +168,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -183,21 +204,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "Tag": { "properties": { "name": { diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index 96c1a1c2..550e588b 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -905,13 +905,13 @@ "KubernetesRelationshipSelectorTemplate": { "properties": { "kind": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "namespace": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" } }, "additionalProperties": false, @@ -962,6 +962,21 @@ "additionalProperties": false, "type": "object" }, + "Lookup": { + "properties": { + "expr": { + "type": "string" + }, + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, "ManagedFieldsEntry": { "properties": { "manager": { @@ -1124,19 +1139,25 @@ "RelationshipConfig": { "properties": { "id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "external_id": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "name": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "namespace": { + "$ref": "#/$defs/Lookup" }, "type": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" }, "agent": { - "$ref": "#/$defs/RelationshipLookup" + "$ref": "#/$defs/Lookup" + }, + "scope": { + "$ref": "#/$defs/Lookup" }, "labels": { "additionalProperties": { @@ -1154,21 +1175,6 @@ "additionalProperties": false, "type": "object" }, - "RelationshipLookup": { - "properties": { - "expr": { - "type": "string" - }, - "value": { - "type": "string" - }, - "label": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, "ResourceSelector": { "properties": { "namespace": { diff --git a/go.mod b/go.mod index 70bef366..d1c152f5 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible github.com/fergusstrange/embedded-postgres v1.25.0 github.com/flanksource/commons v1.24.2 - github.com/flanksource/duty v1.0.539 + github.com/flanksource/duty v1.0.542 github.com/flanksource/is-healthy v1.0.18 github.com/flanksource/ketall v1.1.7 github.com/flanksource/mapstructure v1.6.0 diff --git a/go.sum b/go.sum index e49fdb57..de9e73ad 100644 --- a/go.sum +++ b/go.sum @@ -869,8 +869,8 @@ github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7Ttk github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/flanksource/commons v1.24.2 h1:9PZsmqRTignNu56W/ZoBbmNyeR2l12g7pUHe58t63vQ= github.com/flanksource/commons v1.24.2/go.mod h1:gCg1cmwnLh3nqi9VmMH1MQ6dhFUGUS01wTduf4tHw6c= -github.com/flanksource/duty v1.0.539 h1:GUEGOu5kqM/n/d1EZA8SKTzknlXSMZIe8Zab+oRIwaw= -github.com/flanksource/duty v1.0.539/go.mod h1:MICj5RcXNJdzRG1UWZ7VhNoYJg6kRWjlwqwe5xeqVic= +github.com/flanksource/duty v1.0.542 h1:8+JeTXHvHw9rgq+vYxHGnU/FDtz1B56kUBPWq+BMPXQ= +github.com/flanksource/duty v1.0.542/go.mod h1:MICj5RcXNJdzRG1UWZ7VhNoYJg6kRWjlwqwe5xeqVic= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk= github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo= diff --git a/hack/generate-schemas/go.mod b/hack/generate-schemas/go.mod index a01cda1c..8f630560 100644 --- a/hack/generate-schemas/go.mod +++ b/hack/generate-schemas/go.mod @@ -5,7 +5,7 @@ go 1.22.2 require ( github.com/flanksource/commons v1.24.2 github.com/flanksource/config-db v0.0.65 - github.com/flanksource/duty v1.0.539 + github.com/flanksource/duty v1.0.542 github.com/spf13/cobra v1.7.0 ) diff --git a/hack/generate-schemas/go.sum b/hack/generate-schemas/go.sum index f67b9bbb..380f2ea8 100644 --- a/hack/generate-schemas/go.sum +++ b/hack/generate-schemas/go.sum @@ -734,8 +734,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flanksource/commons v1.24.2 h1:9PZsmqRTignNu56W/ZoBbmNyeR2l12g7pUHe58t63vQ= github.com/flanksource/commons v1.24.2/go.mod h1:gCg1cmwnLh3nqi9VmMH1MQ6dhFUGUS01wTduf4tHw6c= -github.com/flanksource/duty v1.0.539 h1:GUEGOu5kqM/n/d1EZA8SKTzknlXSMZIe8Zab+oRIwaw= -github.com/flanksource/duty v1.0.539/go.mod h1:MICj5RcXNJdzRG1UWZ7VhNoYJg6kRWjlwqwe5xeqVic= +github.com/flanksource/duty v1.0.542 h1:8+JeTXHvHw9rgq+vYxHGnU/FDtz1B56kUBPWq+BMPXQ= +github.com/flanksource/duty v1.0.542/go.mod h1:MICj5RcXNJdzRG1UWZ7VhNoYJg6kRWjlwqwe5xeqVic= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.24.2 h1:WZSriw1MaBhzrDV1IOP9eNsupIPxIHy0yTaMOVhCvsk= github.com/flanksource/gomplate/v3 v3.24.2/go.mod h1:94BxYobZqouGdVezuz6LNto5C+yLMG0LnNnM9CUPyoo= diff --git a/scrapers/processors/json.go b/scrapers/processors/json.go index f0a947cb..5bab43a5 100644 --- a/scrapers/processors/json.go +++ b/scrapers/processors/json.go @@ -11,10 +11,10 @@ import ( "github.com/flanksource/commons/collections" "github.com/flanksource/commons/logger" + "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/config-db/utils" "github.com/flanksource/duty" - "github.com/flanksource/duty/context" "github.com/flanksource/duty/types" "github.com/flanksource/gomplate/v3" "github.com/magiconair/properties" @@ -198,7 +198,7 @@ func (e Extract) String() string { return s } -func getRelationshipsFromRelationshipConfigs(input v1.ScrapeResult, relationshipConfigs []v1.RelationshipConfig) ([]duty.RelationshipSelector, error) { +func getRelationshipsFromRelationshipConfigs(ctx api.ScrapeContext, input v1.ScrapeResult, relationshipConfigs []v1.RelationshipConfig) ([]duty.RelationshipSelector, error) { var output []duty.RelationshipSelector for _, rc := range relationshipConfigs { @@ -226,11 +226,28 @@ func getRelationshipsFromRelationshipConfigs(input v1.ScrapeResult, relationship if err := json.Unmarshal([]byte(celOutput), &output); err != nil { return nil, fmt.Errorf("relationship config expr (%s) did not evaulate to a list of relationship selectors: %w", rc.Expr, err) } + + for i := range output { + switch output[i].Scope { + case "": + output[i].Scope = string(ctx.ScrapeConfig().GetUID()) + case "all": + output[i].Scope = "" + } + } + relationshipSelectors = append(relationshipSelectors, output...) } else { if compiled, err := rc.RelationshipSelectorTemplate.Eval(input.Labels, input.AsMap()); err != nil { return nil, fmt.Errorf("relationship selector is invalid: %w", err) } else if compiled != nil { + switch compiled.Scope { + case "": + compiled.Scope = string(ctx.ScrapeConfig().GetUID()) + case "all": + compiled.Scope = "" + } + relationshipSelectors = append(relationshipSelectors, *compiled) } } @@ -241,7 +258,7 @@ func getRelationshipsFromRelationshipConfigs(input v1.ScrapeResult, relationship return output, nil } -func (e Extract) Extract(ctx context.Context, inputs ...v1.ScrapeResult) ([]v1.ScrapeResult, error) { +func (e Extract) Extract(ctx api.ScrapeContext, inputs ...v1.ScrapeResult) ([]v1.ScrapeResult, error) { var results []v1.ScrapeResult var err error @@ -271,7 +288,7 @@ func (e Extract) Extract(ctx context.Context, inputs ...v1.ScrapeResult) ([]v1.S } // Form new relationships based on the transform configs - if newRelationships, err := getRelationshipsFromRelationshipConfigs(input, e.Transform.Relationship); err != nil { + if newRelationships, err := getRelationshipsFromRelationshipConfigs(ctx, input, e.Transform.Relationship); err != nil { return results, fmt.Errorf("failed to get relationships from relationship configs: %w", err) } else if len(newRelationships) > 0 { input.RelationshipSelectors = append(input.RelationshipSelectors, newRelationships...) diff --git a/scrapers/runscrapers.go b/scrapers/runscrapers.go index 406fccf7..f9073f07 100644 --- a/scrapers/runscrapers.go +++ b/scrapers/runscrapers.go @@ -13,7 +13,6 @@ import ( "github.com/flanksource/config-db/scrapers/kubernetes" "github.com/flanksource/config-db/scrapers/processors" "github.com/flanksource/config-db/utils" - "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -23,7 +22,7 @@ func RunK8ObjScraper(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstru var results v1.ScrapeResults res := kubernetes.ExtractResults(ctx, config, objs, false) for i := range res { - scraped := processScrapeResult(ctx.DutyContext(), ctx.ScrapeConfig().Spec, res[i]) + scraped := processScrapeResult(ctx, res[i]) results = append(results, scraped...) } @@ -35,7 +34,7 @@ func RunK8IncrementalScraper(ctx api.ScrapeContext, config v1.Kubernetes, events var results v1.ScrapeResults var scraper kubernetes.KubernetesScraper for _, result := range scraper.IncrementalScrape(ctx, config, events) { - scraped := processScrapeResult(ctx.DutyContext(), ctx.ScrapeConfig().Spec, result) + scraped := processScrapeResult(ctx, result) results = append(results, scraped...) } @@ -52,7 +51,7 @@ func Run(ctx api.ScrapeContext) ([]v1.ScrapeResult, error) { ctx.DutyContext().Logger.V(3).Infof("Starting scraper") for _, result := range scraper.Scrape(ctx) { - scraped := processScrapeResult(ctx.DutyContext(), ctx.ScrapeConfig().Spec, result) + scraped := processScrapeResult(ctx, result) for i := range scraped { if scraped[i].Error != nil { @@ -98,7 +97,9 @@ func summarizeChanges(changes []v1.ChangeResult) []v1.ChangeResult { } // processScrapeResult extracts possibly more configs from the result -func processScrapeResult(ctx context.Context, config v1.ScraperSpec, result v1.ScrapeResult) v1.ScrapeResults { +func processScrapeResult(ctx api.ScrapeContext, result v1.ScrapeResult) v1.ScrapeResults { + spec := ctx.ScrapeConfig().Spec + if result.AnalysisResult != nil { if rule, ok := analysis.Rules[result.AnalysisResult.Analyzer]; ok { result.AnalysisResult.AnalysisType = models.AnalysisType(rule.Category) @@ -129,7 +130,7 @@ func processScrapeResult(ctx context.Context, config v1.ScraperSpec, result v1.S } // In full mode, we extract all configs and changes from the result. - if config.Full { + if spec.Full { for i := range scraped { extractedConfig, changeRes, err := extractConfigChangesFromConfig(scraped[i].Config) if err != nil {