From b3f81cddf6dd3ffd5ea1b237da2953b550b223b1 Mon Sep 17 00:00:00 2001 From: xchen Date: Tue, 1 Oct 2024 17:41:49 -0700 Subject: [PATCH 1/9] Make batch scraping configurable --- promitor-agent-scraper/README.md | 3 +++ promitor-agent-scraper/templates/configmap.yaml | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/promitor-agent-scraper/README.md b/promitor-agent-scraper/README.md index 675da8e..f28c4d4 100644 --- a/promitor-agent-scraper/README.md +++ b/promitor-agent-scraper/README.md @@ -113,6 +113,9 @@ their default values. | `azureMonitor.logging.enabled` | Indication whether or not to provide logs on the integration with Azure Monitor and its APIs | `false` | | `azureMonitor.logging.logLevel` | Minimum level of logging for Azure Monitor interactions | N/A | | `azureMonitor.integration.useAzureMonitorSdk` | Whether to use the Azure Monitor SDK for scraping; alternative is the legacy Azure Management Library | `true` +| `azureMonitor.integration.metricsBatching.enabled` | Whether to scrape metrics in batch mode | `false` +| `azureMonitor.integration.metricsBatching.maxBatchSize` | Max number of resources within a batch | `50` +| `azureMonitor.integration.metricsBatching.azureRegion` | Azure region of resource targets for batching | | `azureMonitor.integration.history.startingFromInHours` | Defines the amount of hours Promitor will use to define the starting point of the time window used for metric queries. | N/A | | `rbac.create` | If true, create & use RBAC resources | `true` | | `rbac.podSecurityPolicyEnabled` | Create pod security policy resources | `false` | diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index 436cbe7..0340b3d 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -78,8 +78,14 @@ data: {{- end }} isEnabled: {{ .Values.azureMonitor.logging.enabled | quote }} {{- end }} - {{- if .Values.azureMonitor.integration.history.startingFromInHours }} + {{- if or .Values.azureMonitor.logging.enabled .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk}} integration: + {{- if .Values.azureMonitor.integration.metricsBatching.enabled }} + metricsBatching: + enabled: true + maxBatchSize: {{ .Values.azureMonitor.integration.metricsBatching.maxBatchSize }} + azureRegion: {{ .Values.azureMonitor.integration.metricsBatching.azureRegion }} + {{- end }} useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk | quote}} history: startingFromInHours: {{ .Values.azureMonitor.integration.history.startingFromInHours }} From 722c8459c47df078ec4f46ae0bfa9662f2689c2a Mon Sep 17 00:00:00 2001 From: xchen Date: Wed, 23 Oct 2024 23:56:04 -0700 Subject: [PATCH 2/9] simplify if statements --- promitor-agent-scraper/templates/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index 0340b3d..b917579 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -69,7 +69,7 @@ data: verbosity: {{ .Values.telemetry.containerLogs.logLevel | quote }} {{- end }} defaultVerbosity: {{ .Values.telemetry.defaultLogLevel | quote }} - {{- if or .Values.azureMonitor.logging.enabled .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk }} + {{- if hasKey .Values "azureMonitor" }} azureMonitor: {{- if .Values.azureMonitor.logging.enabled }} logging: @@ -78,7 +78,7 @@ data: {{- end }} isEnabled: {{ .Values.azureMonitor.logging.enabled | quote }} {{- end }} - {{- if or .Values.azureMonitor.logging.enabled .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk}} + {{- if hasKey .Values.azureMonitor "integration" }} integration: {{- if .Values.azureMonitor.integration.metricsBatching.enabled }} metricsBatching: From 83d2f6fb1f312b6b68eecc4b7cfca4ccb5a3dac5 Mon Sep 17 00:00:00 2001 From: xchen Date: Wed, 23 Oct 2024 23:58:05 -0700 Subject: [PATCH 3/9] simplify if statements --- promitor-agent-scraper/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index b917579..52647e6 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -80,7 +80,7 @@ data: {{- end }} {{- if hasKey .Values.azureMonitor "integration" }} integration: - {{- if .Values.azureMonitor.integration.metricsBatching.enabled }} + {{- if hasKey .Values.azureMonitor.integration "metricsBatching" }} metricsBatching: enabled: true maxBatchSize: {{ .Values.azureMonitor.integration.metricsBatching.maxBatchSize }} From adbd40ea1b8c9dde5321ff07065fa3f68a4ff200 Mon Sep 17 00:00:00 2001 From: xchen Date: Thu, 24 Oct 2024 17:40:15 -0700 Subject: [PATCH 4/9] Add if checks --- promitor-agent-scraper/templates/configmap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index 52647e6..b40dfcb 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -86,9 +86,13 @@ data: maxBatchSize: {{ .Values.azureMonitor.integration.metricsBatching.maxBatchSize }} azureRegion: {{ .Values.azureMonitor.integration.metricsBatching.azureRegion }} {{- end }} - useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk | quote}} + {{- if hasKey .Values.azureMonitor.integration "useAzureMonitorSdk" }} + useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk}} + {{- end }} + {{- if hasKey .Values.azureMonitor.integration "history" }} history: startingFromInHours: {{ .Values.azureMonitor.integration.history.startingFromInHours }} + {{- end }} {{- end }} {{- end }} metrics-declaration.yaml: |- From bb7a432cd94782f7d5c7f2255097d4c133b1e37b Mon Sep 17 00:00:00 2001 From: xchen Date: Thu, 24 Oct 2024 18:20:56 -0700 Subject: [PATCH 5/9] starting from master version --- .../templates/configmap.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index b40dfcb..c629be4 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -69,7 +69,7 @@ data: verbosity: {{ .Values.telemetry.containerLogs.logLevel | quote }} {{- end }} defaultVerbosity: {{ .Values.telemetry.defaultLogLevel | quote }} - {{- if hasKey .Values "azureMonitor" }} + {{- if or .Values.azureMonitor.logging.enabled .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk .Values.azureMonitor.integration.metricsBatching.enabled }} azureMonitor: {{- if .Values.azureMonitor.logging.enabled }} logging: @@ -78,21 +78,11 @@ data: {{- end }} isEnabled: {{ .Values.azureMonitor.logging.enabled | quote }} {{- end }} - {{- if hasKey .Values.azureMonitor "integration" }} + {{- if or .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk .Values.azureMonitor.integration.metricsBatching.enabled }} integration: - {{- if hasKey .Values.azureMonitor.integration "metricsBatching" }} - metricsBatching: - enabled: true - maxBatchSize: {{ .Values.azureMonitor.integration.metricsBatching.maxBatchSize }} - azureRegion: {{ .Values.azureMonitor.integration.metricsBatching.azureRegion }} - {{- end }} - {{- if hasKey .Values.azureMonitor.integration "useAzureMonitorSdk" }} - useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk}} - {{- end }} - {{- if hasKey .Values.azureMonitor.integration "history" }} + useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk | quote}} history: startingFromInHours: {{ .Values.azureMonitor.integration.history.startingFromInHours }} - {{- end }} {{- end }} {{- end }} metrics-declaration.yaml: |- @@ -109,4 +99,4 @@ data: schedule: {{ .Values.metricDefaults.scraping.schedule | quote }} metrics:{{- if .Values.metrics | len | eq 0 }} []{{- else }} {{ tpl $metrics . | indent 4 }} -{{- end }} +{{- end }} \ No newline at end of file From e5c9ebda6300db528bca780c710c9c1446584bcc Mon Sep 17 00:00:00 2001 From: xchen Date: Thu, 24 Oct 2024 18:27:37 -0700 Subject: [PATCH 6/9] Add proper if logic --- promitor-agent-scraper/templates/configmap.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index c629be4..e9651f6 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -80,11 +80,21 @@ data: {{- end }} {{- if or .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk .Values.azureMonitor.integration.metricsBatching.enabled }} integration: - useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk | quote}} + {{- if hasKey .Values.azureMonitor.integration "metricsBatching" }} + metricsBatching: + enabled: true + maxBatchSize: {{ .Values.azureMonitor.integration.metricsBatching.maxBatchSize }} + azureRegion: {{ .Values.azureMonitor.integration.metricsBatching.azureRegion }} + {{- end }} + {{- if .Values.azureMonitor.integration.useAzureMonitorSdk }} + useAzureMonitorSdk: {{ .Values.azureMonitor.integration.useAzureMonitorSdk }} + {{- end }} + {{- if .Values.azureMonitor.integration.history.startingFromInHours }} history: startingFromInHours: {{ .Values.azureMonitor.integration.history.startingFromInHours }} {{- end }} {{- end }} + {{- end }} metrics-declaration.yaml: |- version: v1 azureMetadata: From 5586abafd23babc7a0d38f01a45deb89fca22aae Mon Sep 17 00:00:00 2001 From: xchen Date: Thu, 24 Oct 2024 18:29:47 -0700 Subject: [PATCH 7/9] Add batching value --- promitor-agent-scraper/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/promitor-agent-scraper/values.yaml b/promitor-agent-scraper/values.yaml index 8f9776e..6dd4bf8 100644 --- a/promitor-agent-scraper/values.yaml +++ b/promitor-agent-scraper/values.yaml @@ -112,6 +112,8 @@ azureMonitor: logging: enabled: false logLevel: + metricsBatching: + enabled: false integration: history: startingFromInHours: From 6b69b0856f5215d1f40a64df9e39083af6bae3ec Mon Sep 17 00:00:00 2001 From: xchen Date: Thu, 24 Oct 2024 18:31:58 -0700 Subject: [PATCH 8/9] Add batching value --- promitor-agent-scraper/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/promitor-agent-scraper/values.yaml b/promitor-agent-scraper/values.yaml index 6dd4bf8..1446231 100644 --- a/promitor-agent-scraper/values.yaml +++ b/promitor-agent-scraper/values.yaml @@ -112,9 +112,9 @@ azureMonitor: logging: enabled: false logLevel: - metricsBatching: - enabled: false integration: + metricsBatching: + enabled: false history: startingFromInHours: From a5ca6f0bbc520a96876ab2e9b7d8fd5d1756813e Mon Sep 17 00:00:00 2001 From: xchen Date: Thu, 24 Oct 2024 18:40:28 -0700 Subject: [PATCH 9/9] Add batching value --- promitor-agent-scraper/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/promitor-agent-scraper/templates/configmap.yaml b/promitor-agent-scraper/templates/configmap.yaml index e9651f6..8b0f0e7 100644 --- a/promitor-agent-scraper/templates/configmap.yaml +++ b/promitor-agent-scraper/templates/configmap.yaml @@ -80,7 +80,7 @@ data: {{- end }} {{- if or .Values.azureMonitor.integration.history.startingFromInHours .Values.azureMonitor.integration.useAzureMonitorSdk .Values.azureMonitor.integration.metricsBatching.enabled }} integration: - {{- if hasKey .Values.azureMonitor.integration "metricsBatching" }} + {{- if .Values.azureMonitor.integration.metricsBatching.enabled }} metricsBatching: enabled: true maxBatchSize: {{ .Values.azureMonitor.integration.metricsBatching.maxBatchSize }}