From 679600998bcf201505002adf512a0c55ef37c65e Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 10 Dec 2024 11:59:39 +0200 Subject: [PATCH] changes following tests --- .../workflows/logzio-apm-collector-test.yaml | 18 +-- charts/logzio-apm-collector/Chart.yaml | 1 + charts/logzio-apm-collector/README.md | 12 +- .../logzio-apm-collector/templates/NOTES.txt | 2 +- .../templates/_helpers-spm.tpl | 2 +- .../templates/_helpers.tpl | 14 -- .../templates/_pod-spm.tpl | 8 +- .../logzio-apm-collector/templates/_pod.tpl | 10 +- .../templates/_validations.tpl | 8 +- .../templates/secret.yaml | 20 +-- charts/logzio-apm-collector/values.yaml | 86 +++++----- tests/apm_metrics_e2e_test.go | 153 ++++++++++++++++++ tests/resources/otel-demo-apm.yaml | 16 ++ tests/resources/tracegen-apm.yaml | 24 +++ 14 files changed, 276 insertions(+), 98 deletions(-) create mode 100644 tests/apm_metrics_e2e_test.go create mode 100644 tests/resources/otel-demo-apm.yaml create mode 100644 tests/resources/tracegen-apm.yaml diff --git a/.github/workflows/logzio-apm-collector-test.yaml b/.github/workflows/logzio-apm-collector-test.yaml index d0c87de9..cdf3a101 100644 --- a/.github/workflows/logzio-apm-collector-test.yaml +++ b/.github/workflows/logzio-apm-collector-test.yaml @@ -47,29 +47,29 @@ jobs: - name: Deploy Helm Chart run: | - cd charts/logzio-telemetry + cd charts/logzio-apm-collector helm upgrade --install \ - --set enabled=true \ \ + --set enabled=true \ --set spm.enabled=true \ --set serviceGraph.enabled=true \ - --set secrets.logzioTracesToken=${{ secrets.LOGZIO_TRACES_TOKEN }} \ - --set secrets.logzioSpmToken=${{ secrets.LOGZIO_METRICS_TOKEN }} \ - --set secrets.logzioRegion="us" \ - --set secrets.env_id=${{ env.ENV_ID }} \ + --set global.logzioTracesToken=${{ secrets.LOGZIO_TRACES_TOKEN }} \ + --set global.logzioSpmToken=${{ secrets.LOGZIO_METRICS_TOKEN }} \ + --set global.logzioRegion="us" \ + --set global.env_id=${{ env.ENV_ID }} \ logzio-apm-collector . kubectl rollout status deployment/logzio-apm-collector --timeout=300s kubectl rollout status deployment/logzio-apm-collector-spm --timeout=300s - name: Run trace generator run: | - kubectl apply -f tests/resources/tracegen.yaml + kubectl apply -f tests/resources/tracegen-apm.yaml kubectl rollout status deployment/trace-gen --timeout=300s - name: Run otel demo for service graph run: | helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts helm repo update - helm install otel-demo -f tests/resources/otel-demo.yaml open-telemetry/opentelemetry-demo --version 0.32.5 + helm install otel-demo -f tests/resources/otel-demo-apm.yaml open-telemetry/opentelemetry-demo --version 0.32.5 kubectl rollout status deployment/otel-demo-loadgenerator --timeout=300s - name: sleep for 3 minutes @@ -82,7 +82,7 @@ jobs: run: | go get go.uber.org/zap go test -v ./tests/traces_e2e_test.go ./tests/common.go - go test -v ./tests/metrics_e2e_test.go ./tests/common.go + go test -v ./tests/apm_metrics_e2e_test.go ./tests/common.go - name: Cleanup Environment run: | diff --git a/charts/logzio-apm-collector/Chart.yaml b/charts/logzio-apm-collector/Chart.yaml index d42dffae..ca3fd627 100644 --- a/charts/logzio-apm-collector/Chart.yaml +++ b/charts/logzio-apm-collector/Chart.yaml @@ -4,6 +4,7 @@ version: 1.0.0 description: Kubernetes APM agent for Logz.io based on OpenTelemetry Collector type: application home: https://logz.io/ +icon: https://logzbucket.s3.eu-west-1.amazonaws.com/logz-io-img/logo400x400.png maintainers: - name: Naama Bendalak email: naama.bendalak@logz.io diff --git a/charts/logzio-apm-collector/README.md b/charts/logzio-apm-collector/README.md index 4eb1e9ab..53545b1e 100644 --- a/charts/logzio-apm-collector/README.md +++ b/charts/logzio-apm-collector/README.md @@ -30,10 +30,10 @@ helm install -n monitoring --create-namespace \ --set enabled=true \ --set spm.enabled=true \ --set serviceGraph.enabled=true \ ---set secrets.logzioTracesToken="<>" \ ---set secrets.logzioSpmToken="<>" \ ---set secrets.logzioRegion="<>" \ ---set secrets.env_id="<>" \ +--set global.logzioTracesToken="<>" \ +--set global.logzioSpmToken="<>" \ +--set global.logzioRegion="<>" \ +--set global.env_id="<>" \ logzio-apm-collector logzio-helm/logzio-apm-collector ``` @@ -66,10 +66,6 @@ logzio-apm-collector.monitoring.svc.cluster.local:<> > > For a complete list, see `values.yaml` >> `traceConfig` >> `receivers`. -> [!WARNING] -> If you're deploying the chart as a sub chart of the `logzio-monitoring` chart, replace: -> `logzio-apm-collector` >> `logzio-monitoring-otel-collector` - ## Custom trace sampling rules To customize the Traces Sampling rules in the OpenTelemetry Collector, you can follow the below steps: diff --git a/charts/logzio-apm-collector/templates/NOTES.txt b/charts/logzio-apm-collector/templates/NOTES.txt index 0a655d63..fbf4bb17 100644 --- a/charts/logzio-apm-collector/templates/NOTES.txt +++ b/charts/logzio-apm-collector/templates/NOTES.txt @@ -15,7 +15,7 @@ {{ fail "[ERROR] The logzio-apm-collector Chart's `otelLogLevel` must be one of 'info', 'warn', 'error' or 'debug'." }} {{- end }} -{{- $region := lower .Values.secrets.logzioRegion }} +{{- $region := lower .Values.global.logzioRegion }} {{- if not (or (eq $region "us") (eq $region "eu") (eq $region "uk") (eq $region "ca") (eq $region "au")) }} {{ print "[WARN] The `logzioRegion` expected value should be one of 'us', 'eu', 'uk', 'ca', 'au'." }} {{- end }} diff --git a/charts/logzio-apm-collector/templates/_helpers-spm.tpl b/charts/logzio-apm-collector/templates/_helpers-spm.tpl index 4206bb90..c583a649 100644 --- a/charts/logzio-apm-collector/templates/_helpers-spm.tpl +++ b/charts/logzio-apm-collector/templates/_helpers-spm.tpl @@ -22,7 +22,7 @@ component: spm-collector Create Logz.io listener address based on region */}} {{- define "spm-collector.listenerAddress" -}} -{{- $region := .Values.secrets.logzioRegion -}} +{{- $region := .Values.global.logzioRegion -}} {{- if or (eq $region "us") (not $region) -}} https://listener.logz.io:8053 {{- else }} diff --git a/charts/logzio-apm-collector/templates/_helpers.tpl b/charts/logzio-apm-collector/templates/_helpers.tpl index d46def68..90212554 100644 --- a/charts/logzio-apm-collector/templates/_helpers.tpl +++ b/charts/logzio-apm-collector/templates/_helpers.tpl @@ -146,17 +146,3 @@ The APM service address {{- $serviceName := include "apm-collector.fullname" .}} {{- printf "http://%s.%s.svc.cluster.local" $serviceName .Release.Namespace }} {{- end }} - -{{/* -Get secret value either from the global section (parent chart is running) or the default sub chart section -*/}} -{{- define "getGlobalOrDefaultValue" -}} -{{- $ctx := index . 0 -}} -{{- $key := index . 1 -}} -{{- $fallback := index . 2 -}} -{{- if hasKey $ctx.Values "global" -}} - {{- $ctx.Values.global | default dict | get $key | default $fallback -}} -{{- else -}} - {{- $fallback -}} -{{- end -}} -{{- end -}} diff --git a/charts/logzio-apm-collector/templates/_pod-spm.tpl b/charts/logzio-apm-collector/templates/_pod-spm.tpl index f5aaa777..cc9e22ec 100644 --- a/charts/logzio-apm-collector/templates/_pod-spm.tpl +++ b/charts/logzio-apm-collector/templates/_pod-spm.tpl @@ -46,13 +46,13 @@ containers: - name: ENV_ID valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: env-id - name: SPM_ENDPOINT - {{- if .Values.secrets.customSpmEndpoint }} + {{- if .Values.global.customSpmEndpoint }} valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: custom-spm-endpoint {{- else }} value: {{ include "spm-collector.listenerAddress" . | quote }} @@ -60,7 +60,7 @@ containers: - name: LOGZIO_SPM_TOKEN valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: logzio-spm-token - name: LOG_LEVEL value: {{ .Values.otelLogLevel | default "info" | quote }} diff --git a/charts/logzio-apm-collector/templates/_pod.tpl b/charts/logzio-apm-collector/templates/_pod.tpl index e89375fe..642fca6b 100644 --- a/charts/logzio-apm-collector/templates/_pod.tpl +++ b/charts/logzio-apm-collector/templates/_pod.tpl @@ -48,23 +48,23 @@ containers: - name: ENV_ID valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: env-id - name: LOGZIO_REGION valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: logzio-listener-region - name: LOGZIO_TRACES_TOKEN valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: logzio-traces-token - {{- if .Values.secrets.customTracesEndpoint }} + {{- if .Values.global.customTracesEndpoint }} - name: CUSTOM_TRACES_ENDPOINT valueFrom: secretKeyRef: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} key: custom-traces-endpoint {{- end }} - name: LOG_LEVEL diff --git a/charts/logzio-apm-collector/templates/_validations.tpl b/charts/logzio-apm-collector/templates/_validations.tpl index f02bdad1..2ed6e9b3 100644 --- a/charts/logzio-apm-collector/templates/_validations.tpl +++ b/charts/logzio-apm-collector/templates/_validations.tpl @@ -3,9 +3,7 @@ Verify tracing token was provided if the chart is enabled */}} {{- define "check-tracing-token" -}} {{- if .Values.enabled }} - {{- $hasGlobalToken := and (hasKey .Values "global") .Values.global.logzioTracesToken -}} - {{- $hasSecretsToken := .Values.secrets.logzioTracesToken -}} - {{- if not (or $hasGlobalToken $hasSecretsToken) }} + {{- if not .Values.global.logzioTracesToken }} {{- fail "Missing Tracing Token" }} {{- end }} {{- end }} @@ -16,9 +14,7 @@ Verify SPM token was provided if SPM is enabled */}} {{- define "check-spm-token" -}} {{- if and (.Values.enabled) (.Values.spm.enabled) }} - {{- $hasGlobalToken := and (hasKey .Values "global") .Values.global.logzioSpmToken -}} - {{- $hasSecretsToken := .Values.secrets.logzioSpmToken -}} - {{- if not (or $hasGlobalToken $hasSecretsToken) }} + {{- if not .Values.global.logzioSpmToken }} {{- fail "Missing SPM Token" }} {{- end }} {{- end }} diff --git a/charts/logzio-apm-collector/templates/secret.yaml b/charts/logzio-apm-collector/templates/secret.yaml index 19fc02f3..f990ccfa 100644 --- a/charts/logzio-apm-collector/templates/secret.yaml +++ b/charts/logzio-apm-collector/templates/secret.yaml @@ -1,23 +1,23 @@ {{ if .Values.enabled}} -{{- if .Values.secrets.enabled }} +{{- if .Values.secret.enabled }} apiVersion: v1 kind: Secret metadata: - name: {{ .Values.secrets.name }} + name: {{ .Values.secret.name }} namespace: {{ .Release.Namespace }} type: Opaque stringData: - env-id: {{ include "getGlobalOrDefaultValue" (list . "env_id" .Values.secrets.env_id) | quote }} - logzio-listener-region: {{ include "getGlobalOrDefaultValue" (list . "logzioRegion" .Values.secrets.logzioRegion) }} + env-id: {{ .Values.global.env_id | quote}} + logzio-listener-region: {{ .Values.global.logzioRegion | quote}} {{- template "check-tracing-token" . }} - logzio-traces-token: {{ include "getGlobalOrDefaultValue" (list . "logzioTracesToken" .Values.secrets.logzioTracesToken) }} + logzio-traces-token: {{ .Values.global.logzioTracesToken }} {{- template "check-spm-token" . }} - logzio-spm-token: {{ include "getGlobalOrDefaultValue" (list . "logzioSpmToken" .Values.secrets.logzioSpmToken) }} - {{- if .Values.secrets.customTracesEndpoint }} - custom-traces-endpoint: {{ .Values.secrets.customTracesEndpoint }} + logzio-spm-token: {{ .Values.global.logzioSpmToken }} + {{- if .Values.global.customTracesEndpoint }} + custom-traces-endpoint: {{ .Values.global.customTracesEndpoint }} {{- end }} - {{- if .Values.secrets.customSpmEndpoint}} - custom-spm-endpoint: {{ .Values.secrets.customSpmEndpoint }} + {{- if .Values.global.customSpmEndpoint}} + custom-spm-endpoint: {{ .Values.global.customSpmEndpoint }} {{- end }} {{- end }} {{ end }} diff --git a/charts/logzio-apm-collector/values.yaml b/charts/logzio-apm-collector/values.yaml index 6908711b..7ad351c8 100644 --- a/charts/logzio-apm-collector/values.yaml +++ b/charts/logzio-apm-collector/values.yaml @@ -21,16 +21,7 @@ namespaceOverride: "" ####################################################################################################################### # Base Configuration Parameters ####################################################################################################################### -secrets: - # When secrets.enabled is true, the logzio secret will be created and managed by this Chart. - # If you're managing the logzio secrets by yourself, set to false. - # Note that in order for the default configuration to work properly, you need to: - # 1. Update secrets.name to your custom secret name - # 2. Include these keys in your secret: env-id, logzio-listener-region, logzio-traces-token, logzio-spm-token - # To use a custom endpoint, include custom-traces-endpoint, custom-spm-endpoint or both parameters in your secret, - # depending on your needs and set secrets.customTracesEndpoint and/or secrets.customSpmEndpoint to `true`. - enabled: true - name: logzio-apm-collector-secret +global: # environment identifier attribute that will be added to all telemetry env_id: "my_env" # Logz.io Tracing Shipping Token @@ -39,7 +30,7 @@ secrets: logzioSpmToken: "" # Logz.io region code logzioRegion: "us" - # Optional - Overrides secrets.LogzioRegion listener address with a custom endpoint. For example: http://endpoint:8080 + # Optional - Overrides global.LogzioRegion listener address with a custom endpoint. For example: http://endpoint:8080 customTracesEndpoint: "" customSpmEndpoint: "" @@ -211,7 +202,7 @@ spmForwarderConfig: pipelines: traces/spm: receivers: [jaeger, zipkin, otlp] - processors: [resourcedetection/all, attributes/env_id, k8sattributes] + processors: [resourcedetection/all, attributes/env_id, k8sattributes, batch] exporters: [otlp] # SPM Collector configuration @@ -233,6 +224,7 @@ spmConfig: grpc: endpoint: "0.0.0.0:4317" processors: + batch: {} metricstransform/metrics-rename: transforms: # rename metric duration.XXX >> latency.XXX @@ -244,6 +236,11 @@ spmConfig: - action: update include: calls new_name: calls_total + # manually add 'seconds' unit to the relevant Service Graph metric names + - include: ^(traces_service_graph_request_(server|client).*)$$ + action: update + match_type: regexp + new_name: $${1}_seconds metricstransform/labels-rename: transforms: # for metrics matching `latencyXXX` or `callsXXX` @@ -303,7 +300,7 @@ spmConfig: exporters: [spanmetrics] metrics/spm-logzio: receivers: [spanmetrics] - processors: [metricstransform/metrics-rename, metricstransform/labels-rename] + processors: [metricstransform/metrics-rename, metricstransform/labels-rename, batch] exporters: [prometheusremotewrite/spm-logzio] # Service Graph configuration @@ -340,11 +337,23 @@ imagePullSecrets: [] # OpenTelemetry Collector executable command: name: otelcol-contrib - extraArgs: [] + extraArgs: + - --feature-gates=connector.spanmetrics.legacyMetricNames # rename traces_span_metrics_xxx >> xxx ####################################################################################################################### # Kubernetes Resources Configuration ####################################################################################################################### +secret: + # When secret.enabled is true, the logzio secret will be created and managed by this Chart. + # If you're managing the logzio secrets by yourself, set to false. + # Note that in order for the default configuration to work properly, you need to: + # 1. Update secrets.name to your custom secret name. + # 2. Include these keys in your secret: env-id, logzio-listener-region, logzio-traces-token, logzio-spm-token. + # To use a custom endpoint, include custom-traces-endpoint, custom-spm-endpoint or both parameters in your secret, + # depending on your needs and set global.customTracesEndpoint and/or global.customSpmEndpoint to `true`. + enabled: true + name: logzio-apm-collector-secret + configMap: # Specifies whether a configMap should be created create: true @@ -403,14 +412,14 @@ service: # Annotations to add to the Service. annotations: {} - # By default, Service will be created setting 'internalTrafficPolicy: Cluster' - # unless other value is explicitly set. - # Setting 'internalTrafficPolicy: Cluster' on a daemonset is not recommended (in such case, use 'internalTrafficPolicy: Local') + ## By default, Service will be created setting 'internalTrafficPolicy: Cluster' + ## unless other value is explicitly set. + ## Setting 'internalTrafficPolicy: Cluster' on a daemonset is not recommended (in such case, use 'internalTrafficPolicy: Local') # internalTrafficPolicy: Cluster - # By default, Service of type 'LoadBalancer' will be created setting 'externalTrafficPolicy: Cluster' - # unless other value is explicitly set. - # Possible values are Cluster or Local (https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip) + ## By default, Service of type 'LoadBalancer' will be created setting 'externalTrafficPolicy: Cluster' + ## unless other value is explicitly set. + ## Possible values are Cluster or Local (https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip) # externalTrafficPolicy: Cluster spmService: @@ -420,14 +429,14 @@ spmService: # Annotations to add to the Service. annotations: {} - # By default, Service will be created setting 'internalTrafficPolicy: Cluster' - # unless other value is explicitly set. - # Setting 'internalTrafficPolicy: Cluster' on a daemonset is not recommended (in such case, use 'internalTrafficPolicy: Local') + ## By default, Service will be created setting 'internalTrafficPolicy: Cluster' + ## unless other value is explicitly set. + ## Setting 'internalTrafficPolicy: Cluster' on a daemonset is not recommended (in such case, use 'internalTrafficPolicy: Local') # internalTrafficPolicy: Cluster - # By default, Service of type 'LoadBalancer' will be created setting 'externalTrafficPolicy: Cluster' - # unless other value is explicitly set. - # Possible values are Cluster or Local (https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip) + ## By default, Service of type 'LoadBalancer' will be created setting 'externalTrafficPolicy: Cluster' + ## unless other value is explicitly set. + ## Possible values are Cluster or Local (https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip) # externalTrafficPolicy: Cluster # Configure HPA for Traces Collector. @@ -533,8 +542,7 @@ extraVolumeMounts: [] # resources.limits.memory # If no resources.limits.memory are defined enabling does nothing. # In a future release this setting will be enabled by default. -# See https://github.com/open-telemetry/opentelemetry-helm-charts/issues/891 -# for more details. +# For more details see https://github.com/open-telemetry/opentelemetry-helm-charts/issues/891 useGOMEMLIMIT: false # Resource allocation. @@ -616,17 +624,16 @@ lifecycleHooks: {} # liveness probe configuration # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ -## livenessProbe: - # Number of seconds after the container has started before startup, liveness or readiness probes are initiated. + ## Number of seconds after the container has started before startup, liveness or readiness probes are initiated. # initialDelaySeconds: 1 - # How often in seconds to perform the probe. + ## How often in seconds to perform the probe. # periodSeconds: 10 - # Number of seconds after which the probe times out. + ## Number of seconds after which the probe times out. # timeoutSeconds: 1 - # Minimum consecutive failures for the probe to be considered failed after having succeeded. + ## Minimum consecutive failures for the probe to be considered failed after having succeeded. # failureThreshold: 1 - # Duration in seconds the pod needs to terminate gracefully upon probe failure. + ## Duration in seconds the pod needs to terminate gracefully upon probe failure. # terminationGracePeriodSeconds: 10 httpGet: port: 13133 @@ -634,17 +641,16 @@ livenessProbe: # readiness probe configuration # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ -## readinessProbe: - # Number of seconds after the container has started before startup, liveness or readiness probes are initiated. + ## Number of seconds after the container has started before startup, liveness or readiness probes are initiated. # initialDelaySeconds: 1 - # How often (in seconds) to perform the probe. + ## How often (in seconds) to perform the probe. # periodSeconds: 10 - # Number of seconds after which the probe times out. + ## Number of seconds after which the probe times out. # timeoutSeconds: 1 - # Minimum consecutive successes for the probe to be considered successful after having failed. + ## Minimum consecutive successes for the probe to be considered successful after having failed. # successThreshold: 1 - # Minimum consecutive failures for the probe to be considered failed after having succeeded. + ## Minimum consecutive failures for the probe to be considered failed after having succeeded. # failureThreshold: 1 httpGet: port: 13133 diff --git a/tests/apm_metrics_e2e_test.go b/tests/apm_metrics_e2e_test.go new file mode 100644 index 00000000..8ba1b8d9 --- /dev/null +++ b/tests/apm_metrics_e2e_test.go @@ -0,0 +1,153 @@ +package tests + +import ( + "encoding/json" + "fmt" + "go.uber.org/zap" + "io" + "net/http" + "os" + "strings" + "testing" +) + +// MetricResponse represents the structure of the API response +type MetricResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +func TestSpmMetricsApm(t *testing.T) { + requiredMetrics := map[string][]string{ + "calls_total": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, + "latency_sum": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, + "latency_count": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, + "latency_bucket": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, + } + envId := os.Getenv("ENV_ID") + query := fmt.Sprintf(`{env_id='%s'}`, envId) + testMetrics(t, requiredMetrics, query) +} + +func TestServiceGraphMetricsApm(t *testing.T) { + requiredMetrics := map[string][]string{ + "traces_service_graph_request_total": {"client", "server"}, + "traces_service_graph_request_failed_total": {"client", "server"}, + "traces_service_graph_request_server_seconds_bucket": {"client", "server"}, + "traces_service_graph_request_server_seconds_count": {"client", "server"}, + "traces_service_graph_request_server_seconds_sum": {"client", "server"}, + "traces_service_graph_request_client_seconds_bucket": {"client", "server"}, + "traces_service_graph_request_client_seconds_count": {"client", "server"}, + "traces_service_graph_request_client_seconds_sum": {"client", "server"}, + } + envId := os.Getenv("ENV_ID") + query := fmt.Sprintf(`{client_env_id='%s'}`, envId) + testMetrics(t, requiredMetrics, query) +} + +func testMetrics(t *testing.T, requiredMetrics map[string][]string, query string) { + metricsApiKey := os.Getenv("LOGZIO_METRICS_API_KEY") + if metricsApiKey == "" { + t.Fatalf("LOGZIO_METRICS_API_KEY environment variable not set") + } + + metricResponse, err := fetchMetrics(metricsApiKey, query) + if err != nil { + t.Fatalf("Failed to fetch metrics: %v", err) + } + + if metricResponse.Status != "success" { + t.Errorf("No metrics found") + } + logger.Info("Found metrics", zap.Int("metrics_count", len(metricResponse.Data.Result))) + // Verify required metrics + missingMetrics := verifyMetrics(metricResponse, requiredMetrics) + if len(missingMetrics) > 0 { + var sb strings.Builder + for _, metric := range missingMetrics { + sb.WriteString(metric + "\n") + } + t.Errorf("Missing metrics or labels:\n%s", sb.String()) + } +} + +// fetchMetrics fetches the metrics from the logz.io API +func fetchMetrics(metricsApiKey string, query string) (*MetricResponse, error) { + url := fmt.Sprintf("%s/metrics/prometheus/api/v1/query?query=%s", BaseLogzioApiUrl, query) + client := &http.Client{} + logger.Info("sending api request", zap.String("url", url)) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("X-API-TOKEN", metricsApiKey) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var metricResponse MetricResponse + err = json.Unmarshal(body, &metricResponse) + if err != nil { + return nil, err + } + + return &metricResponse, nil +} + +// verifyMetrics checks if the required metrics and their labels are present in the response +func verifyMetrics(metricResponse *MetricResponse, requiredMetrics map[string][]string) []string { + missingMetrics := []string{} + + for metricName, requiredLabels := range requiredMetrics { + found := false + for _, result := range metricResponse.Data.Result { + if result.Metric["__name__"] == metricName { + found = true + for _, label := range requiredLabels { + if _, exists := result.Metric[label]; !exists { + missingMetrics = append(missingMetrics, fmt.Sprintf("%s (missing label: %s)", metricName, label)) + } + } + } + } + if !found { + missingMetrics = append(missingMetrics, metricName+" (not found)") + } + } + return deduplicate(missingMetrics) +} + +// deduplicate removes duplicate strings from the input array. +func deduplicate(data []string) []string { + uniqueMap := make(map[string]bool) + var uniqueList []string + + for _, item := range data { + trimmedItem := strings.TrimSpace(item) + if _, exists := uniqueMap[trimmedItem]; !exists { + uniqueMap[trimmedItem] = true + uniqueList = append(uniqueList, trimmedItem) + } + } + + return uniqueList +} diff --git a/tests/resources/otel-demo-apm.yaml b/tests/resources/otel-demo-apm.yaml new file mode 100644 index 00000000..2a1e55da --- /dev/null +++ b/tests/resources/otel-demo-apm.yaml @@ -0,0 +1,16 @@ +default: + envOverrides: + - name: OTEL_COLLECTOR_NAME + value: logzio-apm-collector.default.svc.cluster.local + +opentelemetry-collector: + enabled: false + +jaeger: + enabled: false + +prometheus: + enabled: false + +grafana: + enabled: false diff --git a/tests/resources/tracegen-apm.yaml b/tests/resources/tracegen-apm.yaml new file mode 100644 index 00000000..fdf12d55 --- /dev/null +++ b/tests/resources/tracegen-apm.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trace-gen +spec: + selector: + matchLabels: + app: trace-gen + template: + metadata: + labels: + app: trace-gen + spec: + containers: + - name: trace-gen + image: ghcr.io/frzifus/jaeger-otel-test:latest + args: + [ + '-otel.agent.host=logzio-apm-collector', + '-otel.agent.port=4317', + ] + env: + - name: OTEL_SERVICE_NAME + value: 'local-test-service' \ No newline at end of file