Update Helm chart for flexibility & parameterization

Refactor values for improved Helm chart flexibility, including parameterizing namespace, ports, and secret names. Adjusted configs for sender and aggregator deployments, and revised README instructions.
igor-vovk · Aug 18, 2024 · b36e4c4 · b36e4c4
1 parent 40b4139
commit b36e4c4
Show file tree

Hide file tree

Showing 8 changed files with 70 additions and 59 deletions.
diff --git a/README.md b/README.md
@@ -17,10 +17,12 @@ It is based on Axiom's [instruction](https://axiom.co/docs/send-data/kubernetes)
 ## Installation
 
 ### Helm
-1. Create kubernetes secret with Datadog API key:
+
+1. Create Kubernetes secret with Datadog API key:
    ```shell
-   kubectl create secret generic datadog-api-key --from-literal=DD_API_KEY=<YOUR_DATADOG_API_KEY>
+   kubectl create secret generic datadog-creds --from-literal=DD_API_KEY=<YOUR_DATADOG_API_KEY>
     ```
+   (make sure to create it in the `kube-system` namespace as the chart is deployed there)
 2. Add the Helm repository:
    ```shell
    helm repo add kube-logs-datadog-sender https://igor-vovk.github.io/kube-logs-datadog-sender/
@@ -31,6 +33,7 @@ It is based on Axiom's [instruction](https://axiom.co/docs/send-data/kubernetes)
    ```
 
 ### ArgoCD Application
+Example of ArgoCD Application manifest:
 ```yaml
 apiVersion: argoproj.io/v1alpha1
 kind: Application
@@ -42,10 +45,10 @@ spec:
   source:
     chart: kube-logs-datadog-sender
     repoURL: https://igor-vovk.github.io/kube-logs-datadog-sender/
-    targetRevision: 0.1.0
+    targetRevision: 0.1.2
     helm:
       valuesObject:
-        # configuration of a sender, see chart's values.yaml for all available options
+      # configuration of a sender, see chart's values.yaml for all available options
   destination:
     server: https://kubernetes.default.svc
     namespace: kube-system
@@ -68,11 +71,12 @@ Sender-Aggregator pattern is used to send logs to Datadog:
 ## Points of Improvement
 
 * Consider having thin senders and doing parsing and filtering in the aggregator instance.
-* More flexible Vector configuring from Helm chart.
+* Right now parsing is most optimized for the Logback logs format. Figure out how to make it more generic.
+* More flexible Vector configuration from Helm chart.
 
 ## Referral Links
 
-They help me to pay for the Datadog, so I can see my logs.
+Help me to pay my Datadog bills by following those links:
 
 * [Hetzner Cloud](https://hetzner.cloud/?ref=iAnthJAtoQ8d) – best cloud provider with the cheapest prices in the EU
 * [Bybit](https://www.bybit.nl/invite?ref=EVWANAG) – best crypto exchange
diff --git a/charts/kube-logs-datadog-sender/Chart.yaml b/charts/kube-logs-datadog-sender/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: kube-logs-datadog-sender
-description: Sends logs to Datadog
+description: Kubernetes -> Datadog logs sender
 
 type: application
-version: 0.1.1
-appVersion: 0.1.1
+version: 0.1.2
+appVersion: 0.1.2
diff --git a/charts/kube-logs-datadog-sender/templates/aggregator-configmap.yaml b/charts/kube-logs-datadog-sender/templates/aggregator-configmap.yaml
@@ -5,18 +5,18 @@ kind: ConfigMap
 
 metadata:
   name: vector-logs-aggregator-config
-  namespace: kube-system
+  namespace: {{ .Values.targetNamespace }}
 
 data:
   vector.yaml: |
     api:
       enabled: true
-      address: 0.0.0.0:8686
+      address: "0.0.0.0:{{ .Values.vector.api.port }}"
 
     sources:
       vector:
         type: vector
-        address: 0.0.0.0:8685
+        address: "0.0.0.0:{{ .Values.vector.aggregator.source.port }}"
 
     transforms:
       final_transform:

diff --git a/charts/kube-logs-datadog-sender/templates/aggregator-deployment.yaml b/charts/kube-logs-datadog-sender/templates/aggregator-deployment.yaml
@@ -5,7 +5,7 @@ kind: Deployment
 
 metadata:
   name: vector-logs-aggregator
-  namespace: kube-system
+  namespace: {{ .Values.targetNamespace }}
 
 spec:
   selector:
@@ -22,18 +22,19 @@ spec:
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           args: [ "--config-dir", "/etc/vector/" ]
           ports:
-            - containerPort: 8686
-            - containerPort: 8685
+            - containerPort: {{ .Values.vector.api.port }}
+            - containerPort: {{ .Values.vector.aggregator.source.port }}
           envFrom:
             - secretRef:
-                name: {{ .Values.datadog.secretName | default "datadog-creds" }}
+                name: {{ .Values.datadog.secretName }}
           volumeMounts:
             - mountPath: /etc/vector
               name: config
           livenessProbe:
             httpGet:
+              # Rely on Vector's health endpoint to determine if the container is healthy.
               path: /health
-              port: 8686
+              port: {{ .Values.vector.api.port }}
             initialDelaySeconds: 3
             periodSeconds: 3
           resources:
@@ -57,12 +58,12 @@ apiVersion: v1
 kind: Service
 
 metadata:
-  name: vector-logs-aggregator
+  name: {{ .Values.vector.aggregator.service.name }}
 
 spec:
   selector:
     name: vector-logs-aggregator
   ports:
     - name: source
-      port: 8685
-      targetPort: 8685
+      port: {{ .Values.vector.aggregator.source.port }}
+      targetPort: {{ .Values.vector.aggregator.source.port }}
diff --git a/charts/kube-logs-datadog-sender/templates/sender-configmap.yaml b/charts/kube-logs-datadog-sender/templates/sender-configmap.yaml
@@ -5,25 +5,21 @@ kind: ConfigMap
 
 metadata:
   name: vector-logs-sender-config
-  namespace: kube-system
+  namespace: {{ .Values.targetNamespace }}
 
 data:
   vector.yaml: |
     api:
       enabled: true
-      address: 0.0.0.0:8686
+      address: "0.0.0.0:{{ .Values.vector.api.port }}"
 
     sources:
       logs:
         type: kubernetes_logs
         self_node_name: ${VECTOR_SELF_NODE_NAME}
 
     transforms:
-      sample_logs:
-        type: sample
-        inputs: [ logs ]
-        rate: 500
-      parse_json:
+      parse_logs:
         type: remap
         inputs: [ logs ]
         source: |
@@ -46,11 +42,11 @@ data:
           del(.stream)
       exclude_log_levels:
         type: filter
-        inputs: [ parse_json ]
+        inputs: [ parse_logs ]
         condition: |
           !includes(
             [
-              {{- range .Values.sender.exclude.logLevels }}
+              {{- range .Values.vector.sender.exclude.logLevels }}
                 "{{ . }}",
               {{- end }}
             ],
@@ -63,7 +59,7 @@ data:
         condition: |
           !includes(
             [
-              {{- range .Values.sender.exclude.services }}
+              {{- range .Values.vector.sender.exclude.services }}
                 "{{ . }}",
               {{- end }}
             ],
@@ -80,9 +76,5 @@ data:
       vector_aggregator:
         type: vector
         inputs: [ final_transform ]
-        address: vector-logs-aggregator.kube-system.svc.cluster.local:8685
-      #console:
-      #  type: console
-      #  inputs: [ final_transform ]
-      #  encoding:
-      #    codec: json
+        address: "{{ .Values.vector.aggregator.service.name }}.{{ .Values.targetNamespace }}.svc.cluster.local:{{ .Values.vector.aggregator.source.port }}"
+
diff --git a/charts/kube-logs-datadog-sender/templates/sender-daemonset.yaml b/charts/kube-logs-datadog-sender/templates/sender-daemonset.yaml
@@ -5,7 +5,7 @@ kind: DaemonSet
 
 metadata:
   name: vector-logs-sender
-  namespace: kube-system
+  namespace: {{ .Values.targetNamespace }}
 
 spec:
   selector:
@@ -22,7 +22,7 @@ spec:
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           args: [ "--config-dir", "/etc/vector/", "--require-healthy", "true" ]
           ports:
-            - containerPort: 8686
+            - containerPort: {{ .Values.vector.api.port }}
           env:
             - name: VECTOR_SELF_NODE_NAME
               valueFrom:
@@ -41,8 +41,9 @@ spec:
               readOnly: true
           livenessProbe:
             httpGet:
+              # Rely on Vector's health endpoint to determine if the container is healthy.
               path: /health
-              port: 8686
+              port: {{ .Values.vector.api.port }}
             initialDelaySeconds: 3
             periodSeconds: 3
           resources:

diff --git a/charts/kube-logs-datadog-sender/templates/service-roles.yml b/charts/kube-logs-datadog-sender/templates/service-roles.yml
@@ -4,7 +4,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: vector
-  namespace: kube-system
+  namespace: {{ .Values.targetNamespace }}
 
 ---
 
@@ -32,7 +32,7 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: vector
-    namespace: kube-system
+    namespace: {{ .Values.targetNamespace }}
 roleRef:
   kind: ClusterRole
   name: vector

diff --git a/charts/kube-logs-datadog-sender/values.yaml b/charts/kube-logs-datadog-sender/values.yaml
@@ -2,27 +2,40 @@ image:
   repository: timberio/vector
   tag: latest-alpine
 
+targetNamespace: kube-system
+
 # Name of the secret containing Datadog credentials.
 # It should contain:
-# * DD_API_KEY – (required) API key for your Datadog account
+# * DD_API_KEY – (required) API key from your Datadog account
 datadog:
-  secretName: datadog-secret
+  secretName: datadog-creds
   site: datadoghq.com
 
-sender:
-  exclude:
-    # List of services to exclude from sending. Leave empty to send logs from all services.
-    services:
-      - argocd-application-controller
-      - argocd-applicationset-controller
-      - argocd-notifications-controller
-      - argocd-image-updater
-      - argocd-repo-server
-      - argocd-server
-      - vector-logs-sender
-      - vector-logs-aggregator
+vector:
+  api:
+    port: 8686
+
+  sender:
+    exclude:
+      # List of services to exclude from sending. Leave empty to send logs from all services.
+      services:
+        - argocd-application-controller
+        - argocd-applicationset-controller
+        - argocd-notifications-controller
+        - argocd-image-updater
+        - argocd-repo-server
+        - argocd-server
+        - vector-logs-sender
+        - vector-logs-aggregator
+
+      # List of log levels to exclude from sending. Leave empty to disable filtering by log level.
+      logLevels:
+        - TRACE
+        - DEBUG
+
+  aggregator:
+    service:
+      name: vector-logs-aggregator
 
-    # List of log levels to exclude from sending. Leave empty to disable filtering by log level.
-    logLevels:
-      - TRACE
-      - DEBUG
+    source:
+      port: 8685