Split configuration into smaller files

utilitywarehouse · Jun 21, 2024 · 7eabd47 · 7eabd47
1 parent 394aaae
commit 7eabd47
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 75 deletions.
diff --git a/vector/agent/namespaced/daemonset.yaml b/vector/agent/namespaced/daemonset.yaml
@@ -22,8 +22,10 @@ spec:
       serviceAccountName: vector
       containers:
         - name: vector
-          image: "timberio/vector:0.39.0-debian"
-          imagePullPolicy: IfNotPresent
+          image: "timberio/vector:0.39.X-debian"
+          args:
+            - --config-dir
+            - /etc/vector/
           env:
             - name: VECTOR_LOG
               value: "info"

diff --git a/vector/agent/namespaced/kustomization.yaml b/vector/agent/namespaced/kustomization.yaml
@@ -7,4 +7,6 @@ resources:
 configMapGenerator:
   - name: vector
     files:
-      - vector.yaml=resources/vector.yaml # /etc/vector/vector.yaml is the default config location
+      - audits.yaml=resources/audits.yaml
+      - pods.yaml=resources/pods.yaml
+      - systemd.yaml=resources/systemd.yaml
diff --git a/vector/agent/namespaced/resources/audits.yaml b/vector/agent/namespaced/resources/audits.yaml
@@ -0,0 +1,59 @@
+data_dir: /vector-data-dir
+sources:
+  kubernetes_audit_logs:
+    type: kubernetes_logs
+    extra_namespace_label_selector: "name=sys-falco"
+
+transforms:
+  kubernetes_audits_remap:
+    inputs:
+      - kubernetes_audit_logs
+    type: remap
+    source: |
+      .app, _ = get(.kubernetes.pod_labels, ["app.kubernetes.io/name"])
+      if is_nullish(.app) {
+        .app, _ = get(.kubernetes.pod_labels, ["app"])
+      }
+      if is_nullish(.app) {
+        .app = .kubernetes.container_name
+      }
+      .namespace = .kubernetes.pod_namespace
+      .pod_name = .kubernetes.pod_name
+      .container = .kubernetes.container_name
+      .limit_key = string!(.kubernetes.pod_namespace) + "/" + string!(.app)
+      del(.file)
+      del(.kubernetes)
+      del(.source_type)
+      del(.stream)
+
+  throttle_audits:
+    inputs:
+      - kubernetes_audits_remap
+    type: throttle
+    key_field: '{{ limit_key }}'
+    threshold: 10
+    window_secs: 1
+
+sinks:
+  loki_audits:
+    inputs:
+      - throttle_audits
+    type: loki
+    endpoint: http://loki-audits.sys-log.svc.cluster.aws:3100
+    path: /loki/api/v1/push
+    encoding:
+      codec: text
+    labels:
+      cloud_provider: "${CLOUD_PROVIDER}"
+      uw_environment: "${UW_ENVIRONMENT}"
+      kubernetes_cluster: "${KUBERNETES_CLUSTER}"
+      cluster: "${KUBERNETES_CLUSTER}"
+      provider: "${CLOUD_PROVIDER}"
+      app: '{{ app }}'
+      app_kubernetes_io_name: '{{ app }}'
+      kubernetes_namespace: '{{ namespace }}'
+      namespace: '{{ namespace }}'
+      kubernetes_pod_name: '{{ pod_name }}'
+      pod: '{{ pod_name }}'
+      kubernetes_container: '{{ container }}'
+      container: '{{ container }}'
diff --git a/...or/agent/namespaced/resources/vector.yaml → vector/agent/namespaced/resources/pods.yaml b/...or/agent/namespaced/resources/vector.yaml → vector/agent/namespaced/resources/pods.yaml
@@ -2,22 +2,18 @@ data_dir: /vector-data-dir
 sources:
   kubernetes_logs:
     type: kubernetes_logs
-
-  systemd_logs:
-    type: journald
-    current_boot_only: true
-    journal_directory: /var/log/journal
+    extra_namespace_label_selector: "name!=sys-falco"
 
 transforms:
-  drop_fluentbit:
+  drop_exclude:
     inputs:
       - kubernetes_logs
     type: filter
     condition: '.kubernetes.pod_labels.fluentbit_io_exclude != "true"'
 
   drop_calico:
     inputs:
-      - drop_fluentbit
+      - drop_exclude
     type: filter
     condition: '!(.kubernetes.pod_name == "calico-apiserver" && (contains(string!(.message), "level=info") || contains(string!(.message), "called with key")))'
 
@@ -34,7 +30,7 @@ transforms:
     source: |
       .app, _ = get(.kubernetes.pod_labels, ["app.kubernetes.io/name"])
       if is_nullish(.app) {
-        .app, err = get(.kubernetes.pod_labels, ["app"])
+        .app, _ = get(.kubernetes.pod_labels, ["app"])
       }
       if is_nullish(.app) {
         .app = .kubernetes.container_name
@@ -56,54 +52,10 @@ transforms:
     threshold: 10
     window_secs: 1
 
-  filter_audit_logs:
-    inputs:
-      - throttle
-    type: filter
-    condition: '.namespace == "sys-falco"'
-
-  filter_non_audit:
-    inputs:
-      - throttle
-    type: filter
-    condition: '!(.namespace == "sys-falco")'
-
-  systemd_remap:
-    inputs:
-      - systemd_logs
-    type: remap
-    source: |
-      .systemd_unit = ._SYSTEMD_UNIT
-      .hostname = .host
-      .syslog_identifier = .SYSLOG_IDENTIFIER
-
 sinks:
-  loki_audits:
-    inputs:
-      - filter_audit_logs
-    type: loki
-    endpoint: http://loki-audits.sys-log.svc.cluster.aws:3100
-    path: /loki/api/v1/push
-    encoding:
-      codec: text
-    labels:
-      cloud_provider: "${CLOUD_PROVIDER}"
-      uw_environment: "${UW_ENVIRONMENT}"
-      kubernetes_cluster: "${KUBERNETES_CLUSTER}"
-      cluster: "${KUBERNETES_CLUSTER}"
-      provider: "${CLOUD_PROVIDER}"
-      app: '{{ app }}'
-      app_kubernetes_io_name: '{{ app }}'
-      kubernetes_namespace: '{{ namespace }}'
-      namespace: '{{ namespace }}'
-      kubernetes_pod_name: '{{ pod_name }}'
-      pod: '{{ pod_name }}'
-      kubernetes_container: '{{ container }}'
-      container: '{{ container }}'
-
   loki_kubernetes:
     inputs:
-      - filter_non_audit
+      - throttle
     type: loki
     endpoint: http://loki.sys-log.svc.cluster.aws:3100
     path: /loki/api/v1/push
@@ -123,22 +75,3 @@ sinks:
       pod: '{{ pod_name }}'
       kubernetes_container: '{{ container }}'
       container: '{{ container }}'
-
-  loki_systemd:
-    inputs:
-      - systemd_remap
-    type: loki
-    endpoint: http://loki.sys-log.svc.cluster.aws:3100
-    path: /loki/api/v1/push
-    encoding:
-      codec: text
-    labels:
-      cloud_provider: "${CLOUD_PROVIDER}"
-      uw_environment: "${UW_ENVIRONMENT}"
-      kubernetes_cluster: "${KUBERNETES_CLUSTER}"
-      cluster: "${KUBERNETES_CLUSTER}"
-      provider: "${CLOUD_PROVIDER}"
-      log_source: "systemd"
-      systemd_unit: '{{ systemd_unit }}'
-      hostname: '{{ hostname }}'
-      syslog_identifier: '{{ syslog_identifier }}'
diff --git a/vector/agent/namespaced/resources/systemd.yaml b/vector/agent/namespaced/resources/systemd.yaml
@@ -0,0 +1,36 @@
+data_dir: /vector-data-dir
+sources:
+  systemd_logs:
+    type: journald
+    current_boot_only: true
+    journal_directory: /var/log/journal
+
+transforms:
+  systemd_remap:
+    inputs:
+      - systemd_logs
+    type: remap
+    source: |
+      .systemd_unit = ._SYSTEMD_UNIT
+      .hostname = .host
+      .syslog_identifier = .SYSLOG_IDENTIFIER
+
+sinks:
+  loki_systemd:
+    inputs:
+      - systemd_remap
+    type: loki
+    endpoint: http://loki.sys-log.svc.cluster.aws:3100
+    path: /loki/api/v1/push
+    encoding:
+      codec: text
+    labels:
+      cloud_provider: "${CLOUD_PROVIDER}"
+      uw_environment: "${UW_ENVIRONMENT}"
+      kubernetes_cluster: "${KUBERNETES_CLUSTER}"
+      cluster: "${KUBERNETES_CLUSTER}"
+      provider: "${CLOUD_PROVIDER}"
+      log_source: "systemd"
+      systemd_unit: '{{ systemd_unit }}'
+      hostname: '{{ hostname }}'
+      syslog_identifier: '{{ syslog_identifier }}'