Missing conf worker data plane logs fluentbit add on

.github/workflows/plan-examples.yml

@@ -5,6 +5,7 @@ on:
   pull_request_target:
     branches:
       - main
+      - v4
   workflow_dispatch:
 
 concurrency:

.github/workflows/pre-commit.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - main
+      - v4
     paths:
       - '**.tf'
       - '**.yml'

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: detect-aws-credentials
         args: ['--allow-missing-credentials']
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.78.0
+    rev: v1.80.0
     hooks:
       - id: terraform_fmt
       - id: terraform_docs

modules/kubernetes-addons/README.md

@@ -74,7 +74,7 @@
 | <a name="module_nvidia_device_plugin"></a> [nvidia\_device\_plugin](#module\_nvidia\_device\_plugin) | ./nvidia-device-plugin | n/a |
 | <a name="module_ondat"></a> [ondat](#module\_ondat) | ./ondat | n/a |
 | <a name="module_opentelemetry_operator"></a> [opentelemetry\_operator](#module\_opentelemetry\_operator) | ./opentelemetry-operator | n/a |
-| <a name="module_portworx"></a> [portworx](#module\_portworx) | portworx/portworx-addon/eksblueprints | 0.0.6 |
+| <a name="module_portworx"></a> [portworx](#module\_portworx) | ./portworx | n/a |
 | <a name="module_prometheus"></a> [prometheus](#module\_prometheus) | ./prometheus | n/a |
 | <a name="module_promtail"></a> [promtail](#module\_promtail) | ./promtail | n/a |
 | <a name="module_reloader"></a> [reloader](#module\_reloader) | ./reloader | n/a |
@@ -83,7 +83,7 @@
 | <a name="module_spark_history_server"></a> [spark\_history\_server](#module\_spark\_history\_server) | ./spark-history-server | n/a |
 | <a name="module_spark_k8s_operator"></a> [spark\_k8s\_operator](#module\_spark\_k8s\_operator) | ./spark-k8s-operator | n/a |
 | <a name="module_strimzi_kafka_operator"></a> [strimzi\_kafka\_operator](#module\_strimzi\_kafka\_operator) | ./strimzi-kafka-operator | n/a |
-| <a name="module_sysdig_agent"></a> [sysdig\_agent](#module\_sysdig\_agent) | sysdiglabs/sysdig-addon/eksblueprints | 0.0.3 |
+| <a name="module_sysdig_agent"></a> [sysdig\_agent](#module\_sysdig\_agent) | ./sysdig | n/a |
 | <a name="module_tetrate_istio"></a> [tetrate\_istio](#module\_tetrate\_istio) | ./tetrate-istio | n/a |
 | <a name="module_thanos"></a> [thanos](#module\_thanos) | ./thanos | n/a |
 | <a name="module_traefik"></a> [traefik](#module\_traefik) | ./traefik | n/a |

modules/kubernetes-addons/aws-for-fluentbit/values.yaml

@@ -2,7 +2,134 @@ serviceAccount:
   create: false
   name: ${service_account}
 
+input:
+  enabled: true
+  tag: "kube.app.*"
+  path: "/var/log/containers/*.log"
+  db: "/var/log/flb_kube.db"
+  parser: docker
+  dockerMode: "On"
+  memBufLimit: 5MB
+  skipLongLines: "On"
+  refreshInterval: 10
+  extraInputs: >
+    exclude_path   /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
+additionalInputs: |
+  [INPUT]
+      Name              tail
+      tag               host.messages
+      path              /var/log/messages
+      db                /var/log/flb_messages.db
+      parser            syslog
+      Mem_Buf_Limit     5MB
+      Skip_Long_Lines   On
+      Refresh_Interval  10
+      Read_from_Head    true
+  [INPUT]
+      Name                tail
+      Tag                 host.dmesg
+      Path                /var/log/dmesg
+      Key                 message
+      DB                  /var/log/flb_dmesg.db
+      Mem_Buf_Limit       5MB
+      Skip_Long_Lines     On
+      Refresh_Interval    10
+      Read_from_Head      true
+  [INPUT]
+      Name                tail
+      Tag                 host.secure
+      Path                /var/log/secure
+      Parser              syslog
+      DB                  /var/log/flb_secure.db
+      Mem_Buf_Limit       5MB
+      Skip_Long_Lines     On
+      Refresh_Interval    10
+      Read_from_Head      true
+  [INPUT]
+      Name                systemd
+      Tag                 dataplane.systemd.*
+      Systemd_Filter      _SYSTEMD_UNIT=containerd.service
+      Systemd_Filter      _SYSTEMD_UNIT=kubelet.service
+      DB                  /var/log/systemd.db
+      Path                /var/log/journal
+      Read_From_Tail      true
+
+  [INPUT]
+      Name                tail
+      Tag                 dataplane.tail.*
+      Path                /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
+      parser              docker
+      DB                  /var/log/flb_dataplane_tail.db
+      Mem_Buf_Limit       50MB
+      Skip_Long_Lines     On
+      Refresh_Interval    10
+      Rotate_Wait         30
+      #storage.type        filesystem
+      Read_from_Head      true
+filter:
+  enabled: true
+  match: "kube.app.*"
+  kubeURL: "https://kubernetes.default.svc.cluster.local:443"
+  mergeLog: "On"
+  mergeLogKey: "data"
+  keepLog: "On"
+  k8sLoggingParser: "On"
+  k8sLoggingExclude: "Off"
+  bufferSize: "32k"
+  extraFilters: |
+    Kube_Tag_Prefix     kube.app.var.log.containers.
+additionalFilters: |
+    [FILTER]
+        Name                  aws
+        Match                 host.*
+        imds_version          v2
+        az                    true
+        ec2_instance_id       true
+        ec2_instance_type     true
+        private_ip            true
+        hostname              true
+
+    [FILTER]
+        Name                  modify
+        Match                 dataplane.systemd.*
+        Rename                _HOSTNAME                   hostname
+        Rename                _SYSTEMD_UNIT               systemd_unit
+        Rename                MESSAGE                     message
+        Remove_regex          ^((?!hostname|systemd_unit|message).)*$
+
+    [FILTER]
+        Name                  aws
+        Match                 dataplane.*
+        imds_version          v2
+        az                    true
+        ec2_instance_id       true
+        ec2_instance_type     true
+        private_ip            true
+        hostname              true
 cloudWatchLogs:
   enabled: true
+  match: "kube.app.*"
   region: ${aws_region}
-  logGroupName: ${log_group_name}
+  logGroupName: "/aws/eks/fluentbit-cloudwatch/logs"
+  logGroupTemplate: /aws/containerinsights/application/$kubernetes['namespace_name']
+  logStreamPrefix: "fluentbit-"
+  logStreamTemplate: $kubernetes['pod_name'].$kubernetes['container_name']
+additionalOutputs: |
+  [OUTPUT]
+      Name                cloudwatch_logs
+      match               host.*
+      region              ${aws_region}
+      log_group_name      /aws/containerinsight/host
+      auto_create_group   true
+      log_stream_template $hostname
+      log_stream_prefix   eks
+      extra_user_agent    container-insights
+  [OUTPUT]
+      Name                cloudwatch_logs
+      match               dataplane.*
+      region              ${aws_region}
+      log_group_name      /aws/containerinsights/dataplane
+      auto_create_group   true
+      log_stream_template $hostname
+      log_stream_prefix   eks
+      extra_user_agent    container-insights

modules/kubernetes-addons/main.tf

@@ -389,12 +389,14 @@ module "kube_prometheus_stack" {
 }
 
 module "portworx" {
-  count         = var.enable_portworx ? 1 : 0
-  source        = "portworx/portworx-addon/eksblueprints"
-  version       = "0.0.6"
+  source = "./portworx"
+
+  count = var.enable_portworx ? 1 : 0
+
   helm_config   = var.portworx_helm_config
   addon_context = local.addon_context
 }
+
 module "prometheus" {
   count       = var.enable_prometheus ? 1 : 0
   source      = "./prometheus"
@@ -441,20 +443,14 @@ module "strimzi_kafka_operator" {
 }
 
 module "sysdig_agent" {
-  source  = "sysdiglabs/sysdig-addon/eksblueprints"
-  version = "0.0.3"
+  source = "./sysdig"
 
   count         = var.enable_sysdig_agent ? 1 : 0
   helm_config   = var.sysdig_agent_helm_config
   addon_context = local.addon_context
 }
 
 module "tetrate_istio" {
-  # source  = "tetratelabs/tetrate-istio-addon/eksblueprints"
-  # version = "0.0.7"
-
-  # TODO - remove local source and revert to remote once
-  # https://github.com/tetratelabs/terraform-eksblueprints-tetrate-istio-addon/pull/12  is merged
   source = "./tetrate-istio"
 
   count = var.enable_tetrate_istio ? 1 : 0

modules/kubernetes-addons/portworx/README.md

@@ -0,0 +1,3 @@
+# Portworx add-on for EKS Blueprints
+
+Local copy of https://github.com/portworx/terraform-eksblueprints-portworx-addon

modules/kubernetes-addons/portworx/locals.tf

@@ -0,0 +1,86 @@
+resource "random_string" "id" {
+  length  = 4
+  special = false
+  upper   = false
+}
+
+locals {
+  name                 = "portworx-${random_string.id.result}"
+  namespace            = "kube-system"
+  service_account_name = "${local.name}-sa-${random_string.id.result}"
+
+  aws_marketplace_config = try(var.helm_config["set"][index(var.helm_config.set[*].name, "aws.marketplace")], null)
+  use_aws_marketplace    = local.aws_marketplace_config != null ? local.aws_marketplace_config["value"] : false
+
+  default_helm_config = {
+    name        = local.name
+    description = "A Helm chart for portworx"
+    chart       = "portworx"
+    repository  = "https://raw.githubusercontent.com/portworx/eks-blueprint-helm/main/repo/stable"
+    version     = "2.11.0"
+    namespace   = local.namespace
+    values      = local.default_helm_values
+  }
+
+  helm_config = merge(
+    local.default_helm_config,
+    var.helm_config
+  )
+
+  irsa_iam_policies_list = local.use_aws_marketplace != false ? [aws_iam_policy.portworx_eksblueprint_metering[0].arn] : []
+
+  irsa_config = {
+    create_kubernetes_namespace       = false
+    kubernetes_namespace              = local.namespace
+    create_kubernetes_service_account = true
+    kubernetes_service_account        = local.service_account_name
+    irsa_iam_policies                 = local.irsa_iam_policies_list
+  }
+
+  default_helm_values = [templatefile("${path.module}/values.yaml", {
+    imageVersion           = "2.11.0"
+    clusterName            = local.name
+    drives                 = "type=gp2,size=200"
+    useInternalKVDB        = true
+    kvdbDevice             = "type=gp2,size=150"
+    envVars                = ""
+    maxStorageNodesPerZone = 3
+    useOpenshiftInstall    = false
+    etcdEndPoint           = ""
+    dataInterface          = ""
+    managementInterface    = ""
+    useStork               = true
+    storkVersion           = "2.11.0"
+    customRegistryURL      = ""
+    registrySecret         = ""
+    licenseSecret          = ""
+    monitoring             = false
+    enableCSI              = false
+    enableAutopilot        = false
+    KVDBauthSecretName     = ""
+    eksServiceAccount      = local.service_account_name
+    awsAccessKeyId         = ""
+    awsSecretAccessKey     = ""
+    deleteType             = "UninstallAndWipe"
+    }
+  )]
+}
+
+resource "aws_iam_policy" "portworx_eksblueprint_metering" {
+  count = try(local.use_aws_marketplace, false) ? 1 : 0
+  name  = "portworx_eksblueprint_metering-${random_string.id.result}"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "aws-marketplace:MeterUsage",
+          "aws-marketplace:RegisterUsage"
+        ],
+        Effect   = "Allow",
+        Resource = "*"
+      },
+    ]
+  })
+}

modules/kubernetes-addons/portworx/main.tf

@@ -0,0 +1,7 @@
+module "helm_addon" {
+  source = "../helm-addon"
+
+  addon_context = var.addon_context
+  helm_config   = local.helm_config
+  irsa_config   = local.irsa_config
+}