From adec760208b7ba7201db72b31d3815e928a1c572 Mon Sep 17 00:00:00 2001
From: Fabrizio Cafolla <developer@fabriziocafolla.com>
Date: Fri, 20 Sep 2024 16:00:29 +0200
Subject: [PATCH] refs sparkfabrik-innovation-team/board#3029: Create
 application error log group error (#6)

---
 CHANGELOG.md           |  8 ++++++++
 README.md              |  2 +-
 files/values.yml.tftpl | 42 ++++++++++++++++++++++++++++++++++++++----
 3 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a3c736..a72d58e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,14 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 - Fix typo in outputs: `final_k8s_common_labels` instead of `finale_k8s_common_labels`.
 
+## [0.4.0] - 2024-09-20
+
+[Compare with previous version](https://github.com/sparkfabrik/terraform-helm-fluentbit/compare/0.3.1...0.4.0)
+
+### Changed
+
+- Feat: added a new default log group `application-errors` containing all application errors `4xx` and `5xx`
+
 ## [0.3.1] - 2024-06-03
 
 [Compare with previous version](https://github.com/sparkfabrik/terraform-helm-fluentbit/compare/0.3.0...0.3.1)
diff --git a/README.md b/README.md
index b1b50fd..f3d0e46 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ It also opinionatedly configures FluentBit to send logs to CloudWatch and to exc
 |------|-------------|
 | <a name="output_final_exclude_from_application_log_group"></a> [final\_exclude\_from\_application\_log\_group](#output\_final\_exclude\_from\_application\_log\_group) | The final list of log files to exclude from the application log group. |
 | <a name="output_final_include_in_platform_log_group"></a> [final\_include\_in\_platform\_log\_group](#output\_final\_include\_in\_platform\_log\_group) | The final list of log files to include in the platform log group. |
-| <a name="output_finale_k8s_common_labels"></a> [finale\_k8s\_common\_labels](#output\_finale\_k8s\_common\_labels) | The final list of common labels to apply to the Kubernetes resources. |
+| <a name="output_final_k8s_common_labels"></a> [final\_k8s\_common\_labels](#output\_final\_k8s\_common\_labels) | The final list of common labels to apply to the Kubernetes resources. |
 
 ## Resources
 
diff --git a/files/values.yml.tftpl b/files/values.yml.tftpl
index 6c3f0ec..ea04a16 100644
--- a/files/values.yml.tftpl
+++ b/files/values.yml.tftpl
@@ -73,6 +73,23 @@ additionalInputs: |
       storage.type        filesystem
       Read_from_Head      $${READ_FROM_HEAD}
 
+  [INPUT]
+      Name                tail
+      Tag                 application-errors.*
+      Exclude_Path        ${join(", ", formatlist("/var/log/containers/%s*.log", distinct(additional_exclude_from_application_log_group)))}
+      Path                /var/log/containers/*.log
+      Docker_Mode         On
+      Docker_Mode_Flush   5
+      Docker_Mode_Parser  custom_json
+      Parser              custom_json
+      DB                  /var/fluent-bit/state/flb_container.db
+      Mem_Buf_Limit       50MB
+      Skip_Long_Lines     On
+      Refresh_Interval    10
+      Rotate_Wait         30
+      storage.type        filesystem
+      Read_from_Head      $${READ_FROM_HEAD}
+
   [INPUT]
       Name                tail
       Tag                 platform.*
@@ -118,7 +135,7 @@ filter:
   enabled: false
 
 additionalFilters: |
-  %{~ for tag in ["application", "platform", "fluentbit"] ~}
+  %{~ for tag in ["application", "application-errors", "platform", "fluentbit"] ~}
   [FILTER]
       Name                kubernetes
       Match_regex         ${tag}.*
@@ -134,26 +151,32 @@ additionalFilters: |
 
   [FILTER]
       Name                nest
-      Match_regex         application.*|platform.*|fluentbit.*
+      Match_regex         application.*|application-errors.*|platform.*|fluentbit.*
       Operation           lift
       Nested_under        kubernetes
       Add_prefix          Kube.
 
   [FILTER]
       Name                modify
-      Match_regex         application.*|platform.*|fluentbit.*
+      Match_regex         application.*|application-errors.*|platform.*|fluentbit.*
       Remove              Kube.<Metadata_1>
       Remove              Kube.<Metadata_2>
       Remove              Kube.<Metadata_3>
 
   [FILTER]
       Name                nest
-      Match_regex         application.*|platform.*|fluentbit.*
+      Match_regex         application.*|application-errors.*|platform.*|fluentbit.*
       Operation           nest
       Wildcard            Kube.*
       Nested_under        kubernetes
       Remove_prefix       Kube.
 
+  # Retrive logs with status code 4xx and 5xx
+  [FILTER]
+      Name                grep
+      Match               application-errors.*
+      Regex               $log_processed['status'] [45][0-9]{2}
+
 %{~ if additional_filters != "" }
   ${additional_filters}
 %{~ endif }
@@ -170,6 +193,17 @@ additionalOutputs: |
       log_retention_days  ${application_log_retention_days}
       #extra_user_agent   container-insights
 
+  [OUTPUT]
+      Name                cloudwatch_logs
+      Match               application-errors.*
+      region              $${AWS_REGION}
+      log_group_name      /aws/containerinsights/$${CLUSTER_NAME}/application-errors
+      log_stream_prefix   $${HOST_NAME}-
+      log_format          json
+      auto_create_group   true
+      log_retention_days  ${application_log_retention_days}
+      #extra_user_agent   container-insights
+
   [OUTPUT]
       Name                cloudwatch_logs
       Match               platform.*