refs sparkfabrik-innovation-team/board#3029: Create application error…

… log group error (#6)

CHANGELOG.md

@@ -12,6 +12,14 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 - Fix typo in outputs: `final_k8s_common_labels` instead of `finale_k8s_common_labels`.
 
+## [0.4.0] - 2024-09-20
+
+[Compare with previous version](https://github.com/sparkfabrik/terraform-helm-fluentbit/compare/0.3.1...0.4.0)
+
+### Changed
+
+- Feat: added a new default log group `application-errors` containing all application errors `4xx` and `5xx`
+
 ## [0.3.1] - 2024-06-03
 
 [Compare with previous version](https://github.com/sparkfabrik/terraform-helm-fluentbit/compare/0.3.0...0.3.1)

README.md

@@ -57,7 +57,7 @@ It also opinionatedly configures FluentBit to send logs to CloudWatch and to exc
 |------|-------------|
 | <a name="output_final_exclude_from_application_log_group"></a> [final\_exclude\_from\_application\_log\_group](#output\_final\_exclude\_from\_application\_log\_group) | The final list of log files to exclude from the application log group. |
 | <a name="output_final_include_in_platform_log_group"></a> [final\_include\_in\_platform\_log\_group](#output\_final\_include\_in\_platform\_log\_group) | The final list of log files to include in the platform log group. |
-| <a name="output_finale_k8s_common_labels"></a> [finale\_k8s\_common\_labels](#output\_finale\_k8s\_common\_labels) | The final list of common labels to apply to the Kubernetes resources. |
+| <a name="output_final_k8s_common_labels"></a> [final\_k8s\_common\_labels](#output\_final\_k8s\_common\_labels) | The final list of common labels to apply to the Kubernetes resources. |
 
 ## Resources
 

files/values.yml.tftpl

@@ -73,6 +73,23 @@ additionalInputs: |
  storage.type filesystem
  Read_from_Head $${READ_FROM_HEAD}
 
+ [INPUT]
+ Name tail
+ Tag application-errors.*
+ Exclude_Path ${join(", ", formatlist("/var/log/containers/%s*.log", distinct(additional_exclude_from_application_log_group)))}
+ Path /var/log/containers/*.log
+ Docker_Mode On
+ Docker_Mode_Flush 5
+ Docker_Mode_Parser custom_json
+ Parser custom_json
+ DB /var/fluent-bit/state/flb_container.db
+ Mem_Buf_Limit 50MB
+ Skip_Long_Lines On
+ Refresh_Interval 10
+ Rotate_Wait 30
+ storage.type filesystem
+ Read_from_Head $${READ_FROM_HEAD}
+
  [INPUT]
  Name tail
  Tag platform.*
@@ -118,7 +135,7 @@ filter:
  enabled: false
 
 additionalFilters: |
- %{~ for tag in ["application", "platform", "fluentbit"] ~}
+ %{~ for tag in ["application", "application-errors", "platform", "fluentbit"] ~}
  [FILTER]
  Name kubernetes
  Match_regex ${tag}.*
@@ -134,26 +151,32 @@ additionalFilters: |
 
  [FILTER]
  Name nest
- Match_regex application.*|platform.*|fluentbit.*
+ Match_regex application.*|application-errors.*|platform.*|fluentbit.*
  Operation lift
  Nested_under kubernetes
  Add_prefix Kube.
 
  [FILTER]
  Name modify
- Match_regex application.*|platform.*|fluentbit.*
+ Match_regex application.*|application-errors.*|platform.*|fluentbit.*
  Remove Kube.<Metadata_1>
  Remove Kube.<Metadata_2>
  Remove Kube.<Metadata_3>
 
  [FILTER]
  Name nest
- Match_regex application.*|platform.*|fluentbit.*
+ Match_regex application.*|application-errors.*|platform.*|fluentbit.*
  Operation nest
  Wildcard Kube.*
  Nested_under kubernetes
  Remove_prefix Kube.
 
+ # Retrive logs with status code 4xx and 5xx
+ [FILTER]
+ Name grep
+ Match application-errors.*
+ Regex $log_processed['status'] [45][0-9]{2}
+
 %{~ if additional_filters != "" }
  ${additional_filters}
 %{~ endif }
@@ -170,6 +193,17 @@ additionalOutputs: |
  log_retention_days ${application_log_retention_days}
  #extra_user_agent container-insights
 
+ [OUTPUT]
+ Name cloudwatch_logs
+ Match application-errors.*
+ region $${AWS_REGION}
+ log_group_name /aws/containerinsights/$${CLUSTER_NAME}/application-errors
+ log_stream_prefix $${HOST_NAME}-
+ log_format json
+ auto_create_group true
+ log_retention_days ${application_log_retention_days}
+ #extra_user_agent container-insights
+
  [OUTPUT]
  Name cloudwatch_logs
  Match platform.*