From fec3d4a7f65a2abae6f75e98f4f90620ba4d473d Mon Sep 17 00:00:00 2001 From: Cristian Radu Date: Thu, 6 Jul 2023 07:00:14 +0100 Subject: [PATCH] add a separate configuration option for auto creating a log group --- README.md | 6 ++++-- example/fluentd.conf | 1 + lib/fluent/plugin/out_cloudwatch_logs.rb | 3 ++- test/plugin/test_out_cloudwatch_logs.rb | 23 ++++++++++++++++------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cddd1f9..9033ed1 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ Fetch sample log from CloudWatch Logs: @type cloudwatch_logs log_group_name log-group-name log_stream_name log-stream-name + auto_create_group true auto_create_stream true #message_keys key1,key2,key3,... #max_message_length 32768 @@ -179,7 +180,8 @@ Fetch sample log from CloudWatch Logs: ``` -* `auto_create_stream`: to create log group and stream automatically. (defaults to false) +* `auto_create_group`: to create log group automatically. (defaults to false) +* `auto_create_stream`: to create log stream automatically. (defaults to false) * `aws_key_id`: AWS Access Key. See [Authentication](#authentication) for more information. * `aws_sec_key`: AWS Secret Access Key. See [Authentication](#authentication) for more information. * `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1) @@ -207,7 +209,7 @@ Fetch sample log from CloudWatch Logs: * `remove_log_group_name_key`: remove field specified by `log_group_name_key` * `remove_log_stream_name_key`: remove field specified by `log_stream_name_key` * `remove_retention_in_days_key`: remove field specified by `retention_in_days_key` -* `retention_in_days`: use to set the expiry time for log group when created with `auto_create_stream`. (default to no expiry) +* `retention_in_days`: use to set the expiry time for log group when created with `auto_create_group`. (default to no expiry) * `retention_in_days_key`: use specified field of records as retention period * `use_tag_as_group`: to use tag as a group name * `use_tag_as_stream`: to use tag as a stream name diff --git a/example/fluentd.conf b/example/fluentd.conf index 8f2cde1..45a9968 100644 --- a/example/fluentd.conf +++ b/example/fluentd.conf @@ -14,6 +14,7 @@ @type cloudwatch_logs log_group_name fluent-plugin-cloudwatch-example log_stream_name fluent-plugin-cloudwatch-example + auto_create_group true auto_create_stream true diff --git a/lib/fluent/plugin/out_cloudwatch_logs.rb b/lib/fluent/plugin/out_cloudwatch_logs.rb index ce62913..66bdb23 100644 --- a/lib/fluent/plugin/out_cloudwatch_logs.rb +++ b/lib/fluent/plugin/out_cloudwatch_logs.rb @@ -29,6 +29,7 @@ class TooLargeEventError < Fluent::UnrecoverableError; end config_param :ssl_verify_peer, :bool, :default => true config_param :log_group_name, :string, :default => nil config_param :log_stream_name, :string, :default => nil + config_param :auto_create_group, :bool, default: false config_param :auto_create_stream, :bool, default: false config_param :message_keys, :array, :default => [], value_type: :string config_param :max_message_length, :integer, :default => nil @@ -264,7 +265,7 @@ def write(chunk) end end - if @auto_create_stream + if @auto_create_group create_log_group(group_name, awstags, retention_in_days) else log.warn "Log group '#{group_name}' does not exist" diff --git a/test/plugin/test_out_cloudwatch_logs.rb b/test/plugin/test_out_cloudwatch_logs.rb index d849876..f8e9a2c 100644 --- a/test/plugin/test_out_cloudwatch_logs.rb +++ b/test/plugin/test_out_cloudwatch_logs.rb @@ -22,6 +22,7 @@ def test_configure region us-east-1 log_group_name test_group log_stream_name test_stream + auto_create_group false auto_create_stream false log_group_aws_tags { "tagkey": "tagvalue", "tagkey_2": "tagvalue_2"} retention_in_days 5 @@ -33,6 +34,7 @@ def test_configure assert_equal('us-east-1', d.instance.region) assert_equal('test_group', d.instance.log_group_name) assert_equal('test_stream', d.instance.log_stream_name) + assert_equal(false, d.instance.auto_create_group) assert_equal(false, d.instance.auto_create_stream) assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey")) assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2")) @@ -81,6 +83,7 @@ def test_write new_log_stream config = {'@type' => 'cloudwatch_logs', + 'auto_create_group' => true, 'auto_create_stream' => true, 'log_stream_name' => log_stream_name, 'log_group_name' => log_group_name, @@ -126,6 +129,7 @@ def test_write new_log_stream config = {'@type' => 'cloudwatch_logs', + 'auto_create_group' => true, 'auto_create_stream' => true, 'log_stream_name' => log_stream_name, 'log_group_name' => log_group_name, @@ -171,6 +175,7 @@ def test_write new_log_stream config = {'@type' => 'cloudwatch_logs', + 'auto_create_group' => true, 'auto_create_stream' => true, 'log_stream_name' => log_stream_name, 'log_group_name' => log_group_name, @@ -367,6 +372,7 @@ def test_write_use_placeholders new_log_stream config = {'@type' => 'cloudwatch_logs', + 'auto_create_group' => true, 'auto_create_stream' => true, 'message_keys' => ["message","cloudwatch"], 'log_stream_name' => "${tag}", @@ -405,6 +411,7 @@ def test_write_use_placeholders_parts new_log_stream config = {'@type' => 'cloudwatch_logs', + 'auto_create_group' => true, 'auto_create_stream' => true, 'message_keys' => ["message","cloudwatch"], 'log_stream_name' => "${tag[0]}-${tag[1]}-${tag[2]}-${tag[3]}", @@ -443,6 +450,7 @@ def test_write_use_time_placeholders new_log_stream config = {'@type' => 'cloudwatch_logs', + 'auto_create_group' => true, 'auto_create_stream' => true, 'message_keys' => ["message","cloudwatch"], 'log_stream_name' => "fluent-plugin-cloudwatch-test-%Y%m%d", @@ -605,7 +613,7 @@ def test_log_group_aws_tags d = create_driver(<<-EOC) #{default_config} - auto_create_stream true + auto_create_group true use_tag_as_stream true log_group_name_key group_name_key log_group_aws_tags {"tag1": "value1", "tag2": "value2"} @@ -634,7 +642,7 @@ def test_log_group_aws_tags_with_placeholders config = { "@type" => "cloudwatch_logs", - "auto_create_stream" => true, + "auto_create_group" => true, "use_tag_as_stream" => true, "log_group_name_key" => "group_name_key", "log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}', @@ -675,7 +683,7 @@ def test_retention_in_days d = create_driver(<<-EOC) #{default_config} - auto_create_stream true + auto_create_group true use_tag_as_stream true log_group_name_key group_name_key retention_in_days 7 @@ -703,7 +711,7 @@ def test_invalid_retention_in_days d = create_driver(<<-EOC) #{default_config} - auto_create_stream true + auto_create_group true use_tag_as_stream true log_group_name_key group_name_key retention_in_days 4 @@ -761,7 +769,7 @@ def test_log_group_aws_tags_key d = create_driver(<<-EOC) #{default_config} - auto_create_stream true + auto_create_group true use_tag_as_stream true log_group_name_key group_name_key log_group_aws_tags_key aws_tags @@ -821,7 +829,7 @@ def test_log_group_aws_tags_key_same_group_diff_tags d = create_driver(<<-EOC) #{default_config} - auto_create_stream true + auto_create_group true use_tag_as_stream true log_group_name_key group_name_key log_group_aws_tags_key aws_tags @@ -855,7 +863,7 @@ def test_log_group_aws_tags_key_no_tags d = create_driver(<<-EOC) #{default_config} - auto_create_stream true + auto_create_group true log_group_name_key group_name_key log_stream_name_key stream_name_key remove_log_group_name_key true @@ -1035,6 +1043,7 @@ def test_scrub_record def default_config <<-EOC @type cloudwatch_logs +auto_create_group true auto_create_stream true #{aws_key_id} #{aws_sec_key}