Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a separate configuration option for auto creating a log group #254

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add a separate configuration option for auto creating a log group #254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ Fetch sample log from CloudWatch Logs:
@type cloudwatch_logs
log_group_name log-group-name
log_stream_name log-stream-name
auto_create_group true
auto_create_stream true
#message_keys key1,key2,key3,...
#max_message_length 32768
Expand Down Expand Up @@ -179,7 +180,8 @@ Fetch sample log from CloudWatch Logs:
</match>
```

* `auto_create_stream`: to create log group and stream automatically. (defaults to false)
* `auto_create_group`: to create log group automatically. (defaults to false)
* `auto_create_stream`: to create log stream automatically. (defaults to false)
* `aws_key_id`: AWS Access Key. See [Authentication](#authentication) for more information.
* `aws_sec_key`: AWS Secret Access Key. See [Authentication](#authentication) for more information.
* `concurrency`: use to set the number of threads pushing data to CloudWatch. (default: 1)
Expand Down Expand Up @@ -207,7 +209,7 @@ Fetch sample log from CloudWatch Logs:
* `remove_log_group_name_key`: remove field specified by `log_group_name_key`
* `remove_log_stream_name_key`: remove field specified by `log_stream_name_key`
* `remove_retention_in_days_key`: remove field specified by `retention_in_days_key`
* `retention_in_days`: use to set the expiry time for log group when created with `auto_create_stream`. (default to no expiry)
* `retention_in_days`: use to set the expiry time for log group when created with `auto_create_group`. (default to no expiry)
* `retention_in_days_key`: use specified field of records as retention period
* `use_tag_as_group`: to use tag as a group name
* `use_tag_as_stream`: to use tag as a stream name
Expand Down
1 change: 1 addition & 0 deletions example/fluentd.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
@type cloudwatch_logs
log_group_name fluent-plugin-cloudwatch-example
log_stream_name fluent-plugin-cloudwatch-example
auto_create_group true
auto_create_stream true
</match>

Expand Down
3 changes: 2 additions & 1 deletion lib/fluent/plugin/out_cloudwatch_logs.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class TooLargeEventError < Fluent::UnrecoverableError; end
config_param :ssl_verify_peer, :bool, :default => true
config_param :log_group_name, :string, :default => nil
config_param :log_stream_name, :string, :default => nil
config_param :auto_create_group, :bool, default: false
config_param :auto_create_stream, :bool, default: false
config_param :message_keys, :array, :default => [], value_type: :string
config_param :max_message_length, :integer, :default => nil
Expand Down Expand Up @@ -264,7 +265,7 @@ def write(chunk)
end
end

if @auto_create_stream
if @auto_create_group
create_log_group(group_name, awstags, retention_in_days)
else
log.warn "Log group '#{group_name}' does not exist"
Expand Down
23 changes: 16 additions & 7 deletions test/plugin/test_out_cloudwatch_logs.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ def test_configure
region us-east-1
log_group_name test_group
log_stream_name test_stream
auto_create_group false
auto_create_stream false
log_group_aws_tags { "tagkey": "tagvalue", "tagkey_2": "tagvalue_2"}
retention_in_days 5
Expand All @@ -33,6 +34,7 @@ def test_configure
assert_equal('us-east-1', d.instance.region)
assert_equal('test_group', d.instance.log_group_name)
assert_equal('test_stream', d.instance.log_stream_name)
assert_equal(false, d.instance.auto_create_group)
assert_equal(false, d.instance.auto_create_stream)
assert_equal("tagvalue", d.instance.log_group_aws_tags.fetch("tagkey"))
assert_equal("tagvalue_2", d.instance.log_group_aws_tags.fetch("tagkey_2"))
Expand Down Expand Up @@ -81,6 +83,7 @@ def test_write
new_log_stream

config = {'@type' => 'cloudwatch_logs',
'auto_create_group' => true,
'auto_create_stream' => true,
'log_stream_name' => log_stream_name,
'log_group_name' => log_group_name,
Expand Down Expand Up @@ -126,6 +129,7 @@ def test_write
new_log_stream

config = {'@type' => 'cloudwatch_logs',
'auto_create_group' => true,
'auto_create_stream' => true,
'log_stream_name' => log_stream_name,
'log_group_name' => log_group_name,
Expand Down Expand Up @@ -171,6 +175,7 @@ def test_write
new_log_stream

config = {'@type' => 'cloudwatch_logs',
'auto_create_group' => true,
'auto_create_stream' => true,
'log_stream_name' => log_stream_name,
'log_group_name' => log_group_name,
Expand Down Expand Up @@ -367,6 +372,7 @@ def test_write_use_placeholders
new_log_stream

config = {'@type' => 'cloudwatch_logs',
'auto_create_group' => true,
'auto_create_stream' => true,
'message_keys' => ["message","cloudwatch"],
'log_stream_name' => "${tag}",
Expand Down Expand Up @@ -405,6 +411,7 @@ def test_write_use_placeholders_parts
new_log_stream

config = {'@type' => 'cloudwatch_logs',
'auto_create_group' => true,
'auto_create_stream' => true,
'message_keys' => ["message","cloudwatch"],
'log_stream_name' => "${tag[0]}-${tag[1]}-${tag[2]}-${tag[3]}",
Expand Down Expand Up @@ -443,6 +450,7 @@ def test_write_use_time_placeholders
new_log_stream

config = {'@type' => 'cloudwatch_logs',
'auto_create_group' => true,
'auto_create_stream' => true,
'message_keys' => ["message","cloudwatch"],
'log_stream_name' => "fluent-plugin-cloudwatch-test-%Y%m%d",
Expand Down Expand Up @@ -605,7 +613,7 @@ def test_log_group_aws_tags

d = create_driver(<<-EOC)
#{default_config}
auto_create_stream true
auto_create_group true
use_tag_as_stream true
log_group_name_key group_name_key
log_group_aws_tags {"tag1": "value1", "tag2": "value2"}
Expand Down Expand Up @@ -634,7 +642,7 @@ def test_log_group_aws_tags_with_placeholders

config = {
"@type" => "cloudwatch_logs",
"auto_create_stream" => true,
"auto_create_group" => true,
"use_tag_as_stream" => true,
"log_group_name_key" => "group_name_key",
"log_group_aws_tags" => '{"tag1": "${tag}", "tag2": "${namespace_name}"}',
Expand Down Expand Up @@ -675,7 +683,7 @@ def test_retention_in_days

d = create_driver(<<-EOC)
#{default_config}
auto_create_stream true
auto_create_group true
use_tag_as_stream true
log_group_name_key group_name_key
retention_in_days 7
Expand Down Expand Up @@ -703,7 +711,7 @@ def test_invalid_retention_in_days

d = create_driver(<<-EOC)
#{default_config}
auto_create_stream true
auto_create_group true
use_tag_as_stream true
log_group_name_key group_name_key
retention_in_days 4
Expand Down Expand Up @@ -761,7 +769,7 @@ def test_log_group_aws_tags_key

d = create_driver(<<-EOC)
#{default_config}
auto_create_stream true
auto_create_group true
use_tag_as_stream true
log_group_name_key group_name_key
log_group_aws_tags_key aws_tags
Expand Down Expand Up @@ -821,7 +829,7 @@ def test_log_group_aws_tags_key_same_group_diff_tags

d = create_driver(<<-EOC)
#{default_config}
auto_create_stream true
auto_create_group true
use_tag_as_stream true
log_group_name_key group_name_key
log_group_aws_tags_key aws_tags
Expand Down Expand Up @@ -855,7 +863,7 @@ def test_log_group_aws_tags_key_no_tags

d = create_driver(<<-EOC)
#{default_config}
auto_create_stream true
auto_create_group true
log_group_name_key group_name_key
log_stream_name_key stream_name_key
remove_log_group_name_key true
Expand Down Expand Up @@ -1035,6 +1043,7 @@ def test_scrub_record
def default_config
<<-EOC
@type cloudwatch_logs
auto_create_group true
auto_create_stream true
#{aws_key_id}
#{aws_sec_key}
Expand Down