Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[datadog_logs_index] Add Flex Index Configuration support for Logs Indexes #2569

Merged
merged 7 commits into from
Sep 18, 2024
Merged

[datadog_logs_index] Add Flex Index Configuration support for Logs Indexes #2569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3842696
:sparkles: Add Flex Index Configuration support for Logs Indexes
KillerPaperMice Sep 10, 2024
561730c
:test_tube: enable flex test for logs indexes
KillerPaperMice Sep 10, 2024
c54eed2
:memo: update Logs Index docs for flex retention support
KillerPaperMice Sep 10, 2024
534168c
:white_check_mark: update test cassettes for log indexes
KillerPaperMice Sep 10, 2024
a18a8bf
:rewind: remove api version bumps
KillerPaperMice Sep 10, 2024
cd054cb
Merge branch 'master' into keri/LOGSAC-618-flex-index-support
KillerPaperMice Sep 12, 2024
19d3a53
Merge remote-tracking branch 'origin/master' into keri/LOGSAC-618-fle…
KillerPaperMice Sep 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions datadog/data_source_datadog_logs_indexes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,12 @@ func dataSourceDatadogLogsIndexes() *schema.Resource {
Computed: true,
},
"retention_days": {
Description: "The number of days before logs are deleted from this index.",
Description: "The number of days logs are stored in Standard Tier before aging into the Flex Tier or being deleted from the index.",
Type: schema.TypeInt,
Computed: true,
},
"flex_retention_days": {
Description: "The total number of days logs are stored in Standard and Flex Tier before being deleted from the index.",
Type: schema.TypeInt,
Computed: true,
},
Expand Down Expand Up @@ -151,9 +156,10 @@ func dataSourceDatadogLogsIndexesRead(ctx context.Context, d *schema.ResourceDat
"daily_limit": l.GetDailyLimit(),
"daily_limit_reset": buildTerraformIndexDailyLimitReset(l.GetDailyLimitReset()),
"daily_limit_warning_threshold_percentage": l.GetDailyLimitWarningThresholdPercentage(),
"retention_days": l.GetNumRetentionDays(),
"filter": buildTerraformIndexFilter(l.GetFilter()),
"exclusion_filter": buildTerraformExclusionFilters(l.GetExclusionFilters()),
"retention_days": l.GetNumRetentionDays(),
"flex_retention_days": l.GetNumFlexLogsRetentionDays(),
"filter": buildTerraformIndexFilter(l.GetFilter()),
"exclusion_filter": buildTerraformExclusionFilters(l.GetExclusionFilters()),
}
}
if err := d.Set("logs_indexes", tfLogsIndexes); err != nil {
Expand Down
17 changes: 16 additions & 1 deletion datadog/resource_datadog_logs_index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,13 @@ var indexSchema = map[string]*schema.Schema{
DiffSuppressFunc: suppressDiffWhenDisabledDailyLimit,
},
"retention_days": {
Description: "The number of days before logs are deleted from this index.",
Description: "The number of days logs are stored in Standard Tier before aging into the Flex Tier or being deleted from the index.",
Type: schema.TypeInt,
Optional: true,
Computed: true,
},
"flex_retention_days": {
Description: "The total number of days logs are stored in Standard and Flex Tier before being deleted from the index.",
Type: schema.TypeInt,
Optional: true,
Computed: true,
Expand Down Expand Up @@ -202,6 +208,9 @@ func updateLogsIndexState(d *schema.ResourceData, index *datadogV1.LogsIndex) di
if err := d.Set("retention_days", index.GetNumRetentionDays()); err != nil {
return diag.FromErr(err)
}
if err := d.Set("flex_retention_days", index.GetNumFlexLogsRetentionDays()); err != nil {
return diag.FromErr(err)
}
if err := d.Set("filter", buildTerraformIndexFilter(index.GetFilter())); err != nil {
return diag.FromErr(err)
}
Expand Down Expand Up @@ -275,6 +284,9 @@ func buildDatadogIndexUpdateRequest(d *schema.ResourceData) *datadogV1.LogsIndex
if v, ok := d.GetOk("retention_days"); ok {
ddIndex.SetNumRetentionDays(int64(v.(int)))
}
if v, ok := d.GetOk("flex_retention_days"); ok {
ddIndex.SetNumFlexLogsRetentionDays(int64(v.(int)))
}

ddIndex.ExclusionFilters = *buildDatadogExclusionFilters(d.Get("exclusion_filter").([]interface{}))
return &ddIndex
Expand All @@ -300,6 +312,9 @@ func buildDatadogIndexCreateRequest(d *schema.ResourceData) *datadogV1.LogsIndex
if v, ok := d.GetOk("retention_days"); ok {
ddIndex.SetNumRetentionDays(int64(v.(int)))
}
if v, ok := d.GetOk("flex_retention_days"); ok {
ddIndex.SetNumFlexLogsRetentionDays(int64(v.(int)))
}
ddIndex.ExclusionFilters = *buildDatadogExclusionFilters(d.Get("exclusion_filter").([]interface{}))
return &ddIndex
}
Expand Down
2 changes: 1 addition & 1 deletion datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.freeze
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2024-02-01T16:44:06.859038+01:00
2024-09-10T14:21:35.751535-04:00
58 changes: 29 additions & 29 deletions datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ interactions:
proto: HTTP/1.1
proto_major: 1
proto_minor: 1
content_length: 286
content_length: 321
transfer_encoding: []
trailer: {}
host: api.datadoghq.com
remote_addr: ""
request_uri: ""
body: |
{"daily_limit":200000,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70,"exclusion_filters":[],"filter":{"query":"non-existent-query"},"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","num_retention_days":15}
{"daily_limit":200000,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70,"exclusion_filters":[],"filter":{"query":"non-existent-query"},"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","num_flex_logs_retention_days":180,"num_retention_days":15}
form: {}
headers:
Accept:
Expand All @@ -32,13 +32,13 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 648.244708ms
duration: 267.185125ms
- id: 1
request:
proto: HTTP/1.1
Expand All @@ -55,7 +55,7 @@ interactions:
headers:
Accept:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: GET
response:
proto: HTTP/1.1
Expand All @@ -67,13 +67,13 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 187.354416ms
duration: 125.18425ms
- id: 2
request:
proto: HTTP/1.1
Expand All @@ -90,7 +90,7 @@ interactions:
headers:
Accept:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: GET
response:
proto: HTTP/1.1
Expand All @@ -102,33 +102,33 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 171.845459ms
duration: 133.591916ms
- id: 3
request:
proto: HTTP/1.1
proto_major: 1
proto_minor: 1
content_length: 321
content_length: 356
transfer_encoding: []
trailer: {}
host: api.datadoghq.com
remote_addr: ""
request_uri: ""
body: |
{"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"filter":{"query":"app:coredns","sample_rate":0.97},"is_enabled":true,"name":"Filter coredns logs"}],"filter":{"query":"test:query"},"num_retention_days":15}
{"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"filter":{"query":"app:coredns","sample_rate":0.97},"is_enabled":true,"name":"Filter coredns logs"}],"filter":{"query":"test:query"},"num_flex_logs_retention_days":180,"num_retention_days":15}
form: {}
headers:
Accept:
- application/json
Content-Type:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: PUT
response:
proto: HTTP/1.1
Expand All @@ -140,13 +140,13 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 290.800667ms
duration: 135.232042ms
- id: 4
request:
proto: HTTP/1.1
Expand All @@ -163,7 +163,7 @@ interactions:
headers:
Accept:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: GET
response:
proto: HTTP/1.1
Expand All @@ -175,13 +175,13 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 179.5485ms
duration: 128.066125ms
- id: 5
request:
proto: HTTP/1.1
Expand All @@ -198,7 +198,7 @@ interactions:
headers:
Accept:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: GET
response:
proto: HTTP/1.1
Expand All @@ -210,33 +210,33 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 193.31175ms
duration: 64.196958ms
- id: 6
request:
proto: HTTP/1.1
proto_major: 1
proto_minor: 1
content_length: 248
content_length: 283
transfer_encoding: []
trailer: {}
host: api.datadoghq.com
remote_addr: ""
request_uri: ""
body: |
{"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"disable_daily_limit":true,"exclusion_filters":[],"filter":{"query":"test:query"},"num_retention_days":15}
{"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"disable_daily_limit":true,"exclusion_filters":[],"filter":{"query":"test:query"},"num_flex_logs_retention_days":180,"num_retention_days":15}
form: {}
headers:
Accept:
- application/json
Content-Type:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: PUT
response:
proto: HTTP/1.1
Expand All @@ -248,13 +248,13 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 254.154334ms
duration: 124.066417ms
- id: 7
request:
proto: HTTP/1.1
Expand All @@ -271,7 +271,7 @@ interactions:
headers:
Accept:
- application/json
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246
url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495
method: GET
response:
proto: HTTP/1.1
Expand All @@ -283,10 +283,10 @@ interactions:
content_length: -1
uncompressed: true
body: |
{"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[]}
{"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[],"num_flex_logs_retention_days":180}
headers:
Content-Type:
- application/json
status: 200 OK
code: 200
duration: 179.168042ms
duration: 65.029083ms
4 changes: 4 additions & 0 deletions datadog/tests/resource_datadog_logs_index_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func TestAccDatadogLogsIndex_Basic(t *testing.T) {
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "daily_limit_warning_threshold_percentage", "70"),
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "disable_daily_limit", "false"),
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "retention_days", "15"),
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "flex_retention_days", "180"),
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "filter.#", "1"),
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "filter.0.query", "non-existent-query"),
resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "exclusion_filter.#", "0"),
Expand Down Expand Up @@ -98,6 +99,7 @@ resource "datadog_logs_index" "sample_index" {
}
daily_limit_warning_threshold_percentage = 70
retention_days = 15
flex_retention_days = 180
filter {
query = "non-existent-query"
}
Expand All @@ -117,6 +119,7 @@ resource "datadog_logs_index" "sample_index" {
daily_limit_warning_threshold_percentage = 99.99
disable_daily_limit = false
retention_days = 15
flex_retention_days = 180
filter {
query = "test:query"
}
Expand Down Expand Up @@ -144,6 +147,7 @@ resource "datadog_logs_index" "sample_index" {
daily_limit_warning_threshold_percentage = 70
disable_daily_limit = true
retention_days = 15
flex_retention_days = 180
filter {
query = "test:query"
}
Expand Down
1 change: 1 addition & 0 deletions docs/data-sources/logs_indexes.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ Read-Only:
- `daily_limit_warning_threshold_percentage` (Number)
- `exclusion_filter` (List of Object) (see [below for nested schema](#nestedobjatt--logs_indexes--exclusion_filter))
- `filter` (List of Object) (see [below for nested schema](#nestedobjatt--logs_indexes--filter))
- `flex_retention_days` (Number)
- `name` (String)
- `retention_days` (Number)

Expand Down
Loading
Loading