From d6d81f49c28f7dd90a26a9bab4a2fc935fba68b2 Mon Sep 17 00:00:00 2001 From: Keri Miller Date: Wed, 18 Sep 2024 14:13:59 +0200 Subject: [PATCH] [datadog_logs_index] Add Flex Index Configuration support for Logs Indexes (#2569) * :sparkles: Add Flex Index Configuration support for Logs Indexes * :test_tube: enable flex test for logs indexes * :memo: update Logs Index docs for flex retention support * :white_check_mark: update test cassettes for log indexes * :rewind: remove api version bumps --- datadog/data_source_datadog_logs_indexes.go | 14 +++-- datadog/resource_datadog_logs_index.go | 17 +++++- .../TestAccDatadogLogsIndex_Basic.freeze | 2 +- .../TestAccDatadogLogsIndex_Basic.yaml | 58 +++++++++---------- .../tests/resource_datadog_logs_index_test.go | 4 ++ docs/data-sources/logs_indexes.md | 1 + docs/resources/logs_index.md | 4 +- .../resources/datadog_logs_index/resource.tf | 1 + 8 files changed, 65 insertions(+), 36 deletions(-) diff --git a/datadog/data_source_datadog_logs_indexes.go b/datadog/data_source_datadog_logs_indexes.go index d439108a84..c1ed7bc1e9 100644 --- a/datadog/data_source_datadog_logs_indexes.go +++ b/datadog/data_source_datadog_logs_indexes.go @@ -59,7 +59,12 @@ func dataSourceDatadogLogsIndexes() *schema.Resource { Computed: true, }, "retention_days": { - Description: "The number of days before logs are deleted from this index.", + Description: "The number of days logs are stored in Standard Tier before aging into the Flex Tier or being deleted from the index.", + Type: schema.TypeInt, + Computed: true, + }, + "flex_retention_days": { + Description: "The total number of days logs are stored in Standard and Flex Tier before being deleted from the index.", Type: schema.TypeInt, Computed: true, }, @@ -151,9 +156,10 @@ func dataSourceDatadogLogsIndexesRead(ctx context.Context, d *schema.ResourceDat "daily_limit": l.GetDailyLimit(), "daily_limit_reset": buildTerraformIndexDailyLimitReset(l.GetDailyLimitReset()), "daily_limit_warning_threshold_percentage": l.GetDailyLimitWarningThresholdPercentage(), - "retention_days": l.GetNumRetentionDays(), - "filter": buildTerraformIndexFilter(l.GetFilter()), - "exclusion_filter": buildTerraformExclusionFilters(l.GetExclusionFilters()), + "retention_days": l.GetNumRetentionDays(), + "flex_retention_days": l.GetNumFlexLogsRetentionDays(), + "filter": buildTerraformIndexFilter(l.GetFilter()), + "exclusion_filter": buildTerraformExclusionFilters(l.GetExclusionFilters()), } } if err := d.Set("logs_indexes", tfLogsIndexes); err != nil { diff --git a/datadog/resource_datadog_logs_index.go b/datadog/resource_datadog_logs_index.go index 10ceae8532..378162e439 100644 --- a/datadog/resource_datadog_logs_index.go +++ b/datadog/resource_datadog_logs_index.go @@ -69,7 +69,13 @@ var indexSchema = map[string]*schema.Schema{ DiffSuppressFunc: suppressDiffWhenDisabledDailyLimit, }, "retention_days": { - Description: "The number of days before logs are deleted from this index.", + Description: "The number of days logs are stored in Standard Tier before aging into the Flex Tier or being deleted from the index.", + Type: schema.TypeInt, + Optional: true, + Computed: true, + }, + "flex_retention_days": { + Description: "The total number of days logs are stored in Standard and Flex Tier before being deleted from the index.", Type: schema.TypeInt, Optional: true, Computed: true, @@ -202,6 +208,9 @@ func updateLogsIndexState(d *schema.ResourceData, index *datadogV1.LogsIndex) di if err := d.Set("retention_days", index.GetNumRetentionDays()); err != nil { return diag.FromErr(err) } + if err := d.Set("flex_retention_days", index.GetNumFlexLogsRetentionDays()); err != nil { + return diag.FromErr(err) + } if err := d.Set("filter", buildTerraformIndexFilter(index.GetFilter())); err != nil { return diag.FromErr(err) } @@ -275,6 +284,9 @@ func buildDatadogIndexUpdateRequest(d *schema.ResourceData) *datadogV1.LogsIndex if v, ok := d.GetOk("retention_days"); ok { ddIndex.SetNumRetentionDays(int64(v.(int))) } + if v, ok := d.GetOk("flex_retention_days"); ok { + ddIndex.SetNumFlexLogsRetentionDays(int64(v.(int))) + } ddIndex.ExclusionFilters = *buildDatadogExclusionFilters(d.Get("exclusion_filter").([]interface{})) return &ddIndex @@ -300,6 +312,9 @@ func buildDatadogIndexCreateRequest(d *schema.ResourceData) *datadogV1.LogsIndex if v, ok := d.GetOk("retention_days"); ok { ddIndex.SetNumRetentionDays(int64(v.(int))) } + if v, ok := d.GetOk("flex_retention_days"); ok { + ddIndex.SetNumFlexLogsRetentionDays(int64(v.(int))) + } ddIndex.ExclusionFilters = *buildDatadogExclusionFilters(d.Get("exclusion_filter").([]interface{})) return &ddIndex } diff --git a/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.freeze b/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.freeze index 559381b49b..4906d5d397 100644 --- a/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.freeze +++ b/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.freeze @@ -1 +1 @@ -2024-02-01T16:44:06.859038+01:00 \ No newline at end of file +2024-09-10T14:21:35.751535-04:00 \ No newline at end of file diff --git a/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.yaml b/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.yaml index 0ba50c8853..832612b2a8 100644 --- a/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.yaml +++ b/datadog/tests/cassettes/TestAccDatadogLogsIndex_Basic.yaml @@ -6,14 +6,14 @@ interactions: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 286 + content_length: 321 transfer_encoding: [] trailer: {} host: api.datadoghq.com remote_addr: "" request_uri: "" body: | - {"daily_limit":200000,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70,"exclusion_filters":[],"filter":{"query":"non-existent-query"},"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","num_retention_days":15} + {"daily_limit":200000,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70,"exclusion_filters":[],"filter":{"query":"non-existent-query"},"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","num_flex_logs_retention_days":180,"num_retention_days":15} form: {} headers: Accept: @@ -32,13 +32,13 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 648.244708ms + duration: 267.185125ms - id: 1 request: proto: HTTP/1.1 @@ -55,7 +55,7 @@ interactions: headers: Accept: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: GET response: proto: HTTP/1.1 @@ -67,13 +67,13 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 187.354416ms + duration: 125.18425ms - id: 2 request: proto: HTTP/1.1 @@ -90,7 +90,7 @@ interactions: headers: Accept: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: GET response: proto: HTTP/1.1 @@ -102,33 +102,33 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"non-existent-query"},"num_retention_days":15,"daily_limit":200000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"10:00","reset_utc_offset":"+02:00"},"daily_limit_warning_threshold_percentage":70.0,"exclusion_filters":[],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 171.845459ms + duration: 133.591916ms - id: 3 request: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 321 + content_length: 356 transfer_encoding: [] trailer: {} host: api.datadoghq.com remote_addr: "" request_uri: "" body: | - {"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"filter":{"query":"app:coredns","sample_rate":0.97},"is_enabled":true,"name":"Filter coredns logs"}],"filter":{"query":"test:query"},"num_retention_days":15} + {"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"filter":{"query":"app:coredns","sample_rate":0.97},"is_enabled":true,"name":"Filter coredns logs"}],"filter":{"query":"test:query"},"num_flex_logs_retention_days":180,"num_retention_days":15} form: {} headers: Accept: - application/json Content-Type: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: PUT response: proto: HTTP/1.1 @@ -140,13 +140,13 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 290.800667ms + duration: 135.232042ms - id: 4 request: proto: HTTP/1.1 @@ -163,7 +163,7 @@ interactions: headers: Accept: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: GET response: proto: HTTP/1.1 @@ -175,13 +175,13 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 179.5485ms + duration: 128.066125ms - id: 5 request: proto: HTTP/1.1 @@ -198,7 +198,7 @@ interactions: headers: Accept: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: GET response: proto: HTTP/1.1 @@ -210,33 +210,33 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":20000,"is_rate_limited":false,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"exclusion_filters":[{"name":"Filter coredns logs","is_enabled":true,"filter":{"query":"app:coredns","sample_rate":0.97}}],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 193.31175ms + duration: 64.196958ms - id: 6 request: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 248 + content_length: 283 transfer_encoding: [] trailer: {} host: api.datadoghq.com remote_addr: "" request_uri: "" body: | - {"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"disable_daily_limit":true,"exclusion_filters":[],"filter":{"query":"test:query"},"num_retention_days":15} + {"daily_limit":20000,"daily_limit_reset":{"reset_time":"12:00","reset_utc_offset":"-02:00"},"daily_limit_warning_threshold_percentage":99.99,"disable_daily_limit":true,"exclusion_filters":[],"filter":{"query":"test:query"},"num_flex_logs_retention_days":180,"num_retention_days":15} form: {} headers: Accept: - application/json Content-Type: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: PUT response: proto: HTTP/1.1 @@ -248,13 +248,13 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 254.154334ms + duration: 124.066417ms - id: 7 request: proto: HTTP/1.1 @@ -271,7 +271,7 @@ interactions: headers: Accept: - application/json - url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1706802246 + url: https://api.datadoghq.com/api/v1/logs/config/indexes/tf-testaccdatadoglogsindex-basic-local-1725992495 method: GET response: proto: HTTP/1.1 @@ -283,10 +283,10 @@ interactions: content_length: -1 uncompressed: true body: | - {"name":"tf-testaccdatadoglogsindex-basic-local-1706802246","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[]} + {"name":"tf-testaccdatadoglogsindex-basic-local-1725992495","filter":{"query":"test:query"},"num_retention_days":15,"daily_limit":null,"is_rate_limited":false,"daily_limit_reset":null,"daily_limit_warning_threshold_percentage":null,"exclusion_filters":[],"num_flex_logs_retention_days":180} headers: Content-Type: - application/json status: 200 OK code: 200 - duration: 179.168042ms + duration: 65.029083ms diff --git a/datadog/tests/resource_datadog_logs_index_test.go b/datadog/tests/resource_datadog_logs_index_test.go index 0d48cd40ed..b1a8a8d778 100644 --- a/datadog/tests/resource_datadog_logs_index_test.go +++ b/datadog/tests/resource_datadog_logs_index_test.go @@ -39,6 +39,7 @@ func TestAccDatadogLogsIndex_Basic(t *testing.T) { resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "daily_limit_warning_threshold_percentage", "70"), resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "disable_daily_limit", "false"), resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "retention_days", "15"), + resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "flex_retention_days", "180"), resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "filter.#", "1"), resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "filter.0.query", "non-existent-query"), resource.TestCheckResourceAttr("datadog_logs_index.sample_index", "exclusion_filter.#", "0"), @@ -98,6 +99,7 @@ resource "datadog_logs_index" "sample_index" { } daily_limit_warning_threshold_percentage = 70 retention_days = 15 + flex_retention_days = 180 filter { query = "non-existent-query" } @@ -117,6 +119,7 @@ resource "datadog_logs_index" "sample_index" { daily_limit_warning_threshold_percentage = 99.99 disable_daily_limit = false retention_days = 15 + flex_retention_days = 180 filter { query = "test:query" } @@ -144,6 +147,7 @@ resource "datadog_logs_index" "sample_index" { daily_limit_warning_threshold_percentage = 70 disable_daily_limit = true retention_days = 15 + flex_retention_days = 180 filter { query = "test:query" } diff --git a/docs/data-sources/logs_indexes.md b/docs/data-sources/logs_indexes.md index 62e38240dc..e136b829f7 100644 --- a/docs/data-sources/logs_indexes.md +++ b/docs/data-sources/logs_indexes.md @@ -34,6 +34,7 @@ Read-Only: - `daily_limit_warning_threshold_percentage` (Number) - `exclusion_filter` (List of Object) (see [below for nested schema](#nestedobjatt--logs_indexes--exclusion_filter)) - `filter` (List of Object) (see [below for nested schema](#nestedobjatt--logs_indexes--filter)) +- `flex_retention_days` (Number) - `name` (String) - `retention_days` (Number) diff --git a/docs/resources/logs_index.md b/docs/resources/logs_index.md index d0e24e4011..be45976a21 100644 --- a/docs/resources/logs_index.md +++ b/docs/resources/logs_index.md @@ -25,6 +25,7 @@ resource "datadog_logs_index" "sample_index" { } daily_limit_warning_threshold_percentage = 50 retention_days = 7 + flex_retention_days = 180 filter { query = "*" } @@ -62,7 +63,8 @@ resource "datadog_logs_index" "sample_index" { - `daily_limit_warning_threshold_percentage` (Number) A percentage threshold of the daily quota at which a Datadog warning event is generated. - `disable_daily_limit` (Boolean) If true, sets the daily_limit value to null and the index is not limited on a daily basis (any specified daily_limit value in the request is ignored). If false or omitted, the index's current daily_limit is maintained. - `exclusion_filter` (Block List) List of exclusion filters. (see [below for nested schema](#nestedblock--exclusion_filter)) -- `retention_days` (Number) The number of days before logs are deleted from this index. +- `flex_retention_days` (Number) The total number of days logs are stored in Standard and Flex Tier before being deleted from the index. +- `retention_days` (Number) The number of days logs are stored in Standard Tier before aging into the Flex Tier or being deleted from the index. ### Read-Only diff --git a/examples/resources/datadog_logs_index/resource.tf b/examples/resources/datadog_logs_index/resource.tf index fe2a92057c..c69bf50037 100644 --- a/examples/resources/datadog_logs_index/resource.tf +++ b/examples/resources/datadog_logs_index/resource.tf @@ -9,6 +9,7 @@ resource "datadog_logs_index" "sample_index" { } daily_limit_warning_threshold_percentage = 50 retention_days = 7 + flex_retention_days = 180 filter { query = "*" }