Skip to content

Commit

Permalink
Support flattened label field with downsampling. (#118816) (#118935)
Browse files Browse the repository at this point in the history
If flattened field is configured as non-dimension and non-metric field, then downsampling fails to execute successfully. Downsampling doesn't know how to use the flattened field or how to serialize it. This change addresses this.

Closes #116319
  • Loading branch information
martijnvg authored Dec 18, 2024
1 parent bcaa255 commit 44a05e5
Show file tree
Hide file tree
Showing 7 changed files with 428 additions and 9 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/118816.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 118816
summary: Support flattened field with downsampling
area: Downsampling
type: bug
issues:
- 116319
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import org.elasticsearch.index.mapper.DocumentParserContext;
import org.elasticsearch.index.mapper.DynamicFieldType;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperBuilderContext;
Expand Down Expand Up @@ -666,7 +667,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme
private final boolean isDimension;
private final int ignoreAbove;

public RootFlattenedFieldType(
RootFlattenedFieldType(
String name,
boolean indexed,
boolean hasDocValues,
Expand All @@ -678,7 +679,7 @@ public RootFlattenedFieldType(
this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove);
}

public RootFlattenedFieldType(
RootFlattenedFieldType(
String name,
boolean indexed,
boolean hasDocValues,
Expand Down Expand Up @@ -802,6 +803,10 @@ public MappedFieldType getChildFieldType(String childPath) {
return new KeyedFlattenedFieldType(name(), childPath, this);
}

public MappedFieldType getKeyedFieldType() {
return new KeywordFieldMapper.KeywordFieldType(name() + KEYED_FIELD_SUFFIX);
}

@Override
public boolean isDimension() {
return isDimension;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
* }`
*
*/
class FlattenedFieldSyntheticWriterHelper {
public class FlattenedFieldSyntheticWriterHelper {

private record Prefix(List<String> prefix) {

Expand Down Expand Up @@ -225,17 +225,17 @@ public boolean equals(Object obj) {
}
}

interface SortedKeyedValues {
public interface SortedKeyedValues {
BytesRef next() throws IOException;
}

private final SortedKeyedValues sortedKeyedValues;

FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
public FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
this.sortedKeyedValues = sortedKeyedValues;
}

void write(final XContentBuilder b) throws IOException {
public void write(final XContentBuilder b) throws IOException {
KeyValue curr = new KeyValue(sortedKeyedValues.next());
KeyValue prev = KeyValue.EMPTY;
final List<String> values = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,307 @@
---
"A flattened label field":
- do:
indices.create:
index: source_index
body:
settings:
number_of_shards: 1
index:
mode: time_series
routing_path: [ metricset, k8s.pod.uid ]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
subobjects: false
properties:
"@timestamp":
type: date
metricset:
type: keyword
time_series_dimension: true
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true
name:
type: keyword
agent:
type: flattened
value:
type: long
time_series_metric: gauge

- do:
bulk:
refresh: true
index: source_index
body:
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4" }, "value": 10 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5" }, "value": 20 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6" }, "value": 12 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7" }, "value": 15 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7" }, "value": 9 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8" }, "value": 16 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9" }, "value": 25 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10" }, "value": 17 }}'

- do:
indices.put_settings:
index: source_index
body:
index.blocks.write: true

- do:
indices.downsample:
index: source_index
target_index: target_index
body: >
{
"fixed_interval": "1h"
}
- is_true: acknowledged

- do:
search:
index: target_index
body:
sort: [ "_tsid", "@timestamp" ]

- length: { hits.hits: 4 }
- match: { hits.hits.0._source._doc_count: 2 }
- match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
- match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
- match: { hits.hits.0._source.k8s\.agent: { "id": "second", "version": "2.1.8" } }

- match: { hits.hits.1._source._doc_count: 2 }
- match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
- match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
- match: { hits.hits.1._source.k8s\.agent: { "id": "second", "version": "2.1.10" } }

- match: { hits.hits.2._source._doc_count: 2 }
- match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
- match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
- match: { hits.hits.2._source.k8s\.agent: { "id": "first", "version": "2.0.5" } }

- match: { hits.hits.3._source._doc_count: 2 }
- match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
- match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
- match: { hits.hits.3._source.k8s\.agent: { "id": "first", "version": "2.0.7" } }

---
"A flattened label field with no doc values":
- do:
indices.create:
index: source_index
body:
settings:
number_of_shards: 1
index:
mode: time_series
routing_path: [ metricset, k8s.pod.uid ]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
subobjects: false
properties:
"@timestamp":
type: date
metricset:
type: keyword
time_series_dimension: true
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true
name:
type: keyword
agent:
type: flattened
doc_values: false
value:
type: long
time_series_metric: gauge

- do:
bulk:
refresh: true
index: source_index
body:
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4" }, "value": 10 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5" }, "value": 20 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6" }, "value": 12 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7" }, "value": 15 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7" }, "value": 9 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8" }, "value": 16 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9" }, "value": 25 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10" }, "value": 17 }}'

- do:
indices.put_settings:
index: source_index
body:
index.blocks.write: true

- do:
indices.downsample:
index: source_index
target_index: target_index
body: >
{
"fixed_interval": "1h"
}
- is_true: acknowledged

- do:
search:
index: target_index
body:
sort: [ "_tsid", "@timestamp" ]

- length: { hits.hits: 4 }
- match: { hits.hits.0._source._doc_count: 2 }
- match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
- match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
- is_false: hits.hits.0._source.k8s\.agent

- match: { hits.hits.1._source._doc_count: 2 }
- match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
- match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
- is_false: hits.hits.1._source.k8s\.agent

- match: { hits.hits.2._source._doc_count: 2 }
- match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
- match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
- is_false: hits.hits.2._source.k8s\.agent

- match: { hits.hits.3._source._doc_count: 2 }
- match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
- match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
- is_false: hits.hits.3._source.k8s\.agent

---
"A flattened label field with mixed content":
- do:
indices.create:
index: source_index
body:
settings:
number_of_shards: 1
index:
mode: time_series
routing_path: [ metricset, k8s.pod.uid ]
time_series:
start_time: 2021-04-28T00:00:00Z
end_time: 2021-04-29T00:00:00Z
mappings:
subobjects: false
properties:
"@timestamp":
type: date
metricset:
type: keyword
time_series_dimension: true
k8s:
properties:
pod:
properties:
uid:
type: keyword
time_series_dimension: true
name:
type: keyword
agent:
type: flattened
null_value: my_null_value
value:
type: long
time_series_metric: gauge

- do:
bulk:
refresh: true
index: source_index
body:
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.4", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11 }, "value": 10 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.5", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 20 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.6", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 12 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}, "agent": { "id": "first", "version": "2.0.7", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 15 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.7", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 9 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.8", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 16 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.9", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 25 }}'
- '{"index": {}}'
- '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}, "agent": { "id": "second", "version": "2.1.10", "versions": ["1", "2", "3"], "optional_version": null, "dotted.version": "1.1", "numeric_version": 11}, "value": 17 }}'

- do:
indices.put_settings:
index: source_index
body:
index.blocks.write: true

- do:
indices.downsample:
index: source_index
target_index: target_index
body: >
{
"fixed_interval": "1h"
}
- is_true: acknowledged

- do:
search:
index: target_index
body:
sort: [ "_tsid", "@timestamp" ]

- length: { hits.hits: 4 }
- match: { hits.hits.0._source._doc_count: 2 }
- match: { hits.hits.0._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
- match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
- match: { hits.hits.0._source.k8s\.agent: { "id": "second", "version": "2.1.8", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }

- match: { hits.hits.1._source._doc_count: 2 }
- match: { hits.hits.1._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
- match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
- match: { hits.hits.1._source.k8s\.agent: { "id": "second", "version": "2.1.10", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }

- match: { hits.hits.2._source._doc_count: 2 }
- match: { hits.hits.2._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
- match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
- match: { hits.hits.2._source.k8s\.agent: { "id": "first", "version": "2.0.5", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }

- match: { hits.hits.3._source._doc_count: 2 }
- match: { hits.hits.3._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
- match: { hits.hits.3._source.@timestamp: 2021-04-28T20:00:00.000Z }
- match: { hits.hits.3._source.k8s\.agent: { "id": "first", "version": "2.0.7", "versions": ["1", "2", "3"], "dotted": {"version": "1.1"}, "numeric_version": "11", optional_version: "my_null_value" } }
Loading

0 comments on commit 44a05e5

Please sign in to comment.