diff --git a/http_logs/README.md b/http_logs/README.md index 001f394d..8a7e861d 100644 --- a/http_logs/README.md +++ b/http_logs/README.md @@ -42,6 +42,8 @@ This track allows to overwrite the following parameters with Rally 0.8.0+ using * `number_of_shards` (default: 5) * `source_enabled` (default: true): A boolean defining whether the `_source` field is stored in the index. * `index_settings`: A list of index settings. Index settings defined elsewhere (e.g. `number_of_replicas`) need to be overridden explicitly. +* `index_mode` (default: unset): Set to `logsdb` to enable indexing to [logs data streams](https://www.elastic.co/guide/en/elasticsearch/reference/master/logs-data-stream.html). If not enabled, Rally will not use logs data streams. +* `index_type` (default: unset): Set to `data_stream` to enable indexing to data streams. `index_type` is not required when `index_mode` is set to `logsdb`. * `cluster_health` (default: "green"): The minimum required cluster health. * `ingest_pipeline`: Only applicable for `--challenge=append-index-only-with-ingest-pipeline`, selects which ingest node pipeline to run. Valid options are `'baseline'` (default), `'grok'` and `'geoip'`. For example: `--challenge=append-index-only-with-ingest-pipeline --track-params="ingest_pipeline:'baseline'" ` diff --git a/http_logs/challenges/common/default-schedule.json b/http_logs/challenges/common/default-schedule.json index 91ec7e00..396d27c4 100644 --- a/http_logs/challenges/common/default-schedule.json +++ b/http_logs/challenges/common/default-schedule.json @@ -1,11 +1,44 @@ { - "operation": "delete-index" + "operation": { + "name": "delete-data-stream", + "operation-type": "delete-data-stream", + "only-if-exists": false, + "data-stream": ["logs-181998", "logs-191998", "logs-201998", "logs-211998", "logs-221998", "logs-231998", "logs-241998", "reindexed-logs"] + }, + "tags": ["setup"] + }, + { + "operation": { + "name": "delete-index", + "operation-type": "delete-index", + "only-if-exists": false, + "index": ["logs-181998", "logs-191998", "logs-201998", "logs-211998", "logs-221998", "logs-231998", "logs-241998", "reindexed-logs"] + }, + "tags": ["setup"] + }, + { + "operation" : { + "name": "delete-all-index-templates", + "operation-type": "delete-composable-template" + }, + "tags": ["setup"] }, { "operation": { - "operation-type": "create-index", + "name": "create-all-templates", + "operation-type": "create-composable-template" + }, + "tags": ["setup"] + }, + { + {%- if index_mode == "logsdb" or index_type == "data_stream" %} + {%- set indexing_operation_type = "create-data-stream" %} + {%- endif %} + "operation": { + "operation-type": {{ indexing_operation_type | default("create-index") | tojson }}, "settings": {{index_settings | default({}) | tojson}} - } + }, + "tags": ["setup"] }, { "name": "check-cluster-health", diff --git a/http_logs/challenges/common/setup-schedule.json b/http_logs/challenges/common/setup-schedule.json new file mode 100644 index 00000000..e92dacdd --- /dev/null +++ b/http_logs/challenges/common/setup-schedule.json @@ -0,0 +1,32 @@ +{ + "operation": { + "name": "delete-data-stream", + "operation-type": "delete-data-stream", + "only-if-exists": false, + "data-stream": ["logs-181998", "logs-191998", "logs-201998", "logs-211998", "logs-221998", "logs-231998", "logs-241998", "reindexed-logs"] + }, + "tags": ["setup"] +}, +{ + "operation": { + "name": "delete-index", + "operation-type": "delete-index", + "only-if-exists": false, + "index": ["logs-181998", "logs-191998", "logs-201998", "logs-211998", "logs-221998", "logs-231998", "logs-241998", "reindexed-logs"] + }, + "tags": ["setup"] +}, +{ + "operation" : { + "name": "delete-all-index-templates", + "operation-type": "delete-composable-template" + }, + "tags": ["setup"] +}, +{ + "operation": { + "name": "create-all-templates", + "operation-type": "create-composable-template" + }, + "tags": ["setup"] +} \ No newline at end of file diff --git a/http_logs/challenges/default.json b/http_logs/challenges/default.json index ba59960d..eb861559 100644 --- a/http_logs/challenges/default.json +++ b/http_logs/challenges/default.json @@ -17,14 +17,18 @@ "name": "append-no-conflicts-index-only", "description": "Indexes the whole document corpus using Elasticsearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Rally will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only.", "schedule": [ + {{ rally.collect(parts="common/setup-schedule.json") }}, { - "operation": "delete-index" - }, - { + {%- if index_mode == "logsdb" or index_type == "data_stream" %} + {%- set indexing_operation_type = "create-data-stream" %} + {%- endif %} "operation": { - "operation-type": "create-index", + "operation-type": {{ indexing_operation_type | default("create-index") | tojson }}, "settings": {{index_settings | default({}) | tojson}} - } + }, + "tags": [ + "setup" + ] }, { "name": "check-cluster-health", @@ -77,17 +81,21 @@ "name": "append-sorted-no-conflicts", "description": "Indexes the whole document corpus in an index sorted by timestamp field in descending order (most recent first) and using a setup that will lead to a lower indexing throughput than the default settings. Document ids are unique so all index operations are append only.", "schedule": [ + {{ rally.collect(parts="common/setup-schedule.json") }}, { - "operation": "delete-index" - }, - { + {%- if index_mode == "logsdb" or index_type == "data_stream" %} + {%- set indexing_operation_type = "create-data-stream" %} + {%- endif %} "operation": { - "operation-type": "create-index", + "operation-type": {{ indexing_operation_type | default("create-index") | tojson }}, "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} { "index.sort.field": "@timestamp", "index.sort.order": "desc" }{%- endif %} - } + }, + "tags": [ + "setup" + ] }, { "name": "check-cluster-health", @@ -140,14 +148,21 @@ "name": "append-index-only-with-ingest-pipeline", "description": "Indexes the whole document corpus using Elasticsearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Rally will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only. Runs the documents through an ingest node pipeline to parse the http logs. May require --elasticsearch-plugins='ingest-geoip' ", "schedule": [ + {{ rally.collect(parts="common/setup-schedule.json") }}, { - "operation": "delete-index" - }, - { + {%- if index_mode == "logsdb" or index_type == "data_stream" %} + {%- set indexing_operation_type = "create-data-stream" %} + {%- endif %} "operation": { - "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } + "operation-type": {{ indexing_operation_type | default("create-index") | tojson }}, + "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} { + "index.sort.field": "@timestamp", + "index.sort.order": "desc" + }{%- endif %} + }, + "tags": [ + "setup" + ] }, { "name": "check-cluster-health", @@ -201,10 +216,9 @@ }, { "name": "update", + "description": "Perform bulk update operations. The update challenge is for standard index use only.", "schedule": [ - { - "operation": "delete-index" - }, + {{ rally.collect(parts="common/setup-schedule.json") }}, { "operation": { "operation-type": "create-index", @@ -268,14 +282,21 @@ "name": "append-no-conflicts-index-reindex-only", "description": "Indexes the whole document corpus using Elasticsearch default settings. We only adjust the number of replicas as we benchmark a single node cluster and Rally will only start the benchmark if the cluster turns green. Document ids are unique so all index operations are append only. After indexing, same data are reindexed.", "schedule": [ + {{ rally.collect(parts="common/setup-schedule.json") }}, { - "operation": "delete-index" - }, - { + {%- if index_mode == "logsdb" or index_type == "data_stream" %} + {%- set indexing_operation_type = "create-data-stream" %} + {%- endif %} "operation": { - "operation-type": "create-index", - "settings": {{index_settings | default({}) | tojson}} - } + "operation-type": {{ indexing_operation_type | default("create-index") | tojson }}, + "settings": {%- if index_settings is defined %} {{index_settings | tojson}} {%- else %} { + "index.sort.field": "@timestamp", + "index.sort.order": "desc" + }{%- endif %} + }, + "tags": [ + "setup" + ] }, { "name": "check-cluster-health", diff --git a/http_logs/index-runtime-fields.json b/http_logs/index-template.json similarity index 68% rename from http_logs/index-runtime-fields.json rename to http_logs/index-template.json index d1bdcf04..89bf47a7 100644 --- a/http_logs/index-runtime-fields.json +++ b/http_logs/index-template.json @@ -1,30 +1,84 @@ { - "settings": { - {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%} - "index.number_of_shards": {{ number_of_shards | default(5) }}, - "index.number_of_replicas": {{ number_of_replicas | default(0) }}, - "index.requests.cache.enable": false - {%- endif -%}{# non-serverless-index-settings-marker-end #} - }, - "mappings": { - "dynamic": "strict", - "_source": { - "enabled": {{ source_enabled | default(true) | tojson }} + "priority": 101, + "index_patterns": ["logs-*", "reindexed-logs"], + {%- if index_mode == "logsdb" or index_type == "data_stream" %} + "data_stream": {}, + {%- endif %} + "template": { + "settings": { + {%- if index_mode %} + "mode": {{ index_mode | tojson }}, + {%- endif -%} + {# non-serverless-index-settings-marker-start -#} + {%- if build_flavor != "serverless" %} + "index.number_of_replicas": {{ number_of_replicas | default(0) | tojson }}, + {%- endif -%} + {%- if build_flavor != "serverless" or serverless_operator == true %} + "index.number_of_shards": {{ number_of_shards | default(5) | tojson }}, + "index.requests.cache.enable": false + {%- endif -%} + {# non-serverless-index-settings-marker-end #} }, - "properties": { - "@timestamp": { - "format": "strict_date_optional_time", - "type": "date" - }, - "message": { - "type": "wildcard", - "fields": { - "keyword": { - "type": "keyword" + "mappings": { + "dynamic": "strict", + {%- if index_mode != "logsdb" %} + "_source": { + "enabled": {{ source_enabled | default(true) | tojson }} + }, + {%- endif %} + "properties": { + "@timestamp": { + {%- if (ingest_pipeline is defined and ingest_pipeline == "grok") or runtime_fields is defined %} + "format": "strict_date_optional_time", + {%- else %} + "format": "epoch_second", + {%- endif %} + "type": "date" + }, + {%- if runtime_fields is defined %} + "message": { + "type": "wildcard", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + {%- else %} + "message": { + "type": "keyword", + "index": false, + "doc_values": false + }, + {%- endif %} + "clientip": { + "type": "ip" + }, + "request": { + "type": "match_only_text", + "fields": { + "raw": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "geoip" : { + "properties" : { + "country_name": { "type": "keyword" }, + "city_name": { "type": "keyword" }, + "location" : { "type" : "geo_point" } } } } - }, + } + {%- if runtime_fields is defined %}, "runtime": { {%- set sources = [('source', 'message.source'), ('wildcard', 'message'), ('keyword', 'message.keyword')] %} {%- for source_type, field in sources %} @@ -97,6 +151,6 @@ "type": "keyword", "script": "emit(params._source.message)" } - } + }{% endif %} } } diff --git a/http_logs/index.json b/http_logs/index.json deleted file mode 100644 index c9389c1d..00000000 --- a/http_logs/index.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "settings": { - {# non-serverless-index-settings-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%} - "index.number_of_shards": {{ number_of_shards | default(5) }}, - "index.number_of_replicas": {{ number_of_replicas | default(0) }}, - "index.requests.cache.enable": false - {%- endif -%}{# non-serverless-index-settings-marker-end #} - }, - "mappings": { - "dynamic": "strict", - "_source": { - "enabled": {{ source_enabled | default(true) | tojson }} - }, - "properties": { - "@timestamp": { - {%- if ingest_pipeline is defined and ingest_pipeline == "grok" %} - "format": "strict_date_optional_time", - {%- else %} - "format": "epoch_second", - {%- endif %} - "type": "date" - }, - "message": { - "type": "keyword", - "index": false, - "doc_values": false - }, - "clientip": { - "type": "ip" - }, - "request": { - "type": "match_only_text", - "fields": { - "raw": { - "ignore_above": 256, - "type": "keyword" - } - } - }, - "status": { - "type": "integer" - }, - "size": { - "type": "integer" - }, - "geoip" : { - "properties" : { - "country_name": { "type": "keyword" }, - "city_name": { "type": "keyword" }, - "location" : { "type" : "geo_point" } - } - } - } - } -} diff --git a/http_logs/track.json b/http_logs/track.json index 63cea73f..8dd798ea 100644 --- a/http_logs/track.json +++ b/http_logs/track.json @@ -1,161 +1,162 @@ {% import "rally.helpers" as rally with context %} - - {%- if runtime_fields is defined %} - {% set index_body = 'index-runtime-fields.json' %} - {% set query_range_ts_start = "1998-05-01T00:00:00Z" %} - {% set query_range_ts_end = "1998-05-02T00:00:00Z" %} - {% set search_after_ts = "1998-06-10" %} + {%- set query_range_ts_start = "1998-05-01T00:00:00Z" %} + {%- set query_range_ts_end = "1998-05-02T00:00:00Z" %} + {%- set search_after_ts = "1998-06-10" %} +{% else %} + {%- set query_range_ts_start = "893980800" %} + {%- set query_range_ts_end = "894067200" %} + {%- set search_after_ts = "897436800" %} +{%- endif -%} +{%- if index_mode == "logsdb" or index_type == "data_stream" %} + {%- set target_type = "data-streams" | tojson %} + {%- set target_property = "target-data-stream" | tojson %} {%- else %} - {% set index_body = 'index.json' %} - {% set query_range_ts_start = "893980800" %} - {% set query_range_ts_end = "894067200" %} - {% set search_after_ts = "897436800" %} -{%- endif %} + {%- set target_type = "indices" | tojson %} + {%- set target_property = "target-index" | tojson %} +{%- endif -%} { "version": 2, "description": "HTTP server log data", "#TODO": "Replace index definitions with a template after setting the track version to 2. Explicit index definitions are not necessary anymore.", - "indices": [ + "composable-templates": [ { - "name": "logs-181998", - "body": "{{ index_body }}" + "name": "rally-http_logs", + "index-pattern": "logs-*", + "delete-matching-indices": true, + "template": "index-template.json" + } + ], + {{ target_type }}: [ + { + "name": "logs-181998" }, { - "name": "logs-191998", - "body": "{{ index_body }}" + "name": "logs-191998" }, { - "name": "logs-201998", - "body": "{{ index_body }}" + "name": "logs-201998" }, { - "name": "logs-211998", - "body": "{{ index_body }}" + "name": "logs-211998" }, { - "name": "logs-221998", - "body": "{{ index_body }}" + "name": "logs-221998" }, { - "name": "logs-231998", - "body": "{{ index_body }}" + "name": "logs-231998" }, { - "name": "logs-241998", - "body": "{{ index_body }}" + "name": "logs-241998" }, { - "name": "reindexed-logs", - "body": "{{ index_body }}" + "name": "reindexed-logs" } ], "corpora": [ - {%- if ingest_pipeline is defined and ingest_pipeline == "grok" or runtime_fields is defined %} { "name": "http_logs_unparsed", "base-url": "https://rally-tracks.elastic.co/http_logs", "documents": [ { - "target-index": "logs-181998", + {{ target_property }}: "logs-181998", "source-file": "documents-181998.unparsed.json.bz2", "document-count": 2708746, "compressed-bytes": 13088137, "uncompressed-bytes": 303920342 }, { - "target-index": "logs-191998", + {{ target_property }}: "logs-191998", "source-file": "documents-191998.unparsed.json.bz2", "document-count": 9697882, "compressed-bytes": 47290776, "uncompressed-bytes": 1088378738 }, { - "target-index": "logs-201998", + {{ target_property }}: "logs-201998", "source-file": "documents-201998.unparsed.json.bz2", "document-count": 13053463, "compressed-bytes": 63278452, "uncompressed-bytes": 1456836090 }, { - "target-index": "logs-211998", + {{ target_property }}: "logs-211998", "source-file": "documents-211998.unparsed.json.bz2", "document-count": 17647279, "compressed-bytes": 85739523, "uncompressed-bytes": 1975990671 }, { - "target-index": "logs-221998", + {{ target_property }}: "logs-221998", "source-file": "documents-221998.unparsed.json.bz2", "document-count": 10716760, "compressed-bytes": 53264421, "uncompressed-bytes": 1202551382 }, { - "target-index": "logs-231998", + {{ target_property }}: "logs-231998", "source-file": "documents-231998.unparsed.json.bz2", "document-count": 11961342, "compressed-bytes": 60795929, "uncompressed-bytes": 1334381144 }, { - "target-index": "logs-241998", + {{ target_property }}: "logs-241998", "source-file": "documents-241998.unparsed.json.bz2", "document-count": 181463624, "compressed-bytes": 899190175, "uncompressed-bytes": 20563705716 } ] - } - {%- else %} + }, { "name": "http_logs", "base-url": "https://rally-tracks.elastic.co/http_logs", "documents": [ { - "target-index": "logs-181998", + {{ target_property }}: "logs-181998", "source-file": "documents-181998.json.bz2", "document-count": 2708746, "compressed-bytes": 13843641, "uncompressed-bytes": 363512754 }, { - "target-index": "logs-191998", + {{ target_property }}: "logs-191998", "source-file": "documents-191998.json.bz2", "document-count": 9697882, "compressed-bytes": 49546887, "uncompressed-bytes": 1301732149 }, { - "target-index": "logs-201998", + {{ target_property }}: "logs-201998", "source-file": "documents-201998.json.bz2", "document-count": 13053463, "compressed-bytes": 65759419, "uncompressed-bytes": 1744012279 }, { - "target-index": "logs-211998", + {{ target_property }}: "logs-211998", "source-file": "documents-211998.json.bz2", "document-count": 17647279, "compressed-bytes": 88445049, "uncompressed-bytes": 2364230815 }, { - "target-index": "logs-221998", + {{ target_property }}: "logs-221998", "source-file": "documents-221998.json.bz2", "document-count": 10716760, "compressed-bytes": 54274027, "uncompressed-bytes": 1438320123 }, { - "target-index": "logs-231998", + {{ target_property }}: "logs-231998", "source-file": "documents-231998.json.bz2", "document-count": 11961342, "compressed-bytes": 61043842, "uncompressed-bytes": 1597530673 }, { - "target-index": "logs-241998", + {{ target_property }}: "logs-241998", "source-file": "documents-241998.json.bz2", "document-count": 181463624, "compressed-bytes": 907295259, @@ -163,7 +164,6 @@ } ] } - {%- endif %} ], "operations": [ {{ rally.collect(parts="operations/*.json") }}