From 36a91d03e3d8328085ea2d5b21071d4b7ebc6738 Mon Sep 17 00:00:00 2001 From: Jenkins CI Date: Thu, 23 Feb 2023 21:22:20 +0000 Subject: [PATCH 1/5] updated versioned plugin docs --- .../filters/elasticsearch-index.asciidoc | 2 + .../filters/elasticsearch-v3.14.0.asciidoc | 409 ++++++++++++++++++ 2 files changed, 411 insertions(+) create mode 100644 docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc diff --git a/docs/versioned-plugins/filters/elasticsearch-index.asciidoc b/docs/versioned-plugins/filters/elasticsearch-index.asciidoc index 46dc6382..b6209ae3 100644 --- a/docs/versioned-plugins/filters/elasticsearch-index.asciidoc +++ b/docs/versioned-plugins/filters/elasticsearch-index.asciidoc @@ -5,6 +5,7 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2023-02-23 | <> | 2022-10-26 | <> | 2022-05-24 | <> | 2022-02-08 @@ -31,6 +32,7 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2017-05-03 |======================================================================= +include::elasticsearch-v3.14.0.asciidoc[] include::elasticsearch-v3.13.0.asciidoc[] include::elasticsearch-v3.12.0.asciidoc[] include::elasticsearch-v3.11.1.asciidoc[] diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc new file mode 100644 index 00000000..d8b50880 --- /dev/null +++ b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc @@ -0,0 +1,409 @@ +:plugin: elasticsearch +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v3.14.0 +:release_date: 2023-02-23 +:changelog_url: https://github.com/logstash-plugins/logstash-filter-elasticsearch/blob/v3.14.0/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="{version}-plugins-{type}s-{plugin}"] + +=== Elasticsearch filter plugin {version} + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Search Elasticsearch for a previous log event and copy some fields from it +into the current event. Below are two complete examples of how this filter might +be used. + +The first example uses the legacy 'query' parameter where the user is limited to +an Elasticsearch query_string. +Whenever logstash receives an "end" event, it uses this elasticsearch +filter to find the matching "start" event based on some operation identifier. +Then it copies the `@timestamp` field from the "start" event into a new field on +the "end" event. Finally, using a combination of the "date" filter and the +"ruby" filter, we calculate the time duration in hours between the two events. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query => "type:start AND operation:%{[opid]}" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +The example below reproduces the above example but utilises the query_template. +This query_template represents a full Elasticsearch query DSL and supports the +standard Logstash field substitution syntax. The example below issues +the same query as the first example but uses the template shown. + +[source,ruby] +-------------------------------------------------- +if [type] == "end" { + elasticsearch { + hosts => ["es-server"] + query_template => "template.json" + fields => { "@timestamp" => "started" } + } + + date { + match => ["[started]", "ISO8601"] + target => "[started]" + } + + ruby { + code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)" + } +} +-------------------------------------------------- + +template.json: + +[source,json] +[source,json] +-------------------------------------------------- +{ + "size": 1, + "sort" : [ { "@timestamp" : "desc" } ], + "query": { + "query_string": { + "query": "type:start AND operation:%{[opid]}" + } + }, + "_source": ["@timestamp"] +} +-------------------------------------------------- + +As illustrated above, through the use of 'opid', fields from the Logstash +events can be referenced within the template. +The template will be populated per event prior to being used to query Elasticsearch. + +Notice also that when you use `query_template`, the Logstash attributes `result_size` +and `sort` will be ignored. They should be specified directly in the JSON +template, as shown in the example above. + +[id="{version}-plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <<{version}-plugins-{type}s-{plugin}-user>> AND <<{version}-plugins-{type}s-{plugin}-password>> +* <<{version}-plugins-{type}s-{plugin}-cloud_auth>> +* <<{version}-plugins-{type}s-{plugin}-api_key>> +* <<{version}-plugins-{type}s-{plugin}-keystore>> and/or <<{version}-plugins-{type}s-{plugin}-keystore_password>> + +[id="{version}-plugins-{type}s-{plugin}-autz"] +==== Authorization + +Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level. +The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== Elasticsearch Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-aggregation_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-api_key>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-docinfo_fields>> |{logstash-ref}/configuration-file-structure.html#hash[hash]|No +| <<{version}-plugins-{type}s-{plugin}-enable_sort>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-fields>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-index>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> |{logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-query>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No +| <<{version}-plugins-{type}s-{plugin}-result_on_status_>> |<>|No +| <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No +| <<{version}-plugins-{type}s-{plugin}-keystore_password>> |{logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-tag_on_failure>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-user>> |{logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +Also see <<{version}-plugins-{type}s-{plugin}-common-options>> for a list of options supported by all +filter plugins. + +  + +[id="{version}-plugins-{type}s-{plugin}-aggregation_fields"] +===== `aggregation_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of aggregation names to copy from elasticsearch response into Logstash event fields + +Example: +[source,ruby] + filter { + elasticsearch { + aggregation_fields => { + "my_agg_name" => "my_ls_field" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires +enabling the `ssl` option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the +Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. + +[id="{version}-plugins-{type}s-{plugin}-ca_file"] +===== `ca_file` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +SSL Certificate Authority file + +[id="{version}-plugins-{type}s-{plugin}-ca_trusted_fingerprint"] +===== `ca_trusted_fingerprint` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string], and must contain exactly 64 hexadecimal characters. +* There is no default value for this setting. +* Use of this option _requires_ Logstash 8.3+ + +The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative for the `user`/`password` pair. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + +For more info, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-docinfo_fields"] +===== `docinfo_fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#hash[hash] + * Default value is `{}` + +Hash of docinfo fields to copy from old event (found via elasticsearch) into new event + +Example: +[source,ruby] + filter { + elasticsearch { + docinfo_fields => { + "_id" => "document_id" + "_index" => "document_index" + } + } + } + +[id="{version}-plugins-{type}s-{plugin}-enable_sort"] +===== `enable_sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `true` + +Whether results should be sorted or not + +[id="{version}-plugins-{type}s-{plugin}-fields"] +===== `fields` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `{}` + +An array of fields to copy from the old event (found via elasticsearch) into the +new event, currently being processed. + +In the following example, the values of `@timestamp` and `event_id` on the event +found via elasticsearch are copied to the current event's +`started` and `start_id` fields, respectively: + +[source,ruby] +-------------------------------------------------- +fields => { + "@timestamp" => "started" + "event_id" => "start_id" +} +-------------------------------------------------- + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["localhost:9200"]` + +List of elasticsearch hosts to use for querying. + +[id="{version}-plugins-{type}s-{plugin}-index"] +===== `index` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `""` + +Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices. +Field substitution (e.g. `index-name-%{date_field}`) is available + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Basic Auth - password + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Set the address of a forward HTTP proxy. +An empty string is treated as if proxy was not set, and is useful when using +environment variables e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-query"] +===== `query` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Elasticsearch query string. More information is available in the +{ref}/query-dsl-query-string-query.html#query-string-syntax[Elasticsearch query +string documentation]. + + +[id="{version}-plugins-{type}s-{plugin}-query_template"] +===== `query_template` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +File path to elasticsearch query in DSL format. More information is available in +the {ref}/query-dsl.html[Elasticsearch query documentation]. + +[id="{version}-plugins-{type}s-{plugin}-result_size"] +===== `result_size` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `1` + +How many results to return + +[id="{version}-plugins-{type}s-{plugin}-retry_on_failure"] +===== `retry_on_failure` + +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] +* Default value is `0` (retries disabled) + +How many times to retry an individual failed request. + +When enabled, retry requests that result in connection errors or an HTTP status code included in <<{version}-plugins-{type}s-{plugin}-retry_on_status>> + +[id="{version}-plugins-{type}s-{plugin}-retry_on_status"] +===== `retry_on_status` + +* Value type is <> +* Default value is an empty list `[]` + +Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>, + + +[id="{version}-plugins-{type}s-{plugin}-sort"] +===== `sort` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * Default value is `"@timestamp:desc"` + +Comma-delimited list of `:` pairs that define the sort order + +[id="{version}-plugins-{type}s-{plugin}-ssl"] +===== `ssl` + + * Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] + * Default value is `false` + +SSL + +[id="{version}-plugins-{type}s-{plugin}-keystore"] +===== `keystore` + + * Value type is {logstash-ref}/configuration-file-structure.html#path[path] + * There is no default value for this setting. + +The keystore used to present a certificate to the server. It can be either .jks or .p12 + +[id="{version}-plugins-{type}s-{plugin}-keystore_password"] +===== `keystore_password` + + * Value type is {logstash-ref}/configuration-file-structure.html#password[password] + * There is no default value for this setting. + +Set the keystore password + +[id="{version}-plugins-{type}s-{plugin}-tag_on_failure"] +===== `tag_on_failure` + + * Value type is {logstash-ref}/configuration-file-structure.html#array[array] + * Default value is `["_elasticsearch_lookup_failure"]` + +Tags the event on failure to look up previous log event information. This can be used in later analysis. + +[id="{version}-plugins-{type}s-{plugin}-user"] +===== `user` + + * Value type is {logstash-ref}/configuration-file-structure.html#string[string] + * There is no default value for this setting. + +Basic Auth - username + + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] From 73e53882af909b13a7c3be0f1aa5dadf7d528d92 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Fri, 24 Feb 2023 17:39:03 -0500 Subject: [PATCH 2/5] Fix unresolvable link --- docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc index d8b50880..2e9d2d6f 100644 --- a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc +++ b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc @@ -143,7 +143,7 @@ This plugin supports the following configuration options plus the <<{version}-pl | <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No | <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No | <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No -| <<{version}-plugins-{type}s-{plugin}-result_on_status_>> |<>|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |<>|No | <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No | <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No | <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No From 02b6fcd3d414bbd265f94a61c15f324e893fe7f8 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Fri, 24 Feb 2023 18:05:25 -0500 Subject: [PATCH 3/5] Fix another link formatting error --- docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc index 2e9d2d6f..b0eab769 100644 --- a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc +++ b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc @@ -143,7 +143,7 @@ This plugin supports the following configuration options plus the <<{version}-pl | <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No | <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No | <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No -| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |<>|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |<>|No | <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No | <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No | <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No From cfceb6405c4e9b6b6e51f4d59639d95d465c2002 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Mon, 27 Feb 2023 18:40:28 -0500 Subject: [PATCH 4/5] Update docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc --- docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc index b0eab769..c126e236 100644 --- a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc +++ b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc @@ -143,7 +143,7 @@ This plugin supports the following configuration options plus the <<{version}-pl | <<{version}-plugins-{type}s-{plugin}-query_template>> |{logstash-ref}/configuration-file-structure.html#string[string]|No | <<{version}-plugins-{type}s-{plugin}-result_size>> |{logstash-ref}/configuration-file-structure.html#number[number]|No | <<{version}-plugins-{type}s-{plugin}-retry_on_failure>> |{logstash-ref}/configuration-file-structure.html#number[number]|No -| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |<>|No +| <<{version}-plugins-{type}s-{plugin}-retry_on_status>> |{logstash-ref}/configuration-file-structure.html#number[number]|No | <<{version}-plugins-{type}s-{plugin}-sort>> |{logstash-ref}/configuration-file-structure.html#string[string]|No | <<{version}-plugins-{type}s-{plugin}-ssl>> |{logstash-ref}/configuration-file-structure.html#boolean[boolean]|No | <<{version}-plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No From 22b842d7bf04b01a3169e4b3c5f3d4502668e9ff Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Tue, 7 Mar 2023 18:33:17 -0500 Subject: [PATCH 5/5] Fix link format to work in both LSR and VPR --- docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc index c126e236..765d9cb8 100644 --- a/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc +++ b/docs/versioned-plugins/filters/elasticsearch-v3.14.0.asciidoc @@ -350,7 +350,7 @@ When enabled, retry requests that result in connection errors or an HTTP status [id="{version}-plugins-{type}s-{plugin}-retry_on_status"] ===== `retry_on_status` -* Value type is <> +* Value type is {logstash-ref}/configuration-file-structure.html#number[number] * Default value is an empty list `[]` Which HTTP Status codes to consider for retries (in addition to connection errors) when using <<{version}-plugins-{type}s-{plugin}-retry_on_failure>>,