From 76ab37b35d9b423dfe85b3b21b2fada2826f4994 Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Mon, 6 Nov 2023 08:42:13 +0100 Subject: [PATCH] [DOCS] Uniform formatting for ES|QL commands (#101728) * Source commands * Missing word * Processing commands * Apply suggestions from code review Co-authored-by: Alexander Spies * Review feedback * Add sort detail for mv * More review feedback --------- Co-authored-by: Alexander Spies --- docs/reference/esql/index.asciidoc | 2 +- .../esql/processing-commands/dissect.asciidoc | 10 +-- .../esql/processing-commands/drop.asciidoc | 18 +++++- .../esql/processing-commands/enrich.asciidoc | 10 +-- .../esql/processing-commands/eval.asciidoc | 30 +++++++-- .../esql/processing-commands/grok.asciidoc | 2 +- .../esql/processing-commands/keep.asciidoc | 24 ++++++-- .../esql/processing-commands/limit.asciidoc | 26 ++++++-- .../processing-commands/mv_expand.asciidoc | 19 +++++- .../esql/processing-commands/rename.asciidoc | 21 +++++-- .../esql/processing-commands/sort.asciidoc | 46 ++++++++++---- .../esql/processing-commands/stats.asciidoc | 61 ++++++++++++++----- .../esql/processing-commands/where.asciidoc | 33 ++++++---- .../esql/source-commands/from.asciidoc | 45 ++++++++++++-- .../esql/source-commands/row.asciidoc | 20 ++++++ .../esql/source-commands/show.asciidoc | 29 ++++++++- .../src/main/resources/docs.csv-spec | 2 +- .../src/main/resources/show.csv-spec | 8 ++- 18 files changed, 326 insertions(+), 80 deletions(-) diff --git a/docs/reference/esql/index.asciidoc b/docs/reference/esql/index.asciidoc index 2946f4e61d629..799f95751aa69 100644 --- a/docs/reference/esql/index.asciidoc +++ b/docs/reference/esql/index.asciidoc @@ -62,7 +62,7 @@ An overview of using the <>, <>, and The current limitations of {esql}. <>:: -A few examples of what you can with {esql}. +A few examples of what you can do with {esql}. include::esql-get-started.asciidoc[] diff --git a/docs/reference/esql/processing-commands/dissect.asciidoc b/docs/reference/esql/processing-commands/dissect.asciidoc index eca10c201c968..eb7ab80d6174d 100644 --- a/docs/reference/esql/processing-commands/dissect.asciidoc +++ b/docs/reference/esql/processing-commands/dissect.asciidoc @@ -4,9 +4,9 @@ **Syntax** -[source,txt] +[source,esql] ---- -DISSECT input "pattern" [ append_separator=""] +DISSECT input "pattern" [ APPEND_SEPARATOR=""] ---- *Parameters* @@ -16,9 +16,9 @@ The column that contains the string you want to structure. If the column has multiple values, `DISSECT` will process each value. `pattern`:: -A dissect pattern. +A <>. -`append_separator=""`:: +``:: A string used as the separator between appended values, when using the <>. *Description* @@ -29,7 +29,7 @@ delimiter-based pattern, and extracts the specified keys as columns. Refer to <> for the syntax of dissect patterns. -*Example* +*Examples* // tag::examples[] The following example parses a string that contains a timestamp, some text, and diff --git a/docs/reference/esql/processing-commands/drop.asciidoc b/docs/reference/esql/processing-commands/drop.asciidoc index 50e3b27fb1b28..4787c5f137314 100644 --- a/docs/reference/esql/processing-commands/drop.asciidoc +++ b/docs/reference/esql/processing-commands/drop.asciidoc @@ -2,7 +2,23 @@ [[esql-drop]] === `DROP` -Use `DROP` to remove columns: +**Syntax** + +[source,esql] +---- +DROP columns +---- + +*Parameters* + +`columns`:: +A comma-separated list of columns to remove. Supports wildcards. + +*Description* + +The `DROP` processing command removes one or more columns. + +*Examples* [source,esql] ---- diff --git a/docs/reference/esql/processing-commands/enrich.asciidoc b/docs/reference/esql/processing-commands/enrich.asciidoc index df402f3b1bd50..603683858b8c0 100644 --- a/docs/reference/esql/processing-commands/enrich.asciidoc +++ b/docs/reference/esql/processing-commands/enrich.asciidoc @@ -4,7 +4,7 @@ **Syntax** -[source,txt] +[source,esql] ---- ENRICH policy [ON match_field] [WITH [new_name1 = ]field1, [new_name2 = ]field2, ...] ---- @@ -15,18 +15,18 @@ ENRICH policy [ON match_field] [WITH [new_name1 = ]field1, [new_name2 = ]field2, The name of the enrich policy. You need to <> and <> the enrich policy first. -`ON match_field`:: +`match_field`:: The match field. `ENRICH` uses its value to look for records in the enrich index. If not specified, the match will be performed on the column with the same name as the `match_field` defined in the <>. -`WITH fieldX`:: +`fieldX`:: The enrich fields from the enrich index that are added to the result as new columns. If a column with the same name as the enrich field already exists, the existing column will be replaced by the new column. If not specified, each of the enrich fields defined in the policy is added -`new_nameX =`:: +`new_nameX`:: Enables you to change the name of the column that's added for each of the enrich fields. Defaults to the enrich field name. @@ -74,7 +74,7 @@ include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_on-result] By default, each of the enrich fields defined in the policy is added as a column. To explicitly select the enrich fields that are added, use -`WITH , ...`: +`WITH , , ...`: [source.merge.styled,esql] ---- diff --git a/docs/reference/esql/processing-commands/eval.asciidoc b/docs/reference/esql/processing-commands/eval.asciidoc index a0a78f2a3bf97..eb69a587014ab 100644 --- a/docs/reference/esql/processing-commands/eval.asciidoc +++ b/docs/reference/esql/processing-commands/eval.asciidoc @@ -1,7 +1,30 @@ [discrete] [[esql-eval]] === `EVAL` -`EVAL` enables you to append new columns: + +**Syntax** + +[source,esql] +---- +EVAL column1 = value1[, ..., columnN = valueN] +---- + +*Parameters* + +`columnX`:: +The column name. + +`valueX`:: +The value for the column. Can be a literal, an expression, or a +<>. + +*Description* + +The `EVAL` processing command enables you to append new columns with calculated +values. `EVAL` supports various functions for calculating values. Refer to +<> for more information. + +*Examples* [source.merge.styled,esql] ---- @@ -23,8 +46,3 @@ include::{esql-specs}/docs.csv-spec[tag=evalReplace] |=== include::{esql-specs}/docs.csv-spec[tag=evalReplace-result] |=== - -[discrete] -==== Functions -`EVAL` supports various functions for calculating values. Refer to -<> for more information. diff --git a/docs/reference/esql/processing-commands/grok.asciidoc b/docs/reference/esql/processing-commands/grok.asciidoc index c95fe59f888ce..d5d58a9eaee12 100644 --- a/docs/reference/esql/processing-commands/grok.asciidoc +++ b/docs/reference/esql/processing-commands/grok.asciidoc @@ -4,7 +4,7 @@ **Syntax** -[source,txt] +[source,esql] ---- GROK input "pattern" ---- diff --git a/docs/reference/esql/processing-commands/keep.asciidoc b/docs/reference/esql/processing-commands/keep.asciidoc index 3e54e5a7d1c5c..7515583b1bfd1 100644 --- a/docs/reference/esql/processing-commands/keep.asciidoc +++ b/docs/reference/esql/processing-commands/keep.asciidoc @@ -2,11 +2,25 @@ [[esql-keep]] === `KEEP` -The `KEEP` command enables you to specify what columns are returned and the -order in which they are returned. +**Syntax** -To limit the columns that are returned, use a comma-separated list of column -names. The columns are returned in the specified order: +[source,esql] +---- +KEEP columns +---- + +*Parameters* +`columns`:: +A comma-separated list of columns to keep. Supports wildcards. + +*Description* + +The `KEEP` processing command enables you to specify what columns are returned +and the order in which they are returned. + +*Examples* + +The columns are returned in the specified order: [source.merge.styled,esql] ---- @@ -27,7 +41,7 @@ include::{esql-specs}/docs.csv-spec[tag=keepWildcard] The asterisk wildcard (`*`) by itself translates to all columns that do not match the other arguments. This query will first return all columns with a name -that starts with an h, followed by all other columns: +that starts with `h`, followed by all other columns: [source,esql] ---- diff --git a/docs/reference/esql/processing-commands/limit.asciidoc b/docs/reference/esql/processing-commands/limit.asciidoc index c02b534af59e1..457d5e9e65223 100644 --- a/docs/reference/esql/processing-commands/limit.asciidoc +++ b/docs/reference/esql/processing-commands/limit.asciidoc @@ -2,12 +2,30 @@ [[esql-limit]] === `LIMIT` -The `LIMIT` processing command enables you to limit the number of rows: +**Syntax** [source,esql] ---- -include::{esql-specs}/docs.csv-spec[tag=limit] +LIMIT max_number_of_rows ---- -If not specified, `LIMIT` defaults to `500`. A single query will not return -more than 10,000 rows, regardless of the `LIMIT` value. +*Parameters* + +`max_number_of_rows`:: +The maximum number of rows to return. + +*Description* + +The `LIMIT` processing command enables you to limit the number of rows that are +returned. If not specified, `LIMIT` defaults to `500`. + +A query does not return more than 10,000 rows, regardless of the `LIMIT` value. +You can change this with the `esql.query.result_truncation_max_size` static +cluster setting. + +*Example* + +[source,esql] +---- +include::{esql-specs}/docs.csv-spec[tag=limit] +---- diff --git a/docs/reference/esql/processing-commands/mv_expand.asciidoc b/docs/reference/esql/processing-commands/mv_expand.asciidoc index d62b28aabe440..46dc4fd0a33cf 100644 --- a/docs/reference/esql/processing-commands/mv_expand.asciidoc +++ b/docs/reference/esql/processing-commands/mv_expand.asciidoc @@ -2,7 +2,24 @@ [[esql-mv_expand]] === `MV_EXPAND` -The `MV_EXPAND` processing command expands multivalued fields into one row per value, duplicating other fields: +**Syntax** + +[source,esql] +---- +MV_EXPAND column +---- + +*Parameters* + +`column`:: +The multivalued column to expand. + +*Description* + +The `MV_EXPAND` processing command expands multivalued columns into one row per +value, duplicating other columns. + +*Example* [source.merge.styled,esql] ---- diff --git a/docs/reference/esql/processing-commands/rename.asciidoc b/docs/reference/esql/processing-commands/rename.asciidoc index 1dda424317976..773fe8b640f75 100644 --- a/docs/reference/esql/processing-commands/rename.asciidoc +++ b/docs/reference/esql/processing-commands/rename.asciidoc @@ -2,22 +2,33 @@ [[esql-rename]] === `RENAME` -Use `RENAME` to rename a column using the following syntax: +**Syntax** [source,esql] ---- -RENAME AS +RENAME old_name1 AS new_name1[, ..., old_nameN AS new_nameN] ---- -For example: +*Parameters* + +`old_nameX`:: +The name of a column you want to rename. + +`new_nameX`:: +The new name of the column. + +*Description* + +The `RENAME` processing command renames one or more columns. If a column with +the new name already exists, it will be replaced by the new column. + +*Examples* [source,esql] ---- include::{esql-specs}/docs.csv-spec[tag=rename] ---- -If a column with the new name already exists, it will be replaced by the new -column. Multiple columns can be renamed with a single `RENAME` command: diff --git a/docs/reference/esql/processing-commands/sort.asciidoc b/docs/reference/esql/processing-commands/sort.asciidoc index 76a9193375932..fea7bfaf0c65f 100644 --- a/docs/reference/esql/processing-commands/sort.asciidoc +++ b/docs/reference/esql/processing-commands/sort.asciidoc @@ -1,35 +1,59 @@ [discrete] [[esql-sort]] === `SORT` -Use the `SORT` command to sort rows on one or more fields: + +**Syntax** + +[source,esql] +---- +SORT column1 [ASC/DESC][NULLS FIRST/NULLS LAST][, ..., columnN [ASC/DESC][NULLS FIRST/NULLS LAST]] +---- + +*Parameters* + +`columnX`:: +The column to sort on. + +*Description* + +The `SORT` processing command sorts a table on one or more columns. + +The default sort order is ascending. Use `ASC` or `DESC` to specify an explicit +sort order. + +Two rows with the same sort key are considered equal. You can provide additional +sort expressions to act as tie breakers. + +Sorting on multivalued columns uses the lowest value when sorting ascending and +the highest value when sorting descending. + +By default, `null` values are treated as being larger than any other value. With +an ascending sort order, `null` values are sorted last, and with a descending +sort order, `null` values are sorted first. You can change that by providing +`NULLS FIRST` or `NULLS LAST`. + +*Examples* [source,esql] ---- include::{esql-specs}/docs.csv-spec[tag=sort] ---- -The default sort order is ascending. Set an explicit sort order using `ASC` or -`DESC`: +Explicitly sorting in ascending order with `ASC`: [source,esql] ---- include::{esql-specs}/docs.csv-spec[tag=sortDesc] ---- -Two rows with the same sort key are considered equal. You can provide additional -sort expressions to act as tie breakers: +Providing additional sort expressions to act as tie breakers: [source,esql] ---- include::{esql-specs}/docs.csv-spec[tag=sortTie] ---- -[discrete] -==== `null` values -By default, `null` values are treated as being larger than any other value. With -an ascending sort order, `null` values are sorted last, and with a descending -sort order, `null` values are sorted first. You can change that by providing -`NULLS FIRST` or `NULLS LAST`: +Sorting `null` values first using `NULLS FIRST`: [source,esql] ---- diff --git a/docs/reference/esql/processing-commands/stats.asciidoc b/docs/reference/esql/processing-commands/stats.asciidoc index e0a9bbb52b03e..cbdb74d350fb1 100644 --- a/docs/reference/esql/processing-commands/stats.asciidoc +++ b/docs/reference/esql/processing-commands/stats.asciidoc @@ -1,8 +1,49 @@ [discrete] [[esql-stats-by]] === `STATS ... BY` -Use `STATS ... BY` to group rows according to a common value and calculate one -or more aggregated values over the grouped rows. + +**Syntax** + +[source,esql] +---- +STATS [column1 =] expression1[, ..., [columnN =] expressionN] [BY grouping_column1[, ..., grouping_columnN]] +---- + +*Parameters* + +`columnX`:: +The name by which the aggregated value is returned. If omitted, the name is +equal to the corresponding expression (`expressionX`). + +`expressionX`:: +An expression that computes an aggregated value. + +`grouping_columnX`:: +The column containing the values to group by. + +*Description* + +The `STATS ... BY` processing command groups rows according to a common value +and calculate one or more aggregated values over the grouped rows. If `BY` is +omitted, the output table contains exactly one row with the aggregations applied +over the entire dataset. + +The following aggregation functions are supported: + +include::../functions/aggregation-functions.asciidoc[tag=agg_list] + +NOTE: `STATS` without any groups is much much faster than adding a group. + +NOTE: Grouping on a single column is currently much more optimized than grouping + on many columns. In some tests we have seen grouping on a single `keyword` + column to be five times faster than grouping on two `keyword` columns. Do + not try to work around this by combining the two columns together with + something like <> and then grouping - that is not going to be + faster. + +*Examples* + +Calculating a statistic and grouping by the values of another column: [source.merge.styled,esql] ---- @@ -13,8 +54,8 @@ include::{esql-specs}/docs.csv-spec[tag=stats] include::{esql-specs}/docs.csv-spec[tag=stats-result] |=== -If `BY` is omitted, the output table contains exactly one row with the -aggregations applied over the entire dataset: +Omitting `BY` returns one row with the aggregations applied over the entire +dataset: [source.merge.styled,esql] ---- @@ -39,15 +80,3 @@ keyword family fields): ---- include::{esql-specs}/docs.csv-spec[tag=statsGroupByMultipleValues] ---- - -The following aggregation functions are supported: - -include::../functions/aggregation-functions.asciidoc[tag=agg_list] - -NOTE: `STATS` without any groups is much much faster than adding group. - -NOTE: Grouping on a single field is currently much more optimized than grouping - on many fields. In some tests we've seen grouping on a single `keyword` - field to be five times faster than grouping on two `keyword` fields. Don't - try to work around this combining the two fields together with something - like <> and then grouping - that's not going to be faster. diff --git a/docs/reference/esql/processing-commands/where.asciidoc b/docs/reference/esql/processing-commands/where.asciidoc index 8dd55df12b9e7..e723a977bf99c 100644 --- a/docs/reference/esql/processing-commands/where.asciidoc +++ b/docs/reference/esql/processing-commands/where.asciidoc @@ -2,8 +2,27 @@ [[esql-where]] === `WHERE` -Use `WHERE` to produce a table that contains all the rows from the input table -for which the provided condition evaluates to `true`: +**Syntax** + +[source,esql] +---- +WHERE expression +---- + +*Parameters* + +`expression`:: +A boolean expression. + +*Description* + +The `WHERE` processing command produces a table that contains all the rows from +the input table for which the provided condition evaluates to `true`. + +`WHERE` supports various <> and +<>. + +*Examples* [source,esql] ---- @@ -17,15 +36,7 @@ Which, if `still_hired` is a boolean field, can be simplified to: include::{esql-specs}/docs.csv-spec[tag=whereBoolean] ---- -[discrete] -==== Operators - -Refer to <> for an overview of the supported operators. - -[discrete] -==== Functions -`WHERE` supports various functions for calculating values. Refer to -<> for more information. +Using a function: [source,esql] ---- diff --git a/docs/reference/esql/source-commands/from.asciidoc b/docs/reference/esql/source-commands/from.asciidoc index 5718bfc27ac1c..6f54a42ddad35 100644 --- a/docs/reference/esql/source-commands/from.asciidoc +++ b/docs/reference/esql/source-commands/from.asciidoc @@ -2,10 +2,47 @@ [[esql-from]] === `FROM` -The `FROM` source command returns a table with up to 10,000 documents from a -data stream, index, or alias. Each row in the resulting table represents a -document. Each column corresponds to a field, and can be accessed by the name -of that field. +**Syntax** + +[source,esql] +---- +FROM index_pattern [METADATA fields] +---- + +*Parameters* + +`index_pattern`:: +A list of indices, data streams or aliases. Supports wildcards and date math. + +`fields`:: +A comma-separated list of <> to retrieve. + +*Description* + +The `FROM` source command returns a table with data from a data stream, index, +or alias. Each row in the resulting table represents a document. Each column +corresponds to a field, and can be accessed by the name of that field. + +[NOTE] +==== +By default, an {esql} query without an explicit <> uses an implicit +limit of 500. This applies to `FROM` too. A `FROM` command without `LIMIT`: + +[source,esql] +---- +FROM employees +---- + +is executed as: + +[source,esql] +---- +FROM employees +| LIMIT 500 +---- +==== + +*Examples* [source,esql] ---- diff --git a/docs/reference/esql/source-commands/row.asciidoc b/docs/reference/esql/source-commands/row.asciidoc index edfe5ecbf7cf3..adce844f365b8 100644 --- a/docs/reference/esql/source-commands/row.asciidoc +++ b/docs/reference/esql/source-commands/row.asciidoc @@ -2,9 +2,29 @@ [[esql-row]] === `ROW` +**Syntax** + +[source,esql] +---- +ROW column1 = value1[, ..., columnN = valueN] +---- + +*Parameters* + +`columnX`:: +The column name. + +`valueX`:: +The value for the column. Can be a literal, an expression, or a +<>. + +*Description* + The `ROW` source command produces a row with one or more columns with values that you specify. This can be useful for testing. +*Examples* + [source.merge.styled,esql] ---- include::{esql-specs}/row.csv-spec[tag=example] diff --git a/docs/reference/esql/source-commands/show.asciidoc b/docs/reference/esql/source-commands/show.asciidoc index 956baf628e9f3..ea8c83ceb772a 100644 --- a/docs/reference/esql/source-commands/show.asciidoc +++ b/docs/reference/esql/source-commands/show.asciidoc @@ -1,10 +1,35 @@ [discrete] [[esql-show]] -=== `SHOW ` +=== `SHOW` -The `SHOW ` source command returns information about the deployment and +**Syntax** + +[source,esql] +---- +SHOW item +---- + +*Parameters* + +`item`:: +Can be `INFO` or `FUNCTIONS`. + +*Description* + +The `SHOW` source command returns information about the deployment and its capabilities: * Use `SHOW INFO` to return the deployment's version, build date and hash. * Use `SHOW FUNCTIONS` to return a list of all supported functions and a synopsis of each function. + +*Examples* + +[source.merge.styled,esql] +---- +include::{esql-specs}/show.csv-spec[tag=showFunctionsFiltered] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/show.csv-spec[tag=showFunctionsFiltered-result] +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec index 02d530a2ae835..dbf76033fbe79 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/docs.csv-spec @@ -278,7 +278,7 @@ docsWhereFunction // tag::whereFunction[] FROM employees | KEEP first_name, last_name, height -| WHERE length(first_name) < 4 +| WHERE LENGTH(first_name) < 4 // end::whereFunction[] | SORT first_name ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec index 117bb9646bc5d..0b45f9ac5aea4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/show.csv-spec @@ -184,10 +184,16 @@ synopsis:keyword showFunctionsFiltered -show functions | where starts_with(name, "is_"); +// tag::showFunctionsFiltered[] +SHOW functions +| WHERE STARTS_WITH(name, "is_") +// end::showFunctionsFiltered[] +; +// tag::showFunctionsFiltered-result[] name:keyword | synopsis:keyword | argNames:keyword | argTypes:keyword | argDescriptions:keyword | returnType:keyword | description:keyword | optionalArgs:boolean | variadic:boolean is_finite |? is_finite(arg1:?) |arg1 |? | "" |? | "" | false | false is_infinite |? is_infinite(arg1:?) |arg1 |? | "" |? | "" | false | false is_nan |? is_nan(arg1:?) |arg1 |? | "" |? | "" | false | false +// end::showFunctionsFiltered-result[] ;