diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 792c35e3949b..b1b70d6abb01 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -147,6 +147,7 @@ - [Opensearch dashboards.release notes 1.3.14](../release-notes/opensearch-dashboards.release-notes-1.3.14.md) - [Opensearch dashboards.release notes 1.3.15](../release-notes/opensearch-dashboards.release-notes-1.3.15.md) - [Opensearch dashboards.release notes 1.3.17](../release-notes/opensearch-dashboards.release-notes-1.3.17.md) + - [Opensearch dashboards.release notes 1.3.19](../release-notes/opensearch-dashboards.release-notes-1.3.19.md) - [Opensearch dashboards.release notes 1.3.2](../release-notes/opensearch-dashboards.release-notes-1.3.2.md) - [Opensearch dashboards.release notes 1.3.3](../release-notes/opensearch-dashboards.release-notes-1.3.3.md) - [Opensearch dashboards.release notes 1.3.5](../release-notes/opensearch-dashboards.release-notes-1.3.5.md) diff --git a/src/plugins/data/public/query/query_string/language_service/_recent_query.scss b/src/plugins/data/public/query/query_string/language_service/_recent_query.scss new file mode 100644 index 000000000000..faac658f685f --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/_recent_query.scss @@ -0,0 +1,4 @@ +.recentQuery__table { + padding: $euiSizeXS; + width: 1320px; +} diff --git a/src/plugins/data/public/query/query_string/language_service/flyout_containers.tsx b/src/plugins/data/public/query/query_string/language_service/flyout_containers.tsx new file mode 100644 index 000000000000..4d11d2ed01d7 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/flyout_containers.tsx @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiFlyout } from '@elastic/eui'; +import React from 'react'; + +/* + * "FlyoutContainers" component used to create flyouts + * + * Props taken in as params are: + * flyoutHeader - header JSX element of flyout + * flyoutBody - body JSX element of flyout + * flyoutFooter - footer JSX element of flyout + * ariaLabel - aria-label for focus of flyout + */ + +interface Props { + closeFlyout: () => void; + flyoutHeader: JSX.Element; + flyoutBody: JSX.Element; + flyoutFooter: JSX.Element; + ariaLabel: string; + size?: string; +} + +export const FlyoutContainers = ({ + closeFlyout, + flyoutHeader, + flyoutBody, + flyoutFooter, + ariaLabel, + size, +}: Props) => { + return ( +
+ closeFlyout()} + size={size ? size : 'm'} + aria-labelledby={ariaLabel} + > + {flyoutHeader} + {flyoutBody} + {flyoutFooter} + +
+ ); +}; diff --git a/src/plugins/data/public/query/query_string/language_service/get_query_control_links.tsx b/src/plugins/data/public/query/query_string/language_service/get_query_control_links.tsx new file mode 100644 index 000000000000..9b92d22ae432 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/get_query_control_links.tsx @@ -0,0 +1,172 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import React, { useState } from 'react'; +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiLink, + EuiPopoverTitle, + EuiText, + EuiWrappingPopover, +} from '@elastic/eui'; +import ReactDOM from 'react-dom'; +import { FormattedMessage } from 'react-intl'; +import { + OpenSearchDashboardsContextProvider, + toMountPoint, +} from '../../../../../opensearch_dashboards_react/public'; +import { IDataPluginServices } from '../../../types'; +import { PPLReferenceFlyout } from './ppl_reference_flyout'; + +export interface QueryControl { + id: string; + label: string; + testId: string; + ariaLabel: string; + run: (anchorElement: HTMLElement) => void; + iconType: string; +} + +export const QueryControls = (props: { + services: IDataPluginServices; + queryLanguage: string; + onToggleCollapse: () => void; + savedQueryManagement?: any; + additionalControls?: QueryControl[]; +}) => { + const [isCollapsed, setIsCollapsed] = useState(false); + const [isLanguageReferenceOpen, setIsLanguageReferenceOpen] = useState(false); + + const languageReferenceContainer = document.createElement('div'); + + const onCloseLanguageReference = () => { + ReactDOM.unmountComponentAtNode(languageReferenceContainer); + setIsLanguageReferenceOpen(false); + }; + + const osdDQLDocs = 'https://opensearch.org/docs/2.16/dashboards/dql)'; + const dqlFullName = ( + + ); + + const languageReference: QueryControl = { + id: 'languageReference', + label: i18n.translate('discover.queryControls.languageReference', { + defaultMessage: 'Open', + }), + testId: 'languageReference', + ariaLabel: i18n.translate('discover.queryControls.languageReference', { + defaultMessage: `Language Reference`, + }), + run: async (anchorElement) => { + if (props.queryLanguage === 'PPL' || props.queryLanguage === 'SQL') { + const flyoutSession = props.services.overlays!.openFlyout( + toMountPoint( + + flyoutSession?.close?.().then()} + makeUrl={(searchId: any) => `#/view/${encodeURIComponent(searchId)}`} + /> + + ) + ); + } else { + if (isLanguageReferenceOpen) { + onCloseLanguageReference(); + return; + } + + setIsLanguageReferenceOpen(true); + document.body.appendChild(languageReferenceContainer); + + const element = ( + + + + +
+ +

+ + {dqlFullName} + + ), + }} + /> +

+ +
+ + ); + + ReactDOM.render(element, languageReferenceContainer); + } + }, + iconType: 'iInCircle', + }; + + const languageToggle: QueryControl = { + id: 'languageToggle', + label: i18n.translate('discover.queryControls.languageToggle', { + defaultMessage: 'Toggle', + }), + testId: 'languageToggle', + ariaLabel: i18n.translate('discover.queryControls.languageToggle', { + defaultMessage: `Language Toggle`, + }), + run: () => { + setIsCollapsed(!isCollapsed); + props.onToggleCollapse(); + }, + iconType: isCollapsed ? 'expand' : 'minimize', + }; + + const queryControls = + props.queryLanguage === 'PPL' || props.queryLanguage === 'SQL' + ? [languageReference, languageToggle] + : [languageReference]; + + if (props.additionalControls) { + queryControls.push(...props.additionalControls); + } + + return ( + + {queryControls.map((queryControl) => ( + + queryControl.run(event.currentTarget)} + /> + + ))} + {props.savedQueryManagement} + + ); +}; diff --git a/src/plugins/data/public/query/query_string/language_service/language_service.ts b/src/plugins/data/public/query/query_string/language_service/language_service.ts index 328912a78666..923838c26bb2 100644 --- a/src/plugins/data/public/query/query_string/language_service/language_service.ts +++ b/src/plugins/data/public/query/query_string/language_service/language_service.ts @@ -27,6 +27,10 @@ export class LanguageService { this.queryEditorExtensionMap = {}; } + public createDefaultQueryEditor() { + return createEditor(SingleLineInput, SingleLineInput, DQLBody); + } + public __enhance = (enhancements: UiEnhancements) => { if (enhancements.queryEditorExtension) { this.queryEditorExtensionMap[enhancements.queryEditorExtension.id] = diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/dedup.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/dedup.ts new file mode 100644 index 000000000000..1acde2fb90c9 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/dedup.ts @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const dedupCmd = `## dedup +--- + +### Description + +Use the \'dedup\' command to remove identical documents from the search results, based on the specified field. + +### Syntax + +dedup \[int\] <field-list> \[keepempty=<bool>\] +\[consecutive=<bool>\] + +- \`field-list\`: Required. The comma-delimited field list. At least one field is required. +- \`consecutive\`: Optional. If set to \`true\`, removes duplicate events, where the duplicate events have consecutive timestamps. Default is \`false\`. +- \`int\`: Optional. The \'dedup\' command retains multiple events for each combination when you specify \`<int>\`. The number for \`<int>\` must be greater than 0. If you do not specify a number, only the first occurring event is kept. All other duplicates are removed from the results. Default is \`1\`. +- \`keepempty\`: Optional. If set to \`true\`, keeps the document if any field in the \`field-list\` is null or missing. Default is \`false\`. + +#### Example 1: Dedup by one field + +The following example PPL query shows how to use \`dedup\` to remove duplicate documents based on the \`gender\` field: + + os> source=accounts | dedup gender | fields account_number, gender; + fetched rows / total rows = 2/2 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 13 | F | + +------------------+----------+ + +#### Example 2: Keep two duplicate documents + +The following example PPL query shows how to use \`dedup\` to remove duplicate documents based on the \`gender\` field while keeping two duplicates: + + os> source=accounts | dedup 2 gender | fields account_number, gender; + fetched rows / total rows = 3/3 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 6 | M | + | 13 | F | + +------------------+----------+ + +#### Example 3: Keep or ignore empty fields by default + +The following example PPL query shows how to use \`dedup\` to remove duplicate documents while keeping documents with null values in the specified field: + + os> source=accounts | dedup email keepempty=true | fields account_number, email; + fetched rows / total rows = 4/4 + +------------------+-----------------------+ + | account_number | email | + |------------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 13 | null | + | 18 | daleadams@boink.com | + +------------------+-----------------------+ + +The following example PPL query shows how to use \`dedup\` to remove duplicate documents while ignoring documents with empty values in the specified field: + + os> source=accounts | dedup email | fields account_number, email; + fetched rows / total rows = 3/3 + +------------------+-----------------------+ + | account_number | email | + |------------------+-----------------------| + | 1 | amberduke@pyrami.com | + | 6 | hattiebond@netagy.com | + | 18 | daleadams@boink.com | + +------------------+-----------------------+ + +#### Example 4: Remove duplicate consecutive documents + +The following example PPL query shows how to use \`dedup\` to remove duplicate consecutive documents: + + os> source=accounts | dedup gender consecutive=true | fields account_number, gender; + fetched rows / total rows = 3/3 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 13 | F | + | 18 | M | + +------------------+----------+ + +### Limitation +The \`dedup\` command is not rewritten to [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/). It is only run on the coordinating node. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/eval.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/eval.ts new file mode 100644 index 000000000000..f20a5ee35198 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/eval.ts @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const evalCmd = `## eval +--- + +### Description + +Use the \'eval\' command to evaluate the expression and append the result to the search result. + +### Syntax + +eval <field>=<expression> \["," +<field>=<expression> \]... + +- \`field\`: Required. If the field name does not exist, a new field is created. If the field name exists, the value of the existing field is replaced. +- \`expression\`: Required. Any expression that is supported by the system. + +#### Example 1: Create new fields + +The following example PPL query shows how to use \`eval\` to create a new field for each document. In this example, the new field is \`doubleAge\`. + + os> source=accounts | eval doubleAge = age * 2 | fields age, doubleAge; + fetched rows / total rows = 4/4 + +-------+-------------+ + | age | doubleAge | + |-------+-------------| + | 32 | 64 | + | 36 | 72 | + | 28 | 56 | + | 33 | 66 | + +-------+-------------+ + +#### Example 2: Override existing fields + +The following example PPL query shows how to use \`eval\` to override an existing field. In this example, the existing field \`age\` is overridden by the \`age\` field plus 1. + + os> source=accounts | eval age = age + 1 | fields age; + fetched rows / total rows = 4/4 + +-------+ + | age | + |-------| + | 33 | + | 37 | + | 29 | + | 34 | + +-------+ + +#### Example 3: Create new fields based on the fields defined in the \`eval\` expression + +The following example PPL query shows how to use \`eval\` to create a new field based on the fields defined in the \`eval\` expression. In this example, the new field \`ddAge\` is the evaluation result of the \`doubleAge\` field multiplied by 2. \`doubleAge\` is defined in the \`eval\` command. + + os> source=accounts | eval doubleAge = age * 2, ddAge = doubleAge * 2 | fields age, doubleAge, ddAge; + fetched rows / total rows = 4/4 + +-------+-------------+---------+ + | age | doubleAge | ddAge | + |-------+-------------+---------| + | 32 | 64 | 128 | + | 36 | 72 | 144 | + | 28 | 56 | 112 | + | 33 | 66 | 132 | + +-------+-------------+---------+ + +### Limitation +The \`eval\` command is not rewritten to [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/). It is only run on the coordinating node. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/fields.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/fields.ts new file mode 100644 index 000000000000..2110349b54eb --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/fields.ts @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const fieldsCmd = `## fields +--- +### Description + +Use the \`fields\` command to specify the fields that should be included in or excluded from the search results. + +### Syntax + +fields \[+\|-\] <field-list> + +- \`field-list\`: Required. Comma-separated list of fields to keep or remove. +- \`index\`: Optional. If the plus sign \`+\` is used, only the fields specified in the field list will be included. If the minus \`-\` is used, all the fields specified in the field list will be excluded. Default is \`+\`. + +#### Example 1: Select specified fields from the search result + +The following example PPL query shows how to retrieve the \`account\_number\`, \`firstname\`, and \`lastname\` fields from the search results: + + os> source=accounts | fields account_number, firstname, lastname; + fetched rows / total rows = 4/4 + +------------------+-------------+------------+ + | account_number | firstname | lastname | + |------------------+-------------+------------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +------------------+-------------+------------+ + +#### Example 2: Remove specified fields from the search results + +The following example PPL query shows how to remove the \`account\_number\` field from the search results: + + os> source=accounts | fields account_number, firstname, lastname | fields - account_number; + fetched rows / total rows = 4/4 + +-------------+------------+ + | firstname | lastname | + |-------------+------------| + | Amber | Duke | + | Hattie | Bond | + | Nanette | Bates | + | Dale | Adams | + +-------------+------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/head.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/head.ts new file mode 100644 index 000000000000..468da72d3871 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/head.ts @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const headCmd = `## head +--- + +### Description + +Use the \`head\` command to return the first N number of lines from a search result. + +### Syntax + +head \[N\] + +- \`N\`: Optional. The number of results you want to return. Default is 10. + +#### Example 1: Get the first 10 results + +The following example PPL query shows how to use \`head\` to return the first 10 search results: + + os> source=accounts | fields firstname, age | head; + fetched rows / total rows = 10/10 + +---------------+-----------+ + | firstname | age | + |---------------+-----------| + | Amber | 32 | + | Hattie | 36 | + | Nanette | 28 | + | Dale | 33 | + | Elinor | 36 | + | Virginia | 39 | + | Dillard | 34 | + | Mcgee | 39 | + | Aurelia | 37 | + | Fulton | 23 | + +---------------+-----------+ + +#### Example 2: Get the first N results + +The following example PPL query shows how to use \`head\` to get a specified number of search results. In this example, N is equal to 3: + + os> source=accounts | fields firstname, age | head 3; + fetched rows / total rows = 3/3 + +---------------+-----------+ + | firstname | age | + |---------------+-----------| + | Amber | 32 | + | Hattie | 36 | + | Nanette | 28 | + +---------------+-----------+ + +#### Limitation +The \`head\` command is not rewritten to [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/). It is only run on the coordinating node. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/index.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/index.ts new file mode 100644 index 000000000000..27328a4c2be8 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { dedupCmd } from './dedup'; +export { evalCmd } from './eval'; +export { fieldsCmd } from './fields'; +export { headCmd } from './head'; +export { parseCmd } from './parse'; +export { rareCmd } from './rare'; +export { renameCmd } from './rename'; +export { searchCmd } from './search'; +export { sortCmd } from './sort'; +export { statsCmd } from './stats'; +export { syntaxCmd } from './syntax'; +export { topCmd } from './top'; +export { whereCmd } from './where'; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/parse.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/parse.ts new file mode 100644 index 000000000000..28501cf8555e --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/parse.ts @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const parseCmd = `## parse +--- +### Description + +Use the \`parse\` command to extract information from a text field using a regular expression and add it to the search result. + +### Syntax + +parse <field> <regular-expression> + +- \`field\`: Required. Must be a text field. +- \`regular-expression\`: Required. The regular expression used to extract new fields from a text field. It replaces the original field if a new field name exists. + +### Regular expression + +Use the Java regular expression engine to match the entire text field of each document. Each named capture group in the expression will be converted to a new \`string\` field. + +#### Example 1: Create a new field + +The following example PPL query shows how to create new field \`host\` for each document. \`host\` becomes the hostname after the @ symbol in the \`email\` field. Parsing a null field returns an empty string. + + os> source=accounts | parse email '.+@(?.+)' | fields email, host; + fetched rows / total rows = 4/4 + +-----------------------+------------+ + | email | host | + |-----------------------+------------| + | amberduke@pyrami.com | pyrami.com | + | hattiebond@netagy.com | netagy.com | + | null | | + | daleadams@boink.com | boink.com | + +-----------------------+------------+ + +#### Example 2: Override an existing field + +The following example PPL query shows how to override the existing \`address\` field while excluding the street number: + + os> source=accounts | parse address '\\d+ (?
.+)' | fields address; + fetched rows / total rows = 4/4 + +------------------+ + | address | + |------------------| + | Holmes Lane | + | Bristol Street | + | Madison Street | + | Hutchinson Court | + +------------------+ + +#### Example 3: Filter and sort by casted-parsed field + +The following example PPL query shows how to sort street numbers that are greater than 500 in the \`address\` field: + + os> source=accounts | parse address '(?\d+) (?.+)' | where cast(streetNumber as int) > 500 | sort num(streetNumber) | fields streetNumber, street; + fetched rows / total rows = 3/3 + +----------------+----------------+ + | streetNumber | street | + |----------------+----------------| + | 671 | Bristol Street | + | 789 | Madison Street | + | 880 | Holmes Lane | + +----------------+----------------+ + +### Limitation + +The following limitations apply: + +- Parsed fields cannot be parsed again. For example, the following command is not valid: + + source=accounts | parse address '\\d+ (?.+)' | parse street '\\w+ (?\\w+)'; + +- Other commands cannot overwrite fields created by parsing. For example, in the following query, \`where\` does not match any documents because \`street\` cannot be overridden: + + source=accounts | parse address '\\d+ (?.+)' | eval street='1' | where street='1'; + +- The text field that is parsed cannot be overridden. For example, in the following query, \`street\` is not successfully parsed because \`address\` is overridden: + + source=accounts | parse address '\\d+ (?.+)' | eval address='1'; + +- Fields created by parsing cannot be filtered or sorted after using them in the \`stats\` command. For example, in the following query, \`where\` is not valid: + + source=accounts | parse email '.+@(?.+)' | stats avg(age) by host | where host=pyrami.com; +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/rare.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/rare.ts new file mode 100644 index 000000000000..61f5016f4e9e --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/rare.ts @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const rareCmd = `## rare +--- + +### Description + +Use the \`rare\` command to find the least common tuple of values across all fields in the \`field-list\` field. A maximum of 10 results is returned for each distinct tuple of group-by field values. + +### Syntax + +rare <field-list> \[by-clause\] + +- \`field-list\`: Required. A comma-separated list of field names. +- \`by-clause\`: Optional. One or more fields to group by. + +#### Example 1: Find a field's least common values + +The following example PPL query shows how to find a least common value in the \`gender\` field: + + os> source=accounts | rare gender; + fetched rows / total rows = 2/2 + +------------+ + | gender | + |------------| + | F | + |------------| + | M | + +------------+ + +#### Example 2: Find least common values in group-by fields + +The following example PPL query shows how to find a least common value in the \`age\` field that is grouped by \`gender\`: + + os> source=accounts | rare age by gender; + fetched rows / total rows = 20/20 + +----------+----------+ + | gender | age | + |----------+----------| + | F | 29 | + | F | 20 | + | F | 23 | + | F | 25 | + | F | 37 | + | F | 38 | + | F | 40 | + | F | 27 | + | F | 36 | + | F | 24 | + | M | 27 | + | M | 24 | + | M | 34 | + | M | 38 | + | M | 28 | + | M | 39 | + | M | 21 | + | M | 30 | + | M | 25 | + | M | 29 | + +----------+----------+ + +#### Limitation +The \`rare\` command is not rewritten to [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/). It is only run on the coordinating node. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/rename.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/rename.ts new file mode 100644 index 000000000000..22d9790514a8 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/rename.ts @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const renameCmd = `## rename +--- +### Description + +Use the \`rename\` command to rename one or more fields in the search result. + +### Syntax + +rename <source-field> AS <target-field>\["," +<source-field> AS <target-field>\]... + +- \`source-field\`: Required. The field to rename. +- \`target-field\`: Required. The new field. + +#### Example 1: Rename one field + +The following example PPL query renames a field: + + os> source=accounts | rename account_number as an | fields an; + fetched rows / total rows = 4/4 + +------+ + | an | + |------| + | 1 | + | 6 | + | 13 | + | 18 | + +------+ + +#### Example 2: Rename two or more fields + +The following example PPL query renames two or more fields: + + os> source=accounts | rename account_number as an, employer as emp | fields an, emp; + fetched rows / total rows = 4/4 + +------+---------+ + | an | emp | + |------+---------| + | 1 | Pyrami | + | 6 | Netagy | + | 13 | Quility | + | 18 | null | + +------+---------+ + +#### Limitation +The \`rename\` command is not rewritten to [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/). It is only run on the coordinating node. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/search.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/search.ts new file mode 100644 index 000000000000..2c2e361dd958 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/search.ts @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const searchCmd = `## search +--- + +### Description + +Use the \`search\` command to retrieve a document from the index. The \`search\` +command can only be used as the first command in a PPL query. + +### Syntax + +search source=<index> \[boolean-expression\] + +- \`search\`: Search keywords, which can be ignored. +- \`index\`: Required. Search commands must specify the index to query. +- \`bool-expression\`: Optional. Any expression that can be evaluated to a Boolean value. + +#### Example 1: Fetch all data from an index + +The following example PPL query shows how to fetch all documents from the \`accounts\` index: + + os> source=accounts; + fetched rows / total rows = 4/4 + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + | 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams | + +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ + +#### Example 2: Fetch data with a condition + +The following example PPL query shows how to fetch all documents from the \`accounts\` index by using the \`or\` condition. + + os> source=accounts account_number=1 or gender="F"; + fetched rows / total rows = 2/2 + +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ + | account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname | + |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------| + | 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke | + | 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates | + +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/sort.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/sort.ts new file mode 100644 index 000000000000..d31e84a96e29 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/sort.ts @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const sortCmd = `## sort +--- +### Description + +Use the \`sort\` command to sort search results by a specified field. + +### Syntax + +sort <\[+\|-\] sort-field>... + +- \`sort-field\`: Required. The field to sort by. +- \[+\|-\]: Optional. The symbols used to indicate the sorting order. A plus sign \[+\] indicates ascending order, with null and missing values first. A minus sign \[-\] indicates descending order, with null and missing last. Default is \[+\], with null and missing first. + +#### Example 1: Sort by one field + +The following example PPL query shows how to sort documents by one field in ascending order: + + os> source=accounts | sort age | fields account_number, age; + fetched rows / total rows = 4/4 + +------------------+-------+ + | account_number | age | + |------------------+-------| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +------------------+-------+ + +#### Example 2: Sort by one field and return all results + +The following example PPL query shows how to sort by one field and return all results in ascending order: + + os> source=accounts | sort age | fields account_number, age; + fetched rows / total rows = 4/4 + +------------------+-------+ + | account_number | age | + |------------------+-------| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +------------------+-------+ + +#### Example 3: Sort by one field in descending order + +The following example PPL query shows how to sort by one field in descending order: + + os> source=accounts | sort - age | fields account_number, age; + fetched rows / total rows = 4/4 + +------------------+-------+ + | account_number | age | + |------------------+-------| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +------------------+-------+ + +#### Example 4: Sort multiple fields in both ascending and descending order + +The following example PPL query shows how to sort by multiple fields in both ascending and descending order. In this example, the \`gender\` field is in ascending order and the \`age\` field is in descending order. + + os> source=accounts | sort + gender, - age | fields account_number, gender, age; + fetched rows / total rows = 4/4 + +------------------+----------+-------+ + | account_number | gender | age | + |------------------+----------+-------| + | 13 | F | 28 | + | 6 | M | 36 | + | 18 | M | 33 | + | 1 | M | 32 | + +------------------+----------+-------+ + +#### Example 5: Sort by field, including null values + +The following example PPL query shows how to sort by the \`employer\` field using the default order (\[+\] with null and missing first): + + os> source=accounts | sort employer | fields employer; + fetched rows / total rows = 4/4 + +------------+ + | employer | + |------------| + | null | + | Netagy | + | Pyrami | + | Quility | + +------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/stats.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/stats.ts new file mode 100644 index 000000000000..5392c9ef4b5d --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/stats.ts @@ -0,0 +1,282 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const statsCmd = `## stats +--- + +### Description + +Use the \`stats\` command to calculate the aggregation from the search results. + +The following table catalogs the aggregation functions and defines how the null and missing values are handled. + +| | | | +|----------|-------------|-------------| +| Function | NULL | MISSING | +| COUNT | Not counted | Not counted | +| SUM | Ignore | Ignore | +| AVG | Ignore | Ignore | +| MAX | Ignore | Ignore | +| MIN | Ignore | Ignore | + +### Syntax + +stats <aggregation>... \[by-clause\]... + +- \`aggregation\`: Required. The aggregation function that must be applied to the field. +- \`by-clause\`: Optional. One or more fields to group by. Default: If \`<by-clause>\` is not specified, the \`stats\` command returns one row, which is the aggregation for the entire result set. + +### Aggregation functions +--- +#### COUNT + +The \`count\` function returns the number of rows in the result set. The following is an example PPL query: + + os> source=accounts | stats count(); + fetched rows / total rows = 1/1 + +-----------+ + | count() | + |-----------| + | 4 | + +-----------+ + +#### SUM + +The \`SUM(expr)\` function returns the sum of the values in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats sum(age) by gender; + fetched rows / total rows = 2/2 + +------------+----------+ + | sum(age) | gender | + |------------+----------| + | 28 | F | + | 101 | M | + +------------+----------+ + +#### AVG + +The \`AVG(expr)\` function returns the average of the values in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats avg(age) by gender; + fetched rows / total rows = 2/2 + +--------------------+----------+ + | avg(age) | gender | + |--------------------+----------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+----------+ + +#### MAX + +The \`MAX(expr)\` function returns the largest value in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats max(age); + fetched rows / total rows = 1/1 + +------------+ + | max(age) | + |------------| + | 36 | + +------------+ + +#### MIN + +The \`MIN(expr)\` function returns the smallest value in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats min(age); + fetched rows / total rows = 1/1 + +------------+ + | min(age) | + |------------| + | 28 | + +------------+ + +#### VAR\_SAMP + +The \`VAR\_SAMP(expr)\` function returns the sample variance of a selection of data in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats var_samp(age); + fetched rows / total rows = 1/1 + +--------------------+ + | var_samp(age) | + |--------------------| + | 10.916666666666666 | + +--------------------+ + +#### VAR\_POP + +The \`VAR\_POP(expr)\` function returns the population variance of a selection of data in the expression \`expr\`. See the following example. + + os> source=accounts | stats var_pop(age); + fetched rows / total rows = 1/1 + +----------------+ + | var_pop(age) | + |----------------| + | 8.1875 | + +----------------+ + +#### STDDEV\_SAMP + +The \`STDDEV\_SAMP(expr)\` function returns the sample standard deviation of a set of values in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats stddev_samp(age); + fetched rows / total rows = 1/1 + +--------------------+ + | stddev_samp(age) | + |--------------------| + | 3.304037933599835 | + +--------------------+ + +#### STDDEV\_POP + +The \`STDDEV\_POP(expr)\` function returns the population standard deviation of a set of values in the expression \`expr\`. The following is an example PPL query: + + os> source=accounts | stats stddev_pop(age); + fetched rows / total rows = 1/1 + +--------------------+ + | stddev_pop(age) | + |--------------------| + | 2.8613807855648994 | + +--------------------+ + +### By clause + +The \`by\` clause can contain fields, expressions, scalar functions, or aggregation functions. The \`span\` clause can be used in the \`by\` clause to split specific fields into buckets of the same interval. The \`stats\` command then performs the aggregation on these buckets. + +The span syntax is \`span(field_expr, interval_expr)\`. By default, the interval expression in the \`span\` clause is interpreted in natural units. If the field is a date and time type field and the interval is in date and time units, you must specify the unit in the interval expression. For example, to split the \`age\` field into buckets of 10 years, you would use \`span(age, 10y). To split a timestamp field into hourly intervals, you would use \`span(timestamp, 1h)\`. + +The following table lists the available time units. + +| Span Interval Units | +|----------------------------| +| millisecond (ms) | +| second (s) | +| minute (m, case sensitive) | +| hour (h) | +| day (d) | +| week (w) | +| month (M, case sensitive) | +| quarter (q) | +| year (y) | + +### PPL queries using the stats command + +The following example PPL queries show ways you can use the \`stats\` command in your queries. + +#### Example 1: Calculate event counts + +The following example PPL query calculates event counts: + + os> source=accounts | stats count(); + fetched rows / total rows = 1/1 + +-----------+ + | count() | + |-----------| + | 4 | + +-----------+ + +#### Example 2: Calculate a field's average + +The following example PPL query calculates the average age: + + os> source=accounts | stats avg(age); + fetched rows / total rows = 1/1 + +------------+ + | avg(age) | + |------------| + | 32.25 | + +------------+ + +#### Example 3: Calculate the average of a field by group + +The following example PPL query calculates the average age grouped by gender: + + os> source=accounts | stats avg(age) by gender; + fetched rows / total rows = 2/2 + +--------------------+----------+ + | avg(age) | gender | + |--------------------+----------| + | 28.0 | F | + | 33.666666666666664 | M | + +--------------------+----------+ + +#### Example 4: Calculate the average, sum, and count of a field by group + +The following example PPL query calculates the average age, sum age, and count of events by gender. + + os> source=accounts | stats avg(age), sum(age), count() grouped by gender; + fetched rows / total rows = 2/2 + +--------------------+------------+-----------+----------+ + | avg(age) | sum(age) | count() | gender | + |--------------------+------------+-----------+----------| + | 28.0 | 28 | 1 | F | + | 33.666666666666664 | 101 | 3 | M | + +--------------------+------------+-----------+----------+ + +#### Example 5: Calculate a field's maximum + +The following example PPL query calculates the maximum age: + + os> source=accounts | stats max(age); + fetched rows / total rows = 1/1 + +------------+ + | max(age) | + |------------| + | 36 | + +------------+ + +#### Example 6: Calculate a field's min/max by group + +The following example PPL query calculates the min/max age grouped by gender: + + os> source=accounts | stats max(age), min(age) by gender; + fetched rows / total rows = 2/2 + +------------+------------+----------+ + | max(age) | min(age) | gender | + |------------+------------+----------| + | 28 | 28 | F | + | 36 | 32 | M | + +------------+------------+----------+ + +#### Example 7: Calculate a field's distinct count + +To count the number of distinct values in a field, you can use the \`DISTINCT_COUNT\` or \`DC\` function instead of the \`COUNT\` funtion. + +The following PPL query calculates both the count and distinct count of the \`gender\` field for all accounts. + + os> source=accounts | stats count(gender), distinct_count(gender); + fetched rows / total rows = 1/1 + +-----------------+--------------------------+ + | count(gender) | distinct_count(gender) | + |-----------------+--------------------------| + | 4 | 2 | + +-----------------+--------------------------+ + +#### Example 8: Calculate count by span + +The following PPL query calculates age by span of 10 years. + + os> source=accounts | stats count(age) by span(age, 10) as age_span + fetched rows / total rows = 2/2 + +--------------+------------+ + | count(age) | age_span | + |--------------+------------| + | 1 | 20 | + | 3 | 30 | + +--------------+------------+ + +#### Example 9: Calculate count by gender and span + +The following PPL query calculates age by span of 10 years and groups by gender. + + os> source=accounts | stats count() as cnt by span(age, 5) as age_span, gender + fetched rows / total rows = 3/3 + +-------+------------+----------+ + | cnt | age_span | gender | + |-------+------------+----------| + | 1 | 25 | F | + | 2 | 30 | M | + | 1 | 35 | M | + +-------+------------+----------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/syntax.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/syntax.ts new file mode 100644 index 000000000000..a37cf8be65ce --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/syntax.ts @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const syntaxCmd = `## Syntax +--- +### Command order + +The PPL query starts with a \`search\` command to reference a table to search. +Commands can be in any order. For example, in the following query, the \`search\` command references the \`accounts\` index as the source and then uses fields and a \`where\` command to perform further processing. + +\`\`\` +search source=accounts +| where age > 18 +| fields firstname, lastname +\`\`\` + +### Required and optional arguments + +Required arguments are enclosed in angle brackets \< \>, and optional arguments are enclosed in square brackets \[ \].`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/top.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/top.ts new file mode 100644 index 000000000000..0cf6b7c8b8de --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/top.ts @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const topCmd = `## top +--- +### Description + +Use the \`top\` command to find the most common tuple of values for all +fields in the field list. + +### Syntax + +top \[N\] <field-list> \[by-clause\] + +- \`N\`: The number of results you want to return. Default is 10. +- \`field-list\`: Required. The comma-delimited field list. +- \`by-clause\`: Optional. One or more fields to group by. + +#### Example 1: Find the most common values in a field + +The following example PPL query finds the most common gender. + + os> source=accounts | top 1 gender; + fetched rows / total rows = 1/1 + +------------+ + | gender | + |------------| + | M | + +------------+ + +#### Example 2: Find the most common values grouped by gender + +The following example PPL query finds the most common age grouped by gender. + + os> source=accounts | top 1 age by gender; + fetched rows / total rows = 2/2 + +----------+----------+ + | gender | age | + |----------+----------| + | F | 39 | + | M | 31 | + +----------+----------+ + +#### Limitation +The \`top\` command is not rewritten to [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/). It is only run on the coordinating node. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/where.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/where.ts new file mode 100644 index 000000000000..b92d65b01312 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/commands/where.ts @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const whereCmd = `## where +--- + +### Description + +Use the \`where\` command to filter search results. The \`where\` command only returns the result when the \`bool-expression\` is set to \`true\`. + +### Syntax + +\`where <boolean-expression>\` + +- \`bool-expression\`: Optional. Any expression that can be evaluated to a Boolean expression. + +#### Example 1: Filter the result set with a condition + +The following example PPL query fetches all documents from the \`accounts\` index using an \`or\ condition. + + os> source=accounts | where account_number=1 or gender="F" | fields account_number, gender; + fetched rows / total rows = 2/2 + +------------------+----------+ + | account_number | gender | + |------------------+----------| + | 1 | M | + | 13 | F | + +------------------+----------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/condition.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/condition.ts new file mode 100644 index 000000000000..4824b5aa2300 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/condition.ts @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const conditionFunction = `## Condition +--- + +### Condition functions + +PPL functions use the search capabilities of the OpenSearch engine. However, these functions don't execute directly within the OpenSearch plugin's memory. Instead, they facilitate the global filtering of query results based on specific conditions, such as a \`WHERE\` or \`HAVING\` clause. + +The following sections describe the condition PPL functions. + +### ISNULL + +The \`isnull(field)\` function checks a specific field and returns \`true\` if the field contains no data, that is, it's null. + +**Argument type:** All supported data types + +**Return type:** \`BOOLEAN\` + +#### Example + + os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname + fetched rows / total rows = 4/4 + +----------+------------+-------------+ + | result | employer | firstname | + |----------+------------+-------------| + | False | Pyrami | Amber | + | False | Netagy | Hattie | + | False | Quility | Nanette | + | True | null | Dale | + +----------+------------+-------------+ + +### ISNOTNULL + +The \`isnotnull(field)\` function is the opposite of \`isnull(field)\`. Instead of checking for null values, it checks a specific field and returns \`true\` if the field contains data, that is, it is not null. + +**Argument type:** All supported data types + +**Return type:** \`BOOLEAN\` + +#### Example + + os> source=accounts | where not isnotnull(employer) | fields account_number, employer + fetched rows / total rows = 1/1 + +------------------+------------+ + | account_number | employer | + |------------------+------------| + | 18 | null | + +------------------+------------+ + +### EXISTS + +OpenSearch does not differentiate between null and missing. Thus, a function such as \`ismissing\` or \`isnotmissing\` cannot be used to test if a field exists or not. The \`isnull\` or \`isnotnull\` functions can be used for this purpose. + +#### Example + + os> source=accounts | where isnull(email) | fields account_number, email + fetched rows / total rows = 1/1 + +------------------+---------+ + | account_number | email | + |------------------+---------| + | 13 | null | + +------------------+---------+ + +### IFNULL + +The \`ifnull(field1, field2)\` function returns the value in the first field if it is not null; otherwise, it returns the value in the second field. + +**Argument type:** All supported data types (Note that the semantic check will fail if the parameters are different types.) + +**Return type:** Any + +#### Example + + os> source=accounts | eval result = ifnull(employer, 'default') | fields result, employer, firstname + fetched rows / total rows = 4/4 + +----------+------------+-------------+ + | result | employer | firstname | + |----------+------------+-------------| + | Pyrami | Pyrami | Amber | + | Netagy | Netagy | Hattie | + | Quility | Quility | Nanette | + | default | null | Dale | + +----------+------------+-------------+ + +### NULLIF + +The \`nullif(field1, field2)\` function returns \`null\` if the values in both fields are identical. If the values differ, the function returns the value in the first field (field1). + +**Argument type:** All supported data types (Note that the semantic check will fail if the parameters are different types.) + +**Return type:** Any + +#### Example + + os> source=accounts | eval result = nullif(employer, 'Pyrami') | fields result, employer, firstname + fetched rows / total rows = 4/4 + +----------+------------+-------------+ + | result | employer | firstname | + |----------+------------+-------------| + | null | Pyrami | Amber | + | Netagy | Netagy | Hattie | + | Quility | Quility | Nanette | + | null | null | Dale | + +----------+------------+-------------+ + +### ISNULL + +The \`isnull(field1, field2)\` function checks for null values and returns \`null\` if the values in both fields are identical. If the values differ, the function returns the value in the first field (field1). + +**Argument type:** All supported data types + +**Return type:** Any + +#### Example + + os> source=accounts | eval result = isnull(employer) | fields result, employer, firstname + fetched rows / total rows = 4/4 + +----------+------------+-------------+ + | result | employer | firstname | + |----------+------------+-------------| + | False | Pyrami | Amber | + | False | Netagy | Hattie | + | False | Quility | Nanette | + | True | null | Dale | + +----------+------------+-------------+ + +### IF + +The \`if(condition, expr1, expr2)\` function returns \`expr1\` if \`condition\` is \`true\`, and \`expr2\` otherwise. + +**Argument type:** All supported data types (Note that the semantic check will fail if \`expr1\` and \`expr2\` have different types.) + +**Return type:** Any + +Example: + + os> source=accounts | eval result = if(true, firstname, lastname) | fields result, firstname, lastname + fetched rows / total rows = 4/4 + +----------+-------------+------------+ + | result | firstname | lastname | + |----------+-------------+------------| + | Amber | Amber | Duke | + | Hattie | Hattie | Bond | + | Nanette | Nanette | Bates | + | Dale | Dale | Adams | + +----------+-------------+------------+ + + os> source=accounts | eval result = if(false, firstname, lastname) | fields result, firstname, lastname + fetched rows / total rows = 4/4 + +----------+-------------+------------+ + | result | firstname | lastname | + |----------+-------------+------------| + | Duke | Amber | Duke | + | Bond | Hattie | Bond | + | Bates | Nanette | Bates | + | Adams | Dale | Adams | + +----------+-------------+------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/datetime.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/datetime.ts new file mode 100644 index 000000000000..5b2ca67f50c2 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/datetime.ts @@ -0,0 +1,537 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const datetimeFunction = `## Datetime +--- + +### Datetime functions + +PPL functions use the search capabilities of the OpenSearch engine. However, these functions don't execute directly within the OpenSearch plugin's memory. Instead, they facilitate the global filtering of query results based on specific conditions, such as a \`WHERE\` or \`HAVING\` clause. + +The following sections describe the \`datetime\` PPL functions. + +### ADDDATE + +The \`adddate\` function add a time interval to a date. It supports two forms: adding a specified interval using \`INTERVAL\` keyword or adding an integer number of days directly. + +**Argument type:** \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL/LONG\` + +**Return type:** \`(DATE/DATETIME/TIMESTAMP/STRING, INTERVAL) -> DATETIME\`, \`(DATE, LONG) -> DATE\`, \`(DATETIME/TIMESTAMP/STRING, LONG) -> DATETIME\` + +**Synonyms**: \`[DATE\_ADD](#date_add)\` + +#### Example + + os> source=people | eval \`ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR)\` = ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR), \`ADDDATE(DATE('2020-08-26'), 1)\` = ADDDATE(DATE('2020-08-26'), 1), \`ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1)\` = ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields \`ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR)\`, \`ADDDATE(DATE('2020-08-26'), 1)\`, \`ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1)\` + fetched rows / total rows = 1/1 + +------------------------------------------------+----------------------------------+------------------------------------------------+ + | ADDDATE(DATE('2020-08-26'), INTERVAL 1 HOUR) | ADDDATE(DATE('2020-08-26'), 1) | ADDDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | + |------------------------------------------------+----------------------------------+------------------------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | + +------------------------------------------------+----------------------------------+------------------------------------------------+ + +#### DATE + +The \`date(expr)\` function converts strings to date types and extracts the date portion from existing date, datetime, and timestamp values. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`DATE\` + +#### Example + + >od source=people | eval \`DATE('2020-08-26')\` = DATE('2020-08-26'), \`DATE(TIMESTAMP('2020-08-26 13:49:00'))\` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields \`DATE('2020-08-26')\`, \`DATE(TIMESTAMP('2020-08-26 13:49:00'))\` + fetched rows / total rows = 1/1 + +----------------------+------------------------------------------+ + | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | + |----------------------+------------------------------------------| + | DATE '2020-08-26' | DATE '2020-08-26' | + +----------------------+------------------------------------------+ + +#### DATE\_ADD + +The \`date\_add(date, INTERVAL expr unit)\` or \`date\_add(date, expr)\` adds +the time interval specified by \`expr\` to a given \`date\`. It supports adding a specific interval and adding an integer number of days. + +**Argument type:** \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL/LONG\` + +**Return type:** \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL ->\`, \`DATETIME\`, \`DATE, LONG -> DATE\`, \`DATETIME/TIMESTAMP/STRING, LONG -> DATETIME\` + +**Synonyms:** \`[ADDDATE](#adddate)\` + +#### Example + + os> source=people | eval \`DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR)\` = DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR), \`DATE_ADD(DATE('2020-08-26'), 1)\` = DATE_ADD(DATE('2020-08-26'), 1), \`DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), 1)\` = DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields \`DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR)\`, \`DATE_ADD(DATE('2020-08-26'), 1)\`, \`DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), 1)\` + fetched rows / total rows = 1/1 + +-------------------------------------------------+-----------------------------------+-------------------------------------------------+ + | DATE_ADD(DATE('2020-08-26'), INTERVAL 1 HOUR) | DATE_ADD(DATE('2020-08-26'), 1) | DATE_ADD(TIMESTAMP('2020-08-26 01:01:01'), 1) | + |-------------------------------------------------+-----------------------------------+-------------------------------------------------| + | 2020-08-26 01:00:00 | 2020-08-27 | 2020-08-27 01:01:01 | + +-------------------------------------------------+-----------------------------------+-------------------------------------------------+ + +### DATE\_FORMAT + +The \`date\_format(date, format)\` function takes a date and a format string as arguments and returns the formatted date string according to the specified format. + +The following table lists the available specifier arguments. + +| Specifier | Description | +|-----------|-----------------------------------------------------------| +| %a | Abbreviated weekday name (Sun..Sat) | +| %b | Abbreviated month name (Jan..Dec) | +| %c | Month, numeric (0..12) | +| %D | Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) | +| %d | Day of the month, numeric (00..31) | +| %e | Day of the month, numeric (0..31) | +| %f | Microseconds (000000..999999) | +| %H | Hour (00..23) | +| %h | Hour (01..12) | +| %I | Hour (01..12) | +| %i | Minutes, numeric (00..59) | +| %j | Day of year (001..366) | +| %k | Hour (0..23) | +| %l | Hour (1..12) | +| %M | Month name (January..December) | +| %m | Month, numeric (00..12) | +| %p | AM or PM | +| %r | Time, 12-hour (hh:mm:ss followed by AM or PM) | +| %S | Seconds (00..59) | +| %s | Seconds (00..59) | +| %T | Time, 24-hour (hh:mm:ss) | +| %U | Week (00..53), where Sunday is the first day of the week; WEEK() mode 0 | +| %u | Week (00..53), where Monday is the first day of the week; WEEK() mode 1 | +| %V | Week (01..53), where Sunday is the first day of the week; WEEK() mode 2; used with %X | +| %v | Week (01..53), where Monday is the first day of the week; WEEK() mode 3; used with %x | +| %W | Weekday name (Sunday..Saturday) | +| %w | Day of the week (0=Sunday..6=Saturday) | +| %X | Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V | +| %x | Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v | +| %Y | Year, numeric, four digits | +| %y | Year, numeric (two digits) | +| %% | A literal % character | +| %x | x, for any “x” not listed above | + +**Argument type:** STRING/DATE/DATETIME/TIMESTAMP, STRING + +**Return type:** STRING + +#### Example + + >od source=people | eval \`DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f')\` = DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f'), \`DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r')\` = DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | fields \`DATE_FORMAT('1998-01-31 13:14:15.012345', '%T.%f')\`, \`DATE_FORMAT(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r')\` + fetched rows / total rows = 1/1 + +-----------------------------------------------+----------------------------------------------------------------+ + | DATE('1998-01-31 13:14:15.012345', '%T.%f') | DATE(TIMESTAMP('1998-01-31 13:14:15.012345'), '%Y-%b-%D %r') | + |-----------------------------------------------+----------------------------------------------------------------| + | '13:14:15.012345' | '1998-Jan-31st 01:14:15 PM' | + +-----------------------------------------------+----------------------------------------------------------------+ + +### DATE\_SUB + +**Description** + +Usage: date\_sub(date, INTERVAL expr unit)/ date\_sub(date, expr) +subtracts the time interval expr from date + +Argument type: \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL/LONG\` + +**Return type:** \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL -> DATETIME\`, \`DATE, LONG -> DATE\`, \`DATETIME/TIMESTAMP/STRING, LONG -> DATETIME\` + +**Synonyms:** \`[SUBDATE](#subdate)\` + +#### Example + + os> source=people | eval \`DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY)\` = DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY), \`DATE_SUB(DATE('2020-08-26'), 1)\` = DATE_SUB(DATE('2020-08-26'), 1), \`DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), 1)\` = DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields \`DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY)\`, \`DATE_SUB(DATE('2020-08-26'), 1)\`, \`DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), 1)\` + fetched rows / total rows = 1/1 + +-------------------------------------------------+-----------------------------------+-------------------------------------------------+ + | DATE_SUB(DATE('2008-01-02'), INTERVAL 31 DAY) | DATE_SUB(DATE('2020-08-26'), 1) | DATE_SUB(TIMESTAMP('2020-08-26 01:01:01'), 1) | + |-------------------------------------------------+-----------------------------------+-------------------------------------------------| + | 2007-12-02 | 2020-08-25 | 2020-08-25 01:01:01 | + +-------------------------------------------------+-----------------------------------+-------------------------------------------------+ + +### DAY + +The \`day(date)\` function retrieves the day of the month (1-31) for a provided \`date\`. Note that dated with a value of 0, such as "0000-00-00" or "2008-00-00", are considered invalid. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +**Synonyms:** \`DAYOFMONTH\` + +#### Example + + os> source=people | eval \`DAY(DATE('2020-08-26'))\` = DAY(DATE('2020-08-26')) | fields \`DAY(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +---------------------------+ + | DAY(DATE('2020-08-26')) | + |---------------------------| + | 26 | + +---------------------------+ + +### DAYNAME + +The \`dayname(date)\` function retrieves the full name of the weekday, for example, Monday, Tuesday, and so forth, for a given \`date\`. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`DAYNAME(DATE('2020-08-26'))\` = DAYNAME(DATE('2020-08-26')) | fields \`DAYNAME(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +-------------------------------+ + | DAYNAME(DATE('2020-08-26')) | + |-------------------------------| + | Wednesday | + +-------------------------------+ + +### DAYOFMONTH + +The \`dayofmonth(date)\` function retrieves the day of the month (1-31) for a provided \`date\`. Note that dated with a value of 0, such as "0000-00-00" or "2008-00-00", are considered invalid. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +**Synonyms:** \`DAY\` + +#### Example + + os> source=people | eval \`DAYOFMONTH(DATE('2020-08-26'))\` = DAYOFMONTH(DATE('2020-08-26')) | fields \`DAYOFMONTH(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +----------------------------------+ + | DAYOFMONTH(DATE('2020-08-26')) | + |----------------------------------| + | 26 | + +----------------------------------+ + +### DAYOFWEEK + +The \`dayofweek(date)\` retrieves the numerical index (1-7) representing the weekday for a given \`date\`, where 1 corresponds to Sunday and 7 corresponds to Saturday. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`DAYOFWEEK(DATE('2020-08-26'))\` = DAYOFWEEK(DATE('2020-08-26')) | fields \`DAYOFWEEK(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +---------------------------------+ + | DAYOFWEEK(DATE('2020-08-26')) | + |---------------------------------| + | 4 | + +---------------------------------+ + +### DAYOFYEAR + +The \`dayofyear(date)\` function retrieves the day of the year for a given \`date\`, ranging from 1 to 366. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`DAYOFYEAR(DATE('2020-08-26'))\` = DAYOFYEAR(DATE('2020-08-26')) | fields \`DAYOFYEAR(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +---------------------------------+ + | DAYOFYEAR(DATE('2020-08-26')) | + |---------------------------------| + | 239 | + +---------------------------------+ + +### FROM\_DAYS + +The \`from\_days(N)\` function retrieves the date value corresponding to the provided day number \`N\`. + +**Argument type:** \`INTEGER/LONG\` + +**Return type:** \`DATE\` + +#### Example + + os> source=people | eval \`FROM_DAYS(733687)\` = FROM_DAYS(733687) | fields \`FROM_DAYS(733687)\` + fetched rows / total rows = 1/1 + +---------------------+ + | FROM_DAYS(733687) | + |---------------------| + | 2008-10-07 | + +---------------------+ + +### HOUR + +The \`hour(time)\` function extracts the hour value from a given \`time\`. Unlike the typical time-of-day format wher hours range from 0 to 23, the \`time\` input can have a larger range. Therefore, the \`hour(time)\` function may return values exceeding 23. + +**Argument type:** \`STRING/TIME/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`HOUR(TIME('01:02:03'))\` = HOUR(TIME('01:02:03')) | fields \`HOUR(TIME('01:02:03'))\` + fetched rows / total rows = 1/1 + +--------------------------+ + | HOUR(TIME('01:02:03')) | + |--------------------------| + | 1 | + +--------------------------+ + +### MAKETIME + +**Function signature:** \`MAKETIME(INTEGER, INTEGER, INTEGER) -> DATE\` + +### MICROSECOND + +The \`microsecond(expr)\` function retrieves the microsecond portion (0-999999) from a given \`time\` or \`datetime\` expression. + +**Argument type:** \`STRING/TIME/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`MICROSECOND(TIME('01:02:03.123456'))\` = MICROSECOND(TIME('01:02:03.123456')) | fields \`MICROSECOND(TIME('01:02:03.123456'))\` + fetched rows / total rows = 1/1 + +----------------------------------------+ + | MICROSECOND(TIME('01:02:03.123456')) | + |----------------------------------------| + | 123456 | + +----------------------------------------+ + +### MINUTE + +The \`minute(time)\` extracts the minute value (0-59) from a given \`time\` expression. + +**Argument type:** \`STRING/TIME/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`MINUTE(TIME('01:02:03'))\` = MINUTE(TIME('01:02:03')) | fields \`MINUTE(TIME('01:02:03'))\` + fetched rows / total rows = 1/1 + +----------------------------+ + | MINUTE(TIME('01:02:03')) | + |----------------------------| + | 2 | + +----------------------------+ + +### MONTH + +The \`month(date)\` function extracts the month (1-12) from a valid \`date\` value. However, invalid dates containing 0 values for the month, such as "0000-00-00" or "2008-00-00" are considered invalid. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`MONTH(DATE('2020-08-26'))\` = MONTH(DATE('2020-08-26')) | fields \`MONTH(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +-----------------------------+ + | MONTH(DATE('2020-08-26')) | + |-----------------------------| + | 8 | + +-----------------------------+ + +### MONTHNAME + +The \`monthname(date)\` function retrieves the full name of the month, for example, January, February, and so forth, for a given \`date\`. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`MONTHNAME(DATE('2020-08-26'))\` = MONTHNAME(DATE('2020-08-26')) | fields \`MONTHNAME(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +---------------------------------+ + | MONTHNAME(DATE('2020-08-26')) | + |---------------------------------| + | August | + +---------------------------------+ + +### NOW + +**Function signature:** NOW() -> DATE + +### QUARTER + +The \`quarter(date)\` function retrieves the quarter (1-4) for a given \`date\`. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`QUARTER(DATE('2020-08-26'))\` = QUARTER(DATE('2020-08-26')) | fields \`QUARTER(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +-------------------------------+ + | QUARTER(DATE('2020-08-26')) | + |-------------------------------| + | 3 | + +-------------------------------+ + +### SECOND + +The \`second(time)\` function extracts the second value (0-59) from a given \`time\` expression. + +**Argument type:** \`STRING/TIME/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`SECOND(TIME('01:02:03'))\` = SECOND(TIME('01:02:03')) | fields \`SECOND(TIME('01:02:03'))\` + fetched rows / total rows = 1/1 + +----------------------------+ + | SECOND(TIME('01:02:03')) | + |----------------------------| + | 3 | + +----------------------------+ + +### SUBDATE + +The \`subdate(date, INTERVAL expr unit)\` or \`subdate(date, expr)\` function subtracts a time interval from a date. + +**Argument type:** \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL/LONG\` + +**Return type:** \`DATE/DATETIME/TIMESTAMP/STRING, INTERVAL -> DATETIME\`, \`DATE, LONG -> DATE\`, \`DATETIME/TIMESTAMP/STRING, LONG -> DATETIME\` + +**Synonyms:** \`[DATE\_SUB](#date_sub)\` + +#### Example + + os> source=people | eval \`SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY)\` = SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY), \`SUBDATE(DATE('2020-08-26'), 1)\` = SUBDATE(DATE('2020-08-26'), 1), \`SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1)\` = SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | fields \`SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY)\`, \`SUBDATE(DATE('2020-08-26'), 1)\`, \`SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1)\` + fetched rows / total rows = 1/1 + +------------------------------------------------+----------------------------------+------------------------------------------------+ + | SUBDATE(DATE('2008-01-02'), INTERVAL 31 DAY) | SUBDATE(DATE('2020-08-26'), 1) | SUBDATE(TIMESTAMP('2020-08-26 01:01:01'), 1) | + |------------------------------------------------+----------------------------------+------------------------------------------------| + | 2007-12-02 | 2020-08-25 | 2020-08-25 01:01:01 | + +------------------------------------------------+----------------------------------+------------------------------------------------+ + +### TIME + +The \`time(expr)\` function has dual functionality. If \`expr\` is a string, it contructs a \`time\` object from the provided time value format. Conversly, for input of the type \`date\`, \`datetime\`. \`time\`, or \`timestamp\`, it extracts and returns the pure time component from the given expression. + +**Argument type:** \`STRING/DATE/DATETIME/TIME/TIMESTAMP\` + +**Return type:** \`TIME\` + +#### Example + + >od source=people | eval \`TIME('13:49:00')\` = TIME('13:49:00'), \`TIME(TIMESTAMP('2020-08-26 13:49:00'))\` = TIME(TIMESTAMP('2020-08-26 13:49:00')) | fields \`TIME('13:49:00')\`, \`TIME(TIMESTAMP('2020-08-26 13:49:00'))\` + fetched rows / total rows = 1/1 + +--------------------+------------------------------------------+ + | TIME('13:49:00') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | + |--------------------+------------------------------------------| + | TIME '13:49:00' | TIME '13:49:00' | + +--------------------+------------------------------------------+ + +### TIME\_TO\_SEC + +The \`time\_to\_sec(time)\` function transforms a given \`time\` value into its corresponding number of seconds. + +**Argument type:** \`STRING/TIME/DATETIME/TIMESTAMP\` + +**Return type:** \`LONG\` + +#### Example + + os> source=people | eval \`TIME_TO_SEC(TIME('22:23:00'))\` = TIME_TO_SEC(TIME('22:23:00')) | fields \`TIME_TO_SEC(TIME('22:23:00'))\` + fetched rows / total rows = 1/1 + +---------------------------------+ + | TIME_TO_SEC(TIME('22:23:00')) | + |---------------------------------| + | 80580 | + +---------------------------------+ + +### TIMESTAMP + +The \`timestamp(expr)\` function serves a dual purpose: it can both construct a timestamp object from a string representing a time value or act as a caster, converting exsiting date, datetime, or timestamp objects to a standardized timestamp type with the default UTC time zone. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`TIMESTAMP\` + +#### Example + + >od source=people | eval \`TIMESTAMP('2020-08-26 13:49:00')\` = TIMESTAMP('2020-08-26 13:49:00') | fields \`TIMESTAMP('2020-08-26 13:49:00')\` + fetched rows / total rows = 1/1 + +------------------------------------+ + | TIMESTAMP('2020-08-26 13:49:00') | + |------------------------------------| + | TIMESTAMP '2020-08-26 13:49:00 | + +------------------------------------+ + +### TO\_DAYS + +The \`to\_days(date)\` function calculates the number of days that have elapsed since the year 0 for a given \`date\`. If the provided date is invalid, it returns \`NULL\`. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`LONG\` + +#### Example + + os> source=people | eval \`TO_DAYS(DATE('2008-10-07'))\` = TO_DAYS(DATE('2008-10-07')) | fields \`TO_DAYS(DATE('2008-10-07'))\` + fetched rows / total rows = 1/1 + +-------------------------------+ + | TO_DAYS(DATE('2008-10-07')) | + |-------------------------------| + | 733687 | + +-------------------------------+ + +### WEEK + +The \`week(date\[, mode\])\` function extracts the week number for a given \`date\`. If the mode argument is omitted, the default mode 0 is used. The following table lists the mode arguments. + +| Mode | First day of week | Range | Week 1 is the first week … | +|------|-------------------|-------|-------------------------------| +| 0 | Sunday | 0-53 | with a Sunday in this year | +| 1 | Monday | 0-53 | with 4 or more days this year | +| 2 | Sunday | 1-53 | with a Sunday in this year | +| 3 | Monday | 1-53 | with 4 or more days this year | +| 4 | Sunday | 0-53 | with 4 or more days this year | +| 5 | Monday | 0-53 | with a Monday in this year | +| 6 | Sunday | 1-53 | with 4 or more days this year | +| 7 | Monday | 1-53 | with a Monday in this year | + +**Argument type:** \`DATE/DATETIME/TIMESTAMP/STRING\` + +**Return type:** \`INTEGER\` + +#### Example + + >od source=people | eval \`WEEK(DATE('2008-02-20'))\` = WEEK(DATE('2008-02-20')), \`WEEK(DATE('2008-02-20'), 1)\` = WEEK(DATE('2008-02-20'), 1) | fields \`WEEK(DATE('2008-02-20'))\`, \`WEEK(DATE('2008-02-20'), 1)\` + fetched rows / total rows = 1/1 + +----------------------------+-------------------------------+ + | WEEK(DATE('2008-02-20')) | WEEK(DATE('2008-02-20'), 1) | + |----------------------------|-------------------------------| + | 7 | 8 | + +----------------------------+-------------------------------+ + +### YEAR + +The \`year(date)\` function extracts the year component from a given \`date\` value. However, it only returns valid years within the range of 1000 to 9999. If the provided date is invalid or falls outside this range, the function returns 0. + +**Argument type:** \`STRING/DATE/DATETIME/TIMESTAMP\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`YEAR(DATE('2020-08-26'))\` = YEAR(DATE('2020-08-26')) | fields \`YEAR(DATE('2020-08-26'))\` + fetched rows / total rows = 1/1 + +----------------------------+ + | YEAR(DATE('2020-08-26')) | + |----------------------------| + | 2020 | + +----------------------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/full_text_search.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/full_text_search.ts new file mode 100644 index 000000000000..7f31bae65166 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/full_text_search.ts @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const fullTextSearchFunction = `## Full-text search +--- + +### Full-text search function + +PPL functions use the search capabilities of the OpenSearch engine. However, these functions don't execute directly within the OpenSearch plugin's memory. Instead, they facilitate the global filtering of query results based on specific conditions, such as a \`WHERE\` or \`HAVING\` clause. + +Full-text search allows for searching by full-text queries. For details about full-text search in OpenSearch, see the [Full-text search](https://opensearch.org/docs/latest/search-plugins/sql/full-text/) documentation. + +### MATCH + +The \`match\` function maps user-defined criteria to OpenSearch queries, returning documents that match specific text, number, date, or Boolean values. + +The function signature is \`match(field_expression, query_expression[, option=]*)\`. + +The available parameters are: + +- analyzer +- auto\_generate\_synonyms\_phrase +- fuzziness +- max\_expansions +- prefix\_length +- fuzzy\_transpositions +- fuzzy\_rewrite +- lenient +- operator +- minimum\_should\_match +- zero\_terms\_query +- boost + +**Example 1: Using specific expressions and default values** + +The following example PPL query uses only the \`field\` and \`query\` expressions, with all other parameters set to their default values: + + os> source=accounts | where match(address, 'Street') | fields lastname, address; + fetched rows / total rows = 2/2 + +------------+--------------------+ + | lastname | address | + |------------+--------------------| + | Bond | 671 Bristol Street | + | Bates | 789 Madison Street | + +------------+--------------------+ + +**Example 2: Setting custom values for optional parameters** + +The following example PPL query sets custom values for the optional parameters: + + os> source=accounts | where match(firstname, 'Hattie', operator='AND', boost=2.0) | fields lastname; + fetched rows / total rows = 1/1 + +------------+ + | lastname | + |------------| + | Bond | + +------------+ + +### Limitations + +The full-text search functions can be executed only in [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/), not in-memory. + +To ensure optimal performance and avoid translation issues with complex full-text searhes, place them as clauses within the search command. + +#### Example + +The following is an example complex query that could fail because it is difficult to translate to query DSL: + + \`search source = people | rename firstname as name | dedup account_number | fields name, account_number, balance, employer | where match(employer, 'Open Search') | stats count() by city\` + +To optimize full-text search performance, rewrite the query by placing the \`WHERE\` clause with the full-text search function as the second command after the \`SEARCH\` command. This ensures the full-text search gets pushed down to query DSL. The following is an example query: + + \`search source = people | where match(employer, 'Open Search') | rename firstname as name | dedup account_number | fields name, account_number, balance, employer | stats count() by city\` + +For details about query engine optimization, see the [Optimizations](https://github.com/opensearch-project/sql/blob/22924b13d9cb46759c8d213a7ce903effe06ab47/docs/user/optimization/optimization.rst) developer documentation on GitHub. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/index.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/index.ts new file mode 100644 index 000000000000..e061c6188e43 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { datetimeFunction } from './datetime'; +export { conditionFunction } from './condition'; +export { mathFunction } from './math'; +export { stringFunction } from './string'; +export { fullTextSearchFunction } from './full_text_search'; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/math.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/math.ts new file mode 100644 index 000000000000..ed189c3eb42c --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/math.ts @@ -0,0 +1,518 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const mathFunction = `## Math +--- + +### Math functions + +PPL functions use the search capabilities of the OpenSearch engine. However, these functions don't execute directly within the OpenSearch plugin's memory. Instead, they facilitate the global filtering of query results based on specific conditions, such as a \`WHERE\` or \`HAVING\` clause. + +The following sections describe the \`math\` PPL functions. + +### ABS + +The \`abs\` function is an absolute value function. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +#### Example + + os> source=people | eval \`ABS(-1)\` = ABS(-1) | fields \`ABS(-1)\` + fetched rows / total rows = 1/1 + +-----------+ + | ABS(-1) | + |-----------| + | 1 | + +-----------+ + +### ACOS + +The \`acos(x)\` function is an arc cosine function. The function expects values in the range of \`-1\` to \`1\` and returns \`NULL\` if the values aren't in that range. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`ACOS(0)\` = ACOS(0) | fields \`ACOS(0)\` + fetched rows / total rows = 1/1 + +--------------------+ + | ACOS(0) | + |--------------------| + | 1.5707963267948966 | + +--------------------+ + +### ASIN + +The \`asin(x)\` function is an arc sine function. The function expects values in the range of \`-1\` to \`1\` and returns \`NULL\` if the values aren't in that range. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`ASIN(0)\` = ASIN(0) | fields \`ASIN(0)\` + fetched rows / total rows = 1/1 + +-----------+ + | ASIN(0) | + |-----------| + | 0.0 | + +-----------+ + +### ATAN + +The \`atan(x)\` function is an arc tangent function that returns an arc tangent of a value \`x\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`ATAN(2)\` = ATAN(2), \`ATAN(2, 3)\` = ATAN(2, 3) | fields \`ATAN(2)\`, \`ATAN(2, 3)\` + fetched rows / total rows = 1/1 + +--------------------+--------------------+ + | ATAN(2) | ATAN(2, 3) | + |--------------------+--------------------| + | 1.1071487177940904 | 0.5880026035475675 | + +--------------------+--------------------+ + +### ATAN2 + +The \`atan2(y, x)\` function is an arc tangent function that calculates the angle from a specified point to the coordinate origin as measured from the positive x-axis. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`ATAN2(2, 3)\` = ATAN2(2, 3) | fields \`ATAN2(2, 3)\` + fetched rows / total rows = 1/1 + +--------------------+ + | ATAN2(2, 3) | + |--------------------| + | 0.5880026035475675 | + +--------------------+ + +### CEIL + +The \`ceil(x)\` function returns the smallest integer value that is greater than or equal to the specified value. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`CEIL(2.75)\` = CEIL(2.75) | fields \`CEIL(2.75)\` + fetched rows / total rows = 1/1 + +--------------+ + | CEIL(2.75) | + |--------------| + | 3 | + +--------------+ + +### CONV + +The \`CONV(x, a, b)\` function converts the number \`x\` from \`a\` base to \`b\` base. + +**Argument type:** \`x: STRING, a: INTEGER, b: INTEGER\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`CONV('12', 10, 16)\` = CONV('12', 10, 16), \`CONV('2C', 16, 10)\` = CONV('2C', 16, 10), \`CONV(12, 10, 2)\` = CONV(12, 10, 2), \`CONV(1111, 2, 10)\` = CONV(1111, 2, 10) | fields \`CONV('12', 10, 16)\`, \`CONV('2C', 16, 10)\`, \`CONV(12, 10, 2)\`, \`CONV(1111, 2, 10)\` + fetched rows / total rows = 1/1 + +----------------------+----------------------+-------------------+---------------------+ + | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | + |----------------------+----------------------+-------------------+---------------------| + | c | 44 | 1100 | 15 | + +----------------------+----------------------+-------------------+---------------------+ + +### COS + +The \`cos(x)\` function is a cosine function, with \`x\` in radians. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`COS(0)\` = COS(0) | fields \`COS(0)\` + fetched rows / total rows = 1/1 + +----------+ + | COS(0) | + |----------| + | 1.0 | + +----------+ + +### COT + +The \`cot(x)\` function is a cotangent function. An out-of-range error is returned if \`x\` equals \`0\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`COT(1)\` = COT(1) | fields \`COT(1)\` + fetched rows / total rows = 1/1 + +--------------------+ + | COT(1) | + |--------------------| + | 0.6420926159343306 | + +--------------------+ + +### CRC32 + +The \`crc32\` function calculates the cyclic redundancy check (CRC) value of a given string as a 32-bit unsigned value. + +**Argument type:** \`STRING\` + +**Return type:** \`LONG\` + +#### Example + + os> source=people | eval \`CRC32('MySQL')\` = CRC32('MySQL') | fields \`CRC32('MySQL')\` + fetched rows / total rows = 1/1 + +------------------+ + | CRC32('MySQL') | + |------------------| + | 3259397556 | + +------------------+ + +### DEGREES + +The \`degrees(x)\` function converts \`x\` from radians to degrees. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`DEGREES(1.57)\` = DEGREES(1.57) | fields \`DEGREES(1.57)\` + fetched rows / total rows = 1/1 + +-------------------+ + | DEGREES(1.57) | + |-------------------| + | 89.95437383553924 | + +-------------------+ + +### E + +The \`E()\` function returns Euler's number. + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`E()\` = E() | fields \`E()\` + fetched rows / total rows = 1/1 + +-------------------+ + | E() | + |-------------------| + | 2.718281828459045 | + +-------------------+ + +### EXP + +The \`exp(x)\` function returns \`e\` raised to the power of \`x\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`EXP(2)\` = EXP(2) | fields \`EXP(2)\` + fetched rows / total rows = 1/1 + +------------------+ + | EXP(2) | + |------------------| + | 7.38905609893065 | + +------------------+ + +### FLOOR + +The \`floor(x)\` function returns the largest integer less than or equal to the specified value. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`FLOOR(2.75)\` = FLOOR(2.75) | fields \`FLOOR(2.75)\` + fetched rows / total rows = 1/1 + +---------------+ + | FLOOR(2.75) | + |---------------| + | 2 | + +---------------+ + +### LN + +The \`ln(x)\` function returns the natural logarithm of \`x\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`LN(2)\` = LN(2) | fields \`LN(2)\` + fetched rows / total rows = 1/1 + +--------------------+ + | LN(2) | + |--------------------| + | 0.6931471805599453 | + +--------------------+ + +### LOG + +The \`log(x)\` function returns the natural logarithm of \`x\`. + +**Argument type: \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`LOG(2)\` = LOG(2), \`LOG(2, 8)\` = LOG(2, 8) | fields \`LOG(2)\`, \`LOG(2, 8)\` + fetched rows / total rows = 1/1 + +--------------------+-------------+ + | LOG(2) | LOG(2, 8) | + |--------------------+-------------| + | 0.6931471805599453 | 3.0 | + +--------------------+-------------+ + +### LOG2 + +The \`log2(x)\` function calculates the base-2 logarithm of \`x\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`LOG2(8)\` = LOG2(8) | fields \`LOG2(8)\` + fetched rows / total rows = 1/1 + +-----------+ + | LOG2(8) | + |-----------| + | 3.0 | + +-----------+ + +### LOG10 + +The \`log10(x)\` function calculates the base-10 logarithm of \`x\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`LOG10(100)\` = LOG10(100) | fields \`LOG10(100)\` + fetched rows / total rows = 1/1 + +--------------+ + | LOG10(100) | + |--------------| + | 2.0 | + +--------------+ + +### MOD + +The \`MOD(n, m)\` function calculates the remainder of the number \`n\` divided by \`m\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** If \`m\` is a nonzero value, a type greater in size between types \`n\` and \`m\` is returned. If \`m\` equals \`0\`, \`NULL\` is returned. + +#### Example + + os> source=people | eval \`MOD(3, 2)\` = MOD(3, 2), \`MOD(3.1, 2)\` = MOD(3.1, 2) | fields \`MOD(3, 2)\`, \`MOD(3.1, 2)\` + fetched rows / total rows = 1/1 + +-------------+---------------+ + | MOD(3, 2) | MOD(3.1, 2) | + |-------------+---------------| + | 1 | 1.1 | + +-------------+---------------+ + +### PI + +The \`PI()\` function returns the constant pi. + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`PI()\` = PI() | fields \`PI()\` + fetched rows / total rows = 1/1 + +-------------------+ + | PI() | + |-------------------| + | 3.141592653589793 | + +-------------------+ + +### POW + +The \`POW(x, y)\` function calculates the value of \`x\` raised to the power of \`y\`. Bad inputs return \`NULL\` results. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +**Synonyms:** \`[POWER](#power)\` + +#### Example + + os> source=people | eval \`POW(3, 2)\` = POW(3, 2), \`POW(-3, 2)\` = POW(-3, 2), \`POW(3, -2)\` = POW(3, -2) | fields \`POW(3, 2)\`, \`POW(-3, 2)\`, \`POW(3, -2)\` + fetched rows / total rows = 1/1 + +-------------+--------------+--------------------+ + | POW(3, 2) | POW(-3, 2) | POW(3, -2) | + |-------------+--------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +-------------+--------------+--------------------+ + +### POWER + +The \`POWER(x, y)\` function calculates the value of \`x\` raised to the power of \`y\`. Bad inputs return \`NULL\` results. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +**Synonyms:** \`[POW](#pow)\` + +#### Example + + os> source=people | eval \`POWER(3, 2)\` = POWER(3, 2), \`POWER(-3, 2)\` = POWER(-3, 2), \`POWER(3, -2)\` = POWER(3, -2) | fields \`POWER(3, 2)\`, \`POWER(-3, 2)\`, \`POWER(3, -2)\` + fetched rows / total rows = 1/1 + +---------------+----------------+--------------------+ + | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | + |---------------+----------------+--------------------| + | 9.0 | 9.0 | 0.1111111111111111 | + +---------------+----------------+--------------------+ + +### RADIANS + +The \`radians(x)\` function converts \`x\` from degrees to radians. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`RADIANS(90)\` = RADIANS(90) | fields \`RADIANS(90)\` + fetched rows / total rows = 1/1 + +--------------------+ + | RADIANS(90) | + |--------------------| + | 1.5707963267948966 | + +--------------------+ + +### RAND + +The \`RAND()/RAND(N)\` function returns a random floating-point value in the range \`0 <= value < 1.0\`. If integer \`N\` is specified, the seed is initialized prior to execution. One implication of this behavior is that with identical argument \`N\`, \`rand(N)\` returns the same value each time and thus produces a repeatable sequence of column values. + +**Argument type:** \`INTEGER\` + +**Return type:** \`FLOAT\` + +#### Example + + os> source=people | eval \`RAND(3)\` = RAND(3) | fields \`RAND(3)\` + fetched rows / total rows = 1/1 + +------------+ + | RAND(3) | + |------------| + | 0.73105735 | + +------------+ + +### ROUND + +The \`ROUND(x, d)\` function rounds the argument \`x\` to \`d\` decimal places. \`d\` defaults to \`0\` if a value is not specified. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`(INTEGER/LONG \[,INTEGER\]) -> LONG (FLOAT/DOUBLE \[,INTEGER\]) ->LONG\` + +#### Example + + os> source=people | eval \`ROUND(12.34)\` = ROUND(12.34), \`ROUND(12.34, 1)\` = ROUND(12.34, 1), \`ROUND(12.34, -1)\` = ROUND(12.34, -1), \`ROUND(12, 1)\` = ROUND(12, 1) | fields \`ROUND(12.34)\`, \`ROUND(12.34, 1)\`, \`ROUND(12.34, -1)\`, \`ROUND(12, 1)\` + fetched rows / total rows = 1/1 + +----------------+-------------------+--------------------+----------------+ + | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | + |----------------+-------------------+--------------------+----------------| + | 12.0 | 12.3 | 10.0 | 12 | + +----------------+-------------------+--------------------+----------------+ + +### SIGN + +The \`sign\` function returns the sign of the argument as -1, 0, or 1, depending on whether the number is negative, zero, or positive. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`SIGN(1)\` = SIGN(1), \`SIGN(0)\` = SIGN(0), \`SIGN(-1.1)\` = SIGN(-1.1) | fields \`SIGN(1)\`, \`SIGN(0)\`, \`SIGN(-1.1)\` + fetched rows / total rows = 1/1 + +-----------+-----------+--------------+ + | SIGN(1) | SIGN(0) | SIGN(-1.1) | + |-----------+-----------+--------------| + | 1 | 0 | -1 | + +-----------+-----------+--------------+ + +### SIN + +The \`sin(x)\` function is a sine function, with \`x\` in radians. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** \`DOUBLE\` + +#### Example + + os> source=people | eval \`SIN(0)\` = SIN(0) | fields \`SIN(0)\` + fetched rows / total rows = 1/1 + +----------+ + | SIN(0) | + |----------| + | 0.0 | + +----------+ + +### SQRT + +The \`sqrt\` function calculates the square root of a non-negative value \`x\`. + +**Argument type:** \`INTEGER/LONG/FLOAT/DOUBLE\` + +**Return type:** (Non-negative) \`INTEGER/LONG/FLOAT/DOUBLE -> DOUBLE\` (Negative) \`INTEGER/LONG/FLOAT/DOUBLE -> NULL\` + +#### Example + + os> source=people | eval \`SQRT(4)\` = SQRT(4), \`SQRT(4.41)\` = SQRT(4.41) | fields \`SQRT(4)\`, \`SQRT(4.41)\` + fetched rows / total rows = 1/1 + +-----------+--------------+ + | SQRT(4) | SQRT(4.41) | + |-----------+--------------| + | 2.0 | 2.1 | + +-----------+--------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/string.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/string.ts new file mode 100644 index 000000000000..1667b6bafd89 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/functions/string.ts @@ -0,0 +1,215 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const stringFunction = `## String +--- + +### String functions + +PPL functions use the search capabilities of the OpenSearch engine. However, these functions don't execute directly within the OpenSearch plugin's memory. Instead, they facilitate the global filtering of query results based on specific conditions, such as a \`WHERE\` or \`HAVING\` clause. + +The following sections describe the string functions. + +### CONCAT + +The \`CONCAT(str1, str2)\` function returns \`str1\` and \`str\` concatenated strings. + +**Argument type:** \`STRING, STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`CONCAT('hello', 'world')\` = CONCAT('hello', 'world') | fields \`CONCAT('hello', 'world')\` + fetched rows / total rows = 1/1 + +----------------------------+ + | CONCAT('hello', 'world') | + |----------------------------| + | helloworld | + +----------------------------+ + +### CONCAT\_WS + +The \`CONCAT\_WS(sep, str1, str2)\` function concatenates two strings together, using \`sep\` as a separator between them. + +**Argument type:** \`STRING, STRING, STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`CONCAT_WS(',', 'hello', 'world')\` = CONCAT_WS(',', 'hello', 'world') | fields \`CONCAT_WS(',', 'hello', 'world')\` + fetched rows / total rows = 1/1 + +------------------------------------+ + | CONCAT_WS(',', 'hello', 'world') | + |------------------------------------| + | hello,world | + +------------------------------------+ + +### LENGTH + +The \`length(str)\` function returns the length of a string, measured in number of bytes. + +**Function signature:** \`LENGTH(STRING) -> INTEGER\` + +**Argument type:** \`STRING\` + +**Return type:** \`INTEGER\` + +#### Example + + os> source=people | eval \`LENGTH('helloworld')\` = LENGTH('helloworld') | fields \`LENGTH('helloworld')\` + fetched rows / total rows = 1/1 + +------------------------+ + | LENGTH('helloworld') | + |------------------------| + | 10 | + +------------------------+ + +### LIKE + +The \`like(string, PATTERN)\` function returns \`true\` if the string matches the \`PATTERN\` value. The following two wildcards are commonly used with the \`like\` operator: + +- \`%\`: A percent sign represents zero, one, or multiple characters. +- \`_\`: An underscore represents a single character. + +#### Example + + os> source=people | eval \`LIKE('hello world', '_ello%')\` = LIKE('hello world', '_ello%') | fields \`LIKE('hello world', '_ello%')\` + fetched rows / total rows = 1/1 + +---------------------------------+ + | LIKE('hello world', '_ello%') | + |---------------------------------| + | True | + +---------------------------------+ + +### LOWER + +The \`lower(string)\` function converts a string to lowercase. + +**Argument type:** \`STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`LOWER('helloworld')\` = LOWER('helloworld'), \`LOWER('HELLOWORLD')\` = LOWER('HELLOWORLD') | fields \`LOWER('helloworld')\`, \`LOWER('HELLOWORLD')\` + fetched rows / total rows = 1/1 + +-----------------------+-----------------------+ + | LOWER('helloworld') | LOWER('HELLOWORLD') | + |-----------------------+-----------------------| + | helloworld | helloworld | + +-----------------------+-----------------------+ + +### LTRIM + +The \`ltrim(str)\` function trims leading space characters from a string. + +**Argument type:** \`STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`LTRIM(' hello')\` = LTRIM(' hello'), \`LTRIM('hello ')\` = LTRIM('hello ') | fields \`LTRIM(' hello')\`, \`LTRIM('hello ')\` + fetched rows / total rows = 1/1 + +---------------------+---------------------+ + | LTRIM(' hello') | LTRIM('hello ') | + |---------------------+---------------------| + | hello | hello | + +---------------------+---------------------+ + +### RIGHT + +The \`right(str, len)\` function returns the rightmost \`len\` characters from a \`str\` value. \`NULL\` is returned if any argument is null. + +**Argument type:** \`STRING, INTEGER\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`RIGHT('helloworld', 5)\` = RIGHT('helloworld', 5), \`RIGHT('HELLOWORLD', 0)\` = RIGHT('HELLOWORLD', 0) | fields \`RIGHT('helloworld', 5)\`, \`RIGHT('HELLOWORLD', 0)\` + fetched rows / total rows = 1/1 + +--------------------------+--------------------------+ + | RIGHT('helloworld', 5) | RIGHT('HELLOWORLD', 0) | + |--------------------------+--------------------------| + | world | | + +--------------------------+--------------------------+ + +### RTRIM + +The \`rtrim(str)\` function trims trailing space characters from a string. + +**Argument type:** \`STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`RTRIM(' hello')\` = RTRIM(' hello'), \`RTRIM('hello ')\` = RTRIM('hello ') | fields \`RTRIM(' hello')\`, \`RTRIM('hello ')\` + fetched rows / total rows = 1/1 + +---------------------+---------------------+ + | RTRIM(' hello') | RTRIM('hello ') | + |---------------------+---------------------| + | hello | hello | + +---------------------+---------------------+ + +### SUBSTRING + +The \`substring(str, start)\` or \`substring(str, start, length)\`function returns a substring of the input string \`str\`. If \`length\` is not specified, the function returns the entire string from the \`start\` index. + +**Argument type:** \`STRING, INTEGER, INTEGER\` + +**Return type:** \`STRING\` + +**Synonyms:** \`SUBSTR\` + +#### Example + + os> source=people | eval \`SUBSTRING('helloworld', 5)\` = SUBSTRING('helloworld', 5), \`SUBSTRING('helloworld', 5, 3)\` = SUBSTRING('helloworld', 5, 3) | fields \`SUBSTRING('helloworld', 5)\`, \`SUBSTRING('helloworld', 5, 3)\` + fetched rows / total rows = 1/1 + +------------------------------+---------------------------------+ + | SUBSTRING('helloworld', 5) | SUBSTRING('helloworld', 5, 3) | + |------------------------------+---------------------------------| + | oworld | owo | + +------------------------------+---------------------------------+ + +### TRIM + +The \`trim\` function removes leading and trailing white space from a string. + +**Argument type:** \`STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`TRIM(' hello')\` = TRIM(' hello'), \`TRIM('hello ')\` = TRIM('hello ') | fields \`TRIM(' hello')\`, \`TRIM('hello ')\` + fetched rows / total rows = 1/1 + +--------------------+--------------------+ + | TRIM(' hello') | TRIM('hello ') | + |--------------------+--------------------| + | hello | hello | + +--------------------+--------------------+ + +### UPPER + +The \`upper(string)\` function converts a string to uppercase. + +**Argument type:** \`STRING\` + +**Return type:** \`STRING\` + +#### Example + + os> source=people | eval \`UPPER('helloworld')\` = UPPER('helloworld'), \`UPPER('HELLOWORLD')\` = UPPER('HELLOWORLD') | fields \`UPPER('helloworld')\`, \`UPPER('HELLOWORLD')\` + fetched rows / total rows = 1/1 + +-----------------------+-----------------------+ + | UPPER('helloworld') | UPPER('HELLOWORLD') | + |-----------------------+-----------------------| + | HELLOWORLD | HELLOWORLD | + +-----------------------+-----------------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/groups.tsx b/src/plugins/data/public/query/query_string/language_service/ppl_docs/groups.tsx new file mode 100644 index 000000000000..633def301493 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/groups.tsx @@ -0,0 +1,126 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + dedupCmd, + evalCmd, + fieldsCmd, + headCmd, + parseCmd, + rareCmd, + renameCmd, + searchCmd, + sortCmd, + statsCmd, + syntaxCmd, + topCmd, + whereCmd, +} from './commands'; +import { + mathFunction, + datetimeFunction, + stringFunction, + conditionFunction, + fullTextSearchFunction, +} from './functions'; +import { pplDatatypes, pplIdentifiers } from './language_structure'; + +export const Group1 = { + label: 'Commands', + options: [ + { + label: 'Syntax', + value: syntaxCmd, + }, + { + label: 'dedup', + value: dedupCmd, + }, + { + label: 'eval', + value: evalCmd, + }, + { + label: 'fields', + value: fieldsCmd, + }, + { + label: 'rename', + value: renameCmd, + }, + { + label: 'search', + value: searchCmd, + }, + { + label: 'sort', + value: sortCmd, + }, + { + label: 'stats', + value: statsCmd, + }, + { + label: 'where', + value: whereCmd, + }, + { + label: 'head', + value: headCmd, + }, + { + label: 'parse', + value: parseCmd, + }, + { + label: 'rare', + value: rareCmd, + }, + { + label: 'top', + value: topCmd, + }, + ], +}; + +export const Group2 = { + label: 'Functions', + options: [ + { + label: 'Math', + value: mathFunction, + }, + { + label: 'Date and Time', + value: datetimeFunction, + }, + { + label: 'String', + value: stringFunction, + }, + { + label: 'Condition', + value: conditionFunction, + }, + { + label: 'Full Text Search', + value: fullTextSearchFunction, + }, + ], +}; + +export const Group3 = { + label: 'Language Structure', + options: [ + { + label: 'Identifiers', + value: pplIdentifiers, + }, + { + label: 'Data Types', + value: pplDatatypes, + }, + ], +}; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/datatypes.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/datatypes.ts new file mode 100644 index 000000000000..7914bc628daf --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/datatypes.ts @@ -0,0 +1,326 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const pplDatatypes = `## Data Types +--- + +### Data types + +A data type defines a collection of data type values and a set of predefined operations for those values. PPL supports the following data types: + +.. hlist:: + :columns: 3 + + * array + * binary + * boolean + * byte + * date + * datetime + * double + * float + * geo\_point + * integer + * interval + * ip + * long + * short + * string + * struct + * text + * time + * timestamp + +### Data type mapping + +The following table is a reference guide for the mapping between an OpenSearch data type, a PPL data type, and a SQL data type. + +| OpenSearch type | PPL type | SQL type | +|-----------------|-----------|-----------| +| binary | binary | VARBINARY | +| boolean | boolean | BOOLEAN | +| byte | byte | TINYINT | +| date | timestamp | TIMESTAMP | +| double | double | DOUBLE | +| float | float | REAL | +| half\_float | float | FLOAT | +| integer | integer | INTEGER | +| ip | ip | VARCHAR | +| keyword | string | VARCHAR | +| long | long | BIGINT | +| nested | array | STRUCT | +| object | struct | STRUCT | +| scaled\_float | float | DOUBLE | +| short | byte | SMALLINT | +| text | text | VARCHAR | + +Some PPL types do not correspond to an OpenSearch type. To use functions that require date and time data types, data type conversion must be performed, as described in the following sections. + +### Numeric data types + +Numeric values ranging from -2147483648 to +2147483647 are recognized as +integers, with data type name \`INTEGER\`. For values that fall beyond the specified range, the \`LONG\` data type is assigned during parsing. + +### Date and time data types + +The data types \`date\` and \`time\` represent temporal values. The PPL plugin supports \`date\`, \`time\`, \`datetime\`, \`timestamp\`, and \`interval\`. By default, [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/) uses \`date\` for any date or time types. To integrate with PPL, each data type, excluding \`timestamp\`, contains temporal and time zone information. Use \`datetime\` functions to clarify the date and time types. Note that certain functions may have limitations on the input argument type. See the [Functions](functions.rst) section in this manual for more information. + +#### Date + +The \`date\` data type represents the calendar date, regardless of time zone. A specific date value represents a 24-hour period, but this period differs across time zones and may be subject to variations due to daylight saving time adjustments. Additionally, the date alone does not contain time-specific information. The date values range from '1000-01-01' to '9999-12-31'. + +| Type | Syntax | Range | +|----------|--------------|------------------------------| +| \`date\` | 'yyyy-MM-dd' | '0001-01-01' to '9999-12-31' | + +#### Time + +The \`time\` data type represents the time of day as displayed on a clock or watch, without specifying a particular time zone. It does not include any information about the calendar date. + +| Type | Syntax | Range | +|----------|--------------|------------------------------| +| \`time\` | 'hh:mm:ss\[.fraction\]' | '00:00:00.000000' to '23:59:59.999999' | + +#### Datetime + +The \`datetime\` data type represents a combination of \`date\` and \`time\`. The \`datetime\` data type does not contain time zone information. For an absolute time point that contains both datetime and time zone information, see the [Timestamp](#timestamp) section. + +See the [Conversion between date and time types](#conversion-between-date-and-time-types) section for information about the conversion rule for \`date\` or \`time\` to \`datetime\`. + +| Type | Syntax | Range | +|----------|--------|-------| +| \`datetime\` | 'yyyy-MM-dd hh:mm:ss\[.fraction\]' | '0001-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999' | + +#### Timestamp + +The \`timestamp\` data type represents absolute points in time, unaffected by time zones or conventions. The \`timestamp\` data type differs from other data types in its storage and retrieval behavior. When a timestamp is sorted, it is converted from Coordinated Universal Time (UTC) to the specified time zone. Conversely, when a timestamp is retrieved, it is converted back to UTC before being displayed or used in calculations. This ensures that the timestamp values remain consistent and comparable across different time zones. + +| Type | Syntax | Range | +|-----------|--------|-------| +| Timestamp | 'yyyy-MM-dd hh:mm:ss\[.fraction\]' | '0001-01-01 00:00:01.000000' UTC to '9999-12-31 23:59:59.999999' | + +#### Interval + +The \`interval\` data type represents a span of time encompassing a specified duration or period. + +| Type | Syntax | +|----------|--------------------| +| Interval | INTERVAL expr unit | + +The expression \`expr\` is configured to be repeatedly evaluated to produce a quantitative value. See the [Expressions](expressions.rst) section of this manual for more information. The unit represents the unit used to interpret the quantity, including MICROSECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, and YEAR. The INTERVAL keyword and the unit specifier are not case sensitive. + +Intervals consist of two classes: day-time and year-week. Day-time intervals store days, hours, minutes, seconds, and microseconds. Year-week intervals store years, quarters, months, and weeks. Each type can only be compared to the same type. + +### Date and time conversion + +Date and time types, excluding \`interval\`, can be mutually converted, with some alteration of the value or some information loss, for example, when extracting the \`time\` value from a \`datetime\` value or converting a \`date\` value to a \`datetime\` value. The PPL plugin supports the following conversion rules for each of the types. + +#### \`date\` conversion + +- Because \`date\` does not contain time information, conversion to \`time\` returns a zero time value \`00:00:00\`. +- Converting from \`date\` to \`datetime\` sets the time value to \`00:00:00\` if \`time\` is not provided, for example, \`2020-08-17\` converts to \`2020-08-17 00:00:00\`. +- Converting to \`timestamp\` sets \`time\` to \`00:00:00\` and time zone (UTC by default), for example, \`2020-08-17\` converts to \`2020-08-17 00:00:00 UTC\`. + +#### \`time\` conversion + +- A \`time\` value does not have any date information, so it cannot be converted to other date and time types. + +#### \`datetime\` conversion + +- Converting from \`datetime\` to \`date\` extracts the date component from the \`datetime\` value, for example, \`2020-08-17 14:09:00\` converts to \`2020-08-08\`. +- Converting to \`time\` extracts the time component from the \`datetime\` value, for example, \`2020-08-17 14:09:00\` converts to \`14:09:00\`. +- Because \`datetime\` does not contain time zone information, conversion to \`timestamp\` sets the time zone to the session's time zone, for example, \`2020-08-17 14:09:00\`, with the system time zone set to UTC, for example, \`2020-08-17 14:09:00 UTC\`. + +#### \`timestamp\` conversion + +- Converting from \`timestamp\` to \`date\ extracts the \`date\` and \`time\` values. Converting from \`timestamp\` to \`datetime\` extracts the \`datetime\` value and retains the time zone information. For example, \`2020-08-17 14:09:00 UTC\` converts \`date\` and \`time\` to \`2020-08-17\` and \`14:09:00\` and \`datetime\` to \`2020-08-17 14:09:00\`. + +### String data types + +A \`string\` data type is a series of characters enclosed within single or double quotation marks that serves as a data type for storing text data. + +### Query struct data type + +In PPL, the \`struct\` data type corresponds to the [Object field type in +OpenSearch](https://opensearch.org/docs/latest/field-types/supported-field-types/object-fields/). The \`"."\` is used as the path selector for accessing the inner attribute of the struct data. + +#### Example 1: Struct to store population data + +The following example struct stores population data in an index containing the following fields: deep-nested object field \`city\`, object field of array value \`account\`, and nested field \`projects\`. + + { + "mappings": { + "properties": { + "city": { + "properties": { + "name": { + "type": "keyword" + }, + "location": { + "properties": { + "latitude": { + "type": "double" + } + } + } + } + }, + "account": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "projects": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + } + } + } + } + } + } + +#### Example 2: Struct to store employee data + +The following example struct stores employee data and includes a nested field: + + { + "mappings": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "projects": { + "type": "nested", + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + }, + "fielddata": true + }, + "started_year": { + "type": "long" + } + } + }, + "title": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + +Result set: + + { + "employees_nested" : [ + { + "id" : 3, + "name" : "Bob Smith", + "title" : null, + "projects" : [ + { + "name" : "AWS Redshift Spectrum querying", + "started_year" : 1990 + }, + { + "name" : "AWS Redshift security", + "started_year" : 1999 + }, + { + "name" : "AWS Aurora security", + "started_year" : 2015 + } + ] + }, + { + "id" : 4, + "name" : "Susan Smith", + "title" : "Dev Mgr", + "projects" : [ ] + }, + { + "id" : 6, + "name" : "Jane Smith", + "title" : "Software Eng 2", + "projects" : [ + { + "name" : "AWS Redshift security", + "started_year" : 1998 + }, + { + "name" : "AWS Hello security", + "started_year" : 2015, + "address" : [ + { + "city" : "Dallas", + "state" : "TX" + } + ] + } + ] + } + ] + } + +#### Example 3: Select a struct inner attribute + +The following example PPL query shows how to fetch \`city\` (top level), \`city.name\` (second level), and \`city.location.latitude\` (deeper level) struct data from the results: + + os> source=people | fields city, city.name, city.location.latitude; + fetched rows / total rows = 1/1 + +-----------------------------------------------------+-------------+--------------------------+ + | city | city.name | city.location.latitude | + |-----------------------------------------------------+-------------+--------------------------| + | {'name': 'Seattle', 'location': {'latitude': 10.5}} | Seattle | 10.5 | + +-----------------------------------------------------+-------------+--------------------------+ + +#### Example 4: Group by a struct inner attribute + +The following example PPL query shows how to group by a struct inner attribute: + + os> source=people | stats count() by city.name; + fetched rows / total rows = 1/1 + +-----------+-------------+ + | count() | city.name | + |-----------+-------------| + | 1 | Seattle | + +-----------+-------------+ + +#### Example 5: Select an object field of an array value + +The following example PPL query shows how to select a deeper level for object fields of array values that return the first element in the array. In this example, the document's inner field \`accounts.id\` has three values instead of a tuple: + + os> source = people | fields accounts, accounts.id; + fetched rows / total rows = 1/1 + +------------+---------------+ + | accounts | accounts.id | + |------------+---------------| + | {'id': 1} | 1 | + +------------+---------------+ +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/identifiers.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/identifiers.ts new file mode 100644 index 000000000000..36288736366e --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/identifiers.ts @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const pplIdentifiers = `## Indentifiers +--- + +### Identifiers + +Identifiers are used for naming database objects, such as an index name, a field name, or a custom label. The two types of identifiers are _regular identifiers_ and _delimited identifiers_. + +#### Regular identifiers + +A regular identifier is a string of characters that starts with an ASCII letter (lowercase or uppercase). The subsequent characters can be a combination of letters, digits, or underscores (\`_\`). A regular identifier cannot be a reversed keyword, and white space or other special characters are not allowed. + +The following identifiers are supported by OpenSearch extensions: + +- **Identifiers prefixed by dot \`.\`:** This is called a hidden index. An example is \`.opensearch_dashboards\`. +- **Identifiers prefixed by the \`@\` symbol:** This is common in meta fields generated for data ingestion. +- **Identifiers with \`-\` in the middle:** This is common in index naming conventions with date information. +- **Identifiers with the \`*\` symbol:** This is common in wildcard matches in an index pattern. + +Index names with a date suffix separated by dashes or dots, such as \`cwl-2020.01.11\` or \`logs-7.0-2020.01.11\`, are common in data ingestion. Identifiers used as index names do not need to be enclosed in quotation marks. Additionally, wildcards within date patterns are accepted, enabling data retrieval across indexes covering different date ranges. For example, you can use \`logs-2020.1*\` to search in indexes for October, November, and December 2020. + +#### Example 1: Index pattern without quotes + +The following example PPL query uses an index pattern directly without quotes: + + os> source=accounts | fields account_number, firstname, lastname; + fetched rows / total rows = 4/4 + +------------------+-------------+------------+ + | account_number | firstname | lastname | + |------------------+-------------+------------| + | 1 | Amber | Duke | + | 6 | Hattie | Bond | + | 13 | Nanette | Bates | + | 18 | Dale | Adams | + +------------------+-------------+------------+ + +### Delimited identifiers + +A delimited identifier is an identifier enclosed in backticks \`\` that contains special characters not permitted in regular identifiers. This allows for the use of characters that would otherwise violate the naming rules for identifiers. + +#### Use cases + +Common use cases for delimited identifiers include the following: + +- Identifiers that coincide with reserved keywords. +- Identifiers that contain a dot \`.\` or a dash \`-\` need to be distinguished from regular identifiers with qualifiers. Enclosing such identifiers in backticks \`\` allows for the parser to differentiate them from qualified identifiers and enables date information within index names. +- Identifiers with special characters in index names. Note that OpenSearch permits the use of special characters, including Unicode characters. + +#### Example 2: Index name enclosed in backticks + +The following example PPL query uses an index name enclosed in backticks \`\`: + + os> source=\`accounts\` | fields \`account_number\`; + fetched rows / total rows = 4/4 + +------------------+ + | account_number | + |------------------| + | 1 | + | 6 | + | 13 | + | 18 | + +------------------+ + +### Case sensitivity + +Identifiers are case sensitive and must match what is stored in OpenSearch. For example, if you run \`source=Accounts\`, an error \`index not found\` occurs \` because the index name \`accounts\` is lowercase. +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/index.ts b/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/index.ts new file mode 100644 index 000000000000..75423e184c08 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/language_structure/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { pplDatatypes } from './datatypes'; +export { pplIdentifiers } from './identifiers'; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_docs/overview.tsx b/src/plugins/data/public/query/query_string/language_service/ppl_docs/overview.tsx new file mode 100644 index 000000000000..5c8d92e57df4 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_docs/overview.tsx @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const overview = `## Overview +--- +Piped Processing Language (PPL) is a query language that processes data in a sequential, step-by-step manner. PPL uses a set of commands, connected by pipes (|), to process data and return results (note that the requests and results are read-only in OpenSearch). + +You can query data in OpenSearch using [PPL](https://opensearch.org/docs/latest/search-plugins/sql/ppl/index/), [query domain-specific language (DSL)](https://opensearch.org/docs/latest/query-dsl/index/), or [SQL](https://opensearch.org/docs/latest/search-plugins/sql/sql/index/). + +PPL is the primary language used for observability tasks in OpenSearch. Developers, DevOps engineers, support engineers, site reliability engineers, and IT managers find it useful for exploring and discovering log, monitoring, and observability data. For example, you can use PPL to: + +- Find all log messages that contain a specific error code. +- Identify trends in your data over time. +- Group similar data points. +- Calculate statistics for your data. + +PPL is available in OpenSearch Dashboards and as a standalone command-line tool. Within OpenSearch Dashboards, you can use [Query Workbench](https://opensearch.org/docs/latest/dashboards/query-workbench/) to run on-demand PPL commands and view and save the results as both text and JSON. The [PPL command line interface (CLI)](https://opensearch.org/docs/latest/search-plugins/sql/cli/) is a standalone Python application that you can launch with the \`opensearchsql\` command and then run on-demand PPL commands and view and save the results as both text and JSON. + +Here is an example of a PPL query. This query retrieves the first and last names of all accounts for which the age is greater than 18. + +\`\`\` +source=accounts +| where age > 18 +| fields firstname, lastname +\`\`\` +`; diff --git a/src/plugins/data/public/query/query_string/language_service/ppl_reference_flyout.tsx b/src/plugins/data/public/query/query_string/language_service/ppl_reference_flyout.tsx new file mode 100644 index 000000000000..220dc1ffecc1 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/ppl_reference_flyout.tsx @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + EuiComboBoxOptionOption, + EuiCompressedComboBox, + EuiFlexGroup, + EuiFlexItem, + EuiFlyoutBody, + EuiFlyoutFooter, + EuiFlyoutHeader, + EuiLink, + EuiMarkdownFormat, + EuiSmallButton, + EuiSpacer, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import React, { useState } from 'react'; +import { FlyoutContainers } from './flyout_containers'; +import { Group1, Group2, Group3 } from './ppl_docs/groups'; +import { overview } from './ppl_docs/overview'; + +interface Props { + module: string; + onClose: () => void; +} + +export const PPLReferenceFlyout = ({ module, onClose }: Props) => { + const allOptionsStatic = [{ label: 'Overview', value: overview }, Group1, Group2, Group3]; + const defaultOption = [allOptionsStatic[0]]; + const [selectedOptions, setSelected] = useState(defaultOption); + const [flyoutContent, setFlyoutContent] = useState( + {defaultOption[0].value} + ); + + const onChange = (SelectedOptions: any) => { + setSelected(SelectedOptions); + + const newContent = SelectedOptions.map((option: EuiComboBoxOptionOption) => ( + {option.value} + )); + setFlyoutContent(newContent); + }; + + const flyoutHeader = ( + + +

OpenSearch PPL Reference Manual

+ + + ); + + const PPL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index'; + + const flyoutBody = ( + + + + + + + + + Learn More + + + + + + {flyoutContent} + + ); + + const flyoutFooter = ( + + + + Close + + + + ); + + return ( + + ); +}; diff --git a/src/plugins/data/public/query/query_string/language_service/recent_query.tsx b/src/plugins/data/public/query/query_string/language_service/recent_query.tsx new file mode 100644 index 000000000000..d058cb5c692a --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/recent_query.tsx @@ -0,0 +1,169 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import './_recent_query.scss'; + +import { + EuiBasicTable, + EuiButtonEmpty, + EuiButtonIcon, + EuiCopy, + EuiPopover, + EuiText, +} from '@elastic/eui'; +import moment from 'moment'; + +import React, { useCallback, useEffect, useState } from 'react'; +import { Query, TimeRange } from 'src/plugins/data/common'; +import { QueryStringContract } from '../query_string_manager'; + +// TODO: Need to confirm this number +export const MAX_RECENT_QUERY_SIZE = 10; + +interface RecentQueryItem { + query: Query; + time: number; + timeRange?: TimeRange; +} + +export function RecentQuery(props: { + queryString: QueryStringContract; + query: Query; + onClickRecentQuery: (query: Query, timeRange?: TimeRange) => void; +}) { + const [recentQueries, setRecentQueries] = useState( + props.queryString.getQueryHistory() + ); + const [isPopoverOpen, setPopover] = useState(false); + const onButtonClick = () => { + setPopover(!isPopoverOpen); + }; + + const clearHistory = useCallback(() => { + props.queryString?.clearQueryHistory(); + setRecentQueries(props.queryString?.getQueryHistory()); + }, [props.queryString]); + + const clear = () => { + clearHistory(); + }; + + useEffect(() => { + const done = props.queryString.changeQueryHistory(setRecentQueries); + return () => done(); + }, [props.queryString]); + + const getRowProps = (item: any) => { + const { id } = item; + return { + 'data-test-subj': `row-${id}`, + className: 'customRowClass', + onClick: () => {}, + }; + }; + + const getCellProps = (item: any, column: any) => { + const { id } = item; + const { field } = column; + return { + className: 'customCellClass', + 'data-test-subj': `cell-${id}-${field}`, + textOnly: true, + }; + }; + + const actions = [ + { + name: 'Run', + description: 'Run recent query', + icon: 'play', + type: 'icon', + onClick: (item) => { + props.onClickRecentQuery(recentQueries[item.id].query, recentQueries[item.id].timeRange); + setPopover(false); + }, + 'data-test-subj': 'action-run', + }, + { + render: (item) => { + return ( + + {(copy) => ( + + )} + + ); + }, + }, + ]; + + const tableColumns = [ + { + field: 'query', + name: 'Recent query', + }, + { + field: 'language', + name: 'Language', + }, + { + field: 'time', + name: 'Last run', + }, + { name: 'Actions', actions }, + ]; + + const recentQueryItems = recentQueries + .filter((item, idx) => idx < MAX_RECENT_QUERY_SIZE) + .map((query, idx) => { + const date = moment(query.time); + + const formattedDate = date.format('MMM D, YYYY HH:mm:ss'); + + let queryLanguage = query.query.language; + if (queryLanguage === 'kuery') { + queryLanguage = 'DQL'; + } + + const tableItem = { + id: idx, + query: query.query.query, + timeRange: query.timeRange, + language: queryLanguage, + time: formattedDate, + }; + + return tableItem; + }); + + return ( + + + {'Recent queries'} + + + } + isOpen={isPopoverOpen} + closePopover={() => setPopover(false)} + panelPaddingSize="none" + anchorPosition={'downRight'} + > + + + ); +} diff --git a/src/plugins/data/public/query/query_string/language_service/sql_reference_flyout.tsx b/src/plugins/data/public/query/query_string/language_service/sql_reference_flyout.tsx new file mode 100644 index 000000000000..40b849fb72d7 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/sql_reference_flyout.tsx @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + EuiComboBoxOptionOption, + EuiCompressedComboBox, + EuiFlexGroup, + EuiFlexItem, + EuiFlyoutBody, + EuiFlyoutFooter, + EuiFlyoutHeader, + EuiLink, + EuiMarkdownFormat, + EuiSmallButton, + EuiSpacer, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import React, { useState } from 'react'; +import { FlyoutContainers } from './flyout_containers'; +import { Group1, Group2, Group3 } from './ppl_docs/groups'; +import { overview } from './ppl_docs/overview'; + +interface Props { + module: string; + onClose: () => void; +} + +export const SQLReferenceFlyout = ({ module, onClose }: Props) => { + const flyoutHeader = ( + + +

OpenSearch SQL Reference Manual

+ + + ); + + const flyoutBody = null; + + const flyoutFooter = ( + + + + Close + + + + ); + + return ( + + ); +}; diff --git a/src/plugins/data/public/query/query_string/query_history.ts b/src/plugins/data/public/query/query_string/query_history.ts index 80dfa4b5d560..04e2285add1c 100644 --- a/src/plugins/data/public/query/query_string/query_history.ts +++ b/src/plugins/data/public/query/query_string/query_history.ts @@ -35,7 +35,7 @@ export class QueryHistory { return () => subscription.unsubscribe(); } - addQueryToHistory(dataset: Dataset, query: Query, dateRange?: TimeRange) { + addQueryToHistory(query: Query, dateRange?: TimeRange) { const keys = this.getHistoryKeys(); keys.splice(0, 500); // only maintain most recent X; keys.forEach((key) => { @@ -45,7 +45,6 @@ export class QueryHistory { const timestamp = new Date().getTime(); const k = 'query_' + timestamp; this.storage.set(k, { - dataset, time: timestamp, query, dateRange, diff --git a/src/plugins/data/public/query/query_string/query_string_manager.ts b/src/plugins/data/public/query/query_string/query_string_manager.ts index 6bbc5706b64b..754b444435c0 100644 --- a/src/plugins/data/public/query/query_string/query_string_manager.ts +++ b/src/plugins/data/public/query/query_string/query_string_manager.ts @@ -112,9 +112,9 @@ export class QueryStringManager { }; // Todo: update this function to use the Query object when it is udpated, Query object should include time range and dataset - public addToQueryHistory(dataSet: Dataset, query: Query, timeRange?: TimeRange) { + public addToQueryHistory(query: Query, timeRange?: TimeRange) { if (query.query) { - this.queryHistory.addQueryToHistory(dataSet, query, timeRange); + this.queryHistory.addQueryToHistory(query, timeRange); } } diff --git a/src/plugins/data/public/ui/filter_bar/filter_options.tsx b/src/plugins/data/public/ui/filter_bar/filter_options.tsx index 3edd47e3fddc..131d317ecbc5 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_options.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_options.tsx @@ -55,6 +55,7 @@ import { unpinFilter, UI_SETTINGS, IIndexPattern, + isQueryStringFilter, } from '../../../common'; import { FilterEditor } from './filter_editor'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; @@ -75,6 +76,7 @@ interface Props { onFiltersUpdated?: (filters: Filter[]) => void; loadedSavedQuery?: SavedQuery; useSaveQueryMenu: boolean; + isQueryEditorControl: boolean; } const maxFilterWidth = 600; @@ -371,7 +373,10 @@ const FilterOptionsUI = (props: Props) => { className="osdSavedQueryManagement__popoverButton" title={label} > - + ); @@ -398,6 +403,25 @@ const FilterOptionsUI = (props: Props) => { ); + if (props.isQueryEditorControl) { + return ( + + {saveQueryPanel} + + ); + } + return ( void; editorDidMount: (editor: any) => void; footerItems?: { - start?: Array; - end?: Array; + start?: any[]; + end?: any[]; }; headerRef?: React.RefObject; provideCompletionItems: monaco.languages.CompletionItemProvider['provideCompletionItems']; @@ -72,28 +67,23 @@ export const DefaultInput: React.FC = ({ }} /> {footerItems && ( -
- {footerItems.start?.map((item, index) => ( - - ))} -
- {footerItems.end?.map((item, index) => ( - - ))} -
+ + + {footerItems.start?.map((item) => ( + + {item} + + ))} + + + {footerItems.end?.map((item) => ( + {item} + ))} + + )}
); }; -const FooterItem: React.FC<{ item: FooterItem | string }> = ({ item }) => { - const color = typeof item === 'string' ? ('subdued' as const) : item.color; - const text = typeof item === 'string' ? item : item.text; - return ( - - {text} - - ); -}; - export const createDefaultEditor = createEditor(SingleLineInput, null, DefaultInput); diff --git a/src/plugins/data/public/ui/query_editor/editors/shared.tsx b/src/plugins/data/public/ui/query_editor/editors/shared.tsx index 23b46c22a48c..ad9af1e87e19 100644 --- a/src/plugins/data/public/ui/query_editor/editors/shared.tsx +++ b/src/plugins/data/public/ui/query_editor/editors/shared.tsx @@ -4,6 +4,7 @@ */ import React from 'react'; +import { EuiCompressedFieldText } from '@elastic/eui'; import { monaco } from '@osd/monaco'; import { CodeEditor } from '../../../../../opensearch_dashboards_react/public'; @@ -13,6 +14,7 @@ interface SingleLineInputProps extends React.JSX.IntrinsicAttributes { onChange: (value: string) => void; editorDidMount: (editor: any) => void; provideCompletionItems: monaco.languages.CompletionItemProvider['provideCompletionItems']; + prepend?: React.ComponentProps['prepend']; } type CollapsedComponent = React.ComponentType; @@ -63,51 +65,55 @@ export const SingleLineInput: React.FC = ({ onChange, editorDidMount, provideCompletionItems, + prepend, }) => ( -
- + {prepend} +
+ + overviewRulerLanes: 0, + hideCursorInOverviewRuler: true, + cursorStyle: 'line', + wordBasedSuggestions: false, + }} + suggestionProvider={{ + provideCompletionItems, + }} + languageConfiguration={{ + autoClosingPairs: [ + { + open: '(', + close: ')', + }, + { + open: '"', + close: '"', + }, + ], + }} + /> +
); diff --git a/src/plugins/data/public/ui/query_editor/query_editor.tsx b/src/plugins/data/public/ui/query_editor/query_editor.tsx index ccc616911ed8..619184883220 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor.tsx @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { PopoverAnchorPosition } from '@elastic/eui'; +import { EuiCompressedFieldText, EuiText, PopoverAnchorPosition } from '@elastic/eui'; import classNames from 'classnames'; import { isEqual } from 'lodash'; import React, { Component, createRef, RefObject } from 'react'; @@ -15,9 +15,11 @@ import { fromUser, getQueryLog, PersistedLog, toUser } from '../../query'; import { SuggestionsListSize } from '../typeahead/suggestions_component'; import { QueryLanguageSelector } from './language_selector'; import { QueryEditorExtensions } from './query_editor_extensions'; -import { QueryEditorBtnCollapse } from './query_editor_btn_collapse'; import { getQueryService, getIndexPatterns } from '../../services'; import { DatasetSelector } from '../dataset_selector'; +import { QueryControls } from '../../query/query_string/language_service/get_query_control_links'; +import { RecentQuery } from '../../query/query_string/language_service/recent_query'; +import { DefaultInputProps } from './editors'; const LANGUAGE_ID_SQL = 'SQL'; monaco.languages.register({ id: LANGUAGE_ID_SQL }); @@ -46,6 +48,8 @@ export interface QueryEditorProps { bannerClassName?: string; footerClassName?: string; filterBar?: any; + prepend?: React.ComponentProps['prepend']; + savedQueryManagement?: any; } interface Props extends QueryEditorProps { @@ -157,6 +161,10 @@ export default class QueryEditorUI extends Component { }); }; + private onClickRecentQuery = (query: Query, timeRange?: TimeRange) => { + this.onSubmit(query, timeRange); + }; + private onInputChange = (value: string) => { this.onQueryStringChange(value); @@ -285,6 +293,21 @@ export default class QueryEditorUI extends Component { }; }; + public onToggleCollapse = () => { + this.setState({ isCollapsed: !this.state.isCollapsed }); + }; + + private renderQueryControls = () => { + return ( + + ); + }; + public render() { const className = classNames(this.props.className); @@ -305,7 +328,7 @@ export default class QueryEditorUI extends Component { value: this.getQueryString(), }; - const defaultInputProps = { + const defaultInputProps: DefaultInputProps = { ...baseInputProps, onChange: this.onInputChange, editorDidMount: (editor: monaco.editor.IStandaloneCodeEditor) => { @@ -315,8 +338,19 @@ export default class QueryEditorUI extends Component { }, footerItems: { start: [ - `${this.state.lineCount} ${this.state.lineCount === 1 ? 'line' : 'lines'}`, - this.props.query.dataset?.timeFieldName || '', + + {`${this.state.lineCount} ${this.state.lineCount === 1 ? 'line' : 'lines'}`} + , + + {this.props.query.dataset?.timeFieldName || ''} + , + ], + end: [ + , ], }, provideCompletionItems: this.provideCompletionItems, @@ -350,6 +384,7 @@ export default class QueryEditorUI extends Component { }; }, provideCompletionItems: this.provideCompletionItems, + prepend: this.props.prepend, }; const languageEditorFunc = this.languageManager.getLanguage(this.props.query.language)!.editor; @@ -374,10 +409,6 @@ export default class QueryEditorUI extends Component { className={classNames('osdQueryEditor__banner', this.props.bannerClassName)} />
- this.setState({ isCollapsed: !this.state.isCollapsed })} - isCollapsed={!this.state.isCollapsed} - />
{this.state.isCollapsed @@ -385,7 +416,7 @@ export default class QueryEditorUI extends Component { : languageEditor.TopBar.Expanded && languageEditor.TopBar.Expanded()}
{languageSelector} - {this.props.queryActions} +
{this.renderQueryControls()}
void; - isCollapsed: boolean; -} - -export function QueryEditorBtnCollapse({ onClick, isCollapsed }: Props) { - const label = i18n.translate('queryEditor.collapse', { - defaultMessage: 'Toggle query editor', - }); - return ( -
- - - -
- ); -} diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index 0c15a551a3f8..a5248d3d61cb 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -53,6 +53,7 @@ export interface QueryEditorTopRowProps { timeHistory?: TimeHistoryContract; indicateNoData?: boolean; datePickerRef?: React.RefObject; + savedQueryManagement?: any; } // Needed for React.lazy @@ -174,7 +175,7 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { ); diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index 891cf70379b2..b986a1e11ddd 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -234,19 +234,22 @@ class SearchBarUI extends Component { return this.props.showQueryBar && (showDatePicker || showQueryInput); } - private shouldRenderFilterBar(isEnhancementsEnabledOverride: boolean) { - const language = this.queryStringManager - .getLanguageService() - .getLanguage(this.state.query?.language!); - const isFilterable = language?.fields?.filterable !== false; // Render if undefined or true + private isQueryLanguageFilterable() { + return ( + this.queryStringManager.getLanguageService().getLanguage(this.state.query?.language!)?.fields + ?.filterable ?? true // Render if undefined or true + ); + } + private shouldRenderFilterBar(isEnhancementsEnabledOverride: boolean) { return ( this.props.showFilterBar && this.props.filters && (!this.useNewHeader || this.props.filters.length > 0) && this.props.indexPatterns && compact(this.props.indexPatterns).length > 0 && - (!isEnhancementsEnabledOverride || (isEnhancementsEnabledOverride && isFilterable)) + (!isEnhancementsEnabledOverride || + (isEnhancementsEnabledOverride && this.isQueryLanguageFilterable)) ); } @@ -380,10 +383,9 @@ class SearchBarUI extends Component { } } ); - const dataset = this.queryStringManager.getQuery().dataset; - if (dataset && queryAndDateRange.query) { + + if (queryAndDateRange.query) { this.queryStringManager.addToQueryHistory( - dataset, queryAndDateRange.query, queryAndDateRange.dateRange ); @@ -427,7 +429,10 @@ class SearchBarUI extends Component { .getLanguage(this.state.query?.language!) ?.editorSupportedAppNames?.includes(this.services.appName); - const searchBarMenu = (useSaveQueryMenu: boolean = false) => { + const searchBarMenu = ( + useSaveQueryMenu: boolean = false, + isQueryEditorControl: boolean = false + ) => { return ( this.state.query && this.props.onClearSavedQuery && ( @@ -444,6 +449,7 @@ class SearchBarUI extends Component { savedQueryService={this.savedQueryService} onClearSavedQuery={this.props.onClearSavedQuery} useSaveQueryMenu={useSaveQueryMenu} + isQueryEditorControl={isQueryEditorControl} /> ) ); @@ -480,6 +486,7 @@ class SearchBarUI extends Component { } let queryBar; + if (this.shouldRenderQueryBar(isEnhancementsEnabledOverride)) { queryBar = ( { onSubmit={this.onQueryBarSubmit} indexPatterns={this.props.indexPatterns} isLoading={this.props.isLoading} - prepend={this.props.showFilterBar ? searchBarMenu(!this.useNewHeader) : undefined} + prepend={this.props.showFilterBar ? searchBarMenu(!this.useNewHeader, false) : undefined} showDatePicker={this.props.showDatePicker} dateRangeFrom={this.state.dateRangeFrom} dateRangeTo={this.state.dateRangeTo} @@ -521,7 +528,7 @@ class SearchBarUI extends Component { onSubmit={this.onQueryBarSubmit} indexPatterns={this.props.indexPatterns} isLoading={this.props.isLoading} - prepend={this.props.showFilterBar ? searchBarMenu(!this.useNewHeader) : undefined} + prepend={this.isQueryLanguageFilterable() ? searchBarMenu() : undefined} showDatePicker={this.props.showDatePicker} dateRangeFrom={this.state.dateRangeFrom} dateRangeTo={this.state.dateRangeTo} @@ -540,6 +547,7 @@ class SearchBarUI extends Component { dataTestSubj={this.props.dataTestSubj} indicateNoData={this.props.indicateNoData} datePickerRef={this.props.datePickerRef} + savedQueryManagement={searchBarMenu(false, true)} /> ); }