Remove square brackets in FROM METADATA declaration

packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts

@@ -91,14 +91,14 @@ describe('autocomplete.suggest', () => {
       test('on <kbd>SPACE</kbd> after "METADATA" keyword suggests all metadata fields', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from a, b [METADATA /]', metadataFields);
+        await assertSuggestions('from a, b METADATA /', metadataFields);
         await assertSuggestions('from a, b METADATA /', metadataFields);
       });
 
       test('on <kbd>SPACE</kbd> after "METADATA" column suggests command and pipe operators', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from a, b [metadata _index  /]', [',', '| ']);
+        await assertSuggestions('from a, b metadata _index /', [',', '| ']);
         await assertSuggestions('from a, b metadata _index /', [',', '| ']);
         await assertSuggestions('from a, b metadata _index, _source /', [',', '| ']);
         await assertSuggestions(`from a, b metadata ${METADATA_FIELDS.join(', ')} /`, ['| ']);
@@ -107,7 +107,7 @@ describe('autocomplete.suggest', () => {
       test('filters out already used metadata fields', async () => {
         const { assertSuggestions } = await setup();
 
-        await assertSuggestions('from a, b [metadata _index, /]', metadataFieldsAndIndex);
+        await assertSuggestions('from a, b metadata _index, /', metadataFieldsAndIndex);
         await assertSuggestions('from a, b metadata _index, /', metadataFieldsAndIndex);
       });
     });

packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -102,7 +102,7 @@ describe('autocomplete', () => {
         .map(({ name }) => name.toUpperCase() + ' $0')
     );
     testSuggestions(
-      'from a [metadata _id] | /',
+      'from a metadata _id | /',
       commandDefinitions
         .filter(({ name }) => !sourceCommands.includes(name))
         .map(({ name }) => name.toUpperCase() + ' $0')
@@ -114,7 +114,7 @@ describe('autocomplete', () => {
         .map(({ name }) => name.toUpperCase() + ' $0')
     );
     testSuggestions(
-      'from a [metadata _id] | eval var0 = a | /',
+      'from a metadata _id | eval var0 = a | /',
       commandDefinitions
         .filter(({ name }) => !sourceCommands.includes(name))
         .map(({ name }) => name.toUpperCase() + ' $0')

packages/kbn-esql-validation-autocomplete/src/definitions/options.ts

@@ -39,7 +39,7 @@ export const metadataOption: CommandOptionsDefinition = {
   validate: (option, command, references) => {
     const messages: ESQLMessage[] = [];
     // need to test the parent command here
-    if (/\[metadata/i.test(command.text)) {
+    if (/metadata/i.test(command.text)) {
       messages.push(
         getMessageFromId({
           messageId: 'metadataBracketsDeprecation',

packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json

@@ -10026,4 +10026,4 @@
       "warning": []
     }
   ]
-}
+}

packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts

@@ -1741,7 +1741,7 @@ describe('validation logic', () => {
     it('should basically work when all callbacks are passed', async () => {
       const allErrors = await Promise.all(
         fixtures.testCases
-          .filter(({ query }) => query === 'from index [METADATA _id, _source2]')
+          .filter(({ query }) => query === 'from index METADATA _id, _source2')
           .map(({ query }) =>
             validateQuery(
               query,
@@ -1753,7 +1753,7 @@ describe('validation logic', () => {
       );
       for (const [index, { errors }] of Object.entries(allErrors)) {
         expect(errors.map((e) => ('severity' in e ? e.message : e.text))).toEqual(
-          fixtures.testCases.filter(({ query }) => query === 'from index [METADATA _id, _source2]')[
+          fixtures.testCases.filter(({ query }) => query === 'from index METADATA _id, _source2')[
             Number(index)
           ].error
         );

packages/kbn-language-documentation/src/sections/esql_documentation_sections.tsx

@@ -19,7 +19,7 @@ export const initialSection = (
   <Markdown
     markdownContent={i18n.translate('languageDocumentation.documentationESQL.markdown', {
       defaultMessage: `
-An ES|QL (Elasticsearch query language) query consists of a series of commands, separated by pipe characters: \`|\`. Each query starts with a **source command**, which produces a table, typically with data from Elasticsearch. 
+An ES|QL (Elasticsearch query language) query consists of a series of commands, separated by pipe characters: \`|\`. Each query starts with a **source command**, which produces a table, typically with data from Elasticsearch.
 
 A source command can be followed by one or more **processing commands**. Processing commands can change the output table of the previous command by adding, removing, and changing rows and columns.
 
@@ -29,7 +29,7 @@ source-command
 | processing-command2
 \`\`\`
 
-The result of a query is the table produced by the final processing command.                                  
+The result of a query is the table produced by the final processing command.
                                       `,
     })}
   />
@@ -77,15 +77,15 @@ ES|QL can access the following metadata fields:
 Use the \`METADATA\` directive to enable metadata fields:
 
 \`\`\`
-FROM index [METADATA _index, _id]
+FROM index METADATA _index, _id
 \`\`\`
 
 Metadata fields are only available if the source of the data is an index. Consequently, \`FROM\` is the only source commands that supports the \`METADATA\` directive.
 
 Once enabled, the fields are then available to subsequent processing commands, just like the other index fields:
 
 \`\`\`
-FROM ul_logs, apps [METADATA _index, _version]
+FROM ul_logs, apps METADATA _index, _version
 | WHERE id IN (13, 14) AND _version == 1
 | EVAL key = CONCAT(_index, "_", TO_STR(id))
 | SORT id, _index
@@ -95,7 +95,7 @@ FROM ul_logs, apps [METADATA _index, _version]
 Also, similar to the index fields, once an aggregation is performed, a metadata field will no longer be accessible to subsequent commands, unless used as grouping field:
 
 \`\`\`
-FROM employees [METADATA _index, _id]
+FROM employees METADATA _index, _id
 | STATS max = MAX(emp_no) BY _index
 \`\`\`
             `,
@@ -114,7 +114,7 @@ FROM employees [METADATA _index, _id]
           markdownContent={i18n.translate('languageDocumentation.documentationESQL.row.markdown', {
             defaultMessage: `### ROW
 The \`ROW\` source command produces a row with one or more columns with values that you specify. This can be useful for testing.
-              
+
 \`\`\`
 ROW a = 1, b = "two", c = null
 \`\`\`
@@ -207,7 +207,7 @@ ROW a = "1953-01-23T12:15:00Z - some text - 127.0.0.1"
           markdownContent={i18n.translate('languageDocumentation.documentationESQL.drop.markdown', {
             defaultMessage: `### DROP
 Use \`DROP\` to remove columns from a table:
-              
+
 \`\`\`
 FROM employees
 | DROP height
@@ -347,7 +347,7 @@ ROW a = "12 15.5 15.6 true"
 The \`KEEP\` command enables you to specify what columns are returned and the order in which they are returned.
 
 To limit the columns that are returned, use a comma-separated list of column names. The columns are returned in the specified order:
-              
+
 \`\`\`
 FROM employees
 | KEEP first_name, last_name, height
@@ -384,7 +384,7 @@ FROM employees
             {
               defaultMessage: `### LIMIT
 The \`LIMIT\` processing command enables you to limit the number of rows:
-              
+
 \`\`\`
 FROM employees
 | LIMIT 5
@@ -407,7 +407,7 @@ FROM employees
             'languageDocumentation.documentationESQL.mvExpand.markdown',
             {
               defaultMessage: `### MV_EXPAND
-The \`MV_EXPAND\` processing command expands multivalued fields into one row per value, duplicating other fields:              
+The \`MV_EXPAND\` processing command expands multivalued fields into one row per value, duplicating other fields:
 \`\`\`
 ROW a=[1,2,3], b="b", j=["a","b"]
 | MV_EXPAND a
@@ -607,7 +607,7 @@ FROM employees
             {
               defaultMessage: `### WHERE
 Use \`WHERE\` to produce a table that contains all the rows from the input table for which the provided condition evaluates to \`true\`:
-              
+
 \`\`\`
 FROM employees
 | KEEP first_name, last_name, still_hired
@@ -654,7 +654,7 @@ export const groupingFunctions = {
               defaultMessage: `### BUCKET
 Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range.
 
-\`BUCKET\` works in two modes: 
+\`BUCKET\` works in two modes:
 
 1. Where the size of the bucket is computed based on a buckets count recommendation (four parameters) and a range.
 2. Where the bucket size is provided directly (two parameters).

test/functional/apps/discover/group6/_sidebar_field_stats.ts

@@ -148,7 +148,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
         await discover.selectTextBaseLang();
 
-        const testQuery = `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500`;
+        const testQuery = `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500`;
         await monacoEditor.setCodeEditorValue(testQuery);
         await testSubjects.click('querySubmitButton');
         await header.waitUntilLoadingHasFinished();
@@ -168,7 +168,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await unifiedFieldList.clickFieldListPlusFilter('bytes', '0');
         const editorValue = await monacoEditor.getCodeEditorValue();
         expect(editorValue).to.eql(
-          `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`bytes\`==0`
+          `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`bytes\`==0`
         );
         await unifiedFieldList.closeFieldPopover();
       });
@@ -188,7 +188,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await unifiedFieldList.clickFieldListPlusFilter('extension.raw', 'css');
         const editorValue = await monacoEditor.getCodeEditorValue();
         expect(editorValue).to.eql(
-          `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`extension.raw\`=="css"`
+          `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`extension.raw\`=="css"`
         );
 
         await unifiedFieldList.closeFieldPopover();
@@ -209,7 +209,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await unifiedFieldList.clickFieldListPlusFilter('clientip', '216.126.255.31');
         const editorValue = await monacoEditor.getCodeEditorValue();
         expect(editorValue).to.eql(
-          `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`clientip\`::string=="216.126.255.31"`
+          `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`clientip\`::string=="216.126.255.31"`
         );
 
         await unifiedFieldList.closeFieldPopover();
@@ -234,7 +234,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await unifiedFieldList.clickFieldListExistsFilter('@timestamp');
         const editorValue = await monacoEditor.getCodeEditorValue();
         expect(editorValue).to.eql(
-          `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`@timestamp\` is not null`
+          `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`@timestamp\` is not null`
         );
         await testSubjects.missingOrFail('dscFieldStats-statsFooter');
         await unifiedFieldList.closeFieldPopover();
@@ -269,7 +269,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await unifiedFieldList.clickFieldListPlusFilter('extension', 'css');
         const editorValue = await monacoEditor.getCodeEditorValue();
         expect(editorValue).to.eql(
-          `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`extension\`=="css"`
+          `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`extension\`=="css"`
         );
 
         await unifiedFieldList.closeFieldPopover();

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts

@@ -22,11 +22,11 @@ const getQeryAst = (query: string) => {
 describe('computeHasMetadataOperator', () => {
   it('should be false if query does not have operator', () => {
     expect(computeHasMetadataOperator(getQeryAst('from test*'))).toBe(false);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [metadata]'))).toBe(false);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [metadata id]'))).toBe(false);
+    expect(computeHasMetadataOperator(getQeryAst('from test* metadata'))).toBe(false);
+    expect(computeHasMetadataOperator(getQeryAst('from test* metadata id'))).toBe(false);
     expect(computeHasMetadataOperator(getQeryAst('from metadata*'))).toBe(false);
     expect(computeHasMetadataOperator(getQeryAst('from test* | keep metadata'))).toBe(false);
-    expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="[metadata _id]"'))).toBe(
+    expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="metadata _id"'))).toBe(
       false
     );
   });
@@ -48,19 +48,19 @@ describe('computeHasMetadataOperator', () => {
     ).toBe(true);
 
     // still validates deprecated square bracket syntax
-    expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _id]'))).toBe(true);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _id, _index]'))).toBe(true);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _index, _id]'))).toBe(true);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id ]'))).toBe(true);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [   metadata _id] '))).toBe(true);
-    expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id] | limit 10'))).toBe(
+    expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true);
+    expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true);
+    expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true);
+    expect(computeHasMetadataOperator(getQeryAst('from test*  metadata _id '))).toBe(true);
+    expect(computeHasMetadataOperator(getQeryAst('from test*    metadata _id '))).toBe(true);
+    expect(computeHasMetadataOperator(getQeryAst('from test*  metadata _id | limit 10'))).toBe(
       true
     );
     expect(
       computeHasMetadataOperator(
-        getQeryAst(`from packetbeat* [metadata
+        getQeryAst(`from packetbeat* metadata
 
-        _id ]
+        _id
         | limit 100`)
       )
     ).toBe(true);

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts

@@ -154,7 +154,7 @@ export const parseEsqlQuery = (query: string) => {
   return {
     errors,
     isEsqlQueryAggregating,
-    // non-aggregating query which does not have [metadata], is not a valid one
+    // non-aggregating query which does not have metadata, is not a valid one
     isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(ast),
   };
 };