[ES|QL] Update grammars: change quoting from backtick to quote for in…

…dex names (#187212)

Related to elastic/elasticsearch#108395. This PR
updates the ES|QL grammars (lexer and parser) to match the latest
version in Elasticsearch. The PR adds support for index names wrapped by
`""`, and triple quotes `"""index-name"""` to the ES|QL text editor and
removes backticks.
<img width="1392" alt="Screenshot 2024-07-09 at 09 25 22"
src="https://github.com/elastic/kibana/assets/43350163/4e93f503-9053-42a6-99c5-beda7bb4ba4c">

<img width="1392" alt="Screenshot 2024-07-09 at 09 24 54"
src="https://github.com/elastic/kibana/assets/43350163/eec6292c-f3c0-49f0-9864-cb3e6717e7a7">
<img width="1392" alt="Screenshot 2024-07-09 at 09 25 03"
src="https://github.com/elastic/kibana/assets/43350163/66dca108-a535-4df7-b1c0-42705c7913dd">

---------

Co-authored-by: Quynh Nguyen <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>
(cherry picked from commit 5bc0c78)

packages/kbn-esql-ast/src/__tests__/ast_parser.from.test.ts

@@ -60,6 +60,36 @@ describe('FROM', () => {
       ]);
     });
 
+    it('can parse FROM query with single quote or triple quote', () => {
+      const text = '\tFROM "foo%" \t\t, """bar{{00-00}}""", \n  baz';
+      const { ast, errors } = parse(text);
+
+      expect(errors.length).toBe(0);
+      expect(ast).toMatchObject([
+        {
+          type: 'command',
+          name: 'from',
+          args: [
+            {
+              type: 'source',
+              name: 'foo%',
+              sourceType: 'index',
+            },
+            {
+              type: 'source',
+              name: 'bar{{00-00}}',
+              sourceType: 'index',
+            },
+            {
+              type: 'source',
+              name: 'baz',
+              sourceType: 'index',
+            },
+          ],
+        },
+      ]);
+    });
+
     it('can parse FROM query with a single metadata column', () => {
       const text = 'from foo METADATA bar';
       const { ast, errors } = parse(text);

packages/kbn-esql-ast/src/antlr/esql_lexer.g4

@@ -44,13 +44,15 @@ WS
     : [ \r\n\t]+ -> channel(HIDDEN)
     ;
 
-fragment INDEX_UNQUOTED_IDENTIFIER_PART
-    : ~[=`|,[\]/ \t\r\n]
-    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment
+// in 8.14 ` were not allowed
+// this has been relaxed in 8.15 since " is used for quoting
+fragment UNQUOTED_SOURCE_PART
+    : ~[:"=|,[\]/ \t\r\n]
+    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment -- used in index pattern date spec
     ;
 
-INDEX_UNQUOTED_IDENTIFIER
-    : INDEX_UNQUOTED_IDENTIFIER_PART+
+UNQUOTED_SOURCE
+    : UNQUOTED_SOURCE_PART+
     ;
 
 //
@@ -212,15 +214,13 @@ mode FROM_MODE;
 FROM_PIPE : PIPE -> type(PIPE), popMode;
 FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET);
 FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET);
+FROM_COLON : COLON -> type(COLON);
 FROM_COMMA : COMMA -> type(COMMA);
 FROM_ASSIGN : ASSIGN -> type(ASSIGN);
-FROM_QUOTED_STRING : QUOTED_STRING -> type(QUOTED_STRING);
-
 METADATA : 'metadata';
 
-FROM_INDEX_UNQUOTED_IDENTIFIER
-    : INDEX_UNQUOTED_IDENTIFIER -> type(INDEX_UNQUOTED_IDENTIFIER)
-    ;
+FROM_UNQUOTED_SOURCE : UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE);
+FROM_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING);
 
 FROM_LINE_COMMENT
     : LINE_COMMENT -> channel(HIDDEN)
@@ -311,10 +311,6 @@ ENRICH_POLICY_NAME
     : (ENRICH_POLICY_NAME_BODY+ COLON)? ENRICH_POLICY_NAME_BODY+
     ;
 
-ENRICH_QUOTED_IDENTIFIER
-    : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)
-    ;
-
 ENRICH_MODE_UNQUOTED_VALUE
     : ENRICH_POLICY_NAME -> type(ENRICH_POLICY_NAME)
     ;
@@ -331,7 +327,7 @@ ENRICH_WS
     : WS -> channel(HIDDEN)
     ;
 
-// submode for Enrich to allow different lexing between policy identifier (loose) and field identifiers
+// submode for Enrich to allow different lexing between policy source (loose) and field identifiers
 mode ENRICH_FIELD_MODE;
 ENRICH_FIELD_PIPE : PIPE -> type(PIPE), popMode, popMode;
 ENRICH_FIELD_ASSIGN : ASSIGN -> type(ASSIGN);
@@ -363,13 +359,13 @@ ENRICH_FIELD_WS
 // LOOKUP ON key
 mode LOOKUP_MODE;
 LOOKUP_PIPE : PIPE -> type(PIPE), popMode;
+LOOKUP_COLON : COLON -> type(COLON);
 LOOKUP_COMMA : COMMA -> type(COMMA);
 LOOKUP_DOT: DOT -> type(DOT);
 LOOKUP_ON : ON -> type(ON), pushMode(LOOKUP_FIELD_MODE);
 
-LOOKUP_INDEX_UNQUOTED_IDENTIFIER
-    : INDEX_UNQUOTED_IDENTIFIER -> type(INDEX_UNQUOTED_IDENTIFIER)
-    ;
+LOOKUP_UNQUOTED_SOURCE: UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE);
+LOOKUP_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING);
 
 LOOKUP_LINE_COMMENT
     : LINE_COMMENT -> channel(HIDDEN)
@@ -496,9 +492,8 @@ SETTING_WS
 mode METRICS_MODE;
 METRICS_PIPE : PIPE -> type(PIPE), popMode;
 
-METRICS_INDEX_UNQUOTED_IDENTIFIER
-    : INDEX_UNQUOTED_IDENTIFIER -> type(INDEX_UNQUOTED_IDENTIFIER), popMode, pushMode(CLOSING_METRICS_MODE)
-    ;
+METRICS_UNQUOTED_SOURCE: UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE), popMode, pushMode(CLOSING_METRICS_MODE);
+METRICS_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING), popMode, pushMode(CLOSING_METRICS_MODE);
 
 METRICS_LINE_COMMENT
     : LINE_COMMENT -> channel(HIDDEN)
@@ -515,6 +510,10 @@ METRICS_WS
 // TODO: remove this workaround mode - see https://github.com/elastic/elasticsearch/issues/108528
 mode CLOSING_METRICS_MODE;
 
+CLOSING_METRICS_COLON
+    : COLON -> type(COLON), popMode, pushMode(METRICS_MODE)
+    ;
+
 CLOSING_METRICS_COMMA
     : COMMA -> type(COMMA), popMode, pushMode(METRICS_MODE)
     ;

packages/kbn-esql-ast/src/antlr/esql_lexer.tokens

@@ -22,7 +22,7 @@ UNKNOWN_CMD=21
 LINE_COMMENT=22
 MULTILINE_COMMENT=23
 WS=24
-INDEX_UNQUOTED_IDENTIFIER=25
+UNQUOTED_SOURCE=25
 EXPLAIN_WS=26
 EXPLAIN_LINE_COMMENT=27
 EXPLAIN_MULTILINE_COMMENT=28