[Security GenAI] Remove assistantNaturalLanguageESQLTool feature fl…

…ag and enable by default (elastic#195480)

api_docs/kbn_elastic_assistant_common.devdocs.json

@@ -3443,7 +3443,7 @@
         "label": "ReadKnowledgeBaseResponse",
         "description": [],
         "signature": [
-          "{ elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; security_labs_exists?: boolean | undefined; }"
+          "{ elser_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; security_labs_exists?: boolean | undefined; }"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts",
         "deprecated": false,
@@ -5737,7 +5737,7 @@
         "label": "ReadKnowledgeBaseResponse",
         "description": [],
         "signature": [
-          "Zod.ZodObject<{ elser_exists: Zod.ZodOptional<Zod.ZodBoolean>; esql_exists: Zod.ZodOptional<Zod.ZodBoolean>; index_exists: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_available: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_in_progress: Zod.ZodOptional<Zod.ZodBoolean>; pipeline_exists: Zod.ZodOptional<Zod.ZodBoolean>; security_labs_exists: Zod.ZodOptional<Zod.ZodBoolean>; }, \"strip\", Zod.ZodTypeAny, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; security_labs_exists?: boolean | undefined; }, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; security_labs_exists?: boolean | undefined; }>"
+          "Zod.ZodObject<{ elser_exists: Zod.ZodOptional<Zod.ZodBoolean>; index_exists: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_available: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_in_progress: Zod.ZodOptional<Zod.ZodBoolean>; pipeline_exists: Zod.ZodOptional<Zod.ZodBoolean>; security_labs_exists: Zod.ZodOptional<Zod.ZodBoolean>; }, \"strip\", Zod.ZodTypeAny, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; security_labs_exists?: boolean | undefined; }, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; security_labs_exists?: boolean | undefined; }>"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts",
         "deprecated": false,

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts

@@ -76,7 +76,6 @@ export type ReadKnowledgeBaseRequestParamsInput = z.input<typeof ReadKnowledgeBa
 export type ReadKnowledgeBaseResponse = z.infer<typeof ReadKnowledgeBaseResponse>;
 export const ReadKnowledgeBaseResponse = z.object({
   elser_exists: z.boolean().optional(),
-  esql_exists: z.boolean().optional(),
   index_exists: z.boolean().optional(),
   is_setup_available: z.boolean().optional(),
   is_setup_in_progress: z.boolean().optional(),

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml

@@ -68,8 +68,6 @@ paths:
                 properties:
                   elser_exists:
                     type: boolean
-                  esql_exists:
-                    type: boolean
                   index_exists:
                     type: boolean
                   is_setup_available:

x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_status.test.tsx

@@ -32,7 +32,6 @@ jest.mock('@tanstack/react-query', () => ({
 
 const statusResponse = {
   elser_exists: true,
-  esql_exists: true,
   index_exists: true,
   pipeline_exists: true,
 };

x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_status.tsx

@@ -78,21 +78,3 @@ export const useInvalidateKnowledgeBaseStatus = () => {
     });
   }, [queryClient]);
 };
-
-/**
- * Helper for determining if Knowledge Base setup is complete.
- *
- * Note: Consider moving to API
- *
- * @param kbStatus ReadKnowledgeBaseResponse
- */
-export const isKnowledgeBaseSetup = (kbStatus: ReadKnowledgeBaseResponse | undefined): boolean => {
-  return (
-    (kbStatus?.elser_exists &&
-      kbStatus?.esql_exists &&
-      kbStatus?.security_labs_exists &&
-      kbStatus?.index_exists &&
-      kbStatus?.pipeline_exists) ??
-    false
-  );
-};

x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx

@@ -10,7 +10,6 @@ import { HttpSetup } from '@kbn/core-http-browser';
 import { i18n } from '@kbn/i18n';
 import { Replacements } from '@kbn/elastic-assistant-common';
 import { useKnowledgeBaseStatus } from '../api/knowledge_base/use_knowledge_base_status';
-import { ESQL_RESOURCE } from '../../knowledge_base/setup_knowledge_base_button';
 import { DataStreamApis } from '../use_data_stream_apis';
 import { NEW_CHAT } from '../conversations/conversation_sidepanel/translations';
 import type { ClientMessage } from '../../assistant_context/types';
@@ -58,12 +57,11 @@ export const useChatSend = ({
 
   const { isLoading, sendMessage, abortStream } = useSendMessage();
   const { clearConversation, removeLastMessage } = useConversation();
-  const { data: kbStatus } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+  const { data: kbStatus } = useKnowledgeBaseStatus({ http });
   const isSetupComplete =
     kbStatus?.elser_exists &&
     kbStatus?.index_exists &&
     kbStatus?.pipeline_exists &&
-    kbStatus?.esql_exists &&
     kbStatus?.security_labs_exists;
 
   // Handles sending latest user prompt to API

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx

@@ -69,7 +69,6 @@ jest.mock('../assistant/api/knowledge_base/use_knowledge_base_status', () => ({
     return {
       data: {
         elser_exists: true,
-        esql_exists: true,
         index_exists: true,
         pipeline_exists: true,
       },
@@ -83,22 +82,11 @@ describe('Knowledge base settings', () => {
   beforeEach(() => {
     jest.clearAllMocks();
   });
-  it('Shows correct description when esql is installed', () => {
-    const { getByTestId, queryByTestId } = render(
-      <TestProviders>
-        <KnowledgeBaseSettings {...defaultProps} />
-      </TestProviders>
-    );
-
-    expect(getByTestId('esql-installed')).toBeInTheDocument();
-    expect(queryByTestId('install-esql')).not.toBeInTheDocument();
-  });
   it('On enable knowledge base, call setup knowledge base setup', () => {
     (useKnowledgeBaseStatus as jest.Mock).mockImplementation(() => {
       return {
         data: {
           elser_exists: true,
-          esql_exists: false,
           index_exists: false,
           pipeline_exists: false,
           is_setup_available: true,
@@ -115,14 +103,13 @@ describe('Knowledge base settings', () => {
     expect(queryByTestId('kb-installed')).not.toBeInTheDocument();
     expect(getByTestId('install-kb')).toBeInTheDocument();
     fireEvent.click(getByTestId('setupKnowledgeBaseButton'));
-    expect(mockSetup).toHaveBeenCalledWith('esql');
+    expect(mockSetup).toHaveBeenCalled();
   });
   it('If elser does not exist, do not offer knowledge base', () => {
     (useKnowledgeBaseStatus as jest.Mock).mockImplementation(() => {
       return {
         data: {
           elser_exists: false,
-          esql_exists: false,
           index_exists: false,
           pipeline_exists: false,
         },

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx

@@ -31,7 +31,6 @@ import { useKnowledgeBaseStatus } from '../assistant/api/knowledge_base/use_know
 import { useSetupKnowledgeBase } from '../assistant/api/knowledge_base/use_setup_knowledge_base';
 import { SETUP_KNOWLEDGE_BASE_BUTTON_TOOLTIP } from './translations';
 
-const ESQL_RESOURCE = 'esql';
 const KNOWLEDGE_BASE_INDEX_PATTERN = '.kibana-elastic-ai-assistant-knowledge-base-(SPACE)';
 
 interface Props {
@@ -45,20 +44,14 @@ interface Props {
 export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
   ({ knowledgeBase, setUpdatedKnowledgeBaseSettings }) => {
     const { http, toasts } = useAssistantContext();
-    const {
-      data: kbStatus,
-      isLoading,
-      isFetching,
-    } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+    const { data: kbStatus, isLoading, isFetching } = useKnowledgeBaseStatus({ http });
     const { mutate: setupKB, isLoading: isSettingUpKB } = useSetupKnowledgeBase({ http, toasts });
 
     // Resource enabled state
     const isElserEnabled = kbStatus?.elser_exists ?? false;
-    const isESQLEnabled = kbStatus?.esql_exists ?? false;
     const isSecurityLabsEnabled = kbStatus?.security_labs_exists ?? false;
     const isKnowledgeBaseSetup =
       (isElserEnabled &&
-        isESQLEnabled &&
         isSecurityLabsEnabled &&
         kbStatus?.index_exists &&
         kbStatus?.pipeline_exists) ??
@@ -72,12 +65,11 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
     // Calculated health state for EuiHealth component
     const elserHealth = isElserEnabled ? 'success' : 'subdued';
     const knowledgeBaseHealth = isKnowledgeBaseSetup ? 'success' : 'subdued';
-    const esqlHealth = isESQLEnabled ? 'success' : 'subdued';
 
     //////////////////////////////////////////////////////////////////////////////////////////
     // Main `Knowledge Base` setup button
     const onSetupKnowledgeBaseButtonClick = useCallback(() => {
-      setupKB(ESQL_RESOURCE);
+      setupKB();
     }, [setupKB]);
 
     const toolTipContent = !isSetupAvailable ? SETUP_KNOWLEDGE_BASE_BUTTON_TOOLTIP : undefined;
@@ -119,16 +111,6 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
       );
     }, [isKnowledgeBaseSetup]);
 
-    //////////////////////////////////////////////////////////////////////////////////////////
-    // ESQL Resource
-    const esqlDescription = useMemo(() => {
-      return isESQLEnabled ? (
-        <span data-test-subj="esql-installed">{i18n.ESQL_DESCRIPTION_INSTALLED}</span>
-      ) : (
-        <span data-test-subj="install-esql">{i18n.ESQL_DESCRIPTION}</span>
-      );
-    }, [isESQLEnabled]);
-
     return (
       <>
         <EuiTitle size={'s'}>
@@ -208,20 +190,6 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
               </EuiText>
             </div>
           </EuiFlexItem>
-          <EuiFlexItem grow={false}>
-            <span>
-              <EuiHealth color={esqlHealth}>{i18n.ESQL_LABEL}</EuiHealth>
-              <EuiText
-                size={'xs'}
-                color={'subdued'}
-                css={css`
-                  padding-left: 20px;
-                `}
-              >
-                {esqlDescription}
-              </EuiText>
-            </span>
-          </EuiFlexItem>
         </EuiFlexGroup>
 
         <EuiSpacer size="s" />

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/setup_knowledge_base_button.tsx

@@ -13,8 +13,6 @@ import { useAssistantContext } from '../..';
 import { useSetupKnowledgeBase } from '../assistant/api/knowledge_base/use_setup_knowledge_base';
 import { useKnowledgeBaseStatus } from '../assistant/api/knowledge_base/use_knowledge_base_status';
 
-export const ESQL_RESOURCE = 'esql';
-
 interface Props {
   display?: 'mini';
 }
@@ -26,19 +24,18 @@ interface Props {
 export const SetupKnowledgeBaseButton: React.FC<Props> = React.memo(({ display }: Props) => {
   const { http, toasts } = useAssistantContext();
 
-  const { data: kbStatus } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+  const { data: kbStatus } = useKnowledgeBaseStatus({ http });
   const { mutate: setupKB, isLoading: isSettingUpKB } = useSetupKnowledgeBase({ http, toasts });
 
   const isSetupInProgress = kbStatus?.is_setup_in_progress || isSettingUpKB;
   const isSetupComplete =
     kbStatus?.elser_exists &&
     kbStatus?.index_exists &&
     kbStatus?.pipeline_exists &&
-    kbStatus?.esql_exists &&
     kbStatus?.security_labs_exists;
 
   const onInstallKnowledgeBase = useCallback(() => {
-    setupKB(ESQL_RESOURCE);
+    setupKB();
   }, [setupKB]);
 
   if (isSetupComplete) {

x-pack/plugins/elastic_assistant/server/__mocks__/docs_from_directory_loader.ts

@@ -8,43 +8,7 @@
 import { Document } from 'langchain/document';
 
 /**
- * Mock LangChain `Document`s from `knowledge_base/esql/documentation`, loaded from a LangChain `DirectoryLoader`
- */
-export const mockEsqlDocsFromDirectoryLoader: Document[] = [
-  {
-    pageContent:
-      '[[esql-agg-avg]]\n=== `AVG`\nThe average of a numeric field.\n\n[source.merge.styled,esql]\n----\ninclude::{esql-specs}/stats.csv-spec[tag=avg]\n----\n[%header.monospaced.styled,format=dsv,separator=|]\n|===\ninclude::{esql-specs}/stats.csv-spec[tag=avg-result]\n|===\n\nThe result is always a `double` not matter the input type.\n',
-    metadata: {
-      source:
-        '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/documentation/aggregation_functions/avg.asciidoc',
-    },
-  },
-];
-
-/**
- * Mock LangChain `Document`s from `knowledge_base/esql/language_definition`, loaded from a LangChain `DirectoryLoader`
- */
-export const mockEsqlLanguageDocsFromDirectoryLoader: Document[] = [
-  {
-    pageContent:
-      "lexer grammar EsqlBaseLexer;\n\nDISSECT : 'dissect' -> pushMode(EXPRESSION);\nDROP : 'drop' -> pushMode(SOURCE_IDENTIFIERS);\nENRICH : 'enrich' -> pushMode(SOURCE_IDENTIFIERS);\nEVAL : 'eval' -> pushMode(EXPRESSION);\nEXPLAIN : 'explain' -> pushMode(EXPLAIN_MODE);\nFROM : 'from' -> pushMode(SOURCE_IDENTIFIERS);\nGROK : 'grok' -> pushMode(EXPRESSION);\nINLINESTATS : 'inlinestats' -> pushMode(EXPRESSION);\nKEEP : 'keep' -> pushMode(SOURCE_IDENTIFIERS);\nLIMIT : 'limit' -> pushMode(EXPRESSION);\nMV_EXPAND : 'mv_expand' -> pushMode(SOURCE_IDENTIFIERS);\nPROJECT : 'project' -> pushMode(SOURCE_IDENTIFIERS);\nRENAME : 'rename' -> pushMode(SOURCE_IDENTIFIERS);\nROW : 'row' -> pushMode(EXPRESSION);\nSHOW : 'show' -> pushMode(EXPRESSION);\nSORT : 'sort' -> pushMode(EXPRESSION);\nSTATS : 'stats' -> pushMode(EXPRESSION);\nWHERE : 'where' -> pushMode(EXPRESSION);\nUNKNOWN_CMD : ~[ \\r\\n\\t[\\]/]+ -> pushMode(EXPRESSION);\n\nLINE_COMMENT\n    : '//' ~[\\r\\n]* '\\r'? '\\n'? -> channel(HIDDEN)\n    ;\n\nMULTILINE_COMMENT\n    : '/*' (MULTILINE_COMMENT|.)*? '*/' -> channel(HIDDEN)\n    ;\n\nWS\n    : [ \\r\\n\\t]+ -> channel(HIDDEN)\n    ;\n\n\nmode EXPLAIN_MODE;\nEXPLAIN_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);\nEXPLAIN_PIPE : '|' -> type(PIPE), popMode;\nEXPLAIN_WS : WS -> channel(HIDDEN);\nEXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);\nEXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);\n\nmode EXPRESSION;\n\nPIPE : '|' -> popMode;\n\nfragment DIGIT\n    : [0-9]\n    ;\n\nfragment LETTER\n    : [A-Za-z]\n    ;\n\nfragment ESCAPE_SEQUENCE\n    : '\\\\' [tnr\"\\\\]\n    ;\n\nfragment UNESCAPED_CHARS\n    : ~[\\r\\n\"\\\\]\n    ;\n\nfragment EXPONENT\n    : [Ee] [+-]? DIGIT+\n    ;\n\nSTRING\n    : '\"' (ESCAPE_SEQUENCE | UNESCAPED_CHARS)* '\"'\n    | '\"\"\"' (~[\\r\\n])*? '\"\"\"' '\"'? '\"'?\n    ;\n\nINTEGER_LITERAL\n    : DIGIT+\n    ;\n\nDECIMAL_LITERAL\n    : DIGIT+ DOT DIGIT*\n    | DOT DIGIT+\n    | DIGIT+ (DOT DIGIT*)? EXPONENT\n    | DOT DIGIT+ EXPONENT\n    ;\n\nBY : 'by';\n\nAND : 'and';\nASC : 'asc';\nASSIGN : '=';\nCOMMA : ',';\nDESC : 'desc';\nDOT : '.';\nFALSE : 'false';\nFIRST : 'first';\nLAST : 'last';\nLP : '(';\nIN: 'in';\nIS: 'is';\nLIKE: 'like';\nNOT : 'not';\nNULL : 'null';\nNULLS : 'nulls';\nOR : 'or';\nPARAM: '?';\nRLIKE: 'rlike';\nRP : ')';\nTRUE : 'true';\nINFO : 'info';\nFUNCTIONS : 'functions';\n\nEQ  : '==';\nNEQ : '!=';\nLT  : '<';\nLTE : '<=';\nGT  : '>';\nGTE : '>=';\n\nPLUS : '+';\nMINUS : '-';\nASTERISK : '*';\nSLASH : '/';\nPERCENT : '%';\n\n// Brackets are funny. We can happen upon a CLOSING_BRACKET in two ways - one\n// way is to start in an explain command which then shifts us to expression\n// mode. Thus, the two popModes on CLOSING_BRACKET. The other way could as\n// the start of a multivalued field constant. To line up with the double pop\n// the explain mode needs, we double push when we see that.\nOPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION);\nCLOSING_BRACKET : ']' -> popMode, popMode;\n\n\nUNQUOTED_IDENTIFIER\n    : LETTER (LETTER | DIGIT | '_')*\n    // only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future\n    // also, single `_` and `@` characters are not valid identifiers\n    | ('_' | '@') (LETTER | DIGIT | '_')+\n    ;\n\nQUOTED_IDENTIFIER\n    : '`' ( ~'`' | '``' )* '`'\n    ;\n\nEXPR_LINE_COMMENT\n    : LINE_COMMENT -> channel(HIDDEN)\n    ;\n\nEXPR_MULTILINE_COMMENT\n    : MULTILINE_COMMENT -> channel(HIDDEN)\n    ;\n\nEXPR_WS\n    : WS -> channel(HIDDEN)\n    ;\n\n\n\nmode SOURCE_IDENTIFIERS;\n\nSRC_PIPE : '|' -> type(PIPE), popMode;\nSRC_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(SOURCE_IDENTIFIERS), pushMode(SOURCE_IDENTIFIERS);\nSRC_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);\nSRC_COMMA : ',' -> type(COMMA);\nSRC_ASSIGN : '=' -> type(ASSIGN);\nAS : 'as';\nMETADATA: 'metadata';\nON : 'on';\nWITH : 'with';\n\nSRC_UNQUOTED_IDENTIFIER\n    : SRC_UNQUOTED_IDENTIFIER_PART+\n    ;\n\nfragment SRC_UNQUOTED_IDENTIFIER_PART\n    : ~[=`|,[\\]/ \\t\\r\\n]+\n    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment\n    ;\n\nSRC_QUOTED_IDENTIFIER\n    : QUOTED_IDENTIFIER\n    ;\n\nSRC_LINE_COMMENT\n    : LINE_COMMENT -> channel(HIDDEN)\n    ;\n\nSRC_MULTILINE_COMMENT\n    : MULTILINE_COMMENT -> channel(HIDDEN)\n    ;\n\nSRC_WS\n    : WS -> channel(HIDDEN)\n    ;\n",
-    metadata: {
-      source:
-        '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/language_definition/esql_base_lexer.g4',
-    },
-  },
-  {
-    pageContent:
-      "DISSECT=1\nDROP=2\nENRICH=3\nEVAL=4\nEXPLAIN=5\nFROM=6\nGROK=7\nINLINESTATS=8\nKEEP=9\nLIMIT=10\nMV_EXPAND=11\nPROJECT=12\nRENAME=13\nROW=14\nSHOW=15\nSORT=16\nSTATS=17\nWHERE=18\nUNKNOWN_CMD=19\nLINE_COMMENT=20\nMULTILINE_COMMENT=21\nWS=22\nEXPLAIN_WS=23\nEXPLAIN_LINE_COMMENT=24\nEXPLAIN_MULTILINE_COMMENT=25\nPIPE=26\nSTRING=27\nINTEGER_LITERAL=28\nDECIMAL_LITERAL=29\nBY=30\nAND=31\nASC=32\nASSIGN=33\nCOMMA=34\nDESC=35\nDOT=36\nFALSE=37\nFIRST=38\nLAST=39\nLP=40\nIN=41\nIS=42\nLIKE=43\nNOT=44\nNULL=45\nNULLS=46\nOR=47\nPARAM=48\nRLIKE=49\nRP=50\nTRUE=51\nINFO=52\nFUNCTIONS=53\nEQ=54\nNEQ=55\nLT=56\nLTE=57\nGT=58\nGTE=59\nPLUS=60\nMINUS=61\nASTERISK=62\nSLASH=63\nPERCENT=64\nOPENING_BRACKET=65\nCLOSING_BRACKET=66\nUNQUOTED_IDENTIFIER=67\nQUOTED_IDENTIFIER=68\nEXPR_LINE_COMMENT=69\nEXPR_MULTILINE_COMMENT=70\nEXPR_WS=71\nAS=72\nMETADATA=73\nON=74\nWITH=75\nSRC_UNQUOTED_IDENTIFIER=76\nSRC_QUOTED_IDENTIFIER=77\nSRC_LINE_COMMENT=78\nSRC_MULTILINE_COMMENT=79\nSRC_WS=80\nEXPLAIN_PIPE=81\n'dissect'=1\n'drop'=2\n'enrich'=3\n'eval'=4\n'explain'=5\n'from'=6\n'grok'=7\n'inlinestats'=8\n'keep'=9\n'limit'=10\n'mv_expand'=11\n'project'=12\n'rename'=13\n'row'=14\n'show'=15\n'sort'=16\n'stats'=17\n'where'=18\n'by'=30\n'and'=31\n'asc'=32\n'desc'=35\n'.'=36\n'false'=37\n'first'=38\n'last'=39\n'('=40\n'in'=41\n'is'=42\n'like'=43\n'not'=44\n'null'=45\n'nulls'=46\n'or'=47\n'?'=48\n'rlike'=49\n')'=50\n'true'=51\n'info'=52\n'functions'=53\n'=='=54\n'!='=55\n'<'=56\n'<='=57\n'>'=58\n'>='=59\n'+'=60\n'-'=61\n'*'=62\n'/'=63\n'%'=64\n']'=66\n'as'=72\n'metadata'=73\n'on'=74\n'with'=75\n",
-    metadata: {
-      source:
-        '/Users/andrew.goldstein/Projects/forks/andrew-goldstein/kibana/x-pack/plugins/elastic_assistant/server/knowledge_base/esql/language_definition/esql_base_lexer.tokens',
-    },
-  },
-];
-
-/**
- * Mock LangChain `Document`s from `knowledge_base/esql/example_queries`, loaded from a LangChain `DirectoryLoader`
+ * Mock LangChain `Document`s loaded from a LangChain `DirectoryLoader`
  */
 export const mockExampleQueryDocsFromDirectoryLoader: Document[] = [
   {