[8.x] [Security GenAI] Remove assistantNaturalLanguageESQLTool feat…

…ure flag and enable by default (elastic#195480) (elastic#195674)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Security GenAI] Remove `assistantNaturalLanguageESQLTool` feature
flag and enable by default
(elastic#195480)](elastic#195480)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Steph
Milovic","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-09T14:13:05Z","message":"[Security
GenAI] Remove `assistantNaturalLanguageESQLTool` feature flag and enable
by default
(elastic#195480)","sha":"3dd1ee8ae00a40b582f2d1f784c96269a1d2e016","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement","v9.0.0","Team:
SecuritySolution","backport:prev-minor","Feature:Security
Assistant","Team:Security Generative
AI","v8.16.0"],"number":195480,"url":"https://github.com/elastic/kibana/pull/195480","mergeCommit":{"message":"[Security
GenAI] Remove `assistantNaturalLanguageESQLTool` feature flag and enable
by default
(elastic#195480)","sha":"3dd1ee8ae00a40b582f2d1f784c96269a1d2e016"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195480","number":195480,"mergeCommit":{"message":"[Security
GenAI] Remove `assistantNaturalLanguageESQLTool` feature flag and enable
by default
(elastic#195480)","sha":"3dd1ee8ae00a40b582f2d1f784c96269a1d2e016"}},{"branch":"8.x","label":"v8.16.0","labelRegex":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

api_docs/kbn_elastic_assistant_common.devdocs.json

@@ -3413,7 +3413,7 @@
         "label": "ReadKnowledgeBaseResponse",
         "description": [],
         "signature": [
-          "{ elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; }"
+          "{ elser_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; }"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts",
         "deprecated": false,
@@ -5692,7 +5692,7 @@
         "label": "ReadKnowledgeBaseResponse",
         "description": [],
         "signature": [
-          "Zod.ZodObject<{ elser_exists: Zod.ZodOptional<Zod.ZodBoolean>; esql_exists: Zod.ZodOptional<Zod.ZodBoolean>; index_exists: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_available: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_in_progress: Zod.ZodOptional<Zod.ZodBoolean>; pipeline_exists: Zod.ZodOptional<Zod.ZodBoolean>; }, \"strip\", Zod.ZodTypeAny, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; }, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; }>"
+          "Zod.ZodObject<{ elser_exists: Zod.ZodOptional<Zod.ZodBoolean>; index_exists: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_available: Zod.ZodOptional<Zod.ZodBoolean>; is_setup_in_progress: Zod.ZodOptional<Zod.ZodBoolean>; pipeline_exists: Zod.ZodOptional<Zod.ZodBoolean>; }, \"strip\", Zod.ZodTypeAny, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; }, { elser_exists?: boolean | undefined; esql_exists?: boolean | undefined; index_exists?: boolean | undefined; is_setup_available?: boolean | undefined; is_setup_in_progress?: boolean | undefined; pipeline_exists?: boolean | undefined; }>"
         ],
         "path": "x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts",
         "deprecated": false,

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.gen.ts

@@ -76,7 +76,6 @@ export type ReadKnowledgeBaseRequestParamsInput = z.input<typeof ReadKnowledgeBa
 export type ReadKnowledgeBaseResponse = z.infer<typeof ReadKnowledgeBaseResponse>;
 export const ReadKnowledgeBaseResponse = z.object({
   elser_exists: z.boolean().optional(),
-  esql_exists: z.boolean().optional(),
   index_exists: z.boolean().optional(),
   is_setup_available: z.boolean().optional(),
   is_setup_in_progress: z.boolean().optional(),

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/knowledge_base/crud_kb_route.schema.yaml

@@ -68,8 +68,6 @@ paths:
                 properties:
                   elser_exists:
                     type: boolean
-                  esql_exists:
-                    type: boolean
                   index_exists:
                     type: boolean
                   is_setup_available:

x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_status.test.tsx

@@ -32,7 +32,6 @@ jest.mock('@tanstack/react-query', () => ({
 
 const statusResponse = {
   elser_exists: true,
-  esql_exists: true,
   index_exists: true,
   pipeline_exists: true,
 };

x-pack/packages/kbn-elastic-assistant/impl/assistant/api/knowledge_base/use_knowledge_base_status.tsx

@@ -89,7 +89,6 @@ export const useInvalidateKnowledgeBaseStatus = () => {
 export const isKnowledgeBaseSetup = (kbStatus: ReadKnowledgeBaseResponse | undefined): boolean => {
   return (
     (kbStatus?.elser_exists &&
-      kbStatus?.esql_exists &&
       kbStatus?.security_labs_exists &&
       kbStatus?.index_exists &&
       kbStatus?.pipeline_exists) ??

x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx

@@ -10,7 +10,6 @@ import { HttpSetup } from '@kbn/core-http-browser';
 import { i18n } from '@kbn/i18n';
 import { Replacements } from '@kbn/elastic-assistant-common';
 import { useKnowledgeBaseStatus } from '../api/knowledge_base/use_knowledge_base_status';
-import { ESQL_RESOURCE } from '../../knowledge_base/setup_knowledge_base_button';
 import { DataStreamApis } from '../use_data_stream_apis';
 import { NEW_CHAT } from '../conversations/conversation_sidepanel/translations';
 import type { ClientMessage } from '../../assistant_context/types';
@@ -58,12 +57,11 @@ export const useChatSend = ({
 
   const { isLoading, sendMessage, abortStream } = useSendMessage();
   const { clearConversation, removeLastMessage } = useConversation();
-  const { data: kbStatus } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+  const { data: kbStatus } = useKnowledgeBaseStatus({ http });
   const isSetupComplete =
     kbStatus?.elser_exists &&
     kbStatus?.index_exists &&
     kbStatus?.pipeline_exists &&
-    kbStatus?.esql_exists &&
     kbStatus?.security_labs_exists;
 
   // Handles sending latest user prompt to API

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx

@@ -69,7 +69,6 @@ jest.mock('../assistant/api/knowledge_base/use_knowledge_base_status', () => ({
     return {
       data: {
         elser_exists: true,
-        esql_exists: true,
         index_exists: true,
         pipeline_exists: true,
       },
@@ -83,22 +82,11 @@ describe('Knowledge base settings', () => {
   beforeEach(() => {
     jest.clearAllMocks();
   });
-  it('Shows correct description when esql is installed', () => {
-    const { getByTestId, queryByTestId } = render(
-      <TestProviders>
-        <KnowledgeBaseSettings {...defaultProps} />
-      </TestProviders>
-    );
-
-    expect(getByTestId('esql-installed')).toBeInTheDocument();
-    expect(queryByTestId('install-esql')).not.toBeInTheDocument();
-  });
   it('On enable knowledge base, call setup knowledge base setup', () => {
     (useKnowledgeBaseStatus as jest.Mock).mockImplementation(() => {
       return {
         data: {
           elser_exists: true,
-          esql_exists: false,
           index_exists: false,
           pipeline_exists: false,
           is_setup_available: true,
@@ -115,14 +103,13 @@ describe('Knowledge base settings', () => {
     expect(queryByTestId('kb-installed')).not.toBeInTheDocument();
     expect(getByTestId('install-kb')).toBeInTheDocument();
     fireEvent.click(getByTestId('setupKnowledgeBaseButton'));
-    expect(mockSetup).toHaveBeenCalledWith('esql');
+    expect(mockSetup).toHaveBeenCalled();
   });
   it('If elser does not exist, do not offer knowledge base', () => {
     (useKnowledgeBaseStatus as jest.Mock).mockImplementation(() => {
       return {
         data: {
           elser_exists: false,
-          esql_exists: false,
           index_exists: false,
           pipeline_exists: false,
         },

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx

@@ -31,7 +31,6 @@ import { useKnowledgeBaseStatus } from '../assistant/api/knowledge_base/use_know
 import { useSetupKnowledgeBase } from '../assistant/api/knowledge_base/use_setup_knowledge_base';
 import { SETUP_KNOWLEDGE_BASE_BUTTON_TOOLTIP } from './translations';
 
-const ESQL_RESOURCE = 'esql';
 const KNOWLEDGE_BASE_INDEX_PATTERN = '.kibana-elastic-ai-assistant-knowledge-base-(SPACE)';
 
 interface Props {
@@ -45,20 +44,14 @@ interface Props {
 export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
   ({ knowledgeBase, setUpdatedKnowledgeBaseSettings }) => {
     const { http, toasts } = useAssistantContext();
-    const {
-      data: kbStatus,
-      isLoading,
-      isFetching,
-    } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+    const { data: kbStatus, isLoading, isFetching } = useKnowledgeBaseStatus({ http });
     const { mutate: setupKB, isLoading: isSettingUpKB } = useSetupKnowledgeBase({ http, toasts });
 
     // Resource enabled state
     const isElserEnabled = kbStatus?.elser_exists ?? false;
-    const isESQLEnabled = kbStatus?.esql_exists ?? false;
     const isSecurityLabsEnabled = kbStatus?.security_labs_exists ?? false;
     const isKnowledgeBaseSetup =
       (isElserEnabled &&
-        isESQLEnabled &&
         isSecurityLabsEnabled &&
         kbStatus?.index_exists &&
         kbStatus?.pipeline_exists) ??
@@ -72,12 +65,11 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
     // Calculated health state for EuiHealth component
     const elserHealth = isElserEnabled ? 'success' : 'subdued';
     const knowledgeBaseHealth = isKnowledgeBaseSetup ? 'success' : 'subdued';
-    const esqlHealth = isESQLEnabled ? 'success' : 'subdued';
 
     //////////////////////////////////////////////////////////////////////////////////////////
     // Main `Knowledge Base` setup button
     const onSetupKnowledgeBaseButtonClick = useCallback(() => {
-      setupKB(ESQL_RESOURCE);
+      setupKB();
     }, [setupKB]);
 
     const toolTipContent = !isSetupAvailable ? SETUP_KNOWLEDGE_BASE_BUTTON_TOOLTIP : undefined;
@@ -119,16 +111,6 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
       );
     }, [isKnowledgeBaseSetup]);
 
-    //////////////////////////////////////////////////////////////////////////////////////////
-    // ESQL Resource
-    const esqlDescription = useMemo(() => {
-      return isESQLEnabled ? (
-        <span data-test-subj="esql-installed">{i18n.ESQL_DESCRIPTION_INSTALLED}</span>
-      ) : (
-        <span data-test-subj="install-esql">{i18n.ESQL_DESCRIPTION}</span>
-      );
-    }, [isESQLEnabled]);
-
     return (
       <>
         <EuiTitle size={'s'}>
@@ -208,20 +190,6 @@ export const KnowledgeBaseSettings: React.FC<Props> = React.memo(
               </EuiText>
             </div>
           </EuiFlexItem>
-          <EuiFlexItem grow={false}>
-            <span>
-              <EuiHealth color={esqlHealth}>{i18n.ESQL_LABEL}</EuiHealth>
-              <EuiText
-                size={'xs'}
-                color={'subdued'}
-                css={css`
-                  padding-left: 20px;
-                `}
-              >
-                {esqlDescription}
-              </EuiText>
-            </span>
-          </EuiFlexItem>
         </EuiFlexGroup>
 
         <EuiSpacer size="s" />

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings_management/index.tsx

@@ -46,7 +46,7 @@ import { useFlyoutModalVisibility } from '../../assistant/common/components/assi
 import { IndexEntryEditor } from './index_entry_editor';
 import { DocumentEntryEditor } from './document_entry_editor';
 import { KnowledgeBaseSettings } from '../knowledge_base_settings';
-import { ESQL_RESOURCE, SetupKnowledgeBaseButton } from '../setup_knowledge_base_button';
+import { SetupKnowledgeBaseButton } from '../setup_knowledge_base_button';
 import { useDeleteKnowledgeBaseEntries } from '../../assistant/api/knowledge_base/entries/use_delete_knowledge_base_entries';
 import {
   isSystemEntry,
@@ -73,7 +73,7 @@ export const KnowledgeBaseSettingsManagement: React.FC<Params> = React.memo(({ d
     toasts,
   } = useAssistantContext();
   const [hasPendingChanges, setHasPendingChanges] = useState(false);
-  const { data: kbStatus, isFetched } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+  const { data: kbStatus, isFetched } = useKnowledgeBaseStatus({ http });
   const isKbSetup = isKnowledgeBaseSetup(kbStatus);
 
   // Only needed for legacy settings management

x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/setup_knowledge_base_button.tsx

@@ -13,8 +13,6 @@ import { useAssistantContext } from '../..';
 import { useSetupKnowledgeBase } from '../assistant/api/knowledge_base/use_setup_knowledge_base';
 import { useKnowledgeBaseStatus } from '../assistant/api/knowledge_base/use_knowledge_base_status';
 
-export const ESQL_RESOURCE = 'esql';
-
 interface Props {
   display?: 'mini';
 }
@@ -26,19 +24,18 @@ interface Props {
 export const SetupKnowledgeBaseButton: React.FC<Props> = React.memo(({ display }: Props) => {
   const { http, toasts } = useAssistantContext();
 
-  const { data: kbStatus } = useKnowledgeBaseStatus({ http, resource: ESQL_RESOURCE });
+  const { data: kbStatus } = useKnowledgeBaseStatus({ http });
   const { mutate: setupKB, isLoading: isSettingUpKB } = useSetupKnowledgeBase({ http, toasts });
 
   const isSetupInProgress = kbStatus?.is_setup_in_progress || isSettingUpKB;
   const isSetupComplete =
     kbStatus?.elser_exists &&
     kbStatus?.index_exists &&
     kbStatus?.pipeline_exists &&
-    kbStatus?.esql_exists &&
     kbStatus?.security_labs_exists;
 
   const onInstallKnowledgeBase = useCallback(() => {
-    setupKB(ESQL_RESOURCE);
+    setupKB();
   }, [setupKB]);
 
   if (isSetupComplete) {