Merge branch 'main' into gated-agent-tamper-api

# Conflicts:
#	x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts

docs/settings/alert-action-settings.asciidoc

@@ -142,6 +142,15 @@ A list of action types that are enabled. It defaults to `["*"]`, enabling all ty
 +
 Disabled action types will not appear as an option when creating new connectors, but existing connectors and actions of that type will remain in {kib} and will not function.
 
+`xpack.actions.microsoftExchangeUrl`::
+The URL for the Microsoft Azure Active Directory endpoint to use for MS Exchange email authentication. Default: `https://login.microsoftonline.com`.
+
+`xpack.actions.microsoftGraphApiUrl`::
+The URL for the Microsoft Graph API endpoint to use for MS Exchange email authentication. Default: `https://graph.microsoft.com/v1.0`.
+
+`xpack.actions.microsoftGraphApiScope`::
+The URL for the Microsoft Graph API scope endpoint to use for MS Exchange email authentication. Default: `https://graph.microsoft.com/.default`.
+
 `xpack.actions.proxyUrl` {ess-icon}::
 Specifies the proxy URL to use, if using a proxy for actions. By default, no proxy is used.
 +

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Evaluate API endpoint
+ *   version: 1
+ */
+
+export type GetEvaluateResponse = z.infer<typeof GetEvaluateResponse>;
+export const GetEvaluateResponse = z.object({
+  agentExecutors: z.array(z.string()),
+});

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml

@@ -0,0 +1,40 @@
+openapi: 3.0.0
+info:
+  title: Get Evaluate API endpoint
+  version: '1'
+paths:
+  /internal/elastic_assistant/evaluate:
+    get:
+      operationId: GetEvaluate
+      x-codegen-enabled: true
+      description: Get relevant data for performing an evaluation like available sample data, agents, and evaluators
+      summary: Get relevant data for performing an evaluation
+      tags:
+        - Evaluation API
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  agentExecutors:
+                    type: array
+                    items:
+                      type: string
+                required:
+                  - agentExecutors
+        '400':
+          description: Generic Error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  statusCode:
+                    type: number
+                  error:
+                    type: string
+                  message:
+                    type: string

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Post Evaluate API endpoint
+ *   version: 1
+ */
+
+export type OutputIndex = z.infer<typeof OutputIndex>;
+export const OutputIndex = z.string().regex(/^.kibana-elastic-ai-assistant-/);
+
+export type DatasetItem = z.infer<typeof DatasetItem>;
+export const DatasetItem = z.object({
+  id: z.string().optional(),
+  input: z.string(),
+  prediction: z.string().optional(),
+  reference: z.string(),
+  tags: z.array(z.string()).optional(),
+});
+
+export type Dataset = z.infer<typeof Dataset>;
+export const Dataset = z.array(DatasetItem).default([]);
+
+export type PostEvaluateBody = z.infer<typeof PostEvaluateBody>;
+export const PostEvaluateBody = z.object({
+  dataset: Dataset.optional(),
+  evalPrompt: z.string().optional(),
+});
+
+export type PostEvaluateRequestQuery = z.infer<typeof PostEvaluateRequestQuery>;
+export const PostEvaluateRequestQuery = z.object({
+  /**
+   * Agents parameter description
+   */
+  agents: z.string(),
+  /**
+   * Dataset Name parameter description
+   */
+  datasetName: z.string().optional(),
+  /**
+   * Evaluation Type parameter description
+   */
+  evaluationType: z.string().optional(),
+  /**
+   * Eval Model parameter description
+   */
+  evalModel: z.string().optional(),
+  /**
+   * Models parameter description
+   */
+  models: z.string(),
+  /**
+   * Output Index parameter description
+   */
+  outputIndex: OutputIndex,
+  /**
+   * Project Name parameter description
+   */
+  projectName: z.string().optional(),
+  /**
+   * Run Name parameter description
+   */
+  runName: z.string().optional(),
+});
+export type PostEvaluateRequestQueryInput = z.input<typeof PostEvaluateRequestQuery>;
+
+export type PostEvaluateRequestBody = z.infer<typeof PostEvaluateRequestBody>;
+export const PostEvaluateRequestBody = PostEvaluateBody;
+export type PostEvaluateRequestBodyInput = z.input<typeof PostEvaluateRequestBody>;
+
+export type PostEvaluateResponse = z.infer<typeof PostEvaluateResponse>;
+export const PostEvaluateResponse = z.object({
+  evaluationId: z.string(),
+  success: z.boolean(),
+});

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml

@@ -0,0 +1,126 @@
+openapi: 3.0.0
+info:
+  title: Post Evaluate API endpoint
+  version: '1'
+paths:
+  /internal/elastic_assistant/evaluate:
+    post:
+      operationId: PostEvaluate
+      x-codegen-enabled: true
+      description: Perform an evaluation using sample data against a combination of Agents and Connectors
+      summary: Performs an evaluation of the Elastic Assistant
+      tags:
+        - Evaluation API
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/PostEvaluateBody'
+      parameters:
+        - name: agents
+          in: query
+          description: Agents parameter description
+          required: true
+          schema:
+            type: string
+        - name: datasetName
+          in: query
+          description: Dataset Name parameter description
+          schema:
+            type: string
+        - name: evaluationType
+          in: query
+          description: Evaluation Type parameter description
+          schema:
+            type: string
+        - name: evalModel
+          in: query
+          description: Eval Model parameter description
+          schema:
+            type: string
+        - name: models
+          in: query
+          description: Models parameter description
+          required: true
+          schema:
+            type: string
+        - name: outputIndex
+          in: query
+          description: Output Index parameter description
+          required: true
+          schema:
+            $ref: '#/components/schemas/OutputIndex'
+        - name: projectName
+          in: query
+          description: Project Name parameter description
+          schema:
+            type: string
+        - name: runName
+          in: query
+          description: Run Name parameter description
+          schema:
+            type: string
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  evaluationId:
+                    type: string
+                  success:
+                    type: boolean
+                required:
+                  - evaluationId
+                  - success
+        '400':
+          description: Generic Error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  statusCode:
+                    type: number
+                  error:
+                    type: string
+                  message:
+                    type: string
+components:
+  schemas:
+    OutputIndex:
+      type: string
+      pattern: '^.kibana-elastic-ai-assistant-'
+    DatasetItem:
+      type: object
+      properties:
+        id:
+          type: string
+        input:
+          type: string
+        prediction:
+          type: string
+        reference:
+          type: string
+        tags:
+          type: array
+          items:
+            type: string
+      required:
+        - input
+        - reference
+    Dataset:
+      type: array
+      items:
+        $ref: '#/components/schemas/DatasetItem'
+      default: []
+    PostEvaluateBody:
+      type: object
+      properties:
+        dataset:
+          $ref: '#/components/schemas/Dataset'
+        evalPrompt:
+          type: string

x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// API versioning constants
+export const API_VERSIONS = {
+  public: {
+    v1: '2023-10-31',
+  },
+  internal: {
+    v1: '1',
+  },
+};
+
+export const PUBLIC_API_ACCESS = 'public';
+export const INTERNAL_API_ACCESS = 'internal';
+
+// Evaluation Schemas
+export * from './evaluation/post_evaluate_route.gen';
+export * from './evaluation/get_evaluate_route.gen';
+
+// Capabilities Schemas
+export * from './capabilities/get_capabilities_route.gen';

x-pack/packages/kbn-elastic-assistant-common/index.ts

@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen';
+// Schema constants
+export * from './impl/schemas';
 
 export { defaultAssistantFeatures } from './impl/capabilities';
 export type { AssistantFeatures } from './impl/capabilities';

x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx

@@ -13,7 +13,6 @@ import {
   fetchConnectorExecuteAction,
   FetchConnectorExecuteAction,
   getKnowledgeBaseStatus,
-  postEvaluation,
   postKnowledgeBase,
 } from './api';
 import type { Conversation, Message } from '../assistant_context/types';
@@ -340,52 +339,4 @@ describe('API tests', () => {
       await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
     });
   });
-
-  describe('postEvaluation', () => {
-    it('calls the knowledge base API when correct resource path', async () => {
-      (mockHttp.fetch as jest.Mock).mockResolvedValue({ success: true });
-      const testProps = {
-        http: mockHttp,
-        evalParams: {
-          agents: ['not', 'alphabetical'],
-          dataset: '{}',
-          datasetName: 'Test Dataset',
-          projectName: 'Test Project Name',
-          runName: 'Test Run Name',
-          evalModel: ['not', 'alphabetical'],
-          evalPrompt: 'evalPrompt',
-          evaluationType: ['not', 'alphabetical'],
-          models: ['not', 'alphabetical'],
-          outputIndex: 'outputIndex',
-        },
-      };
-
-      await postEvaluation(testProps);
-
-      expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
-        method: 'POST',
-        body: '{"dataset":{},"evalPrompt":"evalPrompt"}',
-        headers: { 'Content-Type': 'application/json' },
-        query: {
-          models: 'alphabetical,not',
-          agents: 'alphabetical,not',
-          datasetName: 'Test Dataset',
-          evaluationType: 'alphabetical,not',
-          evalModel: 'alphabetical,not',
-          outputIndex: 'outputIndex',
-          projectName: 'Test Project Name',
-          runName: 'Test Run Name',
-        },
-        signal: undefined,
-      });
-    });
-    it('returns error when error is an error', async () => {
-      const error = 'simulated error';
-      (mockHttp.fetch as jest.Mock).mockImplementation(() => {
-        throw new Error(error);
-      });
-
-      await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
-    });
-  });
 });