[Security Solution] [Elastic AI Assistant] Adds internal Get Evaluate…

… API and migrates Post Evaluate API to OAS (elastic#176025) > [!IMPORTANT] > This PR is a reintroduction of elastic#175338, which was [reverted](elastic#175338 (comment)) due to sporadic jest failures within the `security_solution` plugin. Root cause was identified and detailed in elastic#176005 (comment). ## Summary In elastic#174317 we added support for OpenAPI codegen, this PR builds on that functionality by migrating the `Post Evaluate` route `/internal/elastic_assistant/evaluate` to be backed by an OAS, and adds a basic `Get Evaluate` route for rounding out the enhancements outlined in elastic/security-team#8167 (to be in a subsequent PR). Changes include: * Migration of `Post Evaluate` route to OAS * Migration of `Post Evaluate` route to use versioned router * Extracted `evaluate` API calls from `x-pack/packages/kbn-elastic-assistant/impl/assistant/api/api.tsx` to `x-pack/packages/kbn-elastic-assistant/impl/assistant/api/evaluate/evaluate.tsx` * Co-located relevant `use_perform_evaluation` hook * Adds `Get Evaluate` route, and corresponding `use_evaluation_data` hook. Currently only returns `agentExecutors` to be selected for evaluation. * API versioning constants added to `x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts` * Adds new `buildRouteValidationWithZod` function to `x-pack/plugins/elastic_assistant/server/schemas/common.ts` for validating routes against OAS generated zod schemas. ### Checklist Delete any items that are not applicable to this PR. - [X] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [X] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <[email protected]>
fkanout · Mar 4, 2024 · abe8a6e · abe8a6e
1 parent 61a5d3d
commit abe8a6e
Show file tree

Hide file tree

Showing 31 changed files with 901 additions and 425 deletions.
diff --git a/...k/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts b/...k/packages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.gen.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Evaluate API endpoint
+ *   version: 1
+ */
+
+export type GetEvaluateResponse = z.infer<typeof GetEvaluateResponse>;
+export const GetEvaluateResponse = z.object({
+  agentExecutors: z.array(z.string()),
+});
diff --git a/...kages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml b/...kages/kbn-elastic-assistant-common/impl/schemas/evaluation/get_evaluate_route.schema.yaml
@@ -0,0 +1,40 @@
+openapi: 3.0.0
+info:
+  title: Get Evaluate API endpoint
+  version: '1'
+paths:
+  /internal/elastic_assistant/evaluate:
+    get:
+      operationId: GetEvaluate
+      x-codegen-enabled: true
+      description: Get relevant data for performing an evaluation like available sample data, agents, and evaluators
+      summary: Get relevant data for performing an evaluation
+      tags:
+        - Evaluation API
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  agentExecutors:
+                    type: array
+                    items:
+                      type: string
+                required:
+                  - agentExecutors
+        '400':
+          description: Generic Error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  statusCode:
+                    type: number
+                  error:
+                    type: string
+                  message:
+                    type: string
diff --git a/.../packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts b/.../packages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.gen.ts
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from 'zod';
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Post Evaluate API endpoint
+ *   version: 1
+ */
+
+export type OutputIndex = z.infer<typeof OutputIndex>;
+export const OutputIndex = z.string().regex(/^.kibana-elastic-ai-assistant-/);
+
+export type DatasetItem = z.infer<typeof DatasetItem>;
+export const DatasetItem = z.object({
+  id: z.string().optional(),
+  input: z.string(),
+  prediction: z.string().optional(),
+  reference: z.string(),
+  tags: z.array(z.string()).optional(),
+});
+
+export type Dataset = z.infer<typeof Dataset>;
+export const Dataset = z.array(DatasetItem).default([]);
+
+export type PostEvaluateBody = z.infer<typeof PostEvaluateBody>;
+export const PostEvaluateBody = z.object({
+  dataset: Dataset.optional(),
+  evalPrompt: z.string().optional(),
+});
+
+export type PostEvaluateRequestQuery = z.infer<typeof PostEvaluateRequestQuery>;
+export const PostEvaluateRequestQuery = z.object({
+  /**
+   * Agents parameter description
+   */
+  agents: z.string(),
+  /**
+   * Dataset Name parameter description
+   */
+  datasetName: z.string().optional(),
+  /**
+   * Evaluation Type parameter description
+   */
+  evaluationType: z.string().optional(),
+  /**
+   * Eval Model parameter description
+   */
+  evalModel: z.string().optional(),
+  /**
+   * Models parameter description
+   */
+  models: z.string(),
+  /**
+   * Output Index parameter description
+   */
+  outputIndex: OutputIndex,
+  /**
+   * Project Name parameter description
+   */
+  projectName: z.string().optional(),
+  /**
+   * Run Name parameter description
+   */
+  runName: z.string().optional(),
+});
+export type PostEvaluateRequestQueryInput = z.input<typeof PostEvaluateRequestQuery>;
+
+export type PostEvaluateRequestBody = z.infer<typeof PostEvaluateRequestBody>;
+export const PostEvaluateRequestBody = PostEvaluateBody;
+export type PostEvaluateRequestBodyInput = z.input<typeof PostEvaluateRequestBody>;
+
+export type PostEvaluateResponse = z.infer<typeof PostEvaluateResponse>;
+export const PostEvaluateResponse = z.object({
+  evaluationId: z.string(),
+  success: z.boolean(),
+});
diff --git a/...ages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml b/...ages/kbn-elastic-assistant-common/impl/schemas/evaluation/post_evaluate_route.schema.yaml
@@ -0,0 +1,126 @@
+openapi: 3.0.0
+info:
+  title: Post Evaluate API endpoint
+  version: '1'
+paths:
+  /internal/elastic_assistant/evaluate:
+    post:
+      operationId: PostEvaluate
+      x-codegen-enabled: true
+      description: Perform an evaluation using sample data against a combination of Agents and Connectors
+      summary: Performs an evaluation of the Elastic Assistant
+      tags:
+        - Evaluation API
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/PostEvaluateBody'
+      parameters:
+        - name: agents
+          in: query
+          description: Agents parameter description
+          required: true
+          schema:
+            type: string
+        - name: datasetName
+          in: query
+          description: Dataset Name parameter description
+          schema:
+            type: string
+        - name: evaluationType
+          in: query
+          description: Evaluation Type parameter description
+          schema:
+            type: string
+        - name: evalModel
+          in: query
+          description: Eval Model parameter description
+          schema:
+            type: string
+        - name: models
+          in: query
+          description: Models parameter description
+          required: true
+          schema:
+            type: string
+        - name: outputIndex
+          in: query
+          description: Output Index parameter description
+          required: true
+          schema:
+            $ref: '#/components/schemas/OutputIndex'
+        - name: projectName
+          in: query
+          description: Project Name parameter description
+          schema:
+            type: string
+        - name: runName
+          in: query
+          description: Run Name parameter description
+          schema:
+            type: string
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  evaluationId:
+                    type: string
+                  success:
+                    type: boolean
+                required:
+                  - evaluationId
+                  - success
+        '400':
+          description: Generic Error
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  statusCode:
+                    type: number
+                  error:
+                    type: string
+                  message:
+                    type: string
+components:
+  schemas:
+    OutputIndex:
+      type: string
+      pattern: '^.kibana-elastic-ai-assistant-'
+    DatasetItem:
+      type: object
+      properties:
+        id:
+          type: string
+        input:
+          type: string
+        prediction:
+          type: string
+        reference:
+          type: string
+        tags:
+          type: array
+          items:
+            type: string
+      required:
+        - input
+        - reference
+    Dataset:
+      type: array
+      items:
+        $ref: '#/components/schemas/DatasetItem'
+      default: []
+    PostEvaluateBody:
+      type: object
+      properties:
+        dataset:
+          $ref: '#/components/schemas/Dataset'
+        evalPrompt:
+          type: string
diff --git a/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts b/x-pack/packages/kbn-elastic-assistant-common/impl/schemas/index.ts
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// API versioning constants
+export const API_VERSIONS = {
+  public: {
+    v1: '2023-10-31',
+  },
+  internal: {
+    v1: '1',
+  },
+};
+
+export const PUBLIC_API_ACCESS = 'public';
+export const INTERNAL_API_ACCESS = 'internal';
+
+// Evaluation Schemas
+export * from './evaluation/post_evaluate_route.gen';
+export * from './evaluation/get_evaluate_route.gen';
+
+// Capabilities Schemas
+export * from './capabilities/get_capabilities_route.gen';
diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts
@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen';
+// Schema constants
+export * from './impl/schemas';
 
 export { defaultAssistantFeatures } from './impl/capabilities';
 export type { AssistantFeatures } from './impl/capabilities';

diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx
@@ -13,7 +13,6 @@ import {
   fetchConnectorExecuteAction,
   FetchConnectorExecuteAction,
   getKnowledgeBaseStatus,
-  postEvaluation,
   postKnowledgeBase,
 } from './api';
 import type { Conversation, Message } from '../assistant_context/types';
@@ -340,52 +339,4 @@ describe('API tests', () => {
       await expect(deleteKnowledgeBase(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
     });
   });
-
-  describe('postEvaluation', () => {
-    it('calls the knowledge base API when correct resource path', async () => {
-      (mockHttp.fetch as jest.Mock).mockResolvedValue({ success: true });
-      const testProps = {
-        http: mockHttp,
-        evalParams: {
-          agents: ['not', 'alphabetical'],
-          dataset: '{}',
-          datasetName: 'Test Dataset',
-          projectName: 'Test Project Name',
-          runName: 'Test Run Name',
-          evalModel: ['not', 'alphabetical'],
-          evalPrompt: 'evalPrompt',
-          evaluationType: ['not', 'alphabetical'],
-          models: ['not', 'alphabetical'],
-          outputIndex: 'outputIndex',
-        },
-      };
-
-      await postEvaluation(testProps);
-
-      expect(mockHttp.fetch).toHaveBeenCalledWith('/internal/elastic_assistant/evaluate', {
-        method: 'POST',
-        body: '{"dataset":{},"evalPrompt":"evalPrompt"}',
-        headers: { 'Content-Type': 'application/json' },
-        query: {
-          models: 'alphabetical,not',
-          agents: 'alphabetical,not',
-          datasetName: 'Test Dataset',
-          evaluationType: 'alphabetical,not',
-          evalModel: 'alphabetical,not',
-          outputIndex: 'outputIndex',
-          projectName: 'Test Project Name',
-          runName: 'Test Run Name',
-        },
-        signal: undefined,
-      });
-    });
-    it('returns error when error is an error', async () => {
-      const error = 'simulated error';
-      (mockHttp.fetch as jest.Mock).mockImplementation(() => {
-        throw new Error(error);
-      });
-
-      await expect(postEvaluation(knowledgeBaseArgs)).resolves.toThrowError('simulated error');
-    });
-  });
 });