From 1c0ec85b740789ba99c30f09a5b56bc66a9de897 Mon Sep 17 00:00:00 2001
From: Garrett Spong <spong@users.noreply.github.com>
Date: Thu, 26 Sep 2024 12:17:50 -0600
Subject: [PATCH] [Security Assistant] Add support for manually entering eval
 datasets (#194072)

## Summary

On cloud environments we don't send the LangSmith credentials through
the `GET /evaluate` route which returns available datasets for
selection, so the datasets are never populated. Since the Dataset field
didn't allow custom options, this means you couldn't perform evals in
cloud environments.

This PR updates the Dataset field to take custom options so that you can
manually enter the dataset name in cloud environments:

<p align="center">
<img width="500"
src="https://github.com/user-attachments/assets/4828e085-180c-42bb-9656-34bda57b74b5"
/>
</p>


To test, enable the below feature flag to show the evaluation tab under
settings:

```
xpack.securitySolution.enableExperimental:
  - "assistantModelEvaluation"
```
---
 .../evaluation_settings/evaluation_settings.tsx   | 15 +++++++++++++++
 .../settings/evaluation_settings/translations.ts  |  3 ++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
index 319ee812a3cf3..cefc008eba992 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
@@ -95,6 +95,20 @@ export const EvaluationSettings: React.FC = React.memo(() => {
     },
     [setSelectedDatasetOptions]
   );
+  const onDatasetCreateOption = useCallback(
+    (searchValue: string) => {
+      const normalizedSearchValue = searchValue.trim().toLowerCase();
+      if (!normalizedSearchValue) {
+        return;
+      }
+      const newOption = {
+        label: searchValue,
+      };
+
+      setSelectedDatasetOptions([newOption]);
+    },
+    [setSelectedDatasetOptions]
+  );
 
   // Predictions
   // Connectors / Models
@@ -244,6 +258,7 @@ export const EvaluationSettings: React.FC = React.memo(() => {
             options={datasetOptions}
             selectedOptions={selectedDatasetOptions}
             onChange={onDatasetOptionsChange}
+            onCreateOption={onDatasetCreateOption}
             compressed={true}
           />
         </EuiFormRow>
diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts
index b1adb6296b2a1..62902d0f14095 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts
@@ -163,7 +163,8 @@ export const EVALUATOR_DATASET_LABEL = i18n.translate(
 export const LANGSMITH_DATASET_DESCRIPTION = i18n.translate(
   'xpack.elasticAssistant.assistant.settings.evaluationSettings.langsmithDatasetDescription',
   {
-    defaultMessage: 'Name of dataset hosted on LangSmith to evaluate.',
+    defaultMessage:
+      'Name of dataset hosted on LangSmith to evaluate. Must manually enter on cloud environments.',
   }
 );