filter presets and processors based on backend version (#537) (#556)

(cherry picked from commit ea08b7d)

Signed-off-by: Kama Huang <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Kama Huang <[email protected]>

common/constants.ts

@@ -169,6 +169,10 @@ export enum WORKFLOW_TYPE {
   CUSTOM = 'Custom',
   UNKNOWN = 'Unknown',
 }
+// If no datasource version is found, we default to 2.17.0
+export const MIN_SUPPORTED_VERSION = '2.17.0';
+// Min version to support ML processors
+export const MINIMUM_FULL_SUPPORTED_VERSION = '2.19.0';
 
 // the names should be consistent with the underlying implementation. used when generating the
 // final ingest/search pipeline configurations.
@@ -180,6 +184,8 @@ export enum PROCESSOR_TYPE {
   NORMALIZATION = 'normalization-processor',
   COLLAPSE = 'collapse',
   RERANK = 'rerank',
+  TEXT_EMBEDDING = 'text_embedding',
+  TEXT_IMAGE_EMBEDDING = 'text_image_embedding',
 }
 
 export enum MODEL_TYPE {

opensearch_dashboards.json

@@ -5,21 +5,13 @@
   "server": true,
   "ui": true,
   "requiredBundles": [],
-  "requiredPlugins": [
-    "navigation",
-    "opensearchDashboardsUtils"
-  ],
+  "requiredPlugins": ["navigation", "opensearchDashboardsUtils"],
   "optionalPlugins": [
     "dataSource",
     "dataSourceManagement",
     "contentManagement"
   ],
-  "supportedOSDataSourceVersions": ">=2.18.0 <4.0.0",
-  "requiredOSDataSourcePlugins": [
-    "opensearch-ml",
-    "opensearch-flow-framework"
-  ],
-  "configPath": [
-    "flowFrameworkDashboards"
-  ]
-}
+  "supportedOSDataSourceVersions": ">=2.17.0 <4.0.0",
+  "requiredOSDataSourcePlugins": ["opensearch-ml", "opensearch-flow-framework"],
+  "configPath": ["flowFrameworkDashboards"]
+}

public/configs/ingest_processors/index.ts

@@ -7,3 +7,5 @@ export * from './ml_ingest_processor';
 export * from './split_ingest_processor';
 export * from './sort_ingest_processor';
 export * from './text_chunking_ingest_processor';
+export * from './text_embedding_ingest_processor';
+export * from './text_image_embedding_ingest_processor';

public/configs/ingest_processors/text_embedding_ingest_processor.ts

@@ -0,0 +1,37 @@
+import { PROCESSOR_TYPE } from '../../../common';
+import { generateId } from '../../utils';
+import { Processor } from '../processor';
+
+export class TextEmbeddingIngestProcessor extends Processor {
+  constructor() {
+    super();
+    this.name = 'Text Embedding Processor';
+    this.type = PROCESSOR_TYPE.TEXT_EMBEDDING;
+    this.id = generateId('text_embedding_processor_ingest');
+    this.fields = [
+      {
+        id: 'model_id',
+        type: 'string',
+      },
+      {
+        id: 'field_map',
+        type: 'map',
+      },
+    ];
+    this.optionalFields = [
+      {
+        id: 'description',
+        type: 'string',
+      },
+      {
+        id: 'tag',
+        type: 'string',
+      },
+      {
+        id: 'batch_size',
+        type: 'number',
+        value: 1,
+      },
+    ];
+  }
+}

public/configs/ingest_processors/text_image_embedding_ingest_processor.ts

@@ -0,0 +1,36 @@
+import { PROCESSOR_TYPE } from '../../../common';
+import { generateId } from '../../utils';
+import { Processor } from '../processor';
+
+export class TextImageEmbeddingIngestProcessor extends Processor {
+  constructor() {
+    super();
+    this.name = 'Text Image Embedding Processor';
+    this.type = PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING;
+    this.id = generateId('text_image_embedding_processor_ingest');
+    this.fields = [
+      {
+        id: 'model_id',
+        type: 'string',
+      },
+      {
+        id: 'embedding',
+        type: 'string',
+      },
+      {
+        id: 'field_map',
+        type: 'map',
+      },
+    ];
+    this.optionalFields = [
+      {
+        id: 'description',
+        type: 'string',
+      },
+      {
+        id: 'tag',
+        type: 'string',
+      },
+    ];
+  }
+}

public/pages/workflow_detail/components/header.tsx

@@ -115,7 +115,7 @@ export function WorkflowDetailHeader(props: WorkflowDetailHeaderProps) {
 
   // get & render the data source component, if applicable
   let DataSourceComponent: ReactElement | null = null;
-  if (dataSourceEnabled && getDataSourceManagementPlugin()) {
+  if (dataSourceEnabled && getDataSourceManagementPlugin() && dataSourceId) {
     const DataSourceMenu = getDataSourceManagementPlugin().ui.getDataSourceMenu<
       DataSourceViewConfig
     >();

public/pages/workflow_detail/workflow_detail.test.tsx

@@ -13,7 +13,7 @@ import { WorkflowDetailRouterProps } from '../../pages';
 import '@testing-library/jest-dom';
 import { mockStore, resizeObserverMock } from '../../../test/utils';
 import { createMemoryHistory } from 'history';
-import { WORKFLOW_TYPE } from '../../../common';
+import { MINIMUM_FULL_SUPPORTED_VERSION, WORKFLOW_TYPE } from '../../../common';
 
 jest.mock('../../services', () => {
   const { mockCoreServices } = require('../../../test');
@@ -39,15 +39,22 @@ const renderWithRouter = (
     initialEntries: [`/workflow/${workflowId}`],
   });
 
+  const mockInput = {
+    id: workflowId,
+    name: workflowName,
+    type: workflowType,
+    version: [
+      WORKFLOW_TYPE.SEMANTIC_SEARCH,
+      WORKFLOW_TYPE.MULTIMODAL_SEARCH,
+      WORKFLOW_TYPE.HYBRID_SEARCH,
+    ].includes(workflowType)
+      ? MINIMUM_FULL_SUPPORTED_VERSION
+      : undefined,
+  };
+
   return {
     ...render(
-      <Provider
-        store={mockStore({
-          id: workflowId,
-          name: workflowName,
-          type: workflowType,
-        })}
-      >
+      <Provider store={mockStore(mockInput)}>
         <Router history={history}>
           <Switch>
             <Route
@@ -68,6 +75,7 @@ describe('WorkflowDetail Page with create ingestion option', () => {
   beforeEach(() => {
     jest.clearAllMocks();
   });
+
   Object.values(WORKFLOW_TYPE).forEach((type) => {
     test(`renders the WorkflowDetail page with ${type} type`, async () => {
       const {
@@ -110,33 +118,27 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => {
       workflowName,
       WORKFLOW_TYPE.CUSTOM
     );
-
     // Export button opens the export component
     userEvent.click(getByTestId('exportButton'));
     await waitFor(() => {
       expect(getByText(`Export '${workflowName}'`)).toBeInTheDocument();
     });
-
     // Close the export component
     userEvent.click(getByTestId('exportCloseButton'));
-
     // Check workspace button group exists (Visual and JSON)
     getByTestId('visualJSONToggleButtonGroup');
-
-    // Tools panel should collapse and expand on toggle
+    // Tools panel should collapse and expand the toggle
     const toolsPanel = container.querySelector('#tools_panel_id');
     expect(toolsPanel).toBeVisible();
 
     const toggleButton = toolsPanel?.querySelector('button[type="button"]');
     expect(toggleButton).toBeInTheDocument();
     userEvent.click(toggleButton!);
-
     // Tools panel after collapsing
     const collapsedToolsPanel = container.querySelector('#tools_panel_id');
     await waitFor(() => {
       expect(collapsedToolsPanel).toHaveClass('euiResizablePanel-isCollapsed');
     });
-
     // Tools panel after expanding
     userEvent.click(toggleButton!);
     const expandedToolsPanel = container.querySelector('#tools_panel_id');
@@ -153,7 +155,6 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => {
       workflowName,
       WORKFLOW_TYPE.CUSTOM
     );
-
     // The WorkflowDetail Page Close button should navigate back to the workflows list
     userEvent.click(getByTestId('closeButton'));
     await waitFor(() => {
@@ -166,57 +167,57 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow
   beforeEach(() => {
     jest.clearAllMocks();
   });
+
   test(`renders the WorkflowDetail page with skip ingestion option`, async () => {
     const { getByTestId, getAllByText, getAllByTestId } = renderWithRouter(
       workflowId,
       workflowName,
       WORKFLOW_TYPE.HYBRID_SEARCH
     );
-
     // Defining a new ingest pipeline & index is enabled by default
     const enabledCheckbox = getByTestId('switch-ingest.enabled');
-
     // Skipping ingest pipeline and navigating to search
     userEvent.click(enabledCheckbox);
     await waitFor(() => {});
+
     const searchPipelineButton = getByTestId('searchPipelineButton');
     userEvent.click(searchPipelineButton);
-
     // Search pipeline
     await waitFor(() => {
       expect(getAllByText('Define search flow').length).toBeGreaterThan(0);
     });
     expect(getAllByText('Configure query').length).toBeGreaterThan(0);
-
     // Edit Search Query
     const queryEditButton = getByTestId('queryEditButton');
     expect(queryEditButton).toBeInTheDocument();
     userEvent.click(queryEditButton);
+
     await waitFor(() => {
       expect(getAllByText('Edit query definition').length).toBeGreaterThan(0);
     });
+
     const searchQueryPresetButton = getByTestId('searchQueryPresetButton');
     expect(searchQueryPresetButton).toBeInTheDocument();
     const updateSearchQueryButton = getByTestId('updateSearchQueryButton');
     expect(updateSearchQueryButton).toBeInTheDocument();
     userEvent.click(updateSearchQueryButton);
-
     // Add request processor
     const addRequestProcessorButton = await waitFor(
       () => getAllByTestId('addProcessorButton')[0]
     );
     userEvent.click(addRequestProcessorButton);
+
     await waitFor(() => {
-      expect(getAllByText('PROCESSORS').length).toBeGreaterThan(0);
+      const popoverPanel = document.querySelector('.euiPopover__panel');
+      expect(popoverPanel).toBeTruthy();
     });
-
     // Add response processor
     const addResponseProcessorButton = getAllByTestId('addProcessorButton')[1];
     userEvent.click(addResponseProcessorButton);
     await waitFor(() => {
-      expect(getAllByText('PROCESSORS').length).toBeGreaterThan(0);
+      const popoverPanel = document.querySelector('.euiPopover__panel');
+      expect(popoverPanel).toBeTruthy();
     });
-
     // Build and Run query, Back buttons are present
     const searchPipelineBackButton = getByTestId('searchPipelineBackButton');
     userEvent.click(searchPipelineBackButton);

public/pages/workflow_detail/workflow_detail.tsx

@@ -126,7 +126,9 @@ export function WorkflowDetail(props: WorkflowDetailProps) {
   }, [USE_NEW_HOME_PAGE, dataSourceEnabled, dataSourceId, workflowName]);
 
   // form state
-  const [formValues, setFormValues] = useState<WorkflowFormValues>({});
+  const [formValues, setFormValues] = useState<WorkflowFormValues>(
+    {} as WorkflowFormValues
+  );
   const [formSchema, setFormSchema] = useState<WorkflowSchema>(yup.object({}));
 
   // ingest docs state. we need to persist here to update the form values.

public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx

@@ -8,7 +8,6 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import { ProcessorsList } from '../processors_list';
 import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common';
 import { ProcessorsTitle } from '../../../../general_components';
-
 interface EnrichDataProps {
   uiConfig: WorkflowConfig;
   setUiConfig: (uiConfig: WorkflowConfig) => void;