diff --git a/common/constants.ts b/common/constants.ts index d55b6019..f9adec16 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -169,6 +169,10 @@ export enum WORKFLOW_TYPE { CUSTOM = 'Custom', UNKNOWN = 'Unknown', } +// If no datasource version is found, we default to 2.17.0 +export const MIN_SUPPORTED_VERSION = '2.17.0'; +// Min version to support ML processors +export const MINIMUM_FULL_SUPPORTED_VERSION = '2.19.0'; // the names should be consistent with the underlying implementation. used when generating the // final ingest/search pipeline configurations. @@ -180,6 +184,8 @@ export enum PROCESSOR_TYPE { NORMALIZATION = 'normalization-processor', COLLAPSE = 'collapse', RERANK = 'rerank', + TEXT_EMBEDDING = 'text_embedding', + TEXT_IMAGE_EMBEDDING = 'text_image_embedding', } export enum MODEL_TYPE { diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index 47253592..ecf6e31a 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -5,21 +5,13 @@ "server": true, "ui": true, "requiredBundles": [], - "requiredPlugins": [ - "navigation", - "opensearchDashboardsUtils" - ], + "requiredPlugins": ["navigation", "opensearchDashboardsUtils"], "optionalPlugins": [ "dataSource", "dataSourceManagement", "contentManagement" ], - "supportedOSDataSourceVersions": ">=2.18.0 <4.0.0", - "requiredOSDataSourcePlugins": [ - "opensearch-ml", - "opensearch-flow-framework" - ], - "configPath": [ - "flowFrameworkDashboards" - ] -} \ No newline at end of file + "supportedOSDataSourceVersions": ">=2.17.0 <4.0.0", + "requiredOSDataSourcePlugins": ["opensearch-ml", "opensearch-flow-framework"], + "configPath": ["flowFrameworkDashboards"] +} diff --git a/public/configs/ingest_processors/index.ts b/public/configs/ingest_processors/index.ts index 5b4c680f..559def2e 100644 --- a/public/configs/ingest_processors/index.ts +++ b/public/configs/ingest_processors/index.ts @@ -7,3 +7,5 @@ export * from './ml_ingest_processor'; export * from './split_ingest_processor'; export * from './sort_ingest_processor'; export * from './text_chunking_ingest_processor'; +export * from './text_embedding_ingest_processor'; +export * from './text_image_embedding_ingest_processor'; diff --git a/public/configs/ingest_processors/text_embedding_ingest_processor.ts b/public/configs/ingest_processors/text_embedding_ingest_processor.ts new file mode 100644 index 00000000..acf1ef04 --- /dev/null +++ b/public/configs/ingest_processors/text_embedding_ingest_processor.ts @@ -0,0 +1,37 @@ +import { PROCESSOR_TYPE } from '../../../common'; +import { generateId } from '../../utils'; +import { Processor } from '../processor'; + +export class TextEmbeddingIngestProcessor extends Processor { + constructor() { + super(); + this.name = 'Text Embedding Processor'; + this.type = PROCESSOR_TYPE.TEXT_EMBEDDING; + this.id = generateId('text_embedding_processor_ingest'); + this.fields = [ + { + id: 'model_id', + type: 'string', + }, + { + id: 'field_map', + type: 'map', + }, + ]; + this.optionalFields = [ + { + id: 'description', + type: 'string', + }, + { + id: 'tag', + type: 'string', + }, + { + id: 'batch_size', + type: 'number', + value: 1, + }, + ]; + } +} diff --git a/public/configs/ingest_processors/text_image_embedding_ingest_processor.ts b/public/configs/ingest_processors/text_image_embedding_ingest_processor.ts new file mode 100644 index 00000000..64c0cc3f --- /dev/null +++ b/public/configs/ingest_processors/text_image_embedding_ingest_processor.ts @@ -0,0 +1,36 @@ +import { PROCESSOR_TYPE } from '../../../common'; +import { generateId } from '../../utils'; +import { Processor } from '../processor'; + +export class TextImageEmbeddingIngestProcessor extends Processor { + constructor() { + super(); + this.name = 'Text Image Embedding Processor'; + this.type = PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING; + this.id = generateId('text_image_embedding_processor_ingest'); + this.fields = [ + { + id: 'model_id', + type: 'string', + }, + { + id: 'embedding', + type: 'string', + }, + { + id: 'field_map', + type: 'map', + }, + ]; + this.optionalFields = [ + { + id: 'description', + type: 'string', + }, + { + id: 'tag', + type: 'string', + }, + ]; + } +} diff --git a/public/pages/workflow_detail/components/header.tsx b/public/pages/workflow_detail/components/header.tsx index ca1edd29..06fa7250 100644 --- a/public/pages/workflow_detail/components/header.tsx +++ b/public/pages/workflow_detail/components/header.tsx @@ -115,7 +115,7 @@ export function WorkflowDetailHeader(props: WorkflowDetailHeaderProps) { // get & render the data source component, if applicable let DataSourceComponent: ReactElement | null = null; - if (dataSourceEnabled && getDataSourceManagementPlugin()) { + if (dataSourceEnabled && getDataSourceManagementPlugin() && dataSourceId) { const DataSourceMenu = getDataSourceManagementPlugin().ui.getDataSourceMenu< DataSourceViewConfig >(); diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index 3e7bd535..29489ce0 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -13,7 +13,7 @@ import { WorkflowDetailRouterProps } from '../../pages'; import '@testing-library/jest-dom'; import { mockStore, resizeObserverMock } from '../../../test/utils'; import { createMemoryHistory } from 'history'; -import { WORKFLOW_TYPE } from '../../../common'; +import { MINIMUM_FULL_SUPPORTED_VERSION, WORKFLOW_TYPE } from '../../../common'; jest.mock('../../services', () => { const { mockCoreServices } = require('../../../test'); @@ -39,15 +39,22 @@ const renderWithRouter = ( initialEntries: [`/workflow/${workflowId}`], }); + const mockInput = { + id: workflowId, + name: workflowName, + type: workflowType, + version: [ + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, + ].includes(workflowType) + ? MINIMUM_FULL_SUPPORTED_VERSION + : undefined, + }; + return { ...render( - + { beforeEach(() => { jest.clearAllMocks(); }); + Object.values(WORKFLOW_TYPE).forEach((type) => { test(`renders the WorkflowDetail page with ${type} type`, async () => { const { @@ -110,33 +118,27 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { workflowName, WORKFLOW_TYPE.CUSTOM ); - // Export button opens the export component userEvent.click(getByTestId('exportButton')); await waitFor(() => { expect(getByText(`Export '${workflowName}'`)).toBeInTheDocument(); }); - // Close the export component userEvent.click(getByTestId('exportCloseButton')); - // Check workspace button group exists (Visual and JSON) getByTestId('visualJSONToggleButtonGroup'); - - // Tools panel should collapse and expand on toggle + // Tools panel should collapse and expand the toggle const toolsPanel = container.querySelector('#tools_panel_id'); expect(toolsPanel).toBeVisible(); const toggleButton = toolsPanel?.querySelector('button[type="button"]'); expect(toggleButton).toBeInTheDocument(); userEvent.click(toggleButton!); - // Tools panel after collapsing const collapsedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { expect(collapsedToolsPanel).toHaveClass('euiResizablePanel-isCollapsed'); }); - // Tools panel after expanding userEvent.click(toggleButton!); const expandedToolsPanel = container.querySelector('#tools_panel_id'); @@ -153,7 +155,6 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { workflowName, WORKFLOW_TYPE.CUSTOM ); - // The WorkflowDetail Page Close button should navigate back to the workflows list userEvent.click(getByTestId('closeButton')); await waitFor(() => { @@ -166,57 +167,57 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow beforeEach(() => { jest.clearAllMocks(); }); + test(`renders the WorkflowDetail page with skip ingestion option`, async () => { const { getByTestId, getAllByText, getAllByTestId } = renderWithRouter( workflowId, workflowName, WORKFLOW_TYPE.HYBRID_SEARCH ); - // Defining a new ingest pipeline & index is enabled by default const enabledCheckbox = getByTestId('switch-ingest.enabled'); - // Skipping ingest pipeline and navigating to search userEvent.click(enabledCheckbox); await waitFor(() => {}); + const searchPipelineButton = getByTestId('searchPipelineButton'); userEvent.click(searchPipelineButton); - // Search pipeline await waitFor(() => { expect(getAllByText('Define search flow').length).toBeGreaterThan(0); }); expect(getAllByText('Configure query').length).toBeGreaterThan(0); - // Edit Search Query const queryEditButton = getByTestId('queryEditButton'); expect(queryEditButton).toBeInTheDocument(); userEvent.click(queryEditButton); + await waitFor(() => { expect(getAllByText('Edit query definition').length).toBeGreaterThan(0); }); + const searchQueryPresetButton = getByTestId('searchQueryPresetButton'); expect(searchQueryPresetButton).toBeInTheDocument(); const updateSearchQueryButton = getByTestId('updateSearchQueryButton'); expect(updateSearchQueryButton).toBeInTheDocument(); userEvent.click(updateSearchQueryButton); - // Add request processor const addRequestProcessorButton = await waitFor( () => getAllByTestId('addProcessorButton')[0] ); userEvent.click(addRequestProcessorButton); + await waitFor(() => { - expect(getAllByText('PROCESSORS').length).toBeGreaterThan(0); + const popoverPanel = document.querySelector('.euiPopover__panel'); + expect(popoverPanel).toBeTruthy(); }); - // Add response processor const addResponseProcessorButton = getAllByTestId('addProcessorButton')[1]; userEvent.click(addResponseProcessorButton); await waitFor(() => { - expect(getAllByText('PROCESSORS').length).toBeGreaterThan(0); + const popoverPanel = document.querySelector('.euiPopover__panel'); + expect(popoverPanel).toBeTruthy(); }); - // Build and Run query, Back buttons are present const searchPipelineBackButton = getByTestId('searchPipelineBackButton'); userEvent.click(searchPipelineBackButton); diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 80c28ea9..6f46b953 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -126,7 +126,9 @@ export function WorkflowDetail(props: WorkflowDetailProps) { }, [USE_NEW_HOME_PAGE, dataSourceEnabled, dataSourceId, workflowName]); // form state - const [formValues, setFormValues] = useState({}); + const [formValues, setFormValues] = useState( + {} as WorkflowFormValues + ); const [formSchema, setFormSchema] = useState(yup.object({})); // ingest docs state. we need to persist here to update the form values. diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx index ae184a9e..3aca93fc 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx @@ -8,7 +8,6 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsList } from '../processors_list'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsTitle } from '../../../../general_components'; - interface EnrichDataProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index b85909a4..ead3617d 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -4,6 +4,8 @@ */ import React, { useEffect, useState } from 'react'; +import semver from 'semver'; +import { getEffectiveVersion } from '../../../pages/workflows/new_workflow/new_workflow'; import { EuiSmallButtonEmpty, EuiSmallButtonIcon, @@ -23,7 +25,8 @@ import { WorkflowConfig, WorkflowFormValues, } from '../../../../common'; -import { formikToUiConfig } from '../../../utils'; +import { formikToUiConfig, getDataSourceFromURL } from '../../../utils'; + import { CollapseProcessor, MLIngestProcessor, @@ -36,8 +39,16 @@ import { SplitIngestProcessor, SplitSearchResponseProcessor, TextChunkingIngestProcessor, + TextEmbeddingIngestProcessor, + TextImageEmbeddingIngestProcessor, } from '../../../configs'; import { ProcessorInputs } from './processor_inputs'; +import { useLocation } from 'react-router-dom'; +import { getDataSourceEnabled } from '../../../../public/services'; +import { + MIN_SUPPORTED_VERSION, + MINIMUM_FULL_SUPPORTED_VERSION, +} from '../../../../common'; interface ProcessorsListProps { uiConfig: WorkflowConfig; @@ -52,31 +63,184 @@ const PANEL_ID = 0; */ export function ProcessorsList(props: ProcessorsListProps) { const { values } = useFormikContext(); - - // Processor added state. Used to automatically open accordion when a new - // processor is added, assuming users want to immediately configure it. + const [version, setVersion] = useState(''); + const location = useLocation(); const [processorAdded, setProcessorAdded] = useState(false); - - // Popover state when adding new processors const [isPopoverOpen, setPopover] = useState(false); + const [processors, setProcessors] = useState([]); + const closePopover = () => { setPopover(false); }; - // Current processors state - const [processors, setProcessors] = useState([]); + const handlePopoverClick = () => { + setPopover(!isPopoverOpen); + }; + useEffect(() => { - if (props.uiConfig && props.context) { - setProcessors( - props.context === PROCESSOR_CONTEXT.INGEST - ? props.uiConfig.ingest.enrich.processors - : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST - ? props.uiConfig.search.enrichRequest.processors - : props.uiConfig.search.enrichResponse.processors - ); + const dataSourceId = getDataSourceFromURL(location).dataSourceId; + + const enabled = getDataSourceEnabled().enabled; + if (!enabled) { + setVersion(MINIMUM_FULL_SUPPORTED_VERSION); + return; } + + if (dataSourceId) { + getEffectiveVersion(dataSourceId) + .then((ver) => { + setVersion(ver); + }) + .catch(console.error); + } + }, [location]); + + useEffect(() => { + const loadProcessors = async () => { + if (props.uiConfig && props.context) { + let currentProcessors = + props.context === PROCESSOR_CONTEXT.INGEST + ? props.uiConfig.ingest.enrich.processors + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? props.uiConfig.search.enrichRequest.processors + : props.uiConfig.search.enrichResponse.processors; + + setProcessors(currentProcessors || []); + } + }; + + loadProcessors(); }, [props.context, props.uiConfig]); + const getMenuItems = () => { + const isPreV219 = + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + const ingestProcessors = [ + ...(isPreV219 + ? [ + { + name: 'Text Embedding Processor', + onClick: () => { + closePopover(); + addProcessor(new TextEmbeddingIngestProcessor().toObj()); + }, + }, + { + name: 'Text Image Embedding Processor', + onClick: () => { + closePopover(); + addProcessor(new TextImageEmbeddingIngestProcessor().toObj()); + }, + }, + ] + : [ + { + name: 'ML Inference Processor', + onClick: () => { + closePopover(); + addProcessor(new MLIngestProcessor().toObj()); + }, + }, + ]), + { + name: 'Split Processor', + onClick: () => { + closePopover(); + addProcessor(new SplitIngestProcessor().toObj()); + }, + }, + { + name: 'Sort Processor', + onClick: () => { + closePopover(); + addProcessor(new SortIngestProcessor().toObj()); + }, + }, + { + name: 'Text Chunking Processor', + onClick: () => { + closePopover(); + addProcessor(new TextChunkingIngestProcessor().toObj()); + }, + }, + ]; + + const searchRequestProcessors = [ + ...(!isPreV219 + ? [ + { + name: 'ML Inference Processor', + onClick: () => { + closePopover(); + addProcessor(new MLSearchRequestProcessor().toObj()); + }, + }, + ] + : []), + ]; + + const searchResponseProcessors = [ + ...(!isPreV219 + ? [ + { + name: 'ML Inference Processor', + onClick: () => { + closePopover(); + addProcessor(new MLSearchResponseProcessor().toObj()); + }, + }, + ] + : []), + { + name: 'Rerank Processor', + onClick: () => { + closePopover(); + addProcessor(new RerankProcessor().toObj()); + }, + }, + { + name: 'Split Processor', + onClick: () => { + closePopover(); + addProcessor(new SplitSearchResponseProcessor().toObj()); + }, + }, + { + name: 'Sort Processor', + onClick: () => { + closePopover(); + addProcessor(new SortSearchResponseProcessor().toObj()); + }, + }, + { + name: 'Normalization Processor', + onClick: () => { + closePopover(); + addProcessor(new NormalizationProcessor().toObj()); + }, + }, + { + name: 'Collapse Processor', + onClick: () => { + closePopover(); + addProcessor(new CollapseProcessor().toObj()); + }, + }, + ]; + + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: + return ingestProcessors; + case PROCESSOR_CONTEXT.SEARCH_REQUEST: + return searchRequestProcessors; + case PROCESSOR_CONTEXT.SEARCH_RESPONSE: + return searchResponseProcessors; + default: + return []; + } + }; + // Adding a processor to the config. Fetch the existing one // (getting any updated/interim values along the way) and add to // the list of processors @@ -197,9 +361,7 @@ export function ProcessorsList(props: ProcessorsListProps) { { - setPopover(!isPopoverOpen); - }} + onClick={handlePopoverClick} data-testid="addProcessorButton" > {`Add processor`} @@ -210,118 +372,22 @@ export function ProcessorsList(props: ProcessorsListProps) { panelPaddingSize="none" anchorPosition="downLeft" > - { - closePopover(); - addProcessor(new MLIngestProcessor().toObj()); - }, - }, - { - name: 'Split Processor', - onClick: () => { - closePopover(); - addProcessor( - new SplitIngestProcessor().toObj() - ); - }, - }, - { - name: 'Sort Processor', - onClick: () => { - closePopover(); - addProcessor( - new SortIngestProcessor().toObj() - ); - }, - }, - { - name: 'Text Chunking Processor', - onClick: () => { - closePopover(); - addProcessor( - new TextChunkingIngestProcessor().toObj() - ); - }, - }, - ] - : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST - ? [ - { - name: 'ML Inference Processor', - onClick: () => { - closePopover(); - addProcessor( - new MLSearchRequestProcessor().toObj() - ); - }, - }, - ] - : [ - { - name: 'ML Inference Processor', - onClick: () => { - closePopover(); - addProcessor( - new MLSearchResponseProcessor().toObj() - ); - }, - }, - { - name: 'Rerank Processor', - onClick: () => { - closePopover(); - addProcessor(new RerankProcessor().toObj()); - }, - }, - { - name: 'Split Processor', - onClick: () => { - closePopover(); - addProcessor( - new SplitSearchResponseProcessor().toObj() - ); - }, - }, - { - name: 'Sort Processor', - onClick: () => { - closePopover(); - addProcessor( - new SortSearchResponseProcessor().toObj() - ); - }, - }, - { - name: 'Normalization Processor', - onClick: () => { - closePopover(); - addProcessor( - new NormalizationProcessor().toObj() - ); - }, - }, - { - name: 'Collapse Processor', - onClick: () => { - closePopover(); - addProcessor(new CollapseProcessor().toObj()); - }, - }, - ], - }, - ]} - /> + {version && ( + 0 ? 'PROCESSORS' : '', + items: (() => { + const items = getMenuItems(); + return items; + })(), + }, + ]} + /> + )} diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index fc1485ea..7a0490ba 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -53,8 +53,6 @@ import { getDataSourceId, } from '../../../utils'; import { BooleanField } from './input_fields'; - -// styling import '../workspace/workspace-styles.scss'; interface WorkflowInputsProps { diff --git a/public/pages/workflows/new_workflow/new_workflow.test.tsx b/public/pages/workflows/new_workflow/new_workflow.test.tsx index 14a609f9..0a40841a 100644 --- a/public/pages/workflows/new_workflow/new_workflow.test.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.test.tsx @@ -4,8 +4,9 @@ */ import React from 'react'; +import { MemoryRouter as Router } from 'react-router-dom'; // Change this import import { Provider } from 'react-redux'; -import { BrowserRouter as Router, Route, Switch } from 'react-router-dom'; +import { Route, Switch } from 'react-router-dom'; import { NewWorkflow } from './new_workflow'; import { render, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -15,6 +16,11 @@ import '@testing-library/jest-dom'; import { loadPresetWorkflowTemplates } from '../../../../test/utils'; import { INITIAL_ML_STATE } from '../../../../public/store'; +jest.mock('../../../utils', () => ({ + ...jest.requireActual('../../../utils'), + getDataSourceId: () => '123', +})); + jest.mock('../../../services', () => { const { mockCoreServices } = require('../../../../test'); return { @@ -31,71 +37,95 @@ const initialState = { presetWorkflows: loadPresetWorkflowTemplates(), }, }; -const store = mockStore(initialState); const mockDispatch = jest.fn(); -const renderWithRouter = () => +const renderWithRouter = (store: any) => render( - + } /> ); - describe('NewWorkflow', () => { beforeEach(() => { jest.clearAllMocks(); jest.spyOn(ReactReduxHooks, 'useAppDispatch').mockReturnValue(mockDispatch); }); - test('renders the preset workflow names & descriptions', () => { + test('renders the preset workflow names & descriptions', async () => { + const store = mockStore(initialState); const presetWorkflows = loadPresetWorkflowTemplates(); - const { getByPlaceholderText, getAllByText } = renderWithRouter(); + const { getByPlaceholderText, getByText } = renderWithRouter(store); + expect(getByPlaceholderText('Search')).toBeInTheDocument(); - presetWorkflows.forEach((workflow) => { - expect(getAllByText(workflow.name)).toHaveLength(1); - expect(getAllByText(workflow.description)).toHaveLength(1); + + await waitFor(() => { + presetWorkflows.forEach((workflow) => { + if ( + workflow.name === + ['Semantic Search', 'Multimodal Search', 'Hybrid Search'].includes( + workflow.name + ) + ) { + expect(getByText(workflow.name)).toBeInTheDocument(); + expect(getByText(workflow.description)).toBeInTheDocument(); + } + }); }); }); test('renders the quick configure for preset workflow templates', async () => { + const store = mockStore({ + ...initialState, + presets: { + loading: false, + presetWorkflows: loadPresetWorkflowTemplates(), + }, + }); + const { getAllByTestId, getAllByText, getByTestId, queryByText, - } = renderWithRouter(); + } = renderWithRouter(store); + + await waitFor(() => { + expect( + document.querySelector('.euiLoadingSpinner') + ).not.toBeInTheDocument(); + }); - // Click the first "Go" button on the templates and test Quick Configure. const goButtons = getAllByTestId('goButton'); + expect(goButtons.length).toBeGreaterThan(0); userEvent.click(goButtons[0]); + await waitFor(() => { expect(getAllByText('Quick configure')).toHaveLength(1); }); - // Verify that the create button is present in the Quick Configure pop-up. expect(getByTestId('quickConfigureCreateButton')).toBeInTheDocument(); - // Click the "Cancel" button in the Quick Configure pop-up. const quickConfigureCancelButton = getByTestId( 'quickConfigureCancelButton' ); userEvent.click(quickConfigureCancelButton); - // Ensure the quick configure pop-up is closed after canceling. await waitFor(() => { expect(queryByText('quickConfigureCreateButton')).toBeNull(); }); }); test('search functionality ', async () => { - const { getByText, getByPlaceholderText, queryByText } = renderWithRouter(); + const store = mockStore(initialState); + const { getByText, getByPlaceholderText, queryByText } = renderWithRouter( + store + ); - // Search by Template Name userEvent.type(getByPlaceholderText('Search'), 'hybrid'); await waitFor(() => { expect(getByText('Hybrid Search')).toBeInTheDocument(); diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index a8e8c6c3..fc5c940b 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -29,9 +29,77 @@ import { import { enrichPresetWorkflowWithUiMetadata } from './utils'; import { getDataSourceId, isDataSourceReady } from '../../../utils'; import { getDataSourceEnabled } from '../../../services'; +import semver from 'semver'; +import { DataSourceAttributes } from '../../../../../../src/plugins/data_source/common/data_sources'; +import { getSavedObjectsClient } from '../../../../public/services'; +import { + WORKFLOW_TYPE, + MIN_SUPPORTED_VERSION, + MINIMUM_FULL_SUPPORTED_VERSION, +} from '../../../../common/constants'; interface NewWorkflowProps {} +export const getEffectiveVersion = async ( + dataSourceId: string | undefined +): Promise => { + try { + if (dataSourceId === undefined) { + throw new Error('Data source is required'); + } + + const dataSource = await getSavedObjectsClient().get( + 'data-source', + dataSourceId + ); + const version = + dataSource?.attributes?.dataSourceVersion || MIN_SUPPORTED_VERSION; + return version; + } catch (error) { + console.error('Error getting version:', error); + return MIN_SUPPORTED_VERSION; + } +}; + +const filterPresetsByVersion = async ( + workflows: WorkflowTemplate[], + dataSourceId: string | undefined +): Promise => { + // if MDS is disabled, skip the version check and assume it is version 2.19+ + const dataSourceEnabled = getDataSourceEnabled().enabled; + if (!dataSourceEnabled) { + return workflows; + } + + if (!dataSourceId) { + return []; + } + + const allowedPresetsFor217 = [ + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, + ]; + + const version = await getEffectiveVersion(dataSourceId); + + if (semver.lt(version, MIN_SUPPORTED_VERSION)) { + return []; + } + + if ( + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION) + ) { + return workflows.filter((workflow) => { + const workflowType = workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; + return allowedPresetsFor217.includes(workflowType as WORKFLOW_TYPE); + }); + } + + return workflows; +}; + /** * Contains the searchable library of templated workflows based * on a variety of use cases. Can click on them to load in a pre-configured @@ -41,7 +109,6 @@ export function NewWorkflow(props: NewWorkflowProps) { const dispatch = useAppDispatch(); const dataSourceId = getDataSourceId(); const dataSourceEnabled = getDataSourceEnabled().enabled; - // workflows state const { presetWorkflows, loading } = useSelector( (state: AppState) => state.presets @@ -50,6 +117,7 @@ export function NewWorkflow(props: NewWorkflowProps) { const [filteredWorkflows, setFilteredWorkflows] = useState< WorkflowTemplate[] >([]); + const [isVersionLoading, setIsVersionLoading] = useState(false); // search bar state const [searchQuery, setSearchQuery] = useState(''); @@ -72,25 +140,63 @@ export function NewWorkflow(props: NewWorkflowProps) { // initial hook to populate all workflows // enrich them with dynamically-generated UI flows based on use case useEffect(() => { - if (presetWorkflows) { - setAllWorkflows( - presetWorkflows.map((presetWorkflow) => - enrichPresetWorkflowWithUiMetadata(presetWorkflow) - ) + const loadWorkflows = async () => { + if (!presetWorkflows || presetWorkflows.length === 0) { + return; + } + + const dataSourceEnabled = getDataSourceEnabled().enabled; + + if (!dataSourceEnabled) { + const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata( + presetWorkflow, + MINIMUM_FULL_SUPPORTED_VERSION + ) + ); + setAllWorkflows(enrichedWorkflows); + setFilteredWorkflows(enrichedWorkflows); + setIsVersionLoading(false); + return; + } + + if (!dataSourceId) { + setAllWorkflows([]); + setFilteredWorkflows([]); + setIsVersionLoading(true); + return; + } + + setIsVersionLoading(true); + + const version = await getEffectiveVersion(dataSourceId); + + const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) ); - } - }, [presetWorkflows]); - // initial hook to populate filtered workflows - useEffect(() => { - setFilteredWorkflows(allWorkflows); - }, [allWorkflows]); + const versionFilteredWorkflows = await filterPresetsByVersion( + enrichedWorkflows, + dataSourceId + ); + + setAllWorkflows(versionFilteredWorkflows); + setFilteredWorkflows(versionFilteredWorkflows); + setIsVersionLoading(false); + }; + + loadWorkflows(); + }, [presetWorkflows, dataSourceId, dataSourceEnabled]); // When search query updated, re-filter preset list useEffect(() => { setFilteredWorkflows(fetchFilteredWorkflows(allWorkflows, searchQuery)); }, [searchQuery]); + useEffect(() => { + setFilteredWorkflows(allWorkflows); + }, [allWorkflows]); + return ( @@ -101,8 +207,16 @@ export function NewWorkflow(props: NewWorkflowProps) { /> - {loading ? ( - + {loading || isVersionLoading ? ( + + + + + ) : ( {filteredWorkflows.map((workflow: Workflow, index) => { diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index f0c05ef5..59749ecb 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -10,6 +10,8 @@ import { MLSearchRequestProcessor, MLSearchResponseProcessor, NormalizationProcessor, + TextEmbeddingIngestProcessor, + TextImageEmbeddingIngestProcessor, } from '../../../configs'; import { WorkflowTemplate, @@ -19,7 +21,6 @@ import { WORKFLOW_TYPE, FETCH_ALL_QUERY, customStringify, - TERM_QUERY_TEXT, MULTIMODAL_SEARCH_QUERY_BOOL, IProcessorConfig, VECTOR_TEMPLATE_PLACEHOLDER, @@ -28,29 +29,39 @@ import { HYBRID_SEARCH_QUERY_MATCH_KNN, WorkflowConfig, UI_METADATA_SCHEMA_VERSION, + SEMANTIC_SEARCH_QUERY_NEURAL, + MULTIMODAL_SEARCH_QUERY_NEURAL, + HYBRID_SEARCH_QUERY_MATCH_NEURAL, + TERM_QUERY_TEXT, } from '../../../../common'; import { generateId } from '../../../utils'; +import semver from 'semver'; +import { MINIMUM_FULL_SUPPORTED_VERSION } from '../../../../common'; // Fn to produce the complete preset template with all necessary UI metadata. export function enrichPresetWorkflowWithUiMetadata( - presetWorkflow: Partial + presetWorkflow: Partial, + version: string ): WorkflowTemplate { + const defaultVersion = MINIMUM_FULL_SUPPORTED_VERSION; + const workflowVersion = version ?? defaultVersion; + let uiMetadata = {} as UIState; switch (presetWorkflow.ui_metadata?.type || WORKFLOW_TYPE.CUSTOM) { case WORKFLOW_TYPE.SEMANTIC_SEARCH: { - uiMetadata = fetchSemanticSearchMetadata(); + uiMetadata = fetchSemanticSearchMetadata(workflowVersion); break; } case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { - uiMetadata = fetchMultimodalSearchMetadata(); + uiMetadata = fetchMultimodalSearchMetadata(workflowVersion); break; } case WORKFLOW_TYPE.HYBRID_SEARCH: { - uiMetadata = fetchHybridSearchMetadata(); + uiMetadata = fetchHybridSearchMetadata(workflowVersion); break; } case WORKFLOW_TYPE.RAG: { - uiMetadata = fetchRAGMetadata(); + uiMetadata = fetchRAGMetadata(workflowVersion); break; } default: { @@ -138,68 +149,103 @@ export function fetchEmptyUIConfig(): WorkflowConfig { }; } -export function fetchSemanticSearchMetadata(): UIState { +export function fetchSemanticSearchMetadata(version: string): UIState { + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.SEMANTIC_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + + baseState.config.ingest.enrich.processors = isPreV219 + ? [new TextEmbeddingIngestProcessor().toObj()] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - KNN_QUERY - ), - ]; + + baseState.config.search.request.value = customStringify( + isPreV219 ? SEMANTIC_SEARCH_QUERY_NEURAL : TERM_QUERY_TEXT + ); + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), + ]; + return baseState; } -export function fetchMultimodalSearchMetadata(): UIState { +export function fetchMultimodalSearchMetadata(version: string): UIState { + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.MULTIMODAL_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + + baseState.config.ingest.enrich.processors = isPreV219 + ? [new TextImageEmbeddingIngestProcessor().toObj()] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); + baseState.config.search.request.value = customStringify( - MULTIMODAL_SEARCH_QUERY_BOOL + isPreV219 ? MULTIMODAL_SEARCH_QUERY_NEURAL : MULTIMODAL_SEARCH_QUERY_BOOL ); - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - KNN_QUERY - ), - ]; + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), + ]; + return baseState; } -export function fetchHybridSearchMetadata(): UIState { +export function fetchHybridSearchMetadata(version: string): UIState { + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + + baseState.config.ingest.enrich.processors = isPreV219 + ? [new TextEmbeddingIngestProcessor().toObj()] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); + + baseState.config.search.request.value = customStringify( + isPreV219 ? HYBRID_SEARCH_QUERY_MATCH_NEURAL : TERM_QUERY_TEXT + ); + baseState.config.search.enrichResponse.processors = [ injectDefaultWeightsInNormalizationProcessor( new NormalizationProcessor().toObj() ), ]; - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - HYBRID_SEARCH_QUERY_MATCH_KNN - ), - ]; + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + HYBRID_SEARCH_QUERY_MATCH_KNN + ), + ]; + return baseState; } -export function fetchRAGMetadata(): UIState { +export function fetchRAGMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.RAG; baseState.config.ingest.index.name.value = generateId('my_index', 6); diff --git a/test/interfaces.ts b/test/interfaces.ts index d9c3e20e..17455169 100644 --- a/test/interfaces.ts +++ b/test/interfaces.ts @@ -9,4 +9,5 @@ export type WorkflowInput = { id: string; name: string; type: WORKFLOW_TYPE; + version?: string; }; diff --git a/test/utils.ts b/test/utils.ts index 8953288d..de87cb89 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -10,7 +10,10 @@ import { INITIAL_WORKFLOWS_STATE, } from '../public/store'; import { WorkflowInput } from '../test/interfaces'; -import { WORKFLOW_TYPE } from '../common/constants'; +import { + MINIMUM_FULL_SUPPORTED_VERSION, + WORKFLOW_TYPE, +} from '../common/constants'; import { UIState, Workflow, WorkflowDict } from '../common/interfaces'; import { fetchEmptyMetadata, @@ -44,27 +47,42 @@ export function mockStore(...workflowSets: WorkflowInput[]) { } function generateWorkflow({ id, name, type }: WorkflowInput): Workflow { + const isSearchWorkflow = [ + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, + ].includes(type); + + const version = { + template: '1.0.0', + compatibility: isSearchWorkflow + ? [MINIMUM_FULL_SUPPORTED_VERSION] + : ['2.18.0', '3.0.0'], + }; + return { id, name, - version: { template: '1.0.0', compatibility: ['2.18.0', '3.0.0'] }, - ui_metadata: getConfig(type), + version, + ui_metadata: getConfig(type, version.compatibility[0]), }; } -function getConfig(workflowType: WORKFLOW_TYPE) { +function getConfig(workflowType: WORKFLOW_TYPE, version?: string) { let uiMetadata = {} as UIState; + const searchVersion = version || MINIMUM_FULL_SUPPORTED_VERSION; + switch (workflowType) { case WORKFLOW_TYPE.SEMANTIC_SEARCH: { - uiMetadata = fetchSemanticSearchMetadata(); + uiMetadata = fetchSemanticSearchMetadata(searchVersion); break; } case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { - uiMetadata = fetchMultimodalSearchMetadata(); + uiMetadata = fetchMultimodalSearchMetadata(searchVersion); break; } case WORKFLOW_TYPE.HYBRID_SEARCH: { - uiMetadata = fetchHybridSearchMetadata(); + uiMetadata = fetchHybridSearchMetadata(searchVersion); break; } default: {