From 40be91ba052730bc53b83f08759f1682a9a51e90 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Wed, 4 Dec 2024 21:24:19 -0800 Subject: [PATCH 1/9] filter preset based on version get from backend and change enrich data section processors based on the version. Signed-off-by: Kama Huang --- common/constants.ts | 26 +- common/interfaces.ts | 2 +- opensearch_dashboards.json | 18 +- public/configs/ingest_processors/index.ts | 2 + .../text_embedding_ingest_processor.ts | 37 ++ .../text_image_embedding_ingest_processor.ts | 36 ++ .../workflow_detail/workflow_detail.test.tsx | 25 +- .../pages/workflow_detail/workflow_detail.tsx | 4 +- .../ingest_inputs/enrich_data.tsx | 4 + .../ingest_inputs/ingest_inputs.tsx | 6 +- .../workflow_inputs/processors_list.tsx | 331 +++++++++++------- .../search_inputs/enrich_search_request.tsx | 4 + .../search_inputs/enrich_search_response.tsx | 4 + .../search_inputs/search_inputs.tsx | 5 + .../workflow_inputs/workflow_inputs.tsx | 2 - .../new_workflow/new_workflow.test.tsx | 64 +++- .../workflows/new_workflow/new_workflow.tsx | 105 +++++- public/pages/workflows/new_workflow/utils.ts | 38 +- public/plugin.ts | 1 + 19 files changed, 500 insertions(+), 214 deletions(-) create mode 100644 public/configs/ingest_processors/text_embedding_ingest_processor.ts create mode 100644 public/configs/ingest_processors/text_image_embedding_ingest_processor.ts diff --git a/common/constants.ts b/common/constants.ts index d55b6019..b3ec59f5 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -180,6 +180,8 @@ export enum PROCESSOR_TYPE { NORMALIZATION = 'normalization-processor', COLLAPSE = 'collapse', RERANK = 'rerank', + TEXT_EMBEDDING = 'text_embedding', + TEXT_IMAGE_EMBEDDING = 'text_image_embedding', } export enum MODEL_TYPE { @@ -311,7 +313,6 @@ export const SEMANTIC_SEARCH_QUERY_NEURAL = { neural: { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, - model_id: MODEL_ID_PATTERN, k: DEFAULT_K, }, }, @@ -333,20 +334,16 @@ export const MULTIMODAL_SEARCH_QUERY_NEURAL = { }, }; export const MULTIMODAL_SEARCH_QUERY_BOOL = { + _source: { + excludes: [VECTOR_FIELD_PATTERN], + }, query: { - bool: { - must: [ - { - match: { - [TEXT_FIELD_PATTERN]: QUERY_TEXT_PATTERN, - }, - }, - { - match: { - [IMAGE_FIELD_PATTERN]: QUERY_IMAGE_PATTERN, - }, - }, - ], + neural: { + [VECTOR_FIELD_PATTERN]: { + query_text: QUERY_TEXT_PATTERN, + query_image: QUERY_IMAGE_PATTERN, + k: DEFAULT_K, + }, }, }, }; @@ -394,7 +391,6 @@ export const HYBRID_SEARCH_QUERY_MATCH_NEURAL = { neural: { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, - model_id: MODEL_ID_PATTERN, k: DEFAULT_K, }, }, diff --git a/common/interfaces.ts b/common/interfaces.ts index cdc02fd3..145d0228 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -22,7 +22,7 @@ export type Index = { ********** WORKFLOW TYPES/INTERFACES ********** */ export type MDSQueryParams = { - dataSourceId?: string; + dataSourceId?: string | undefined; }; export type ConfigFieldType = diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index 47253592..b228e54e 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -1,25 +1,17 @@ { "id": "flowFrameworkDashboards", "version": "2.19.0.0", - "opensearchDashboardsVersion": "2.19.0", + "opensearchDashboardsVersion": "2.18.0", "server": true, "ui": true, "requiredBundles": [], - "requiredPlugins": [ - "navigation", - "opensearchDashboardsUtils" - ], + "requiredPlugins": ["navigation", "opensearchDashboardsUtils"], "optionalPlugins": [ "dataSource", "dataSourceManagement", "contentManagement" ], "supportedOSDataSourceVersions": ">=2.18.0 <4.0.0", - "requiredOSDataSourcePlugins": [ - "opensearch-ml", - "opensearch-flow-framework" - ], - "configPath": [ - "flowFrameworkDashboards" - ] -} \ No newline at end of file + "requiredOSDataSourcePlugins": ["opensearch-ml", "opensearch-flow-framework"], + "configPath": ["flowFrameworkDashboards"] +} diff --git a/public/configs/ingest_processors/index.ts b/public/configs/ingest_processors/index.ts index 5b4c680f..559def2e 100644 --- a/public/configs/ingest_processors/index.ts +++ b/public/configs/ingest_processors/index.ts @@ -7,3 +7,5 @@ export * from './ml_ingest_processor'; export * from './split_ingest_processor'; export * from './sort_ingest_processor'; export * from './text_chunking_ingest_processor'; +export * from './text_embedding_ingest_processor'; +export * from './text_image_embedding_ingest_processor'; diff --git a/public/configs/ingest_processors/text_embedding_ingest_processor.ts b/public/configs/ingest_processors/text_embedding_ingest_processor.ts new file mode 100644 index 00000000..acf1ef04 --- /dev/null +++ b/public/configs/ingest_processors/text_embedding_ingest_processor.ts @@ -0,0 +1,37 @@ +import { PROCESSOR_TYPE } from '../../../common'; +import { generateId } from '../../utils'; +import { Processor } from '../processor'; + +export class TextEmbeddingIngestProcessor extends Processor { + constructor() { + super(); + this.name = 'Text Embedding Processor'; + this.type = PROCESSOR_TYPE.TEXT_EMBEDDING; + this.id = generateId('text_embedding_processor_ingest'); + this.fields = [ + { + id: 'model_id', + type: 'string', + }, + { + id: 'field_map', + type: 'map', + }, + ]; + this.optionalFields = [ + { + id: 'description', + type: 'string', + }, + { + id: 'tag', + type: 'string', + }, + { + id: 'batch_size', + type: 'number', + value: 1, + }, + ]; + } +} diff --git a/public/configs/ingest_processors/text_image_embedding_ingest_processor.ts b/public/configs/ingest_processors/text_image_embedding_ingest_processor.ts new file mode 100644 index 00000000..64c0cc3f --- /dev/null +++ b/public/configs/ingest_processors/text_image_embedding_ingest_processor.ts @@ -0,0 +1,36 @@ +import { PROCESSOR_TYPE } from '../../../common'; +import { generateId } from '../../utils'; +import { Processor } from '../processor'; + +export class TextImageEmbeddingIngestProcessor extends Processor { + constructor() { + super(); + this.name = 'Text Image Embedding Processor'; + this.type = PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING; + this.id = generateId('text_image_embedding_processor_ingest'); + this.fields = [ + { + id: 'model_id', + type: 'string', + }, + { + id: 'embedding', + type: 'string', + }, + { + id: 'field_map', + type: 'map', + }, + ]; + this.optionalFields = [ + { + id: 'description', + type: 'string', + }, + { + id: 'tag', + type: 'string', + }, + ]; + } +} diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index 3e7bd535..8c2f70f8 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -111,19 +111,15 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { WORKFLOW_TYPE.CUSTOM ); - // Export button opens the export component userEvent.click(getByTestId('exportButton')); await waitFor(() => { expect(getByText(`Export '${workflowName}'`)).toBeInTheDocument(); }); - // Close the export component userEvent.click(getByTestId('exportCloseButton')); - // Check workspace button group exists (Visual and JSON) getByTestId('visualJSONToggleButtonGroup'); - // Tools panel should collapse and expand on toggle const toolsPanel = container.querySelector('#tools_panel_id'); expect(toolsPanel).toBeVisible(); @@ -131,13 +127,11 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { expect(toggleButton).toBeInTheDocument(); userEvent.click(toggleButton!); - // Tools panel after collapsing const collapsedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { expect(collapsedToolsPanel).toHaveClass('euiResizablePanel-isCollapsed'); }); - // Tools panel after expanding userEvent.click(toggleButton!); const expandedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { @@ -154,7 +148,6 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { WORKFLOW_TYPE.CUSTOM ); - // The WorkflowDetail Page Close button should navigate back to the workflows list userEvent.click(getByTestId('closeButton')); await waitFor(() => { expect(history.location.pathname).toBe('/workflows'); @@ -166,6 +159,7 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow beforeEach(() => { jest.clearAllMocks(); }); + test(`renders the WorkflowDetail page with skip ingestion option`, async () => { const { getByTestId, getAllByText, getAllByTestId } = renderWithRouter( workflowId, @@ -173,51 +167,50 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow WORKFLOW_TYPE.HYBRID_SEARCH ); - // Defining a new ingest pipeline & index is enabled by default const enabledCheckbox = getByTestId('switch-ingest.enabled'); - // Skipping ingest pipeline and navigating to search userEvent.click(enabledCheckbox); await waitFor(() => {}); + const searchPipelineButton = getByTestId('searchPipelineButton'); userEvent.click(searchPipelineButton); - // Search pipeline await waitFor(() => { expect(getAllByText('Define search flow').length).toBeGreaterThan(0); }); expect(getAllByText('Configure query').length).toBeGreaterThan(0); - // Edit Search Query const queryEditButton = getByTestId('queryEditButton'); expect(queryEditButton).toBeInTheDocument(); userEvent.click(queryEditButton); + await waitFor(() => { expect(getAllByText('Edit query definition').length).toBeGreaterThan(0); }); + const searchQueryPresetButton = getByTestId('searchQueryPresetButton'); expect(searchQueryPresetButton).toBeInTheDocument(); const updateSearchQueryButton = getByTestId('updateSearchQueryButton'); expect(updateSearchQueryButton).toBeInTheDocument(); userEvent.click(updateSearchQueryButton); - // Add request processor const addRequestProcessorButton = await waitFor( () => getAllByTestId('addProcessorButton')[0] ); userEvent.click(addRequestProcessorButton); + await waitFor(() => { - expect(getAllByText('PROCESSORS').length).toBeGreaterThan(0); + const popoverPanel = document.querySelector('.euiPopover__panel'); + expect(popoverPanel).toBeTruthy(); }); - // Add response processor const addResponseProcessorButton = getAllByTestId('addProcessorButton')[1]; userEvent.click(addResponseProcessorButton); await waitFor(() => { - expect(getAllByText('PROCESSORS').length).toBeGreaterThan(0); + const popoverPanel = document.querySelector('.euiPopover__panel'); + expect(popoverPanel).toBeTruthy(); }); - // Build and Run query, Back buttons are present const searchPipelineBackButton = getByTestId('searchPipelineBackButton'); userEvent.click(searchPipelineBackButton); diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 80c28ea9..6f46b953 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -126,7 +126,9 @@ export function WorkflowDetail(props: WorkflowDetailProps) { }, [USE_NEW_HOME_PAGE, dataSourceEnabled, dataSourceId, workflowName]); // form state - const [formValues, setFormValues] = useState({}); + const [formValues, setFormValues] = useState( + {} as WorkflowFormValues + ); const [formSchema, setFormSchema] = useState(yup.object({})); // ingest docs state. we need to persist here to update the form values. diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx index ae184a9e..d20e1f47 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx @@ -8,10 +8,13 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsList } from '../processors_list'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsTitle } from '../../../../general_components'; +import { SavedObject } from '../../../../../../../src/core/public'; +import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface EnrichDataProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; + dataSource?: SavedObject; } /** @@ -30,6 +33,7 @@ export function EnrichData(props: EnrichDataProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} context={PROCESSOR_CONTEXT.INGEST} + dataSource={props.dataSource} /> diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx index 2358003f..1496142d 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx @@ -36,7 +36,11 @@ export function IngestInputs(props: IngestInputsProps) { - + diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index b85909a4..6299a6ad 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -4,6 +4,8 @@ */ import React, { useEffect, useState } from 'react'; +import semver from 'semver'; +import { getEffectiveVersion } from '../../../pages/workflows/new_workflow/new_workflow'; import { EuiSmallButtonEmpty, EuiSmallButtonIcon, @@ -23,7 +25,8 @@ import { WorkflowConfig, WorkflowFormValues, } from '../../../../common'; -import { formikToUiConfig } from '../../../utils'; +import { formikToUiConfig, getDataSourceFromURL } from '../../../utils'; + import { CollapseProcessor, MLIngestProcessor, @@ -36,13 +39,20 @@ import { SplitIngestProcessor, SplitSearchResponseProcessor, TextChunkingIngestProcessor, + TextEmbeddingIngestProcessor, + TextImageEmbeddingIngestProcessor, } from '../../../configs'; import { ProcessorInputs } from './processor_inputs'; +import { SavedObject } from '../../../../../../src/core/public'; +import { DataSourceAttributes } from '../../../../../../src/plugins/data_source/common/data_sources'; +import { useLocation } from 'react-router-dom'; interface ProcessorsListProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; context: PROCESSOR_CONTEXT; + dataSource?: SavedObject; + onProcessorsChange?: (count: number) => void; } const PANEL_ID = 0; @@ -52,30 +62,188 @@ const PANEL_ID = 0; */ export function ProcessorsList(props: ProcessorsListProps) { const { values } = useFormikContext(); - - // Processor added state. Used to automatically open accordion when a new - // processor is added, assuming users want to immediately configure it. + const [version, setVersion] = useState(''); + const location = useLocation(); const [processorAdded, setProcessorAdded] = useState(false); - - // Popover state when adding new processors const [isPopoverOpen, setPopover] = useState(false); + const [processors, setProcessors] = useState([]); + const closePopover = () => { setPopover(false); }; - // Current processors state - const [processors, setProcessors] = useState([]); + const handlePopoverClick = () => { + setPopover(!isPopoverOpen); + }; + + useEffect(() => { + const dataSourceId = getDataSourceFromURL(location).dataSourceId; + if (dataSourceId) { + getEffectiveVersion(dataSourceId) + .then((ver) => { + setVersion(ver); + }) + .catch(console.error); + } + }, [location]); + useEffect(() => { - if (props.uiConfig && props.context) { - setProcessors( - props.context === PROCESSOR_CONTEXT.INGEST - ? props.uiConfig.ingest.enrich.processors - : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST - ? props.uiConfig.search.enrichRequest.processors - : props.uiConfig.search.enrichResponse.processors - ); + const loadProcessors = async () => { + if (props.uiConfig && props.context) { + let currentProcessors = + props.context === PROCESSOR_CONTEXT.INGEST + ? props.uiConfig.ingest.enrich.processors + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? props.uiConfig.search.enrichRequest.processors + : props.uiConfig.search.enrichResponse.processors; + + // If version is 2.17 or 2.18, filter out ML processors + if ( + version && + semver.gte(version, '2.17.0') && + semver.lt(version, '2.19.0') + ) { + currentProcessors = + currentProcessors?.filter( + (processor) => processor.type !== 'ml_inference' + ) || []; + } + setProcessors(currentProcessors || []); + props.onProcessorsChange?.(currentProcessors?.length || 0); + } + }; + + loadProcessors(); + }, [props.context, props.uiConfig, version]); + + const getMenuItems = () => { + const isPreV219 = + semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0'); + + const ingestProcessors = [ + ...(isPreV219 + ? [ + { + name: 'Text Embedding Processor', + onClick: () => { + closePopover(); + addProcessor(new TextEmbeddingIngestProcessor().toObj()); + }, + }, + { + name: 'Text Image Embedding Processor', + onClick: () => { + closePopover(); + addProcessor(new TextImageEmbeddingIngestProcessor().toObj()); + }, + }, + ] + : [ + { + name: 'ML Inference Processor', + onClick: () => { + closePopover(); + addProcessor(new MLIngestProcessor().toObj()); + }, + }, + ]), + { + name: 'Split Processor', + onClick: () => { + closePopover(); + addProcessor(new SplitIngestProcessor().toObj()); + }, + }, + { + name: 'Sort Processor', + onClick: () => { + closePopover(); + addProcessor(new SortIngestProcessor().toObj()); + }, + }, + { + name: 'Text Chunking Processor', + onClick: () => { + closePopover(); + addProcessor(new TextChunkingIngestProcessor().toObj()); + }, + }, + ]; + + const searchRequestProcessors = [ + ...(!isPreV219 + ? [ + { + name: 'ML Inference Processor', + onClick: () => { + closePopover(); + addProcessor(new MLSearchRequestProcessor().toObj()); + }, + }, + ] + : []), + ]; + + const searchResponseProcessors = [ + ...(!isPreV219 + ? [ + { + name: 'ML Inference Processor', + onClick: () => { + closePopover(); + addProcessor(new MLSearchResponseProcessor().toObj()); + }, + }, + ] + : []), + { + name: 'Rerank Processor', + onClick: () => { + closePopover(); + addProcessor(new RerankProcessor().toObj()); + }, + }, + { + name: 'Split Processor', + onClick: () => { + closePopover(); + addProcessor(new SplitSearchResponseProcessor().toObj()); + }, + }, + { + name: 'Sort Processor', + onClick: () => { + closePopover(); + addProcessor(new SortSearchResponseProcessor().toObj()); + }, + }, + { + name: 'Normalization Processor', + onClick: () => { + closePopover(); + addProcessor(new NormalizationProcessor().toObj()); + }, + }, + { + name: 'Collapse Processor', + onClick: () => { + closePopover(); + addProcessor(new CollapseProcessor().toObj()); + }, + }, + ]; + + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: + return ingestProcessors; + case PROCESSOR_CONTEXT.SEARCH_REQUEST: + return searchRequestProcessors; + case PROCESSOR_CONTEXT.SEARCH_RESPONSE: + return searchResponseProcessors; + default: + return []; } - }, [props.context, props.uiConfig]); + }; // Adding a processor to the config. Fetch the existing one // (getting any updated/interim values along the way) and add to @@ -197,9 +365,7 @@ export function ProcessorsList(props: ProcessorsListProps) { { - setPopover(!isPopoverOpen); - }} + onClick={handlePopoverClick} data-testid="addProcessorButton" > {`Add processor`} @@ -210,118 +376,19 @@ export function ProcessorsList(props: ProcessorsListProps) { panelPaddingSize="none" anchorPosition="downLeft" > - { - closePopover(); - addProcessor(new MLIngestProcessor().toObj()); - }, - }, - { - name: 'Split Processor', - onClick: () => { - closePopover(); - addProcessor( - new SplitIngestProcessor().toObj() - ); - }, - }, - { - name: 'Sort Processor', - onClick: () => { - closePopover(); - addProcessor( - new SortIngestProcessor().toObj() - ); - }, - }, - { - name: 'Text Chunking Processor', - onClick: () => { - closePopover(); - addProcessor( - new TextChunkingIngestProcessor().toObj() - ); - }, - }, - ] - : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST - ? [ - { - name: 'ML Inference Processor', - onClick: () => { - closePopover(); - addProcessor( - new MLSearchRequestProcessor().toObj() - ); - }, - }, - ] - : [ - { - name: 'ML Inference Processor', - onClick: () => { - closePopover(); - addProcessor( - new MLSearchResponseProcessor().toObj() - ); - }, - }, - { - name: 'Rerank Processor', - onClick: () => { - closePopover(); - addProcessor(new RerankProcessor().toObj()); - }, - }, - { - name: 'Split Processor', - onClick: () => { - closePopover(); - addProcessor( - new SplitSearchResponseProcessor().toObj() - ); - }, - }, - { - name: 'Sort Processor', - onClick: () => { - closePopover(); - addProcessor( - new SortSearchResponseProcessor().toObj() - ); - }, - }, - { - name: 'Normalization Processor', - onClick: () => { - closePopover(); - addProcessor( - new NormalizationProcessor().toObj() - ); - }, - }, - { - name: 'Collapse Processor', - onClick: () => { - closePopover(); - addProcessor(new CollapseProcessor().toObj()); - }, - }, - ], - }, - ]} - /> + {version && ( + 0 ? 'PROCESSORS' : '', + items: getMenuItems(), + }, + ]} + /> + )} diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx index d2fd3488..d2623a89 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx @@ -8,10 +8,13 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsTitle } from '../../../../general_components'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsList } from '../processors_list'; +import { SavedObject } from '../../../../../../../src/core/public'; +import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface EnrichSearchRequestProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; + dataSource?: SavedObject; } /** @@ -32,6 +35,7 @@ export function EnrichSearchRequest(props: EnrichSearchRequestProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} context={PROCESSOR_CONTEXT.SEARCH_REQUEST} + dataSource={props.dataSource} /> diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx index 4ad3b288..da41a772 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx @@ -8,10 +8,13 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsTitle } from '../../../../general_components'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsList } from '../processors_list'; +import { SavedObject } from '../../../../../../../src/core/public'; +import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface EnrichSearchResponseProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; + dataSource?: SavedObject; } /** @@ -32,6 +35,7 @@ export function EnrichSearchResponse(props: EnrichSearchResponseProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} context={PROCESSOR_CONTEXT.SEARCH_RESPONSE} + dataSource={props.dataSource} /> diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx index c922a6ac..9eeb7403 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx @@ -14,10 +14,13 @@ import { } from '../../../../../common'; import { catIndices, useAppDispatch } from '../../../../store'; import { getDataSourceId } from '../../../../utils'; +import { SavedObject } from '../../../../../../../src/core/public'; +import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface SearchInputsProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; + dataSource?: SavedObject; } /** @@ -45,6 +48,7 @@ export function SearchInputs(props: SearchInputsProps) { @@ -54,6 +58,7 @@ export function SearchInputs(props: SearchInputsProps) { diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index fc1485ea..7a0490ba 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -53,8 +53,6 @@ import { getDataSourceId, } from '../../../utils'; import { BooleanField } from './input_fields'; - -// styling import '../workspace/workspace-styles.scss'; interface WorkflowInputsProps { diff --git a/public/pages/workflows/new_workflow/new_workflow.test.tsx b/public/pages/workflows/new_workflow/new_workflow.test.tsx index 14a609f9..0a40841a 100644 --- a/public/pages/workflows/new_workflow/new_workflow.test.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.test.tsx @@ -4,8 +4,9 @@ */ import React from 'react'; +import { MemoryRouter as Router } from 'react-router-dom'; // Change this import import { Provider } from 'react-redux'; -import { BrowserRouter as Router, Route, Switch } from 'react-router-dom'; +import { Route, Switch } from 'react-router-dom'; import { NewWorkflow } from './new_workflow'; import { render, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -15,6 +16,11 @@ import '@testing-library/jest-dom'; import { loadPresetWorkflowTemplates } from '../../../../test/utils'; import { INITIAL_ML_STATE } from '../../../../public/store'; +jest.mock('../../../utils', () => ({ + ...jest.requireActual('../../../utils'), + getDataSourceId: () => '123', +})); + jest.mock('../../../services', () => { const { mockCoreServices } = require('../../../../test'); return { @@ -31,71 +37,95 @@ const initialState = { presetWorkflows: loadPresetWorkflowTemplates(), }, }; -const store = mockStore(initialState); const mockDispatch = jest.fn(); -const renderWithRouter = () => +const renderWithRouter = (store: any) => render( - + } /> ); - describe('NewWorkflow', () => { beforeEach(() => { jest.clearAllMocks(); jest.spyOn(ReactReduxHooks, 'useAppDispatch').mockReturnValue(mockDispatch); }); - test('renders the preset workflow names & descriptions', () => { + test('renders the preset workflow names & descriptions', async () => { + const store = mockStore(initialState); const presetWorkflows = loadPresetWorkflowTemplates(); - const { getByPlaceholderText, getAllByText } = renderWithRouter(); + const { getByPlaceholderText, getByText } = renderWithRouter(store); + expect(getByPlaceholderText('Search')).toBeInTheDocument(); - presetWorkflows.forEach((workflow) => { - expect(getAllByText(workflow.name)).toHaveLength(1); - expect(getAllByText(workflow.description)).toHaveLength(1); + + await waitFor(() => { + presetWorkflows.forEach((workflow) => { + if ( + workflow.name === + ['Semantic Search', 'Multimodal Search', 'Hybrid Search'].includes( + workflow.name + ) + ) { + expect(getByText(workflow.name)).toBeInTheDocument(); + expect(getByText(workflow.description)).toBeInTheDocument(); + } + }); }); }); test('renders the quick configure for preset workflow templates', async () => { + const store = mockStore({ + ...initialState, + presets: { + loading: false, + presetWorkflows: loadPresetWorkflowTemplates(), + }, + }); + const { getAllByTestId, getAllByText, getByTestId, queryByText, - } = renderWithRouter(); + } = renderWithRouter(store); + + await waitFor(() => { + expect( + document.querySelector('.euiLoadingSpinner') + ).not.toBeInTheDocument(); + }); - // Click the first "Go" button on the templates and test Quick Configure. const goButtons = getAllByTestId('goButton'); + expect(goButtons.length).toBeGreaterThan(0); userEvent.click(goButtons[0]); + await waitFor(() => { expect(getAllByText('Quick configure')).toHaveLength(1); }); - // Verify that the create button is present in the Quick Configure pop-up. expect(getByTestId('quickConfigureCreateButton')).toBeInTheDocument(); - // Click the "Cancel" button in the Quick Configure pop-up. const quickConfigureCancelButton = getByTestId( 'quickConfigureCancelButton' ); userEvent.click(quickConfigureCancelButton); - // Ensure the quick configure pop-up is closed after canceling. await waitFor(() => { expect(queryByText('quickConfigureCreateButton')).toBeNull(); }); }); test('search functionality ', async () => { - const { getByText, getByPlaceholderText, queryByText } = renderWithRouter(); + const store = mockStore(initialState); + const { getByText, getByPlaceholderText, queryByText } = renderWithRouter( + store + ); - // Search by Template Name userEvent.type(getByPlaceholderText('Search'), 'hybrid'); await waitFor(() => { expect(getByText('Hybrid Search')).toBeInTheDocument(); diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index a8e8c6c3..a0fdb6d9 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -29,9 +29,68 @@ import { import { enrichPresetWorkflowWithUiMetadata } from './utils'; import { getDataSourceId, isDataSourceReady } from '../../../utils'; import { getDataSourceEnabled } from '../../../services'; +import semver from 'semver'; +import { DataSourceAttributes } from '../../../../../../src/plugins/data_source/common/data_sources'; +import { getSavedObjectsClient } from '../../../../public/services'; interface NewWorkflowProps {} +export const getEffectiveVersion = async ( + dataSourceId: string | undefined +): Promise => { + try { + // Don't make the API call if no dataSourceId + if (dataSourceId === undefined) { + console.log('cannot find data source'); + throw new Error('Data source is required'); + } + + const dataSource = await getSavedObjectsClient().get( + 'data-source', + dataSourceId + ); + const version = dataSource?.attributes?.dataSourceVersion || '2.17.0'; + return version; + } catch (error) { + console.error('Error getting version:', error); + return '2.17.0'; + } +}; + +const filterWorkflowsByVersion = async ( + workflows: WorkflowTemplate[], + dataSourceId: string +): Promise => { + const allowedPresetsFor217 = [ + 'Semantic Search', + 'Multimodal Search', + 'Hybrid Search', + ]; + + try { + const version = await getEffectiveVersion(dataSourceId); + if (version === undefined) { + return []; + } + + if (semver.lt(version, '2.17.0')) { + return []; + } + + if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { + return workflows.filter((workflow) => + allowedPresetsFor217.includes(workflow.name) + ); + } + + return workflows; + } catch (error) { + return workflows.filter((workflow) => + allowedPresetsFor217.includes(workflow.name) + ); + } +}; + /** * Contains the searchable library of templated workflows based * on a variety of use cases. Can click on them to load in a pre-configured @@ -69,28 +128,36 @@ export function NewWorkflow(props: NewWorkflowProps) { } }, [dataSourceId, dataSourceEnabled]); - // initial hook to populate all workflows - // enrich them with dynamically-generated UI flows based on use case useEffect(() => { - if (presetWorkflows) { - setAllWorkflows( - presetWorkflows.map((presetWorkflow) => - enrichPresetWorkflowWithUiMetadata(presetWorkflow) - ) + const loadWorkflows = async () => { + if (!presetWorkflows) return; + if (!dataSourceId) return; + + const version = await getEffectiveVersion(dataSourceId); + const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) ); - } - }, [presetWorkflows]); - // initial hook to populate filtered workflows - useEffect(() => { - setFilteredWorkflows(allWorkflows); - }, [allWorkflows]); + const versionFilteredWorkflows = await filterWorkflowsByVersion( + enrichedWorkflows, + dataSourceId + ); + + setAllWorkflows(versionFilteredWorkflows); + }; + + loadWorkflows(); + }, [presetWorkflows, dataSourceId]); // When search query updated, re-filter preset list useEffect(() => { setFilteredWorkflows(fetchFilteredWorkflows(allWorkflows, searchQuery)); }, [searchQuery]); + useEffect(() => { + setFilteredWorkflows(allWorkflows); + }, [allWorkflows]); + return ( @@ -101,8 +168,16 @@ export function NewWorkflow(props: NewWorkflowProps) { /> - {loading ? ( - + {!dataSourceId || loading ? ( + + + + + ) : ( {filteredWorkflows.map((workflow: Workflow, index) => { diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index f0c05ef5..819ad56a 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -28,12 +28,15 @@ import { HYBRID_SEARCH_QUERY_MATCH_KNN, WorkflowConfig, UI_METADATA_SCHEMA_VERSION, + PROCESSOR_TYPE, } from '../../../../common'; import { generateId } from '../../../utils'; +import semver from 'semver'; // Fn to produce the complete preset template with all necessary UI metadata. export function enrichPresetWorkflowWithUiMetadata( - presetWorkflow: Partial + presetWorkflow: Partial, + version: string ): WorkflowTemplate { let uiMetadata = {} as UIState; switch (presetWorkflow.ui_metadata?.type || WORKFLOW_TYPE.CUSTOM) { @@ -59,6 +62,39 @@ export function enrichPresetWorkflowWithUiMetadata( } } + if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { + uiMetadata.config.ingest.enrich.processors = uiMetadata.config.ingest.enrich.processors.filter( + (p) => p.type !== PROCESSOR_TYPE.ML + ); + uiMetadata.config.search.enrichRequest.processors = uiMetadata.config.search.enrichRequest.processors.filter( + (p) => p.type !== PROCESSOR_TYPE.ML + ); + uiMetadata.config.search.enrichResponse.processors = uiMetadata.config.search.enrichResponse.processors.filter( + (p) => p.type !== PROCESSOR_TYPE.ML + ); + } + + if (semver.eq(version, '2.19.0')) { + uiMetadata.config.ingest.enrich.processors = uiMetadata.config.ingest.enrich.processors.filter( + (p) => + p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && + p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && + p.type !== PROCESSOR_TYPE.NORMALIZATION + ); + uiMetadata.config.search.enrichRequest.processors = uiMetadata.config.search.enrichRequest.processors.filter( + (p) => + p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && + p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && + p.type !== PROCESSOR_TYPE.NORMALIZATION + ); + uiMetadata.config.search.enrichResponse.processors = uiMetadata.config.search.enrichResponse.processors.filter( + (p) => + p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && + p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && + p.type !== PROCESSOR_TYPE.NORMALIZATION + ); + } + return { ...presetWorkflow, ui_metadata: { diff --git a/public/plugin.ts b/public/plugin.ts index dfe4d198..711d98dc 100644 --- a/public/plugin.ts +++ b/public/plugin.ts @@ -31,6 +31,7 @@ import { setHeaderActionMenu, } from './services'; import { configureRoutes } from './route_service'; +import { dataSourceFilterFn } from './utils'; export class FlowFrameworkDashboardsPlugin implements From 119b38f98e964bab208c270565f63388a0aaceb3 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Wed, 18 Dec 2024 22:39:38 -0800 Subject: [PATCH 2/9] address comments made on 12/17 Signed-off-by: Kama Huang --- common/constants.ts | 4 ++ common/interfaces.ts | 2 +- opensearch_dashboards.json | 2 +- .../workflow_detail/workflow_detail.test.tsx | 2 +- .../ingest_inputs/enrich_data.tsx | 1 - .../workflow_inputs/processors_list.tsx | 16 +------- .../search_inputs/enrich_search_request.tsx | 1 - .../search_inputs/enrich_search_response.tsx | 1 - .../workflows/new_workflow/new_workflow.tsx | 37 ++++++++++++------- 9 files changed, 31 insertions(+), 35 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index b3ec59f5..15954d3d 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -169,6 +169,8 @@ export enum WORKFLOW_TYPE { CUSTOM = 'Custom', UNKNOWN = 'Unknown', } +// If no datasource version is found, we default to 2.17.0 +export const MIN_SUPPORTED_VERSION = '2.17.0'; // the names should be consistent with the underlying implementation. used when generating the // final ingest/search pipeline configurations. @@ -342,6 +344,7 @@ export const MULTIMODAL_SEARCH_QUERY_BOOL = { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, query_image: QUERY_IMAGE_PATTERN, + model_id: MODEL_ID_PATTERN, k: DEFAULT_K, }, }, @@ -391,6 +394,7 @@ export const HYBRID_SEARCH_QUERY_MATCH_NEURAL = { neural: { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, + model_id: MODEL_ID_PATTERN, k: DEFAULT_K, }, }, diff --git a/common/interfaces.ts b/common/interfaces.ts index 145d0228..cdc02fd3 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -22,7 +22,7 @@ export type Index = { ********** WORKFLOW TYPES/INTERFACES ********** */ export type MDSQueryParams = { - dataSourceId?: string | undefined; + dataSourceId?: string; }; export type ConfigFieldType = diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index b228e54e..a1409aa6 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -11,7 +11,7 @@ "dataSourceManagement", "contentManagement" ], - "supportedOSDataSourceVersions": ">=2.18.0 <4.0.0", + "supportedOSDataSourceVersions": ">=2.17.0 <4.0.0", "requiredOSDataSourcePlugins": ["opensearch-ml", "opensearch-flow-framework"], "configPath": ["flowFrameworkDashboards"] } diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index 8c2f70f8..adb43839 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -115,7 +115,7 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { await waitFor(() => { expect(getByText(`Export '${workflowName}'`)).toBeInTheDocument(); }); - + // Close the export component userEvent.click(getByTestId('exportCloseButton')); getByTestId('visualJSONToggleButtonGroup'); diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx index d20e1f47..90ce0600 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx @@ -33,7 +33,6 @@ export function EnrichData(props: EnrichDataProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} context={PROCESSOR_CONTEXT.INGEST} - dataSource={props.dataSource} /> diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index 6299a6ad..ec541d72 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -43,15 +43,12 @@ import { TextImageEmbeddingIngestProcessor, } from '../../../configs'; import { ProcessorInputs } from './processor_inputs'; -import { SavedObject } from '../../../../../../src/core/public'; -import { DataSourceAttributes } from '../../../../../../src/plugins/data_source/common/data_sources'; import { useLocation } from 'react-router-dom'; interface ProcessorsListProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; context: PROCESSOR_CONTEXT; - dataSource?: SavedObject; onProcessorsChange?: (count: number) => void; } @@ -97,24 +94,13 @@ export function ProcessorsList(props: ProcessorsListProps) { ? props.uiConfig.search.enrichRequest.processors : props.uiConfig.search.enrichResponse.processors; - // If version is 2.17 or 2.18, filter out ML processors - if ( - version && - semver.gte(version, '2.17.0') && - semver.lt(version, '2.19.0') - ) { - currentProcessors = - currentProcessors?.filter( - (processor) => processor.type !== 'ml_inference' - ) || []; - } setProcessors(currentProcessors || []); props.onProcessorsChange?.(currentProcessors?.length || 0); } }; loadProcessors(); - }, [props.context, props.uiConfig, version]); + }, [props.context, props.uiConfig]); const getMenuItems = () => { const isPreV219 = diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx index d2623a89..fd38f43d 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx @@ -35,7 +35,6 @@ export function EnrichSearchRequest(props: EnrichSearchRequestProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} context={PROCESSOR_CONTEXT.SEARCH_REQUEST} - dataSource={props.dataSource} /> diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx index da41a772..68ac7d0d 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx @@ -35,7 +35,6 @@ export function EnrichSearchResponse(props: EnrichSearchResponseProps) { uiConfig={props.uiConfig} setUiConfig={props.setUiConfig} context={PROCESSOR_CONTEXT.SEARCH_RESPONSE} - dataSource={props.dataSource} /> diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index a0fdb6d9..d7d6a6b1 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -32,6 +32,10 @@ import { getDataSourceEnabled } from '../../../services'; import semver from 'semver'; import { DataSourceAttributes } from '../../../../../../src/plugins/data_source/common/data_sources'; import { getSavedObjectsClient } from '../../../../public/services'; +import { + WORKFLOW_TYPE, + MIN_SUPPORTED_VERSION, +} from '../../../../common/constants'; interface NewWorkflowProps {} @@ -39,7 +43,6 @@ export const getEffectiveVersion = async ( dataSourceId: string | undefined ): Promise => { try { - // Don't make the API call if no dataSourceId if (dataSourceId === undefined) { console.log('cannot find data source'); throw new Error('Data source is required'); @@ -49,22 +52,23 @@ export const getEffectiveVersion = async ( 'data-source', dataSourceId ); - const version = dataSource?.attributes?.dataSourceVersion || '2.17.0'; + const version = + dataSource?.attributes?.dataSourceVersion || MIN_SUPPORTED_VERSION; return version; } catch (error) { console.error('Error getting version:', error); - return '2.17.0'; + return MIN_SUPPORTED_VERSION; } }; -const filterWorkflowsByVersion = async ( +const filterPresetsByVersion = async ( workflows: WorkflowTemplate[], dataSourceId: string ): Promise => { const allowedPresetsFor217 = [ - 'Semantic Search', - 'Multimodal Search', - 'Hybrid Search', + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, ]; try { @@ -78,16 +82,19 @@ const filterWorkflowsByVersion = async ( } if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { - return workflows.filter((workflow) => - allowedPresetsFor217.includes(workflow.name) - ); + return workflows.filter((workflow) => { + const workflowType = + workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; + return allowedPresetsFor217.includes(workflowType as WORKFLOW_TYPE); + }); } return workflows; } catch (error) { - return workflows.filter((workflow) => - allowedPresetsFor217.includes(workflow.name) - ); + return workflows.filter((workflow) => { + const workflowType = workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; + return allowedPresetsFor217.includes(workflowType as WORKFLOW_TYPE); + }); } }; @@ -128,6 +135,8 @@ export function NewWorkflow(props: NewWorkflowProps) { } }, [dataSourceId, dataSourceEnabled]); + // initial hook to populate all workflows + // enrich them with dynamically-generated UI flows based on use case useEffect(() => { const loadWorkflows = async () => { if (!presetWorkflows) return; @@ -138,7 +147,7 @@ export function NewWorkflow(props: NewWorkflowProps) { enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) ); - const versionFilteredWorkflows = await filterWorkflowsByVersion( + const versionFilteredWorkflows = await filterPresetsByVersion( enrichedWorkflows, dataSourceId ); From d612c1face1948fb13abdbcde591119db506de20 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Thu, 19 Dec 2024 20:09:58 -0800 Subject: [PATCH 3/9] addressing comments made on 12/19. Check if MDS is configured when loading the page. If disabled, the version should be set to 2.19+ Signed-off-by: Kama Huang --- common/constants.ts | 24 +++++++++++-------- opensearch_dashboards.json | 2 +- .../workflow_detail/components/header.tsx | 2 +- .../ingest_inputs/enrich_data.tsx | 4 ---- .../ingest_inputs/ingest_inputs.tsx | 6 +---- .../workflow_inputs/processors_list.tsx | 2 -- .../search_inputs/enrich_search_request.tsx | 3 --- .../search_inputs/enrich_search_response.tsx | 3 --- .../search_inputs/search_inputs.tsx | 5 ---- .../workflows/new_workflow/new_workflow.tsx | 14 +++++++++-- public/plugin.ts | 1 - 11 files changed, 29 insertions(+), 37 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 15954d3d..be26bc38 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -315,6 +315,7 @@ export const SEMANTIC_SEARCH_QUERY_NEURAL = { neural: { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, + model_id: MODEL_ID_PATTERN, k: DEFAULT_K, }, }, @@ -336,17 +337,20 @@ export const MULTIMODAL_SEARCH_QUERY_NEURAL = { }, }; export const MULTIMODAL_SEARCH_QUERY_BOOL = { - _source: { - excludes: [VECTOR_FIELD_PATTERN], - }, query: { - neural: { - [VECTOR_FIELD_PATTERN]: { - query_text: QUERY_TEXT_PATTERN, - query_image: QUERY_IMAGE_PATTERN, - model_id: MODEL_ID_PATTERN, - k: DEFAULT_K, - }, + bool: { + must: [ + { + match: { + [TEXT_FIELD_PATTERN]: QUERY_TEXT_PATTERN, + }, + }, + { + match: { + [IMAGE_FIELD_PATTERN]: QUERY_IMAGE_PATTERN, + }, + }, + ], }, }, }; diff --git a/opensearch_dashboards.json b/opensearch_dashboards.json index a1409aa6..ecf6e31a 100644 --- a/opensearch_dashboards.json +++ b/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "flowFrameworkDashboards", "version": "2.19.0.0", - "opensearchDashboardsVersion": "2.18.0", + "opensearchDashboardsVersion": "2.19.0", "server": true, "ui": true, "requiredBundles": [], diff --git a/public/pages/workflow_detail/components/header.tsx b/public/pages/workflow_detail/components/header.tsx index ca1edd29..06fa7250 100644 --- a/public/pages/workflow_detail/components/header.tsx +++ b/public/pages/workflow_detail/components/header.tsx @@ -115,7 +115,7 @@ export function WorkflowDetailHeader(props: WorkflowDetailHeaderProps) { // get & render the data source component, if applicable let DataSourceComponent: ReactElement | null = null; - if (dataSourceEnabled && getDataSourceManagementPlugin()) { + if (dataSourceEnabled && getDataSourceManagementPlugin() && dataSourceId) { const DataSourceMenu = getDataSourceManagementPlugin().ui.getDataSourceMenu< DataSourceViewConfig >(); diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx index 90ce0600..3aca93fc 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/enrich_data.tsx @@ -8,13 +8,9 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsList } from '../processors_list'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsTitle } from '../../../../general_components'; -import { SavedObject } from '../../../../../../../src/core/public'; -import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; - interface EnrichDataProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; - dataSource?: SavedObject; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx index 1496142d..2358003f 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/ingest_inputs.tsx @@ -36,11 +36,7 @@ export function IngestInputs(props: IngestInputsProps) { - + diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index ec541d72..c875ac8e 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -49,7 +49,6 @@ interface ProcessorsListProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; context: PROCESSOR_CONTEXT; - onProcessorsChange?: (count: number) => void; } const PANEL_ID = 0; @@ -95,7 +94,6 @@ export function ProcessorsList(props: ProcessorsListProps) { : props.uiConfig.search.enrichResponse.processors; setProcessors(currentProcessors || []); - props.onProcessorsChange?.(currentProcessors?.length || 0); } }; diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx index fd38f43d..d2fd3488 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_request.tsx @@ -8,13 +8,10 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsTitle } from '../../../../general_components'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsList } from '../processors_list'; -import { SavedObject } from '../../../../../../../src/core/public'; -import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface EnrichSearchRequestProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; - dataSource?: SavedObject; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx index 68ac7d0d..4ad3b288 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/enrich_search_response.tsx @@ -8,13 +8,10 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { ProcessorsTitle } from '../../../../general_components'; import { PROCESSOR_CONTEXT, WorkflowConfig } from '../../../../../common'; import { ProcessorsList } from '../processors_list'; -import { SavedObject } from '../../../../../../../src/core/public'; -import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface EnrichSearchResponseProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; - dataSource?: SavedObject; } /** diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx index 9eeb7403..c922a6ac 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/search_inputs.tsx @@ -14,13 +14,10 @@ import { } from '../../../../../common'; import { catIndices, useAppDispatch } from '../../../../store'; import { getDataSourceId } from '../../../../utils'; -import { SavedObject } from '../../../../../../../src/core/public'; -import { DataSourceAttributes } from '../../../../../../../src/plugins/data_source/common/data_sources'; interface SearchInputsProps { uiConfig: WorkflowConfig; setUiConfig: (uiConfig: WorkflowConfig) => void; - dataSource?: SavedObject; } /** @@ -48,7 +45,6 @@ export function SearchInputs(props: SearchInputsProps) { @@ -58,7 +54,6 @@ export function SearchInputs(props: SearchInputsProps) { diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index d7d6a6b1..1387db2e 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -63,8 +63,13 @@ export const getEffectiveVersion = async ( const filterPresetsByVersion = async ( workflows: WorkflowTemplate[], - dataSourceId: string + dataSourceId: string | undefined ): Promise => { + // if MDS is disabled, skip the version check and assume it is version 2.19+ + const dataSourceEnabled = getDataSourceEnabled().enabled; + if (!dataSourceEnabled) { + return workflows; + } const allowedPresetsFor217 = [ WORKFLOW_TYPE.SEMANTIC_SEARCH, WORKFLOW_TYPE.MULTIMODAL_SEARCH, @@ -72,6 +77,10 @@ const filterPresetsByVersion = async ( ]; try { + if (!dataSourceId) { + return []; + } + const version = await getEffectiveVersion(dataSourceId); if (version === undefined) { return []; @@ -140,7 +149,8 @@ export function NewWorkflow(props: NewWorkflowProps) { useEffect(() => { const loadWorkflows = async () => { if (!presetWorkflows) return; - if (!dataSourceId) return; + const dataSourceEnabled = getDataSourceEnabled().enabled; + if (dataSourceEnabled && !dataSourceId) return; const version = await getEffectiveVersion(dataSourceId); const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => diff --git a/public/plugin.ts b/public/plugin.ts index 711d98dc..dfe4d198 100644 --- a/public/plugin.ts +++ b/public/plugin.ts @@ -31,7 +31,6 @@ import { setHeaderActionMenu, } from './services'; import { configureRoutes } from './route_service'; -import { dataSourceFilterFn } from './utils'; export class FlowFrameworkDashboardsPlugin implements From 61a4c6f50c78e270e12c36b5bdb9db785d2eab38 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Fri, 20 Dec 2024 15:46:19 -0800 Subject: [PATCH 4/9] add filtering logic. If MDS is configured but disabled, set default version to 2.19 Signed-off-by: Kama Huang --- .../workflow_inputs/processors_list.tsx | 20 ++- .../workflows/new_workflow/new_workflow.tsx | 143 +++++++++++++++--- 2 files changed, 138 insertions(+), 25 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index c875ac8e..ad4d1dfc 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -44,6 +44,7 @@ import { } from '../../../configs'; import { ProcessorInputs } from './processor_inputs'; import { useLocation } from 'react-router-dom'; +import { getDataSourceEnabled } from '../../../../public/services'; interface ProcessorsListProps { uiConfig: WorkflowConfig; @@ -74,9 +75,19 @@ export function ProcessorsList(props: ProcessorsListProps) { useEffect(() => { const dataSourceId = getDataSourceFromURL(location).dataSourceId; + console.log('DataSourceId:', dataSourceId); + + const enabled = getDataSourceEnabled().enabled; + if (!enabled) { + console.log('MDS disabled, setting version to 2.19.0'); + setVersion('2.19.0'); + return; + } + if (dataSourceId) { getEffectiveVersion(dataSourceId) .then((ver) => { + console.log('Current version:', ver); setVersion(ver); }) .catch(console.error); @@ -103,7 +114,8 @@ export function ProcessorsList(props: ProcessorsListProps) { const getMenuItems = () => { const isPreV219 = semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0'); - + console.log('Version:', version); + console.log('Is pre-2.19:', isPreV219); const ingestProcessors = [ ...(isPreV219 ? [ @@ -368,7 +380,11 @@ export function ProcessorsList(props: ProcessorsListProps) { { id: PANEL_ID, title: getMenuItems().length > 0 ? 'PROCESSORS' : '', - items: getMenuItems(), + items: (() => { + const items = getMenuItems(); + console.log('Menu items:', items); + return items; + })(), }, ]} /> diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 1387db2e..1ad779b6 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -42,6 +42,16 @@ interface NewWorkflowProps {} export const getEffectiveVersion = async ( dataSourceId: string | undefined ): Promise => { + // console.log('Checking data source enabled status...'); + // const dataSourceEnabled = getDataSourceEnabled().enabled; + // console.log('Data source enabled:', dataSourceEnabled); + + // // If MDS is disabled, return a default version + // if (!dataSourceEnabled) { + // console.log('MDS is disabled, returning latest version'); + // return '2.19.0'; + // } + try { if (dataSourceId === undefined) { console.log('cannot find data source'); @@ -52,9 +62,10 @@ export const getEffectiveVersion = async ( 'data-source', dataSourceId ); - const version = - dataSource?.attributes?.dataSourceVersion || MIN_SUPPORTED_VERSION; - return version; + // const version = + // dataSource?.attributes?.dataSourceVersion || MIN_SUPPORTED_VERSION; + // return version; + return '2.19.0'; } catch (error) { console.error('Error getting version:', error); return MIN_SUPPORTED_VERSION; @@ -65,11 +76,19 @@ const filterPresetsByVersion = async ( workflows: WorkflowTemplate[], dataSourceId: string | undefined ): Promise => { + console.log('Initial workflows count:', workflows.length); // if MDS is disabled, skip the version check and assume it is version 2.19+ const dataSourceEnabled = getDataSourceEnabled().enabled; + console.log('MDS enabled:', dataSourceEnabled); if (!dataSourceEnabled) { + console.log('MDS is disabled, returning all workflows'); return workflows; } + + if (!dataSourceId) { + return []; + } + const allowedPresetsFor217 = [ WORKFLOW_TYPE.SEMANTIC_SEARCH, WORKFLOW_TYPE.MULTIMODAL_SEARCH, @@ -77,14 +96,7 @@ const filterPresetsByVersion = async ( ]; try { - if (!dataSourceId) { - return []; - } - const version = await getEffectiveVersion(dataSourceId); - if (version === undefined) { - return []; - } if (semver.lt(version, '2.17.0')) { return []; @@ -117,6 +129,8 @@ export function NewWorkflow(props: NewWorkflowProps) { const dataSourceId = getDataSourceId(); const dataSourceEnabled = getDataSourceEnabled().enabled; + console.log('Initial render:', { dataSourceId, dataSourceEnabled }); + // workflows state const { presetWorkflows, loading } = useSelector( (state: AppState) => state.presets @@ -125,6 +139,7 @@ export function NewWorkflow(props: NewWorkflowProps) { const [filteredWorkflows, setFilteredWorkflows] = useState< WorkflowTemplate[] >([]); + const [isVersionLoading, setIsVersionLoading] = useState(false); // search bar state const [searchQuery, setSearchQuery] = useState(''); @@ -144,29 +159,111 @@ export function NewWorkflow(props: NewWorkflowProps) { } }, [dataSourceId, dataSourceEnabled]); + useEffect(() => { + console.log('State changed:', { + loading, + isVersionLoading, + presetWorkflowsLength: presetWorkflows?.length, + allWorkflowsLength: allWorkflows.length, + filteredWorkflowsLength: filteredWorkflows.length, + }); + }, [ + loading, + isVersionLoading, + presetWorkflows, + allWorkflows, + filteredWorkflows, + ]); + // initial hook to populate all workflows // enrich them with dynamically-generated UI flows based on use case useEffect(() => { const loadWorkflows = async () => { - if (!presetWorkflows) return; + console.log('loadWorkflows called:', { + hasPresetWorkflows: Boolean(presetWorkflows?.length), + dataSourceId, + }); + + if (!presetWorkflows || presetWorkflows.length === 0) { + console.log('No preset workflows, returning early'); + + return; + } + const dataSourceEnabled = getDataSourceEnabled().enabled; - if (dataSourceEnabled && !dataSourceId) return; + console.log('MDS enabled:', dataSourceEnabled); - const version = await getEffectiveVersion(dataSourceId); - const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => - enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) - ); + if (!dataSourceEnabled) { + console.log('MDS disabled, setting all workflows'); - const versionFilteredWorkflows = await filterPresetsByVersion( - enrichedWorkflows, - dataSourceId - ); + const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow, '2.19.0') + ); + setAllWorkflows(enrichedWorkflows); + setFilteredWorkflows(enrichedWorkflows); + setIsVersionLoading(false); + return; + } + + if (!dataSourceId) { + console.log('No datasource ID, clearing workflows and setting loading'); + + setAllWorkflows([]); + setFilteredWorkflows([]); + setIsVersionLoading(true); + return; + } + console.log('Starting version check'); - setAllWorkflows(versionFilteredWorkflows); + setIsVersionLoading(true); + + // const version = await getEffectiveVersion(dataSourceId); + // const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + // enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) + // ); + + // const versionFilteredWorkflows = await filterPresetsByVersion( + // enrichedWorkflows, + // dataSourceId + // ); + + // setAllWorkflows(versionFilteredWorkflows); + // setFilteredWorkflows(versionFilteredWorkflows); + // }; + try { + const version = await getEffectiveVersion(dataSourceId); + console.log('Got version:', version); + + const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) + ); + + const versionFilteredWorkflows = await filterPresetsByVersion( + enrichedWorkflows, + dataSourceId + ); + console.log( + 'Setting filtered workflows:', + versionFilteredWorkflows.length + ); + + setAllWorkflows(versionFilteredWorkflows); + setFilteredWorkflows(versionFilteredWorkflows); + setIsVersionLoading(false); + } catch (error) { + console.error('Error loading workflows:', error); + setAllWorkflows([]); + setFilteredWorkflows([]); + if (dataSourceId) { + setIsVersionLoading(false); + } + } }; loadWorkflows(); - }, [presetWorkflows, dataSourceId]); + }, [presetWorkflows, dataSourceId, dataSourceEnabled]); + + console.log('Render - loading states:', { loading, isVersionLoading }); // When search query updated, re-filter preset list useEffect(() => { @@ -187,7 +284,7 @@ export function NewWorkflow(props: NewWorkflowProps) { /> - {!dataSourceId || loading ? ( + {loading || isVersionLoading ? ( Date: Wed, 25 Dec 2024 17:26:54 -0800 Subject: [PATCH 5/9] clean up logs and format code Signed-off-by: Kama Huang --- .../workflow_inputs/processors_list.tsx | 5 -- .../workflows/new_workflow/new_workflow.tsx | 72 +------------------ 2 files changed, 3 insertions(+), 74 deletions(-) diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index ad4d1dfc..ea029e7a 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -75,11 +75,9 @@ export function ProcessorsList(props: ProcessorsListProps) { useEffect(() => { const dataSourceId = getDataSourceFromURL(location).dataSourceId; - console.log('DataSourceId:', dataSourceId); const enabled = getDataSourceEnabled().enabled; if (!enabled) { - console.log('MDS disabled, setting version to 2.19.0'); setVersion('2.19.0'); return; } @@ -87,7 +85,6 @@ export function ProcessorsList(props: ProcessorsListProps) { if (dataSourceId) { getEffectiveVersion(dataSourceId) .then((ver) => { - console.log('Current version:', ver); setVersion(ver); }) .catch(console.error); @@ -114,8 +111,6 @@ export function ProcessorsList(props: ProcessorsListProps) { const getMenuItems = () => { const isPreV219 = semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0'); - console.log('Version:', version); - console.log('Is pre-2.19:', isPreV219); const ingestProcessors = [ ...(isPreV219 ? [ diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 1ad779b6..4005fd5a 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -42,19 +42,8 @@ interface NewWorkflowProps {} export const getEffectiveVersion = async ( dataSourceId: string | undefined ): Promise => { - // console.log('Checking data source enabled status...'); - // const dataSourceEnabled = getDataSourceEnabled().enabled; - // console.log('Data source enabled:', dataSourceEnabled); - - // // If MDS is disabled, return a default version - // if (!dataSourceEnabled) { - // console.log('MDS is disabled, returning latest version'); - // return '2.19.0'; - // } - try { if (dataSourceId === undefined) { - console.log('cannot find data source'); throw new Error('Data source is required'); } @@ -62,10 +51,9 @@ export const getEffectiveVersion = async ( 'data-source', dataSourceId ); - // const version = - // dataSource?.attributes?.dataSourceVersion || MIN_SUPPORTED_VERSION; - // return version; - return '2.19.0'; + const version = + dataSource?.attributes?.dataSourceVersion || MIN_SUPPORTED_VERSION; + return version; } catch (error) { console.error('Error getting version:', error); return MIN_SUPPORTED_VERSION; @@ -79,9 +67,7 @@ const filterPresetsByVersion = async ( console.log('Initial workflows count:', workflows.length); // if MDS is disabled, skip the version check and assume it is version 2.19+ const dataSourceEnabled = getDataSourceEnabled().enabled; - console.log('MDS enabled:', dataSourceEnabled); if (!dataSourceEnabled) { - console.log('MDS is disabled, returning all workflows'); return workflows; } @@ -128,9 +114,6 @@ export function NewWorkflow(props: NewWorkflowProps) { const dispatch = useAppDispatch(); const dataSourceId = getDataSourceId(); const dataSourceEnabled = getDataSourceEnabled().enabled; - - console.log('Initial render:', { dataSourceId, dataSourceEnabled }); - // workflows state const { presetWorkflows, loading } = useSelector( (state: AppState) => state.presets @@ -159,43 +142,17 @@ export function NewWorkflow(props: NewWorkflowProps) { } }, [dataSourceId, dataSourceEnabled]); - useEffect(() => { - console.log('State changed:', { - loading, - isVersionLoading, - presetWorkflowsLength: presetWorkflows?.length, - allWorkflowsLength: allWorkflows.length, - filteredWorkflowsLength: filteredWorkflows.length, - }); - }, [ - loading, - isVersionLoading, - presetWorkflows, - allWorkflows, - filteredWorkflows, - ]); - // initial hook to populate all workflows // enrich them with dynamically-generated UI flows based on use case useEffect(() => { const loadWorkflows = async () => { - console.log('loadWorkflows called:', { - hasPresetWorkflows: Boolean(presetWorkflows?.length), - dataSourceId, - }); - if (!presetWorkflows || presetWorkflows.length === 0) { - console.log('No preset workflows, returning early'); - return; } const dataSourceEnabled = getDataSourceEnabled().enabled; - console.log('MDS enabled:', dataSourceEnabled); if (!dataSourceEnabled) { - console.log('MDS disabled, setting all workflows'); - const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => enrichPresetWorkflowWithUiMetadata(presetWorkflow, '2.19.0') ); @@ -206,33 +163,16 @@ export function NewWorkflow(props: NewWorkflowProps) { } if (!dataSourceId) { - console.log('No datasource ID, clearing workflows and setting loading'); - setAllWorkflows([]); setFilteredWorkflows([]); setIsVersionLoading(true); return; } - console.log('Starting version check'); setIsVersionLoading(true); - // const version = await getEffectiveVersion(dataSourceId); - // const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => - // enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) - // ); - - // const versionFilteredWorkflows = await filterPresetsByVersion( - // enrichedWorkflows, - // dataSourceId - // ); - - // setAllWorkflows(versionFilteredWorkflows); - // setFilteredWorkflows(versionFilteredWorkflows); - // }; try { const version = await getEffectiveVersion(dataSourceId); - console.log('Got version:', version); const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) @@ -242,10 +182,6 @@ export function NewWorkflow(props: NewWorkflowProps) { enrichedWorkflows, dataSourceId ); - console.log( - 'Setting filtered workflows:', - versionFilteredWorkflows.length - ); setAllWorkflows(versionFilteredWorkflows); setFilteredWorkflows(versionFilteredWorkflows); @@ -263,8 +199,6 @@ export function NewWorkflow(props: NewWorkflowProps) { loadWorkflows(); }, [presetWorkflows, dataSourceId, dataSourceEnabled]); - console.log('Render - loading states:', { loading, isVersionLoading }); - // When search query updated, re-filter preset list useEffect(() => { setFilteredWorkflows(fetchFilteredWorkflows(allWorkflows, searchQuery)); From 200b2d97128eaac9859c587cb2c4198db4bdb6bf Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Fri, 27 Dec 2024 16:42:14 -0800 Subject: [PATCH 6/9] address comments on 12/26 Signed-off-by: Kama Huang --- common/constants.ts | 2 + .../workflow_detail/workflow_detail.test.tsx | 26 +-- .../workflow_inputs/processors_list.tsx | 10 +- .../workflows/new_workflow/new_workflow.tsx | 14 +- public/pages/workflows/new_workflow/utils.ts | 150 ++++++++++-------- 5 files changed, 115 insertions(+), 87 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index be26bc38..f9adec16 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -171,6 +171,8 @@ export enum WORKFLOW_TYPE { } // If no datasource version is found, we default to 2.17.0 export const MIN_SUPPORTED_VERSION = '2.17.0'; +// Min version to support ML processors +export const MINIMUM_FULL_SUPPORTED_VERSION = '2.19.0'; // the names should be consistent with the underlying implementation. used when generating the // final ingest/search pipeline configurations. diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index adb43839..531de490 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -110,28 +110,28 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { workflowName, WORKFLOW_TYPE.CUSTOM ); - + // Export button opens the export component userEvent.click(getByTestId('exportButton')); await waitFor(() => { expect(getByText(`Export '${workflowName}'`)).toBeInTheDocument(); }); // Close the export component userEvent.click(getByTestId('exportCloseButton')); - + // Check workspace button group exists (Visual and JSON) getByTestId('visualJSONToggleButtonGroup'); - + // Tools panel should collapse and expand the toggle const toolsPanel = container.querySelector('#tools_panel_id'); expect(toolsPanel).toBeVisible(); const toggleButton = toolsPanel?.querySelector('button[type="button"]'); expect(toggleButton).toBeInTheDocument(); userEvent.click(toggleButton!); - + // Tools panel after collapsing const collapsedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { expect(collapsedToolsPanel).toHaveClass('euiResizablePanel-isCollapsed'); }); - + // Tools panel after expanding userEvent.click(toggleButton!); const expandedToolsPanel = container.querySelector('#tools_panel_id'); await waitFor(() => { @@ -147,7 +147,7 @@ describe('WorkflowDetail Page Functionality (Custom Workflow)', () => { workflowName, WORKFLOW_TYPE.CUSTOM ); - + // The WorkflowDetail Page Close button should navigate back to the workflows list userEvent.click(getByTestId('closeButton')); await waitFor(() => { expect(history.location.pathname).toBe('/workflows'); @@ -166,20 +166,20 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow workflowName, WORKFLOW_TYPE.HYBRID_SEARCH ); - + // Defining a new ingest pipeline & index is enabled by default const enabledCheckbox = getByTestId('switch-ingest.enabled'); - + // Skipping ingest pipeline and navigating to search userEvent.click(enabledCheckbox); await waitFor(() => {}); const searchPipelineButton = getByTestId('searchPipelineButton'); userEvent.click(searchPipelineButton); - + // Search pipeline await waitFor(() => { expect(getAllByText('Define search flow').length).toBeGreaterThan(0); }); expect(getAllByText('Configure query').length).toBeGreaterThan(0); - + // Edit Search Query const queryEditButton = getByTestId('queryEditButton'); expect(queryEditButton).toBeInTheDocument(); userEvent.click(queryEditButton); @@ -193,7 +193,7 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow const updateSearchQueryButton = getByTestId('updateSearchQueryButton'); expect(updateSearchQueryButton).toBeInTheDocument(); userEvent.click(updateSearchQueryButton); - + // Add request processor const addRequestProcessorButton = await waitFor( () => getAllByTestId('addProcessorButton')[0] ); @@ -203,14 +203,14 @@ describe('WorkflowDetail Page with skip ingestion option (Hybrid Search Workflow const popoverPanel = document.querySelector('.euiPopover__panel'); expect(popoverPanel).toBeTruthy(); }); - + // Add response processor const addResponseProcessorButton = getAllByTestId('addProcessorButton')[1]; userEvent.click(addResponseProcessorButton); await waitFor(() => { const popoverPanel = document.querySelector('.euiPopover__panel'); expect(popoverPanel).toBeTruthy(); }); - + // Build and Run query, Back buttons are present const searchPipelineBackButton = getByTestId('searchPipelineBackButton'); userEvent.click(searchPipelineBackButton); diff --git a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx index ea029e7a..ead3617d 100644 --- a/public/pages/workflow_detail/workflow_inputs/processors_list.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processors_list.tsx @@ -45,6 +45,10 @@ import { import { ProcessorInputs } from './processor_inputs'; import { useLocation } from 'react-router-dom'; import { getDataSourceEnabled } from '../../../../public/services'; +import { + MIN_SUPPORTED_VERSION, + MINIMUM_FULL_SUPPORTED_VERSION, +} from '../../../../common'; interface ProcessorsListProps { uiConfig: WorkflowConfig; @@ -78,7 +82,7 @@ export function ProcessorsList(props: ProcessorsListProps) { const enabled = getDataSourceEnabled().enabled; if (!enabled) { - setVersion('2.19.0'); + setVersion(MINIMUM_FULL_SUPPORTED_VERSION); return; } @@ -110,7 +114,8 @@ export function ProcessorsList(props: ProcessorsListProps) { const getMenuItems = () => { const isPreV219 = - semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0'); + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); const ingestProcessors = [ ...(isPreV219 ? [ @@ -377,7 +382,6 @@ export function ProcessorsList(props: ProcessorsListProps) { title: getMenuItems().length > 0 ? 'PROCESSORS' : '', items: (() => { const items = getMenuItems(); - console.log('Menu items:', items); return items; })(), }, diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 4005fd5a..520dcdf8 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -35,6 +35,7 @@ import { getSavedObjectsClient } from '../../../../public/services'; import { WORKFLOW_TYPE, MIN_SUPPORTED_VERSION, + MINIMUM_FULL_SUPPORTED_VERSION, } from '../../../../common/constants'; interface NewWorkflowProps {} @@ -64,7 +65,6 @@ const filterPresetsByVersion = async ( workflows: WorkflowTemplate[], dataSourceId: string | undefined ): Promise => { - console.log('Initial workflows count:', workflows.length); // if MDS is disabled, skip the version check and assume it is version 2.19+ const dataSourceEnabled = getDataSourceEnabled().enabled; if (!dataSourceEnabled) { @@ -84,11 +84,14 @@ const filterPresetsByVersion = async ( try { const version = await getEffectiveVersion(dataSourceId); - if (semver.lt(version, '2.17.0')) { + if (semver.lt(version, MIN_SUPPORTED_VERSION)) { return []; } - if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { + if ( + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION) + ) { return workflows.filter((workflow) => { const workflowType = workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; @@ -154,7 +157,10 @@ export function NewWorkflow(props: NewWorkflowProps) { if (!dataSourceEnabled) { const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => - enrichPresetWorkflowWithUiMetadata(presetWorkflow, '2.19.0') + enrichPresetWorkflowWithUiMetadata( + presetWorkflow, + MINIMUM_FULL_SUPPORTED_VERSION + ) ); setAllWorkflows(enrichedWorkflows); setFilteredWorkflows(enrichedWorkflows); diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 819ad56a..4f84a220 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -10,6 +10,8 @@ import { MLSearchRequestProcessor, MLSearchResponseProcessor, NormalizationProcessor, + TextEmbeddingIngestProcessor, + TextImageEmbeddingIngestProcessor, } from '../../../configs'; import { WorkflowTemplate, @@ -19,7 +21,6 @@ import { WORKFLOW_TYPE, FETCH_ALL_QUERY, customStringify, - TERM_QUERY_TEXT, MULTIMODAL_SEARCH_QUERY_BOOL, IProcessorConfig, VECTOR_TEMPLATE_PLACEHOLDER, @@ -28,10 +29,13 @@ import { HYBRID_SEARCH_QUERY_MATCH_KNN, WorkflowConfig, UI_METADATA_SCHEMA_VERSION, - PROCESSOR_TYPE, + SEMANTIC_SEARCH_QUERY_NEURAL, + MULTIMODAL_SEARCH_QUERY_NEURAL, + HYBRID_SEARCH_QUERY_MATCH_NEURAL, } from '../../../../common'; import { generateId } from '../../../utils'; import semver from 'semver'; +import { MINIMUM_FULL_SUPPORTED_VERSION } from '../../../../common'; // Fn to produce the complete preset template with all necessary UI metadata. export function enrichPresetWorkflowWithUiMetadata( @@ -41,19 +45,19 @@ export function enrichPresetWorkflowWithUiMetadata( let uiMetadata = {} as UIState; switch (presetWorkflow.ui_metadata?.type || WORKFLOW_TYPE.CUSTOM) { case WORKFLOW_TYPE.SEMANTIC_SEARCH: { - uiMetadata = fetchSemanticSearchMetadata(); + uiMetadata = fetchSemanticSearchMetadata(version); break; } case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { - uiMetadata = fetchMultimodalSearchMetadata(); + uiMetadata = fetchMultimodalSearchMetadata(version); break; } case WORKFLOW_TYPE.HYBRID_SEARCH: { - uiMetadata = fetchHybridSearchMetadata(); + uiMetadata = fetchHybridSearchMetadata(version); break; } case WORKFLOW_TYPE.RAG: { - uiMetadata = fetchRAGMetadata(); + uiMetadata = fetchRAGMetadata(version); break; } default: { @@ -62,39 +66,6 @@ export function enrichPresetWorkflowWithUiMetadata( } } - if (semver.gte(version, '2.17.0') && semver.lt(version, '2.19.0')) { - uiMetadata.config.ingest.enrich.processors = uiMetadata.config.ingest.enrich.processors.filter( - (p) => p.type !== PROCESSOR_TYPE.ML - ); - uiMetadata.config.search.enrichRequest.processors = uiMetadata.config.search.enrichRequest.processors.filter( - (p) => p.type !== PROCESSOR_TYPE.ML - ); - uiMetadata.config.search.enrichResponse.processors = uiMetadata.config.search.enrichResponse.processors.filter( - (p) => p.type !== PROCESSOR_TYPE.ML - ); - } - - if (semver.eq(version, '2.19.0')) { - uiMetadata.config.ingest.enrich.processors = uiMetadata.config.ingest.enrich.processors.filter( - (p) => - p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && - p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && - p.type !== PROCESSOR_TYPE.NORMALIZATION - ); - uiMetadata.config.search.enrichRequest.processors = uiMetadata.config.search.enrichRequest.processors.filter( - (p) => - p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && - p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && - p.type !== PROCESSOR_TYPE.NORMALIZATION - ); - uiMetadata.config.search.enrichResponse.processors = uiMetadata.config.search.enrichResponse.processors.filter( - (p) => - p.type !== PROCESSOR_TYPE.TEXT_EMBEDDING && - p.type !== PROCESSOR_TYPE.TEXT_IMAGE_EMBEDDING && - p.type !== PROCESSOR_TYPE.NORMALIZATION - ); - } - return { ...presetWorkflow, ui_metadata: { @@ -174,68 +145,113 @@ export function fetchEmptyUIConfig(): WorkflowConfig { }; } -export function fetchSemanticSearchMetadata(): UIState { +export function fetchSemanticSearchMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.SEMANTIC_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + + baseState.config.ingest.enrich.processors = isPreV219 + ? [ + new TextEmbeddingIngestProcessor().toObj(), + new TextImageEmbeddingIngestProcessor().toObj(), + ] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - KNN_QUERY - ), - ]; + + baseState.config.search.request.value = customStringify( + isPreV219 ? SEMANTIC_SEARCH_QUERY_NEURAL : KNN_QUERY + ); + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), + ]; + return baseState; } -export function fetchMultimodalSearchMetadata(): UIState { +export function fetchMultimodalSearchMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.MULTIMODAL_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + + baseState.config.ingest.enrich.processors = isPreV219 + ? [ + new TextEmbeddingIngestProcessor().toObj(), + new TextImageEmbeddingIngestProcessor().toObj(), + ] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); + baseState.config.search.request.value = customStringify( - MULTIMODAL_SEARCH_QUERY_BOOL + isPreV219 ? MULTIMODAL_SEARCH_QUERY_NEURAL : MULTIMODAL_SEARCH_QUERY_BOOL ); - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - KNN_QUERY - ), - ]; + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), + ]; + return baseState; } -export function fetchHybridSearchMetadata(): UIState { +export function fetchHybridSearchMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH; - baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); + + baseState.config.ingest.enrich.processors = isPreV219 + ? [ + new TextEmbeddingIngestProcessor().toObj(), + new TextImageEmbeddingIngestProcessor().toObj(), + ] + : [new MLIngestProcessor().toObj()]; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); - baseState.config.search.request.value = customStringify(TERM_QUERY_TEXT); + + baseState.config.search.request.value = customStringify( + isPreV219 ? HYBRID_SEARCH_QUERY_MATCH_NEURAL : HYBRID_SEARCH_QUERY_MATCH_KNN + ); + baseState.config.search.enrichResponse.processors = [ injectDefaultWeightsInNormalizationProcessor( new NormalizationProcessor().toObj() ), ]; - baseState.config.search.enrichRequest.processors = [ - injectQueryTemplateInProcessor( - new MLSearchRequestProcessor().toObj(), - HYBRID_SEARCH_QUERY_MATCH_KNN - ), - ]; + + baseState.config.search.enrichRequest.processors = isPreV219 + ? [] + : [ + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + HYBRID_SEARCH_QUERY_MATCH_KNN + ), + ]; + return baseState; } -export function fetchRAGMetadata(): UIState { +export function fetchRAGMetadata(version: string): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.RAG; baseState.config.ingest.index.name.value = generateId('my_index', 6); From 71bf71f1aa75fd3f74f3370a20e3393f97db8189 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Sat, 28 Dec 2024 00:04:48 -0800 Subject: [PATCH 7/9] fix unit tests after address comments on 12/27 Signed-off-by: Kama Huang --- .../workflow_detail/workflow_detail.test.tsx | 10 +++++- public/pages/workflows/new_workflow/utils.ts | 23 +++++++++++++ test/interfaces.ts | 1 + test/utils.ts | 32 +++++++++++++++---- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index 531de490..49d7cee4 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -13,7 +13,7 @@ import { WorkflowDetailRouterProps } from '../../pages'; import '@testing-library/jest-dom'; import { mockStore, resizeObserverMock } from '../../../test/utils'; import { createMemoryHistory } from 'history'; -import { WORKFLOW_TYPE } from '../../../common'; +import { MINIMUM_FULL_SUPPORTED_VERSION, WORKFLOW_TYPE } from '../../../common'; jest.mock('../../services', () => { const { mockCoreServices } = require('../../../test'); @@ -39,6 +39,12 @@ const renderWithRouter = ( initialEntries: [`/workflow/${workflowId}`], }); + const needsVersion = [ + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, + ].includes(workflowType); + return { ...render( @@ -68,6 +75,7 @@ describe('WorkflowDetail Page with create ingestion option', () => { beforeEach(() => { jest.clearAllMocks(); }); + Object.values(WORKFLOW_TYPE).forEach((type) => { test(`renders the WorkflowDetail page with ${type} type`, async () => { const { diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 4f84a220..7acbfa0f 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -146,6 +146,11 @@ export function fetchEmptyUIConfig(): WorkflowConfig { } export function fetchSemanticSearchMetadata(version: string): UIState { + if (!version) { + throw new Error( + 'Version parameter is required for fetchSemanticSearchMetadata' + ); + } let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.SEMANTIC_SEARCH; const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); @@ -179,6 +184,12 @@ export function fetchSemanticSearchMetadata(version: string): UIState { } export function fetchMultimodalSearchMetadata(version: string): UIState { + if (!version) { + throw new Error( + 'Version parameter is required for fetchSemanticSearchMetadata' + ); + } + let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.MULTIMODAL_SEARCH; @@ -213,6 +224,12 @@ export function fetchMultimodalSearchMetadata(version: string): UIState { } export function fetchHybridSearchMetadata(version: string): UIState { + if (!version) { + throw new Error( + 'Version parameter is required for fetchSemanticSearchMetadata' + ); + } + let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH; const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); @@ -252,6 +269,12 @@ export function fetchHybridSearchMetadata(version: string): UIState { } export function fetchRAGMetadata(version: string): UIState { + if (!version) { + throw new Error( + 'Version parameter is required for fetchSemanticSearchMetadata' + ); + } + let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.RAG; baseState.config.ingest.index.name.value = generateId('my_index', 6); diff --git a/test/interfaces.ts b/test/interfaces.ts index d9c3e20e..17455169 100644 --- a/test/interfaces.ts +++ b/test/interfaces.ts @@ -9,4 +9,5 @@ export type WorkflowInput = { id: string; name: string; type: WORKFLOW_TYPE; + version?: string; }; diff --git a/test/utils.ts b/test/utils.ts index 8953288d..de87cb89 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -10,7 +10,10 @@ import { INITIAL_WORKFLOWS_STATE, } from '../public/store'; import { WorkflowInput } from '../test/interfaces'; -import { WORKFLOW_TYPE } from '../common/constants'; +import { + MINIMUM_FULL_SUPPORTED_VERSION, + WORKFLOW_TYPE, +} from '../common/constants'; import { UIState, Workflow, WorkflowDict } from '../common/interfaces'; import { fetchEmptyMetadata, @@ -44,27 +47,42 @@ export function mockStore(...workflowSets: WorkflowInput[]) { } function generateWorkflow({ id, name, type }: WorkflowInput): Workflow { + const isSearchWorkflow = [ + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, + ].includes(type); + + const version = { + template: '1.0.0', + compatibility: isSearchWorkflow + ? [MINIMUM_FULL_SUPPORTED_VERSION] + : ['2.18.0', '3.0.0'], + }; + return { id, name, - version: { template: '1.0.0', compatibility: ['2.18.0', '3.0.0'] }, - ui_metadata: getConfig(type), + version, + ui_metadata: getConfig(type, version.compatibility[0]), }; } -function getConfig(workflowType: WORKFLOW_TYPE) { +function getConfig(workflowType: WORKFLOW_TYPE, version?: string) { let uiMetadata = {} as UIState; + const searchVersion = version || MINIMUM_FULL_SUPPORTED_VERSION; + switch (workflowType) { case WORKFLOW_TYPE.SEMANTIC_SEARCH: { - uiMetadata = fetchSemanticSearchMetadata(); + uiMetadata = fetchSemanticSearchMetadata(searchVersion); break; } case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { - uiMetadata = fetchMultimodalSearchMetadata(); + uiMetadata = fetchMultimodalSearchMetadata(searchVersion); break; } case WORKFLOW_TYPE.HYBRID_SEARCH: { - uiMetadata = fetchHybridSearchMetadata(); + uiMetadata = fetchHybridSearchMetadata(searchVersion); break; } default: { From b57af4331ef67b146e24325c2ee135920b18a104 Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Mon, 30 Dec 2024 15:57:19 -0800 Subject: [PATCH 8/9] address comments on 12/30. remove unnecessary try-catch blocks in newworkflows file and modify processors for each preset in the fetch function Signed-off-by: Kama Huang --- .../workflow_detail/workflow_detail.test.tsx | 26 ++++---- .../workflows/new_workflow/new_workflow.tsx | 59 +++++++------------ public/pages/workflows/new_workflow/utils.ts | 56 ++++-------------- 3 files changed, 47 insertions(+), 94 deletions(-) diff --git a/public/pages/workflow_detail/workflow_detail.test.tsx b/public/pages/workflow_detail/workflow_detail.test.tsx index 49d7cee4..29489ce0 100644 --- a/public/pages/workflow_detail/workflow_detail.test.tsx +++ b/public/pages/workflow_detail/workflow_detail.test.tsx @@ -39,22 +39,22 @@ const renderWithRouter = ( initialEntries: [`/workflow/${workflowId}`], }); - const needsVersion = [ - WORKFLOW_TYPE.SEMANTIC_SEARCH, - WORKFLOW_TYPE.MULTIMODAL_SEARCH, - WORKFLOW_TYPE.HYBRID_SEARCH, - ].includes(workflowType); + const mockInput = { + id: workflowId, + name: workflowName, + type: workflowType, + version: [ + WORKFLOW_TYPE.SEMANTIC_SEARCH, + WORKFLOW_TYPE.MULTIMODAL_SEARCH, + WORKFLOW_TYPE.HYBRID_SEARCH, + ].includes(workflowType) + ? MINIMUM_FULL_SUPPORTED_VERSION + : undefined, + }; return { ...render( - + { - const workflowType = - workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; - return allowedPresetsFor217.includes(workflowType as WORKFLOW_TYPE); - }); - } + if (semver.lt(version, MIN_SUPPORTED_VERSION)) { + return []; + } - return workflows; - } catch (error) { + if ( + semver.gte(version, MIN_SUPPORTED_VERSION) && + semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION) + ) { return workflows.filter((workflow) => { const workflowType = workflow.ui_metadata?.type ?? WORKFLOW_TYPE.UNKNOWN; return allowedPresetsFor217.includes(workflowType as WORKFLOW_TYPE); }); } + + return workflows; }; /** @@ -177,29 +169,20 @@ export function NewWorkflow(props: NewWorkflowProps) { setIsVersionLoading(true); - try { - const version = await getEffectiveVersion(dataSourceId); + const version = await getEffectiveVersion(dataSourceId); - const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => - enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) - ); + const enrichedWorkflows = presetWorkflows.map((presetWorkflow) => + enrichPresetWorkflowWithUiMetadata(presetWorkflow, version) + ); - const versionFilteredWorkflows = await filterPresetsByVersion( - enrichedWorkflows, - dataSourceId - ); + const versionFilteredWorkflows = await filterPresetsByVersion( + enrichedWorkflows, + dataSourceId + ); - setAllWorkflows(versionFilteredWorkflows); - setFilteredWorkflows(versionFilteredWorkflows); - setIsVersionLoading(false); - } catch (error) { - console.error('Error loading workflows:', error); - setAllWorkflows([]); - setFilteredWorkflows([]); - if (dataSourceId) { - setIsVersionLoading(false); - } - } + setAllWorkflows(versionFilteredWorkflows); + setFilteredWorkflows(versionFilteredWorkflows); + setIsVersionLoading(false); }; loadWorkflows(); diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 7acbfa0f..308703c5 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -42,22 +42,25 @@ export function enrichPresetWorkflowWithUiMetadata( presetWorkflow: Partial, version: string ): WorkflowTemplate { + const defaultVersion = MINIMUM_FULL_SUPPORTED_VERSION; + const workflowVersion = version ?? defaultVersion; + let uiMetadata = {} as UIState; switch (presetWorkflow.ui_metadata?.type || WORKFLOW_TYPE.CUSTOM) { case WORKFLOW_TYPE.SEMANTIC_SEARCH: { - uiMetadata = fetchSemanticSearchMetadata(version); + uiMetadata = fetchSemanticSearchMetadata(workflowVersion); break; } case WORKFLOW_TYPE.MULTIMODAL_SEARCH: { - uiMetadata = fetchMultimodalSearchMetadata(version); + uiMetadata = fetchMultimodalSearchMetadata(workflowVersion); break; } case WORKFLOW_TYPE.HYBRID_SEARCH: { - uiMetadata = fetchHybridSearchMetadata(version); + uiMetadata = fetchHybridSearchMetadata(workflowVersion); break; } case WORKFLOW_TYPE.RAG: { - uiMetadata = fetchRAGMetadata(version); + uiMetadata = fetchRAGMetadata(workflowVersion); break; } default: { @@ -146,20 +149,12 @@ export function fetchEmptyUIConfig(): WorkflowConfig { } export function fetchSemanticSearchMetadata(version: string): UIState { - if (!version) { - throw new Error( - 'Version parameter is required for fetchSemanticSearchMetadata' - ); - } + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.SEMANTIC_SEARCH; - const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); baseState.config.ingest.enrich.processors = isPreV219 - ? [ - new TextEmbeddingIngestProcessor().toObj(), - new TextImageEmbeddingIngestProcessor().toObj(), - ] + ? [new TextEmbeddingIngestProcessor().toObj()] : [new MLIngestProcessor().toObj()]; baseState.config.ingest.index.name.value = generateId('knn_index', 6); @@ -184,22 +179,12 @@ export function fetchSemanticSearchMetadata(version: string): UIState { } export function fetchMultimodalSearchMetadata(version: string): UIState { - if (!version) { - throw new Error( - 'Version parameter is required for fetchSemanticSearchMetadata' - ); - } - + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.MULTIMODAL_SEARCH; - const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); - baseState.config.ingest.enrich.processors = isPreV219 - ? [ - new TextEmbeddingIngestProcessor().toObj(), - new TextImageEmbeddingIngestProcessor().toObj(), - ] + ? [new TextImageEmbeddingIngestProcessor().toObj()] : [new MLIngestProcessor().toObj()]; baseState.config.ingest.index.name.value = generateId('knn_index', 6); @@ -224,21 +209,12 @@ export function fetchMultimodalSearchMetadata(version: string): UIState { } export function fetchHybridSearchMetadata(version: string): UIState { - if (!version) { - throw new Error( - 'Version parameter is required for fetchSemanticSearchMetadata' - ); - } - + const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH; - const isPreV219 = semver.lt(version, MINIMUM_FULL_SUPPORTED_VERSION); baseState.config.ingest.enrich.processors = isPreV219 - ? [ - new TextEmbeddingIngestProcessor().toObj(), - new TextImageEmbeddingIngestProcessor().toObj(), - ] + ? [new TextEmbeddingIngestProcessor().toObj()] : [new MLIngestProcessor().toObj()]; baseState.config.ingest.index.name.value = generateId('knn_index', 6); @@ -269,12 +245,6 @@ export function fetchHybridSearchMetadata(version: string): UIState { } export function fetchRAGMetadata(version: string): UIState { - if (!version) { - throw new Error( - 'Version parameter is required for fetchSemanticSearchMetadata' - ); - } - let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.RAG; baseState.config.ingest.index.name.value = generateId('my_index', 6); From c2cf94bacd1398dee98c00963118c7f9cdf13cec Mon Sep 17 00:00:00 2001 From: Kama Huang Date: Tue, 31 Dec 2024 14:50:00 -0800 Subject: [PATCH 9/9] address comments on 12/31 Signed-off-by: Kama Huang --- public/pages/workflows/new_workflow/utils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index 308703c5..59749ecb 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -32,6 +32,7 @@ import { SEMANTIC_SEARCH_QUERY_NEURAL, MULTIMODAL_SEARCH_QUERY_NEURAL, HYBRID_SEARCH_QUERY_MATCH_NEURAL, + TERM_QUERY_TEXT, } from '../../../../common'; import { generateId } from '../../../utils'; import semver from 'semver'; @@ -163,7 +164,7 @@ export function fetchSemanticSearchMetadata(version: string): UIState { }); baseState.config.search.request.value = customStringify( - isPreV219 ? SEMANTIC_SEARCH_QUERY_NEURAL : KNN_QUERY + isPreV219 ? SEMANTIC_SEARCH_QUERY_NEURAL : TERM_QUERY_TEXT ); baseState.config.search.enrichRequest.processors = isPreV219 @@ -223,7 +224,7 @@ export function fetchHybridSearchMetadata(version: string): UIState { }); baseState.config.search.request.value = customStringify( - isPreV219 ? HYBRID_SEARCH_QUERY_MATCH_NEURAL : HYBRID_SEARCH_QUERY_MATCH_KNN + isPreV219 ? HYBRID_SEARCH_QUERY_MATCH_NEURAL : TERM_QUERY_TEXT ); baseState.config.search.enrichResponse.processors = [