diff --git a/package.json b/package.json index 1c2a7f61d09b..4a9484a8d840 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "start": "scripts/use_node scripts/opensearch_dashboards --dev", "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST", "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security", - "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true --uiSettings.overrides['state:storeInSessionStorage']=true", + "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true", "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev", "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev", "lint": "yarn run lint:es && yarn run lint:style", diff --git a/packages/osd-stylelint-config/config/global_selectors.json b/packages/osd-stylelint-config/config/global_selectors.json index 1da8137e9a7a..ca442760f731 100644 --- a/packages/osd-stylelint-config/config/global_selectors.json +++ b/packages/osd-stylelint-config/config/global_selectors.json @@ -27,8 +27,7 @@ "src/plugins/discover/public/application/view_components/canvas/discover_canvas.scss", "src/plugins/discover/public/application/components/sidebar/discover_sidebar.scss", "src/plugins/data/public/ui/query_string_input/_query_bar.scss", - "src/plugins/data/public/ui/query_editor/_query_editor.scss", - "src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss" + "src/plugins/data/public/ui/query_editor/_query_editor.scss" ] } } \ No newline at end of file diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index ad934e0a73ae..9dc6a84b484a 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -457,14 +457,14 @@ export class SavedObjectsClient { * { id: 'foo', type: 'index-pattern' } * ]) */ - public bulkGet = (objects: Array<{ id: string; type: string }> = []) => { + public bulkGet = (objects: Array<{ id: string; type: string }> = []) => { const filteredObjects = objects.map((obj) => pick(obj, ['id', 'type'])); return this.performBulkGet(filteredObjects).then((resp) => { resp.saved_objects = resp.saved_objects.map((d) => this.createSavedObject(d)); return renameKeys< PromiseType>, - SavedObjectsBatchResponse - >({ saved_objects: 'savedObjects' }, resp) as SavedObjectsBatchResponse; + SavedObjectsBatchResponse + >({ saved_objects: 'savedObjects' }, resp) as SavedObjectsBatchResponse; }); }; diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts index af6dbef00c37..f81f63066ea7 100644 --- a/src/plugins/data/common/constants.ts +++ b/src/plugins/data/common/constants.ts @@ -9,6 +9,8 @@ * GitHub history for details. */ +import { DATA_STRUCTURE_META_TYPES, DataStructure } from './types'; + /* * Licensed to Elasticsearch B.V. under one or more contributor * license agreements. See the NOTICE file distributed with @@ -28,10 +30,41 @@ * under the License. */ +export const DEFAULT_DATA = { + STRUCTURES: { + ROOT: { + id: 'ROOT', + title: 'Data', + type: 'ROOT', + meta: { + type: DATA_STRUCTURE_META_TYPES.FEATURE, + icon: 'folderOpen', + tooltip: 'Root Data Structure', + }, + } as DataStructure, + }, + + SET_TYPES: { + INDEX_PATTERN: 'INDEX_PATTERN', + INDEX: 'INDEXES', + }, + + SOURCE_TYPES: { + OPENSEARCH: 'OPENSEARCH', + LEGACY: 'LEGACY', + }, +}; + +export const DEFAULT_QUERY_LANGUAGE = 'kuery'; + export const DEFAULT_QUERY = { - LANGUAGE: 'kuery', - DATASET_TYPE: 'INDEX_PATTERN', - ENGINE_TYPE: 'OPENSEARCH', + LANGUAGE: DEFAULT_QUERY_LANGUAGE, + DATASET: { + TYPE: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN, + DATASOURCE: { + TYPE: DEFAULT_DATA.SOURCE_TYPES.OPENSEARCH, + }, + }, }; export const UI_SETTINGS = { diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts index c5303e0260b4..8c9f63b0f0d3 100644 --- a/src/plugins/data/common/data_frames/utils.ts +++ b/src/plugins/data/common/data_frames/utils.ts @@ -18,7 +18,7 @@ import { import { IFieldType } from './fields'; import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns'; import { IOpenSearchDashboardsSearchRequest } from '../search'; -import { GetAggTypeFn, GetDataFrameAggQsFn, TimeRange } from '../types'; +import { GetAggTypeFn, GetDataFrameAggQsFn, Query, TimeRange } from '../types'; /** * Returns the raw data frame from the search request. @@ -380,25 +380,25 @@ export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFr */ export const updateDataFrameMeta = ({ dataFrame, - qs, + query, aggConfig, timeField, timeFilter, getAggQsFn, }: { dataFrame: IDataFrame; - qs: string; + query: Query; aggConfig: DataFrameAggConfig; timeField: any; timeFilter: string; getAggQsFn: GetDataFrameAggQsFn; }) => { dataFrame.meta = { - ...dataFrame.meta, + ...dataFrame?.meta, aggs: aggConfig, aggsQs: { [aggConfig.id]: getAggQsFn({ - qs, + query, aggConfig, timeField, timeFilter, @@ -411,7 +411,7 @@ export const updateDataFrameMeta = ({ for (const [key, subAgg] of Object.entries(subAggs)) { const subAggConfig: Record = { [key]: subAgg }; dataFrame.meta.aggsQs[subAgg.id] = getAggQsFn({ - qs, + query, aggConfig: subAggConfig as DataFrameAggConfig, timeField, timeFilter, diff --git a/src/plugins/data/common/datasets/_structure_cache.ts b/src/plugins/data/common/datasets/_structure_cache.ts new file mode 100644 index 000000000000..1799d4638134 --- /dev/null +++ b/src/plugins/data/common/datasets/_structure_cache.ts @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataStructure, CachedDataStructure } from './types'; + +export interface DataStructureCache { + get: (id: string) => CachedDataStructure | undefined; + set: (id: string, value: CachedDataStructure) => CachedDataStructure; + clear: (id: string) => void; + clearAll: () => void; +} + +export function createDataStructureCache(): DataStructureCache { + const cache: Record = {}; + + const dataStructureCache: DataStructureCache = { + get: (id: string) => { + return cache[id]; + }, + set: (id: string, value: CachedDataStructure) => { + cache[id] = value; + return value; + }, + clear: (id: string) => { + delete cache[id]; + }, + // TODO: call this on log out + clearAll: () => { + Object.keys(cache).forEach((key) => delete cache[key]); + }, + }; + + return dataStructureCache; +} + +export function toCachedDataStructure(dataStructure: DataStructure): CachedDataStructure { + return { + id: dataStructure.id, + title: dataStructure.title, + type: dataStructure.type, + parent: dataStructure.parent?.id || '', + children: dataStructure.children?.map((child) => child.id) || [], + }; +} diff --git a/src/plugins/data/common/datasets/types.ts b/src/plugins/data/common/datasets/types.ts index 65f523d40fea..69549a631ab9 100644 --- a/src/plugins/data/common/datasets/types.ts +++ b/src/plugins/data/common/datasets/types.ts @@ -3,6 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { EuiIconProps } from '@elastic/eui'; +export * from './_structure_cache'; + /** * Describes a data source with its properties. */ @@ -116,42 +119,55 @@ export interface DataStructure { parent?: DataStructure; /** Optional array of child data structures */ children?: DataStructure[]; + hasNext?: boolean; + columnHeader?: string; /** Optional metadata for the data structure */ meta?: DataStructureMeta; } -/** - * Metadata for a data structure, used for additional properties like icons or tooltips. - */ -export interface DataStructureMeta { - type: DATA_STRUCTURE_META_TYPES; -} - /** * DataStructureMeta types */ export enum DATA_STRUCTURE_META_TYPES { FEATURE = 'FEATURE', + TYPE = 'TYPE', CUSTOM = 'CUSTOM', } /** * Metadata for a data structure, used for additional properties like icons or tooltips. */ -export interface DataStructureFeatureMeta extends DataStructureMeta { +export interface DataStructureFeatureMeta { type: DATA_STRUCTURE_META_TYPES.FEATURE; - icon: string; + icon?: string; + tooltip?: string; +} + +/** + * Metadata for dataset type + */ +export interface DataStructureDataTypeMeta { + type: DATA_STRUCTURE_META_TYPES.TYPE; + icon: EuiIconProps; tooltip: string; } /** * Metadata for a data structure with CUSTOM type, allowing any additional fields. */ -export interface DataStructureCustomMeta extends DataStructureMeta { +export interface DataStructureCustomMeta { type: DATA_STRUCTURE_META_TYPES.CUSTOM; [key: string]: any; } +/** + * Union type for DataStructureMeta + */ +export type DataStructureMeta = + | DataStructureFeatureMeta + | DataStructureDataTypeMeta + | DataStructureCustomMeta; + /** * Represents a cached version of DataStructure with string references instead of object references. * @@ -168,13 +184,24 @@ export interface DataStructureCustomMeta extends DataStructureMeta { * ] * }; */ -export interface CachedDataStructure extends Omit { +export interface CachedDataStructure extends Omit { /** ID of the parent data structure */ parent: string; /** Array of child data structure IDs */ children: string[]; } +export interface BaseDataset { + /** Unique identifier for the dataset, for non-index pattern based datasets, we will append the data source ID if present */ + id: string; + /** Human-readable name of the dataset that is used to query */ + title: string; + /** The type of the dataset, registered by other classes */ + type: string; + /** Optional reference to the data source */ + dataSource?: DataSource; +} + /** * Defines the structure of a dataset, including its type and reference to a data source. * NOTE: For non-index pattern datasets we will append the data source ID to the front of @@ -189,10 +216,10 @@ export interface CachedDataStructure extends Omit { + // TODO: This behaviour will cause the index pattern title to be resolved differently depending on how its fetched since the get method in this service will not append the datasource title if (obj.type === 'index-pattern') { const result = { ...obj }; result.attributes.title = await getIndexPatternTitle( diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index e6ce014a7835..84a094637481 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -320,16 +320,9 @@ export class SearchSource { * @return {undefined|IDataFrame} */ async createDataFrame(searchRequest: SearchRequest) { - const rawQueryString = this.getRawQueryStringFromRequest(searchRequest); const dataFrame = createDataFrame({ name: searchRequest.index.title || searchRequest.index, fields: [], - ...(rawQueryString && { - meta: { - queryConfig: { qs: rawQueryString }, - ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }), - }, - }), }); await this.setDataFrame(dataFrame); return this.getDataFrame(); diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts index 57dfa2bde7a8..eb8b6500b6fa 100644 --- a/src/plugins/data/common/types.ts +++ b/src/plugins/data/common/types.ts @@ -29,6 +29,7 @@ */ import { DataFrameAggConfig, IDataFrame } from './data_frames'; +import { Query } from './query'; import { BucketAggType, MetricAggType } from './search'; export * from './query/types'; @@ -50,12 +51,12 @@ export * from './datasets/types'; export type GetConfigFn = (key: string, defaultOverride?: T) => T; export type GetDataFrameFn = () => IDataFrame | undefined; export type GetDataFrameAggQsFn = ({ - qs, + query, aggConfig, timeField, timeFilter, }: { - qs: string; + query: Query; aggConfig: DataFrameAggConfig; timeField: any; timeFilter: any; diff --git a/src/plugins/data/public/antlr/opensearch_sql/code_completion.ts b/src/plugins/data/public/antlr/opensearch_sql/code_completion.ts index d9aeedf727cf..bfd0d9d2da9c 100644 --- a/src/plugins/data/public/antlr/opensearch_sql/code_completion.ts +++ b/src/plugins/data/public/antlr/opensearch_sql/code_completion.ts @@ -47,7 +47,7 @@ export const getSuggestions = async ({ services, }: QuerySuggestionGetFnArgs): Promise => { const { api } = services.uiSettings; - const dataSetManager = services.data.query.dataSetManager; + const queryString = services.data.query.queryString; const { lineNumber, column } = position || {}; const suggestions = getOpenSearchSqlAutoCompleteSuggestions(query, { line: lineNumber || selectionStart, @@ -60,7 +60,7 @@ export const getSuggestions = async ({ // Fetch columns and values if (suggestions.suggestColumns?.tables?.length) { const tableNames = suggestions.suggestColumns.tables.map((table) => table.name); - const schemas = await fetchTableSchemas(tableNames, api, dataSetManager); + const schemas = await fetchTableSchemas(tableNames, api, queryString); (schemas as IDataFrameResponse[]).forEach((schema: IDataFrameResponse) => { if ('body' in schema && schema.body && 'fields' in schema.body) { diff --git a/src/plugins/data/public/antlr/shared/utils.test.ts b/src/plugins/data/public/antlr/shared/utils.test.ts index f255542a0184..905634e0940d 100644 --- a/src/plugins/data/public/antlr/shared/utils.test.ts +++ b/src/plugins/data/public/antlr/shared/utils.test.ts @@ -5,7 +5,7 @@ import { of } from 'rxjs'; import { fetchData } from './utils'; -import { DataSetManager } from '../../query'; +import { QueryStringManager } from '../../query'; describe('fetchData', () => { it('should fetch data using the dataSourceRequestHandler', async () => { @@ -26,18 +26,20 @@ describe('fetchData', () => { fetch: jest.fn().mockResolvedValue('fetchedData'), }, }; - const mockDataSetManager: Partial = { + const mockQueryString: Partial = { getUpdates$: jest .fn() .mockReturnValue(of({ dataSourceRef: { id: 'testId', name: 'testTitle' } })), - getDataSet: jest.fn().mockReturnValue({ dataSourceRef: { id: 'testId', name: 'testTitle' } }), + getDatasetService: jest + .fn() + .mockReturnValue({ dataSourceRef: { id: 'testId', name: 'testTitle' } }), }; const result = await fetchData( mockTables, mockQueryFormatter, mockApi, - mockDataSetManager as DataSetManager + mockQueryString as QueryStringManager ); expect(result).toEqual(['fetchedData', 'fetchedData']); expect(mockQueryFormatter).toHaveBeenCalledWith('table1', 'testId', 'testTitle'); @@ -59,16 +61,16 @@ describe('fetchData', () => { fetch: jest.fn().mockResolvedValue('fetchedData'), }, }; - const mockDataSetManager: Partial = { + const mockQueryString: Partial = { getUpdates$: jest.fn().mockReturnValue(of(undefined)), - getDataSet: jest.fn().mockReturnValue(undefined), + getDatasetService: jest.fn().mockReturnValue(undefined), }; const result = await fetchData( mockTables, mockQueryFormatter, mockApi, - mockDataSetManager as DataSetManager + mockQueryString as QueryStringManager ); expect(result).toEqual(['fetchedData', 'fetchedData']); expect(mockQueryFormatter).toHaveBeenCalledWith('table1'); diff --git a/src/plugins/data/public/antlr/shared/utils.ts b/src/plugins/data/public/antlr/shared/utils.ts index 50897a408523..7ae9dd18940b 100644 --- a/src/plugins/data/public/antlr/shared/utils.ts +++ b/src/plugins/data/public/antlr/shared/utils.ts @@ -5,7 +5,7 @@ import { from } from 'rxjs'; import { distinctUntilChanged, startWith, switchMap } from 'rxjs/operators'; -import { DataSetManager } from '../../query'; +import { QueryStringContract } from '../../query'; export interface IDataSourceRequestHandlerParams { dataSourceId: string; @@ -14,22 +14,22 @@ export interface IDataSourceRequestHandlerParams { // Function to get raw suggestion data export const getRawSuggestionData$ = ( - dataSetManager: DataSetManager, + queryString: QueryStringContract, dataSourceRequestHandler: ({ dataSourceId, title, }: IDataSourceRequestHandlerParams) => Promise, defaultRequestHandler: () => Promise ) => - dataSetManager.getUpdates$().pipe( - startWith(dataSetManager.getDataSet()), + queryString.getUpdates$().pipe( + startWith(queryString.getQuery()), distinctUntilChanged(), - switchMap((dataSet) => { - if (!dataSet) { + switchMap((query) => { + if (!query) { return from(defaultRequestHandler()); } - const dataSourceId = dataSet?.dataSourceRef?.id; - const title = dataSet?.dataSourceRef?.name; + const dataSourceId = query.dataset?.dataSource?.id; + const title = query.dataset?.dataSource?.title; return from(dataSourceRequestHandler({ dataSourceId, title })); }) ); @@ -52,11 +52,11 @@ export const fetchData = ( tables: string[], queryFormatter: (table: string, dataSourceId?: string, title?: string) => any, api: any, - dataSetManager: DataSetManager + queryString: QueryStringContract ) => { return new Promise((resolve, reject) => { getRawSuggestionData$( - dataSetManager, + queryString, ({ dataSourceId, title }) => { const requests = tables.map(async (table) => { const body = JSON.stringify(queryFormatter(table, dataSourceId, title)); @@ -82,11 +82,11 @@ export const fetchData = ( }; // Specific fetch function for table schemas -export const fetchTableSchemas = (tables: string[], api: any, dataSetManager: DataSetManager) => { +export const fetchTableSchemas = (tables: string[], api: any, queryString: QueryStringContract) => { return fetchData( tables, (table, dataSourceId, title) => ({ - query: { qs: `DESCRIBE TABLES LIKE ${table}`, format: 'jdbc' }, + query: { query: `DESCRIBE TABLES LIKE ${table}`, format: 'jdbc' }, df: { meta: { queryConfig: { @@ -97,6 +97,6 @@ export const fetchTableSchemas = (tables: string[], api: any, dataSetManager: Da }, }), api, - dataSetManager + queryString ); }; diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 208359352e4b..a4f93ff2fb41 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -445,7 +445,7 @@ export { QueryEditorTopRow, // for BWC, keeping the old name IUiStart as DataPublicPluginStartUi, - DataSetNavigator, + useQueryStringManager, } from './ui'; /** @@ -462,6 +462,14 @@ export { QueryState, getDefaultQuery, FilterManager, + QueryStringContract, + QueryStringManager, + DatasetTypeConfig, + DatasetService, + DatasetServiceContract, + LanguageConfig, + LanguageService, + LanguageServiceContract, SavedQuery, SavedQueryService, SavedQueryTimeFilter, diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 09bdd96d58e2..ec267a20eee1 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -134,9 +134,15 @@ export class DataPublicPlugin expressions.registerFunction(opensearchaggs); expressions.registerFunction(indexPatternLoad); + const searchService = this.searchService.setup(core, { + usageCollection, + expressions, + }); + const queryService = this.queryService.setup({ uiSettings: core.uiSettings, storage: this.storage, + defaultSearchInterceptor: searchService.getDefaultSearchInterceptor(), }); uiActions.registerAction( @@ -157,11 +163,6 @@ export class DataPublicPlugin })) ); - const searchService = this.searchService.setup(core, { - usageCollection, - expressions, - }); - const uiService = this.uiService.setup(core, {}); const ac = this.autocomplete.setup(core); diff --git a/src/plugins/data/public/query/index.tsx b/src/plugins/data/public/query/index.tsx index 37cd52471db0..d5dda197917a 100644 --- a/src/plugins/data/public/query/index.tsx +++ b/src/plugins/data/public/query/index.tsx @@ -29,10 +29,10 @@ */ export * from './lib'; - +export * from './types'; export * from './query_service'; export * from './filter_manager'; -export * from './query_string/dataset_manager'; +export * from './query_string'; export * from './timefilter'; export * from './saved_query'; export * from './persisted_log'; diff --git a/src/plugins/data/public/query/mocks.ts b/src/plugins/data/public/query/mocks.ts index f1377ad8ea0d..3e47bc92752c 100644 --- a/src/plugins/data/public/query/mocks.ts +++ b/src/plugins/data/public/query/mocks.ts @@ -33,7 +33,6 @@ import { QueryService, QuerySetup, QueryStart } from '.'; import { timefilterServiceMock } from './timefilter/timefilter_service.mock'; import { createFilterManagerMock } from './filter_manager/filter_manager.mock'; import { queryStringManagerMock } from './query_string/query_string_manager.mock'; -import { dataSetManagerMock } from './query_string/dataset_manager/dataset_manager.mock'; type QueryServiceClientContract = PublicMethodsOf; @@ -42,7 +41,6 @@ const createSetupContractMock = () => { filterManager: createFilterManagerMock(), timefilter: timefilterServiceMock.createSetupContract(), queryString: queryStringManagerMock.createSetupContract(), - dataSetManager: dataSetManagerMock.createSetupContract(), state$: new Observable(), }; @@ -57,7 +55,6 @@ const createStartContractMock = () => { savedQueries: jest.fn() as any, state$: new Observable(), timefilter: timefilterServiceMock.createStartContract(), - dataSetManager: dataSetManagerMock.createStartContract(), getOpenSearchQuery: jest.fn(), }; diff --git a/src/plugins/data/public/query/query_service.ts b/src/plugins/data/public/query/query_service.ts index 0f902aa4bcde..95d1bd136f53 100644 --- a/src/plugins/data/public/query/query_service.ts +++ b/src/plugins/data/public/query/query_service.ts @@ -29,39 +29,27 @@ */ import { share } from 'rxjs/operators'; -import { IUiSettingsClient, SavedObjectsClientContract } from 'src/core/public'; import { FilterManager } from './filter_manager'; import { createAddToQueryLog } from './lib'; import { TimefilterService, TimefilterSetup } from './timefilter'; import { createSavedQueryService } from './saved_query/saved_query_service'; import { createQueryStateObservable } from './state_sync/create_global_query_observable'; import { QueryStringManager, QueryStringContract } from './query_string'; -import { - buildOpenSearchQuery, - DataStorage, - getOpenSearchQueryConfig, - IndexPatternsService, -} from '../../common'; +import { buildOpenSearchQuery, getOpenSearchQueryConfig } from '../../common'; import { getUiSettings } from '../services'; import { IndexPattern } from '..'; +import { + IQuerySetup, + IQueryStart, + QueryServiceSetupDependencies, + QueryServiceStartDependencies, +} from './types'; /** * Query Service * @internal */ -interface QueryServiceSetupDependencies { - storage: DataStorage; - uiSettings: IUiSettingsClient; -} - -interface QueryServiceStartDependencies { - savedObjectsClient: SavedObjectsClientContract; - storage: DataStorage; - uiSettings: IUiSettingsClient; - indexPatterns: IndexPatternsService; -} - export class QueryService { filterManager!: FilterManager; timefilter!: TimefilterSetup; @@ -69,7 +57,11 @@ export class QueryService { state$!: ReturnType; - public setup({ storage, uiSettings }: QueryServiceSetupDependencies) { + public setup({ + storage, + uiSettings, + defaultSearchInterceptor, + }: QueryServiceSetupDependencies): IQuerySetup { this.filterManager = new FilterManager(uiSettings); const timefilterService = new TimefilterService(); @@ -78,20 +70,18 @@ export class QueryService { storage, }); - this.queryStringManager = new QueryStringManager(storage, uiSettings); + this.queryStringManager = new QueryStringManager(storage, uiSettings, defaultSearchInterceptor); this.state$ = createQueryStateObservable({ filterManager: this.filterManager, timefilter: this.timefilter, queryString: this.queryStringManager, - datasetManager: this.queryStringManager.getDatasetManager(), }).pipe(share()); return { filterManager: this.filterManager, timefilter: this.timefilter, queryString: this.queryStringManager, - datasetManager: this.queryStringManager.getDatasetManager(), state$: this.state$, }; } @@ -101,8 +91,8 @@ export class QueryService { storage, uiSettings, indexPatterns, - }: QueryServiceStartDependencies) { - this.queryStringManager.getDatasetManager().init(indexPatterns); + }: QueryServiceStartDependencies): IQueryStart { + this.queryStringManager.getDatasetService().init(); return { addToQueryLog: createAddToQueryLog({ storage, @@ -110,7 +100,6 @@ export class QueryService { }), filterManager: this.filterManager, queryString: this.queryStringManager, - dataSetManager: this.queryStringManager.getDatasetManager(), savedQueries: createSavedQueryService(savedObjectsClient), state$: this.state$, timefilter: this.timefilter, diff --git a/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.mock.ts b/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.mock.ts deleted file mode 100644 index fc3c52aa11ca..000000000000 --- a/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.mock.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { DatasetContract } from '.'; - -const createSetupContractMock = () => { - const datasetManagerMock: jest.Mocked = { - init: jest.fn(), - getDataset: jest.fn(), - setDataset: jest.fn(), - getUpdates$: jest.fn(), - getDefaultDataset: jest.fn(), - fetchDefaultDataset: jest.fn(), - initWithIndexPattern: jest.fn(), - }; - return datasetManagerMock; -}; - -export const datasetManagerMock = { - createSetupContract: createSetupContractMock, - createStartContract: createSetupContractMock, -}; diff --git a/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.test.ts b/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.test.ts deleted file mode 100644 index 597f7418af37..000000000000 --- a/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { DatasetManager } from './dataset_manager'; -import { coreMock } from '../../../../../../core/public/mocks'; -import { Dataset } from '../../../../common'; - -describe('DatasetManager', () => { - let service: DatasetManager; - - beforeEach(() => { - const uiSettingsMock = coreMock.createSetup().uiSettings; - uiSettingsMock.get.mockReturnValue(true); - service = new DatasetManager(uiSettingsMock); - }); - - test('getUpdates$ is a cold emits only after dataset changes', () => { - const obs$ = service.getUpdates$(); - const emittedValues: Dataset[] = []; - obs$.subscribe((v) => { - emittedValues.push(v!); - }); - expect(emittedValues).toHaveLength(0); - expect(emittedValues[0]).toEqual(undefined); - - const newDataset: Dataset = { - id: 'test_dataset', - title: 'Test Dataset', - type: 'INDEX_PATTERN', - }; - service.setDataset(newDataset); - expect(emittedValues).toHaveLength(1); - expect(emittedValues[0]).toEqual(newDataset); - - service.setDataset({ ...newDataset }); - expect(emittedValues).toHaveLength(2); - }); -}); diff --git a/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.ts b/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.ts deleted file mode 100644 index af43fccacc50..000000000000 --- a/src/plugins/data/public/query/query_string/dataset_manager/dataset_manager.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { BehaviorSubject } from 'rxjs'; -import { CoreStart } from 'opensearch-dashboards/public'; -import { skip } from 'rxjs/operators'; -import { DEFAULT_QUERY, Dataset, DataSource, IndexPattern, UI_SETTINGS } from '../../../../common'; -import { IndexPatternsContract } from '../../../index_patterns'; - -export class DatasetManager { - private dataset$: BehaviorSubject; - private indexPatterns?: IndexPatternsContract; - private defaultDataset?: Dataset; - - constructor(private readonly uiSettings: CoreStart['uiSettings']) { - this.dataset$ = new BehaviorSubject(undefined); - } - - public init = async (indexPatterns: IndexPatternsContract) => { - if (!this.uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED)) return; - this.indexPatterns = indexPatterns; - this.defaultDataset = await this.fetchDefaultDataset(); - }; - - public initWithIndexPattern = (indexPattern: IndexPattern | null) => { - if (!this.uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED)) return; - if (!indexPattern || !indexPattern.id) { - return undefined; - } - - this.defaultDataset = { - id: indexPattern.id, - title: indexPattern.title, - type: DEFAULT_QUERY.DATASET_TYPE, - timeFieldName: indexPattern.timeFieldName, - ...(indexPattern.dataSourceRef - ? { - dataSource: { - id: indexPattern.dataSourceRef?.id, - title: indexPattern.dataSourceRef?.name, - type: indexPattern.dataSourceRef?.type, - } as DataSource, - } - : {}), - }; - }; - - public getUpdates$ = () => { - return this.dataset$.asObservable().pipe(skip(1)); - }; - - public getDataset = () => { - return this.dataset$.getValue(); - }; - - /** - * Updates the query. - * @param {Query} query - */ - public setDataset = (dataSet: Dataset | undefined) => { - if (!this.uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED)) return; - this.dataset$.next(dataSet); - }; - - public getDefaultDataset = () => { - return this.defaultDataset; - }; - - public fetchDefaultDataset = async (): Promise => { - const defaultIndexPatternId = this.uiSettings.get('defaultIndex'); - if (!defaultIndexPatternId) { - return undefined; - } - - const indexPattern = await this.indexPatterns?.get(defaultIndexPatternId); - if (!indexPattern || !indexPattern.id) { - return undefined; - } - - return { - id: indexPattern.id, - title: indexPattern.title, - type: DEFAULT_QUERY.DATASET_TYPE, - timeFieldName: indexPattern.timeFieldName, - ...(indexPattern.dataSourceRef - ? { - dataSource: { - id: indexPattern.dataSourceRef?.id, - title: indexPattern.dataSourceRef?.name, - type: indexPattern.dataSourceRef?.type, - } as DataSource, - } - : {}), - }; - }; -} - -export type DatasetContract = PublicMethodsOf; diff --git a/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts new file mode 100644 index 000000000000..7dd10c2e5245 --- /dev/null +++ b/src/plugins/data/public/query/query_string/dataset_service/dataset_service.ts @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CoreStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { + Dataset, + DataStructure, + IndexPatternSpec, + DEFAULT_DATA, + IFieldType, +} from '../../../../common'; +import { getIndexPatterns } from '../../../services'; +import { DatasetTypeConfig } from './types'; +import { indexPatternTypeConfig, indexTypeConfig } from './lib'; + +export class DatasetService { + private defaultDataset?: Dataset; + private typesRegistry: Map = new Map(); + + constructor(private readonly uiSettings: CoreStart['uiSettings']) { + this.registerDefaultTypes(); + } + + /** + * Registers default handlers for index patterns and indices. + */ + private registerDefaultTypes() { + this.registerType(indexPatternTypeConfig); + this.registerType(indexTypeConfig); + } + + public async init(): Promise { + this.defaultDataset = await this.fetchDefaultDataset(); + } + + public registerType(handlerConfig: DatasetTypeConfig): void { + this.typesRegistry.set(handlerConfig.id, handlerConfig); + } + + public getType(type: string): DatasetTypeConfig | undefined { + return this.typesRegistry.get(type); + } + + public getTypes(): DatasetTypeConfig[] { + return Array.from(this.typesRegistry.values()); + } + + public getDefault(): Dataset | undefined { + return this.defaultDataset; + } + + public async cacheDataset(dataset: Dataset): Promise { + const type = this.getType(dataset.type); + if (dataset) { + const spec = { + id: dataset.id, + title: dataset.title, + timeFieldName: { + name: dataset.timeFieldName, + type: 'date', + } as Partial, + fields: await type?.fetchFields(dataset), + dataSourceRef: dataset.dataSource + ? { + id: dataset.dataSource.id!, + name: dataset.dataSource.title, + type: dataset.dataSource.type, + } + : undefined, + } as IndexPatternSpec; + const temporaryIndexPattern = await getIndexPatterns().create(spec); + getIndexPatterns().saveToCache(dataset.id, temporaryIndexPattern); + } + } + + public fetchOptions( + savedObjects: SavedObjectsClientContract, + path: DataStructure[], + dataType: string + ): Promise { + const type = this.typesRegistry.get(dataType); + if (!type) { + throw new Error(`No handler found for type: ${path[0]}`); + } + return type.fetch(savedObjects, path); + } + + private async fetchDefaultDataset(): Promise { + const defaultIndexPatternId = this.uiSettings.get('defaultIndex'); + if (!defaultIndexPatternId) { + return undefined; + } + + const indexPattern = await getIndexPatterns().get(defaultIndexPatternId); + if (!indexPattern || !indexPattern.id) { + return undefined; + } + + const dataType = this.typesRegistry.get(DEFAULT_DATA.SET_TYPES.INDEX_PATTERN); + if (dataType) { + const dataset = dataType.toDataset([ + { + id: indexPattern.id, + title: indexPattern.title, + type: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN, + }, + ]); + return { ...dataset, timeFieldName: indexPattern.timeFieldName }; + } + + return undefined; + } +} + +export type DatasetServiceContract = PublicMethodsOf; diff --git a/src/plugins/data/public/query/query_string/dataset_service/index.ts b/src/plugins/data/public/query/query_string/dataset_service/index.ts new file mode 100644 index 000000000000..8becfe4126ac --- /dev/null +++ b/src/plugins/data/public/query/query_string/dataset_service/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './types'; +export { DatasetServiceContract, DatasetService } from './dataset_service'; diff --git a/src/plugins/data/public/query/query_string/dataset_manager/index.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index.ts similarity index 54% rename from src/plugins/data/public/query/query_string/dataset_manager/index.ts rename to src/plugins/data/public/query/query_string/dataset_service/lib/index.ts index 01291faa1fd1..68ba7d2d09b8 100644 --- a/src/plugins/data/public/query/query_string/dataset_manager/index.ts +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index.ts @@ -3,4 +3,5 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { DatasetContract, DatasetManager } from './dataset_manager'; +export * from './index_type'; +export * from './index_pattern_type'; diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_pattern_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_pattern_type.ts new file mode 100644 index 000000000000..eeb17cb7a146 --- /dev/null +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_pattern_type.ts @@ -0,0 +1,125 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { DataSourceAttributes } from '../../../../../../data_source/common/data_sources'; +import { + DEFAULT_DATA, + DataStructure, + DatasetField, + Dataset, + IIndexPattern, + DATA_STRUCTURE_META_TYPES, +} from '../../../../../common'; +import { DatasetTypeConfig } from '../types'; +import { getIndexPatterns } from '../../../../services'; + +export const indexPatternTypeConfig: DatasetTypeConfig = { + id: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN, + title: 'Index Patterns', + meta: { + icon: { type: 'indexPatternApp' }, + tooltip: 'OpenSearch Index Patterns', + }, + + toDataset: (path) => { + const pattern = path[path.length - 1]; + return { + id: pattern.id, + title: pattern.title, + type: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN, + dataSource: pattern.parent + ? { + id: pattern.parent.id, + title: pattern.parent.title, + type: pattern.parent.type, + } + : undefined, + } as Dataset; + }, + + fetch: async (savedObjects, path) => { + const dataStructure = path[path.length - 1]; + const indexPatterns = await fetchIndexPatterns(savedObjects); + return { + ...dataStructure, + columnHeader: 'Index patterns', + children: indexPatterns, + hasNext: false, + }; + }, + + fetchFields: async (dataset: Dataset): Promise => { + const indexPattern = await getIndexPatterns().get(dataset.id); + return indexPattern.fields.map((field: any) => ({ + name: field.name, + type: field.type, + })); + }, + + supportedLanguages: (dataset): string[] => { + if (dataset.dataSource?.type === 'OpenSearch Serverless') { + return ['DQL', 'Lucene']; + } + return ['DQL', 'Lucene', 'PPL', 'SQL']; + }, +}; + +const fetchIndexPatterns = async (client: SavedObjectsClientContract): Promise => { + const resp = await client.find({ + type: 'index-pattern', + fields: ['title', 'timeFieldName', 'references'], + search: `*`, + searchFields: ['title'], + perPage: 100, + }); + + // Get all unique data source ids + const datasourceIds = Array.from( + new Set( + resp.savedObjects + .filter((savedObject) => savedObject.references.length > 0) + .map((savedObject) => savedObject.references.find((ref) => ref.type === 'data-source')?.id) + .filter(Boolean) + ) + ) as string[]; + + const dataSourceMap: Record = {}; + if (datasourceIds.length > 0) { + const dataSourceResp = await client.bulkGet( + datasourceIds.map((id) => ({ id, type: 'data-source' })) + ); + + dataSourceResp.savedObjects.forEach((savedObject) => { + dataSourceMap[savedObject.id] = savedObject.attributes; + }); + } + + return resp.savedObjects.map( + (savedObject): DataStructure => { + const dataSourceId = savedObject.references.find((ref) => ref.type === 'data-source')?.id; + const dataSource = dataSourceId ? dataSourceMap[dataSourceId] : undefined; + + const indexPatternDataStructure: DataStructure = { + id: savedObject.id, + title: savedObject.attributes.title, + type: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN, + meta: { + type: DATA_STRUCTURE_META_TYPES.CUSTOM, + timeFieldName: savedObject.attributes.timeFieldName, + }, + }; + + if (dataSource) { + indexPatternDataStructure.parent = { + id: dataSourceId!, // Since we know it exists + title: dataSource.title, + type: dataSource.dataSourceEngineType ?? 'OpenSearch', + }; + } + return indexPatternDataStructure; + } + ); +}; diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts new file mode 100644 index 000000000000..fcc531c56f80 --- /dev/null +++ b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { map } from 'rxjs/operators'; +import { DEFAULT_DATA, DataStructure, Dataset } from '../../../../../common'; +import { DatasetTypeConfig } from '../types'; +import { getSearchService, getIndexPatterns } from '../../../../services'; + +const INDEX_INFO = { + LOCAL_DATASOURCE: { + id: '', + title: 'Local Cluster', + type: 'DATA_SOURCE', + }, +}; + +export const indexTypeConfig: DatasetTypeConfig = { + id: DEFAULT_DATA.SET_TYPES.INDEX, + title: 'Indexes', + meta: { + icon: { type: 'logoOpenSearch' }, + tooltip: 'OpenSearch Indexes', + }, + + toDataset: (path) => { + const index = path[path.length - 1]; + const dataSource = path.find((ds) => ds.type === 'DATA_SOURCE'); + + return { + id: index.id, + title: index.title, + type: DEFAULT_DATA.SET_TYPES.INDEX, + dataSource: dataSource + ? { + id: dataSource.id, + title: dataSource.title, + type: dataSource.type, + } + : INDEX_INFO.LOCAL_DATASOURCE, + }; + }, + + fetch: async (savedObjects, path) => { + const dataStructure = path[path.length - 1]; + switch (dataStructure.type) { + case 'DATA_SOURCE': { + const indices = await fetchIndices(dataStructure); + return { + ...dataStructure, + hasNext: false, + columnHeader: 'Indexes', + children: indices.map((indexName) => ({ + id: `${dataStructure.id}::${indexName}`, + title: indexName, + type: 'INDEX', + })), + }; + } + + default: { + const dataSources = await fetchDataSources(savedObjects); + return { + ...dataStructure, + columnHeader: 'Cluster', + hasNext: true, + children: dataSources, + }; + } + } + }, + + fetchFields: async (dataset) => { + const fields = await getIndexPatterns().getFieldsForWildcard({ + pattern: dataset.title, + dataSourceId: dataset.dataSource?.id, + }); + return fields.map((field: any) => ({ + name: field.name, + type: field.type, + })); + }, + + supportedLanguages: (dataset: Dataset): string[] => { + return ['SQL', 'PPL', 'DQL', 'Lucene']; + }, +}; + +const fetchDataSources = async (client: SavedObjectsClientContract) => { + const resp = await client.find({ + type: 'data-source', + perPage: 10000, + }); + const dataSources: DataStructure[] = [INDEX_INFO.LOCAL_DATASOURCE]; + return dataSources.concat( + resp.savedObjects.map((savedObject) => ({ + id: savedObject.id, + title: savedObject.attributes.title, + type: 'DATA_SOURCE', + })) + ); +}; + +const fetchIndices = async (dataStructure: DataStructure): Promise => { + const search = getSearchService(); + const buildSearchRequest = () => ({ + params: { + ignoreUnavailable: true, + expand_wildcards: 'all', + index: '*', + body: { + size: 0, + aggs: { + indices: { + terms: { + field: '_index', + size: 100, + }, + }, + }, + }, + }, + dataSourceId: dataStructure.id, + }); + + const searchResponseToArray = (response: any) => { + const { rawResponse } = response; + return rawResponse.aggregations + ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) + : []; + }; + + return search + .getDefaultSearchInterceptor() + .search(buildSearchRequest()) + .pipe(map(searchResponseToArray)) + .toPromise(); +}; diff --git a/src/plugins/data/public/query/query_string/dataset_service/types.ts b/src/plugins/data/public/query/query_string/dataset_service/types.ts new file mode 100644 index 000000000000..8d597694ac49 --- /dev/null +++ b/src/plugins/data/public/query/query_string/dataset_service/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { EuiIconProps } from '@elastic/eui'; +import { Dataset, DatasetField, DataStructure } from '../../../../common'; + +/** + * Configuration for handling dataset operations. + */ +export interface DatasetTypeConfig { + /** Unique identifier for the dataset handler */ + id: string; + /** Human-readable title for the dataset type */ + title: string; + /** Metadata for UI representation */ + meta: { + /** Icon to represent the dataset type */ + icon: EuiIconProps; + /** Optional tooltip text */ + tooltip?: string; + }; + /** + * Converts a DataStructure to a Dataset. + * @param {DataStructure} dataStructure - The data structure to convert. + * @returns {Dataset} Dataset. + */ + toDataset: (path: DataStructure[]) => Dataset; + /** + * Fetches child options for a given DataStructure. + * @param {SavedObjectsClientContract} client - The saved objects client. + * @param {DataStructure} dataStructure - The parent DataStructure. + * @returns {Promise} A promise that resolves to a DatasetHandlerFetchResponse. + */ + fetch: (client: SavedObjectsClientContract, path: DataStructure[]) => Promise; + /** + * Fetches fields for the dataset. + * @returns {Promise} A promise that resolves to an array of DatasetFields. + */ + fetchFields: (dataset: Dataset) => Promise; + /** + * Retrieves the supported query languages for this dataset type. + * @returns {Promise} A promise that resolves to an array of supported language names. + */ + supportedLanguages: (dataset: Dataset) => string[]; +} diff --git a/src/plugins/data/public/query/query_string/index.ts b/src/plugins/data/public/query/query_string/index.ts index 21d250e3fbb1..61d25349fbcd 100644 --- a/src/plugins/data/public/query/query_string/index.ts +++ b/src/plugins/data/public/query/query_string/index.ts @@ -29,3 +29,5 @@ */ export { QueryStringContract, QueryStringManager } from './query_string_manager'; +export { DatasetServiceContract, DatasetService, DatasetTypeConfig } from './dataset_service'; +export { LanguageServiceContract, LanguageService, LanguageConfig } from './language_service'; diff --git a/src/plugins/data/public/query/query_string/language_service/index.ts b/src/plugins/data/public/query/query_string/language_service/index.ts new file mode 100644 index 000000000000..79aea071de3f --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './types'; +export { LanguageServiceContract, LanguageService } from './language_service'; diff --git a/src/plugins/data/public/query/query_string/language_service/language_service.ts b/src/plugins/data/public/query/query_string/language_service/language_service.ts new file mode 100644 index 000000000000..c4d2d85b8482 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/language_service.ts @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { LanguageConfig } from './types'; +import { getDQLLanguageConfig, getLuceneLanguageConfig } from './lib'; +import { ISearchInterceptor } from '../../../search'; + +export class LanguageService { + private languages: Map = new Map(); + + constructor(private readonly defaultSearchInterceptor: ISearchInterceptor) { + this.registerDefaultLanguages(); + } + + /** + * Registers default handlers for index patterns and indices. + */ + private registerDefaultLanguages() { + this.registerLanguage(getDQLLanguageConfig(this.defaultSearchInterceptor)); + this.registerLanguage(getLuceneLanguageConfig(this.defaultSearchInterceptor)); + } + + public registerLanguage(config: LanguageConfig): void { + this.languages.set(config.id, config); + } + + public getLanguage(language: string): LanguageConfig | undefined { + return this.languages.get(language); + } + + public getLanguages(): LanguageConfig[] { + return Array.from(this.languages.values()); + } + + public getDefaultLanguage(): LanguageConfig { + return this.languages.get('kuery') || this.languages.values().next().value; + } +} + +export type LanguageServiceContract = PublicMethodsOf; diff --git a/src/plugins/data/public/query/query_string/language_service/lib/dql_language.ts b/src/plugins/data/public/query/query_string/language_service/lib/dql_language.ts new file mode 100644 index 000000000000..865c6aae84f5 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/lib/dql_language.ts @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { LanguageConfig } from '../types'; +import { ISearchInterceptor } from '../../../../search'; + +export const getDQLLanguageConfig = (search: ISearchInterceptor): LanguageConfig => { + return { + id: 'kuery', + title: 'DQL', + search, + getQueryString(_) { + return ''; + }, + searchBar: { + showQueryInput: true, + showFilterBar: true, + showDatePicker: true, + showAutoRefreshOnly: false, + }, + fields: { + filterable: true, + visualizable: true, + }, + showDocLinks: true, + supportedAppNames: ['discover'], + }; +}; diff --git a/src/plugins/data/public/query/query_string/language_service/lib/index.ts b/src/plugins/data/public/query/query_string/language_service/lib/index.ts new file mode 100644 index 000000000000..42b8f7f2fb18 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/lib/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './dql_language'; +export * from './lucene_language'; diff --git a/src/plugins/data/public/query/query_string/language_service/lib/lucene_language.ts b/src/plugins/data/public/query/query_string/language_service/lib/lucene_language.ts new file mode 100644 index 000000000000..3b9643e92739 --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/lib/lucene_language.ts @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { LanguageConfig } from '../types'; +import { ISearchInterceptor } from '../../../../search'; + +export const getLuceneLanguageConfig = (search: ISearchInterceptor): LanguageConfig => { + return { + id: 'lucene', + title: 'Lucene', + search, + getQueryString(_) { + return ''; + }, + searchBar: { + showQueryInput: true, + showFilterBar: true, + showDatePicker: true, + showAutoRefreshOnly: false, + }, + fields: { + filterable: true, + visualizable: true, + }, + showDocLinks: true, + supportedAppNames: ['discover'], + }; +}; diff --git a/src/plugins/data/public/query/query_string/language_service/types.ts b/src/plugins/data/public/query/query_string/language_service/types.ts new file mode 100644 index 000000000000..53bf397da3fa --- /dev/null +++ b/src/plugins/data/public/query/query_string/language_service/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ISearchInterceptor } from '../../../search'; +import { Query } from '../../../../public'; + +export interface LanguageConfig { + id: string; + title: string; + search: ISearchInterceptor; + getQueryString: (query: Query) => string; + searchBar?: { + showQueryInput?: boolean; + showFilterBar?: boolean; + showDatePicker?: boolean; + showAutoRefreshOnly?: boolean; + dateRange?: { + initialFrom?: string; + initialTo?: string; + }; + }; + fields?: { + filterable?: boolean; + visualizable?: boolean; + }; + showDocLinks?: boolean; + supportedAppNames: string[]; +} diff --git a/src/plugins/data/public/query/query_string/query_history.ts b/src/plugins/data/public/query/query_string/query_history.ts index 17e5f2cd1494..80dfa4b5d560 100644 --- a/src/plugins/data/public/query/query_string/query_history.ts +++ b/src/plugins/data/public/query/query_string/query_history.ts @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { DataStorage, SimpleDataSet } from 'src/plugins/data/common'; import { BehaviorSubject } from 'rxjs'; +import { DataStorage, Dataset } from '../../../common'; import { Query, TimeRange } from '../..'; // Todo: Implement a more advanced QueryHistory class when needed for recent query history @@ -35,7 +35,7 @@ export class QueryHistory { return () => subscription.unsubscribe(); } - addQueryToHistory(dataSet: SimpleDataSet, query: Query, dateRange?: TimeRange) { + addQueryToHistory(dataset: Dataset, query: Query, dateRange?: TimeRange) { const keys = this.getHistoryKeys(); keys.splice(0, 500); // only maintain most recent X; keys.forEach((key) => { @@ -45,7 +45,7 @@ export class QueryHistory { const timestamp = new Date().getTime(); const k = 'query_' + timestamp; this.storage.set(k, { - dataSet, + dataset, time: timestamp, query, dateRange, diff --git a/src/plugins/data/public/query/query_string/query_string_manager.mock.ts b/src/plugins/data/public/query/query_string/query_string_manager.mock.ts index 82ebf586ab3d..4d2f389a7f71 100644 --- a/src/plugins/data/public/query/query_string/query_string_manager.mock.ts +++ b/src/plugins/data/public/query/query_string/query_string_manager.mock.ts @@ -38,6 +38,15 @@ const createSetupContractMock = () => { getDefaultQuery: jest.fn(), formatQuery: jest.fn(), clearQuery: jest.fn(), + addToQueryHistory: jest.fn(), + getQueryHistory: jest.fn(), + clearQueryHistory: jest.fn(), + changeQueryHistory: jest.fn(), + getInitialQuery: jest.fn(), + getInitialQueryByLanguage: jest.fn(), + getDatasetService: jest.fn(), + getLanguageService: jest.fn(), + getInitialQueryByDataset: jest.fn(), }; return queryStringManagerMock; }; diff --git a/src/plugins/data/public/query/query_string/query_string_manager.ts b/src/plugins/data/public/query/query_string/query_string_manager.ts index bde708cbf698..38bab9a8132c 100644 --- a/src/plugins/data/public/query/query_string/query_string_manager.ts +++ b/src/plugins/data/public/query/query_string/query_string_manager.ts @@ -33,37 +33,39 @@ import { skip } from 'rxjs/operators'; import { CoreStart } from 'opensearch-dashboards/public'; import { Dataset, DataStorage, Query, TimeRange, UI_SETTINGS } from '../../../common'; import { createHistory, QueryHistory } from './query_history'; -import { DatasetContract, DatasetManager } from './dataset_manager'; +import { DatasetService, DatasetServiceContract } from './dataset_service'; +import { LanguageService, LanguageServiceContract } from './language_service'; +import { ISearchInterceptor } from '../../search'; export class QueryStringManager { private query$: BehaviorSubject; private queryHistory: QueryHistory; - private datasetManager!: DatasetContract; + private datasetService!: DatasetServiceContract; + private languageService!: LanguageServiceContract; constructor( private readonly storage: DataStorage, - private readonly uiSettings: CoreStart['uiSettings'] + private readonly uiSettings: CoreStart['uiSettings'], + private readonly defaultSearchInterceptor: ISearchInterceptor ) { this.query$ = new BehaviorSubject(this.getDefaultQuery()); this.queryHistory = createHistory({ storage }); - this.datasetManager = new DatasetManager(uiSettings); + this.datasetService = new DatasetService(uiSettings); + this.languageService = new LanguageService(this.defaultSearchInterceptor); } private getDefaultQueryString() { return this.storage.get('userQueryString') || ''; } - private getDefaultLanguage() { - return ( - this.storage.get('userQueryLanguage') || - this.uiSettings.get(UI_SETTINGS.SEARCH_QUERY_LANGUAGE) - ); - } - public getDefaultQuery() { return { query: this.getDefaultQueryString(), language: this.getDefaultLanguage(), + ...(this.uiSettings && + this.uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED) && { + dataset: this.datasetService?.getDefault(), + }), }; } @@ -74,6 +76,7 @@ export class QueryStringManager { return { query, language: this.getDefaultLanguage(), + dataset: this.datasetService?.getDefault(), }; } else { return query; @@ -92,11 +95,10 @@ export class QueryStringManager { * Updates the query. * @param {Query} query */ - public setQuery = (query: Query) => { + public setQuery = (query: Partial) => { const curQuery = this.query$.getValue(); - if (query?.language !== curQuery.language || query?.query !== curQuery.query) { - this.query$.next(query); - } + const newQuery = { ...curQuery, ...query }; + this.query$.next(newQuery); }; /** @@ -124,41 +126,51 @@ export class QueryStringManager { public changeQueryHistory(listener: (reqs: any[]) => void) { return this.queryHistory.change(listener); } - /** - * TODO: verify if we want to just access the dataset manager directly or access each function - */ - public getDatasetManager = () => { - return this.datasetManager; + + public getDatasetService = () => { + return this.datasetService; }; - // TODO: uncomment or use based on decision above - // public initDataset = async (indexPatterns: IndexPatternsContract) => { - // return this.datasetManager.init(indexPatterns); - // }; + public getLanguageService = () => { + return this.languageService; + }; - // public initDatasetWithIndexPattern = (indexPattern: IndexPattern | null) => { - // return this.datasetManager.initWithIndexPattern(indexPattern); - // }; + public getInitialQuery = () => { + return this.getInitialQueryByLanguage(this.query$.getValue().language); + }; - // public getDatasetUpdates$ = () => { - // return this.datasetManager.getUpdates$(); - // }; + public getInitialQueryByLanguage = (languageId: string) => { + const curQuery = this.query$.getValue(); + const language = this.languageService.getLanguage(languageId); + const dataset = curQuery.dataset; + const input = language?.getQueryString(curQuery) || ''; - // public getDataset = () => { - // return this.datasetManager.getDataset(); - // }; + return { + query: input, + language: languageId, + dataset, + }; + }; - // public setDataset = (dataset: Dataset | undefined) => { - // return this.datasetManager.setDataset(dataset); - // }; + public getInitialQueryByDataset = (newDataset: Dataset) => { + const curQuery = this.query$.getValue(); + const languageId = curQuery.language; + const language = this.languageService.getLanguage(languageId); + const newQuery = { ...curQuery, dataset: newDataset }; + const input = language?.getQueryString(newQuery) || ''; - // public getDefaultDataset = () => { - // return this.datasetManager.getDefaultDataset(); - // }; + return { + ...newQuery, + query: input, + }; + }; - // public fetchDefaultDataset = async (): Promise => { - // return this.datasetManager.fetchDefaultDataset(); - // }; + private getDefaultLanguage() { + return ( + this.storage.get('userQueryLanguage') || + this.uiSettings.get(UI_SETTINGS.SEARCH_QUERY_LANGUAGE) + ); + } } export type QueryStringContract = PublicMethodsOf; diff --git a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts index 55fe08fa9106..66e212bd6468 100644 --- a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts +++ b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts @@ -38,12 +38,7 @@ import { } from '../../../../opensearch_dashboards_utils/public'; import { QuerySetup, QueryStart } from '../query_service'; import { QueryState, QueryStateChange } from './types'; -import { - FilterStateStore, - COMPARE_ALL_OPTIONS, - compareFilters, - UI_SETTINGS, -} from '../../../common'; +import { FilterStateStore, COMPARE_ALL_OPTIONS, compareFilters } from '../../../common'; import { validateTimeRange } from '../timefilter'; /** @@ -56,14 +51,10 @@ import { validateTimeRange } from '../timefilter'; */ export const connectStorageToQueryState = async ( { - dataSetManager, filterManager, queryString, state$, - }: Pick< - QueryStart | QuerySetup, - 'timefilter' | 'filterManager' | 'queryString' | 'dataSetManager' | 'state$' - >, + }: Pick, OsdUrlStateStorage: IOsdUrlStateStorage, syncConfig: { filters: FilterStateStore; @@ -80,17 +71,10 @@ export const connectStorageToQueryState = async ( if (syncConfig.filters === FilterStateStore.APP_STATE) { syncKeys.push('appFilters'); } - if (syncConfig.dataSet) { - syncKeys.push('dataSet'); - } const initialStateFromURL: QueryState = OsdUrlStateStorage.get('_q') ?? { query: queryString.getDefaultQuery(), filters: filterManager.getAppFilters(), - ...(uiSettings && - uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED) && { - dataSet: dataSetManager.getDataSet(), - }), }; // set up initial '_q' flag in the URL to sync query and filter changes @@ -106,20 +90,6 @@ export const connectStorageToQueryState = async ( } } - if ( - syncConfig.dataSet && - !_.isEqual(initialStateFromURL.dataSet, dataSetManager.getDataSet()) - ) { - if (initialStateFromURL.dataSet) { - dataSetManager.setDataSet(_.cloneDeep(initialStateFromURL.dataSet)); - } else { - const defaultDataSet = await dataSetManager.getDefaultDataSet(); - if (defaultDataSet) { - dataSetManager.setDataSet(defaultDataSet); - } - } - } - if (syncConfig.filters === FilterStateStore.APP_STATE) { if ( !initialStateFromURL.filters || @@ -152,10 +122,6 @@ export const connectStorageToQueryState = async ( newState.filters = filterManager.getAppFilters(); } - if (syncConfig.dataSet && changes.dataSet) { - newState.dataSet = dataSetManager.getDataSet(); - } - return newState; }) ) @@ -185,12 +151,8 @@ export const connectToQueryState = ( timefilter: { timefilter }, filterManager, queryString, - dataSetManager, state$, - }: Pick< - QueryStart | QuerySetup, - 'timefilter' | 'filterManager' | 'dataSetManager' | 'queryString' | 'state$' - >, + }: Pick, stateContainer: BaseStateContainer, syncConfig: { time?: boolean; @@ -223,9 +185,6 @@ export const connectToQueryState = ( break; } } - if (syncConfig.dataSet) { - syncKeys.push('dataSet'); - } // initial syncing // TODO: @@ -280,11 +239,6 @@ export const connectToQueryState = ( } } - if (syncConfig.dataSet && !initialState.dataSet) { - initialState.dataSet = dataSetManager.getDefaultDataSet(); - initialDirty = true; - } - if (initialDirty) { stateContainer.set({ ...stateContainer.get(), ...initialState }); } @@ -322,9 +276,6 @@ export const connectToQueryState = ( newState.filters = filterManager.getAppFilters(); } } - if (syncConfig.dataSet && changes.dataSet) { - newState.dataSet = dataSetManager.getDataSet(); - } return newState; }) ) @@ -384,21 +335,6 @@ export const connectToQueryState = ( } } - if (syncConfig.dataSet) { - const currentDataSet = dataSetManager.getDataSet(); - if (!_.isEqual(state.dataSet, currentDataSet)) { - if (state.dataSet) { - dataSetManager.setDataSet(state.dataSet); - } else { - const defaultDataSet = dataSetManager.getDefaultDataSet(); - if (defaultDataSet) { - dataSetManager.setDataSet(defaultDataSet); - stateContainer.set({ ...stateContainer.get(), dataSet: defaultDataSet }); - } - } - } - } - updateInProgress = false; }), ]; diff --git a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts index 2a7e18ae8416..8abcb3ece18d 100644 --- a/src/plugins/data/public/query/state_sync/create_global_query_observable.ts +++ b/src/plugins/data/public/query/state_sync/create_global_query_observable.ts @@ -36,18 +36,15 @@ import { QueryState, QueryStateChange } from './index'; import { createStateContainer } from '../../../../opensearch_dashboards_utils/public'; import { isFilterPinned, compareFilters, COMPARE_ALL_OPTIONS } from '../../../common'; import { QueryStringContract } from '../query_string'; -import { DatasetContract } from '../query_string/dataset_manager'; export function createQueryStateObservable({ timefilter: { timefilter }, filterManager, queryString, - datasetManager, }: { timefilter: TimefilterSetup; filterManager: FilterManager; queryString: QueryStringContract; - datasetManager: DatasetContract; }): Observable<{ changes: QueryStateChange; state: QueryState }> { return new Observable((subscriber) => { const state = createStateContainer({ @@ -55,7 +52,6 @@ export function createQueryStateObservable({ refreshInterval: timefilter.getRefreshInterval(), filters: filterManager.getFilters(), query: queryString.getQuery(), - dataset: queryString.getDatasetManager().getDataset(), }); let currentChange: QueryStateChange = {}; @@ -64,13 +60,6 @@ export function createQueryStateObservable({ currentChange.query = true; state.set({ ...state.get(), query: queryString.getQuery() }); }), - queryString - .getDatasetManager() - .getUpdates$() - .subscribe(() => { - currentChange.dataset = true; - state.set({ ...state.get(), dataset: queryString.getDatasetManager().getDataset() }); - }), timefilter.getTimeUpdate$().subscribe(() => { currentChange.time = true; state.set({ ...state.get(), time: timefilter.getTime() }); diff --git a/src/plugins/data/public/query/state_sync/sync_state_with_url.ts b/src/plugins/data/public/query/state_sync/sync_state_with_url.ts index f34c0912200c..e8041a8c016b 100644 --- a/src/plugins/data/public/query/state_sync/sync_state_with_url.ts +++ b/src/plugins/data/public/query/state_sync/sync_state_with_url.ts @@ -38,7 +38,6 @@ import { QuerySetup, QueryStart } from '../query_service'; import { connectToQueryState } from './connect_to_query_state'; import { QueryState } from './types'; import { FilterStateStore } from '../../../common/opensearch_query/filters'; -import { UI_SETTINGS } from '../../../common'; const GLOBAL_STATE_STORAGE_KEY = '_g'; @@ -48,26 +47,18 @@ const GLOBAL_STATE_STORAGE_KEY = '_g'; * @param osdUrlStateStorage to use for syncing */ export const syncQueryStateWithUrl = ( - query: Pick< - QueryStart | QuerySetup, - 'filterManager' | 'timefilter' | 'queryString' | 'dataSetManager' | 'state$' - >, + query: Pick, osdUrlStateStorage: IOsdUrlStateStorage, uiSettings?: CoreStart['uiSettings'] ) => { const { timefilter: { timefilter }, filterManager, - dataSetManager, } = query; const defaultState: QueryState = { time: timefilter.getTime(), refreshInterval: timefilter.getRefreshInterval(), filters: filterManager.getGlobalFilters(), - ...(uiSettings && - uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED) && { - dataSet: dataSetManager.getDataSet(), - }), }; // retrieve current state from `_g` url @@ -89,7 +80,6 @@ export const syncQueryStateWithUrl = ( refreshInterval: true, time: true, filters: FilterStateStore.GLOBAL_STATE, - dataSet: true, }); // if there weren't any initial state in url, diff --git a/src/plugins/data/public/query/types.ts b/src/plugins/data/public/query/types.ts new file mode 100644 index 000000000000..ee359f0939a2 --- /dev/null +++ b/src/plugins/data/public/query/types.ts @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IUiSettingsClient, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { Observable } from 'rxjs'; +import { DataStorage } from '../../common'; +import { IndexPattern, IndexPatternsService } from '../index_patterns'; +import { ISearchInterceptor } from '../search'; +import { FilterManager } from './filter_manager'; +import { QueryStringContract } from './query_string'; +import { QueryStateChange, QueryState } from './state_sync'; +import { createAddToQueryLog } from './lib'; +import { createSavedQueryService } from './saved_query'; +import { TimefilterSetup } from './timefilter'; + +export interface IQuerySetup { + filterManager: FilterManager; + timefilter: TimefilterSetup; + queryString: QueryStringContract; + state$: Observable<{ changes: QueryStateChange; state: QueryState }>; +} + +export interface IQueryStart { + addToQueryLog: ReturnType; + filterManager: FilterManager; + queryString: QueryStringContract; + savedQueries: ReturnType; + state$: Observable<{ changes: QueryStateChange; state: QueryState }>; + timefilter: TimefilterSetup; + getOpenSearchQuery: (indexPattern: IndexPattern) => any; +} + +/** @internal */ +export interface QueryServiceSetupDependencies { + storage: DataStorage; + uiSettings: IUiSettingsClient; + defaultSearchInterceptor: ISearchInterceptor; +} + +/** @internal */ +export interface QueryServiceStartDependencies { + savedObjectsClient: SavedObjectsClientContract; + storage: DataStorage; + uiSettings: IUiSettingsClient; + indexPatterns: IndexPatternsService; +} diff --git a/src/plugins/data/public/search/index.ts b/src/plugins/data/public/search/index.ts index bfd0bc1ed780..f5ff39c40c4f 100644 --- a/src/plugins/data/public/search/index.ts +++ b/src/plugins/data/public/search/index.ts @@ -56,5 +56,5 @@ export { export { getOpenSearchPreference } from './opensearch_search'; -export { SearchInterceptor, SearchInterceptorDeps } from './search_interceptor'; +export { SearchInterceptor, SearchInterceptorDeps, ISearchInterceptor } from './search_interceptor'; export * from './errors'; diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 5516b4abc79d..297cb3342c5c 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -62,7 +62,6 @@ import { IDataFrame, IDataFrameResponse, createDataFrameCache, - dataFrameToSpec, } from '../../common/data_frames'; import { getQueryService, getUiService } from '../services'; import { UI_SETTINGS } from '../../common'; @@ -127,6 +126,7 @@ export class SearchService implements Plugin { __enhance: (enhancements: SearchEnhancements) => { this.searchInterceptor = enhancements.searchInterceptor; }, + getDefaultSearchInterceptor: () => this.defaultSearchInterceptor, }; } @@ -135,19 +135,19 @@ export class SearchService implements Plugin { { fieldFormats, indexPatterns }: SearchServiceStartDependencies ): ISearchStart { const search = ((request, options) => { - const selectedLanguage = getQueryService().queryString.getQuery().language; - const uiService = getUiService(); - const enhancement = uiService.Settings.getQueryEnhancements(selectedLanguage); - uiService.Settings.setUiOverridesByUserQueryLanguage(selectedLanguage); + const queryStringManager = getQueryService().queryString; + const language = queryStringManager.getQuery().language; + const languageConfig = queryStringManager.getLanguageService().getLanguage(language); + getUiService().Settings.setUiOverridesByUserQueryLanguage(language); const isEnhancedEnabled = uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED); - if (enhancement) { + if (languageConfig) { if (!isEnhancedEnabled) { notifications.toasts.addWarning( - `Query enhancements are disabled. Please enable to use: ${selectedLanguage}.` + `Query enhancements are disabled. Please enable to use: ${languageConfig.id}.` ); } - return enhancement.search.search(request, options); + return languageConfig.search.search(request, options); } return this.defaultSearchInterceptor.search(request, options); }) as ISearchGeneric; @@ -158,28 +158,7 @@ export class SearchService implements Plugin { const dfService: DataFrameService = { get: () => this.dfCache.get(), set: async (dataFrame: IDataFrame) => { - if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) { - indexPatterns.clearCache(this.dfCache.get()!.name, false); - } - - if ( - dataFrame.meta && - dataFrame.meta.queryConfig && - 'dataSource' in dataFrame.meta.queryConfig - ) { - const dataSource = await indexPatterns.findDataSourceByTitle( - dataFrame.meta.queryConfig.dataSource - ); - dataFrame.meta.queryConfig.dataSourceId = dataSource?.id; - } this.dfCache.set(dataFrame); - const dataSetName = `${dataFrame.meta?.queryConfig?.dataSourceId ?? ''}.${dataFrame.name}`; - const existingIndexPattern = await indexPatterns.get(dataSetName, true); - const dataSet = await indexPatterns.create( - dataFrameToSpec(dataFrame, existingIndexPattern?.id ?? dataSetName), - !existingIndexPattern?.id - ); - indexPatterns.saveToCache(dataSetName, dataSet); }, clear: () => { if (this.dfCache.get() === undefined) return; diff --git a/src/plugins/data/public/search/types.ts b/src/plugins/data/public/search/types.ts index 29dc37b41c91..d34271ef7568 100644 --- a/src/plugins/data/public/search/types.ts +++ b/src/plugins/data/public/search/types.ts @@ -54,6 +54,7 @@ export interface ISearchSetup { * @internal */ __enhance: (enhancements: SearchEnhancements) => void; + getDefaultSearchInterceptor: () => ISearchInterceptor; } /** diff --git a/src/plugins/data/public/ui/_index.scss b/src/plugins/data/public/ui/_index.scss index 4aa425041f58..9ab4ca672b38 100644 --- a/src/plugins/data/public/ui/_index.scss +++ b/src/plugins/data/public/ui/_index.scss @@ -2,6 +2,6 @@ @import "./typeahead/index"; @import "./saved_query_management/index"; @import "./query_string_input/index"; -@import "./dataset_navigator/index"; +@import "./dataset_selector/index"; @import "./query_editor/index"; @import "./shard_failure_modal/shard_failure_modal"; diff --git a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss b/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss deleted file mode 100644 index 875a2d62635f..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -.dataSetNavigator { - padding: $euiSizeXS; - color: $euiColorPrimaryText; - - &__icon { - margin-right: 4px; - } - - &__loading { - padding: $euiSizeS; - } - - &__menu { - width: 365px; - - .euiContextMenu__panel > div { - @include euiYScrollWithShadows; - - max-height: 60vh; - } - } -} - -.dataSetNavigatorFormWrapper { - padding: $euiSizeS; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/_index.scss b/src/plugins/data/public/ui/dataset_navigator/_index.scss deleted file mode 100644 index 53acdffad43d..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/_index.scss +++ /dev/null @@ -1 +0,0 @@ -@import "./dataset_navigator"; diff --git a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx deleted file mode 100644 index b6a6274e13cf..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/create_dataset_navigator.tsx +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { DataSetNavigator, DataSetNavigatorProps } from './'; -import { DataSetContract } from '../../query'; - -// Updated function signature to include additional dependencies -export function createDataSetNavigator( - savedObjectsClient: SavedObjectsClientContract, - http: HttpStart, - dataSetManager: DataSetContract -) { - // Return a function that takes props, omitting the dependencies from the props type - return (props: Omit) => ( - - ); -} diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx deleted file mode 100644 index 7844475c100f..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx +++ /dev/null @@ -1,685 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { - EuiButton, - EuiButtonEmpty, - EuiContextMenu, - EuiForm, - EuiFormRow, - EuiIcon, - EuiLoadingSpinner, - EuiPanel, - EuiPopover, - EuiSelect, - EuiToolTip, -} from '@elastic/eui'; -import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { i18n } from '@osd/i18n'; -import { Dataset, DataSource, DataStructure } from '../../../common'; -import { - useLoadDatabasesToCache, - useLoadExternalDataSourcesToCache, - useLoadTablesToCache, -} from './lib/catalog_cache/cache_loader'; -import { CatalogCacheManager } from './lib/catalog_cache/cache_manager'; -import { CachedDataSourceStatus, CachedDatabase, DirectQueryLoadingStatus } from './lib/types'; -import { - getIndexPatterns, - getNotifications, - getQueryService, - getSearchService, - getUiService, -} from '../../services'; -import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib'; -import { useDataSetManager } from '../search_bar/lib/use_dataset_manager'; -import { DataSetContract } from '../../query'; - -export interface DataSetNavigatorProps { - savedObjectsClient?: SavedObjectsClientContract; - http?: HttpStart; - dataSetManager?: DataSetContract; -} - -interface SelectedDataSetState extends Dataset { - database?: DataStructure | undefined; -} - -export const DataSetNavigator: React.FC = ({ - savedObjectsClient, - http, - dataSetManager: initialDataSet, -}) => { - const searchService = getSearchService(); - const queryService = getQueryService(); - const uiService = getUiService(); - const indexPatternsService = getIndexPatterns(); - const notifications = getNotifications(); - - const { dataSet } = useDataSetManager({ dataSetManager: initialDataSet! }); - - const isInitialized = useRef(false); - const [isOpen, setIsOpen] = useState(false); - const [isLoading, setIsLoading] = useState(false); - const [dataSources, setDataSources] = useState([]); - const [externalDataSources, setExternalDataSources] = useState([]); - const [indexPatterns, setIndexPatterns] = useState([]); - const [cachedDatabases, setCachedDatabases] = useState([]); - const [cachedTables, setCachedTables] = useState([]); - const [selectedDataSetState, setSelectedDataSetState] = useState< - SelectedDataSetState | undefined - >(undefined); - const isExternalDataSourcesEnabled = externalDataSources.length > 0; - - const { - loadStatus: dataSourcesLoadStatus, - loadExternalDataSources: startLoadingDataSources, - } = useLoadExternalDataSourcesToCache(http!, notifications); - const { - loadStatus: databasesLoadStatus, - startLoading: startLoadingDatabases, - } = useLoadDatabasesToCache(http!, notifications); - const { loadStatus: tablesLoadStatus, startLoading: startLoadingTables } = useLoadTablesToCache( - http!, - notifications - ); - - const togglePopover = () => setIsOpen((prev) => !prev); - const closePopover = () => setIsOpen(false); - - const onRefresh = useCallback(() => { - if (!isCatalogCacheFetching(dataSourcesLoadStatus) && dataSources.length > 0) { - startLoadingDataSources(dataSources); - } - }, [dataSourcesLoadStatus, dataSources, startLoadingDataSources]); - - const handleSelectedDataSet = useCallback( - async (ds?: Dataset) => { - const selectedDataSet = ds ?? selectedDataSetState; - if (!selectedDataSet || !selectedDataSet.id) return; - - const language = uiService.Settings.getUserQueryLanguage(); - const queryEnhancements = uiService.Settings.getQueryEnhancements(language); - const initialInput = queryEnhancements?.searchBar?.queryStringInput?.initialValue; - - const query = initialInput - ? initialInput.replace('', selectedDataSet.title!) - : ''; - uiService.Settings.setUserQueryString(query); - queryService.queryString.setQuery({ query, language }); - - queryService.dataSetManager.setDataSet(selectedDataSet); - - CatalogCacheManager.addRecentDataSet({ - id: selectedDataSet.id, - title: selectedDataSet.title ?? selectedDataSet.id!, - dataSource: selectedDataSet.dataSource, - timeFieldName: selectedDataSet.timeFieldName, - type: selectedDataSet.type, - }); - - setSelectedDataSetState({ - id: selectedDataSet.id, - title: selectedDataSet.title, - ...(selectedDataSet.dataSource && { - dataSource: { - id: selectedDataSet.dataSource?.id, - title: selectedDataSet.dataSource?.title, - type: selectedDataSet.dataSource?.type, - }, - }), - timeFieldName: selectedDataSet.timeFieldName, - type: selectedDataSet.type, - }); - - closePopover(); - }, - [queryService, setSelectedDataSetState, selectedDataSetState, uiService.Settings] - ); - - useEffect(() => { - const initializeData = async () => { - if (isInitialized.current) return; - - setIsLoading(true); - try { - const [fetchedIndexPatterns, fetchedDataSources] = await Promise.all([ - fetchIndexPatterns(savedObjectsClient!, ''), - fetchDataSources(savedObjectsClient!), - ]); - - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) { - setExternalDataSources( - externalDataSourcesCache.dataSources.map((ds) => ({ - id: ds.id, - title: ds.title, - type: ds.type, - })) - ); - } else if (fetchedDataSources.length > 0) { - setExternalDataSources(await startLoadingDataSources(fetchedDataSources)); - } - - setIndexPatterns(fetchedIndexPatterns); - setDataSources(fetchedDataSources); - - if (dataSet) { - setSelectedDataSetState({ - id: dataSet.id, - title: dataSet.title, - type: dataSet.type, - timeFieldName: dataSet.timeFieldName, - ...(dataSet.dataSource - ? { - dataSource: { - id: dataSet.dataSource.id, - title: dataSet.dataSource.title, - type: dataSet.dataSource.type, - }, - } - : { dataSource: undefined }), - database: undefined, - }); - } - - isInitialized.current = true; - } finally { - setIsLoading(false); - } - }; - - initializeData(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - useEffect(() => { - const status = dataSourcesLoadStatus.toLowerCase(); - const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache(); - if (status === DirectQueryLoadingStatus.SUCCESS) { - setExternalDataSources( - externalDataSourcesCache.dataSources.map((ds) => ({ - id: ds.id, - title: ds.title, - type: ds.type, - })) - ); - } else if ( - status === DirectQueryLoadingStatus.CANCELED || - status === DirectQueryLoadingStatus.FAILED - ) { - notifications.toasts.addWarning('Error loading external data sources'); - } - }, [dataSourcesLoadStatus, notifications.toasts]); - - const handleSelectExternalDataSource = useCallback( - async (dataSource: DataSource) => { - if (dataSource && dataSource.type === 'EXTERNAL') { - const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( - dataSource.title, - dataSource.id! - ); - if ( - (dataSourceCache.status === CachedDataSourceStatus.Empty || - dataSourceCache.status === CachedDataSourceStatus.Failed) && - !isCatalogCacheFetching(databasesLoadStatus) - ) { - await startLoadingDatabases({ - dataSourceName: dataSource.title, - dataSourceMDSId: dataSource.id!, - }); - } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) { - setCachedDatabases( - dataSourceCache.databases.map((db) => ({ - id: db.id, - title: db.title, - type: 'DATABASE', - })) - ); - } - setSelectedDataSetState((prevState) => ({ - ...prevState!, - dataSource, - })); - } - }, - [databasesLoadStatus, startLoadingDatabases] - ); - - const handleSelectExternalDatabase = useCallback( - (externalDatabase: DataStructure) => { - if (selectedDataSetState?.dataSource && externalDatabase && externalDatabase.title) { - let databaseCache: CachedDatabase; - try { - databaseCache = CatalogCacheManager.getDatabase( - selectedDataSetState.dataSource.title, - externalDatabase.title, - selectedDataSetState.dataSource.id! - ); - } catch (error) { - return; - } - if ( - databaseCache.status === CachedDataSourceStatus.Empty || - (databaseCache.status === CachedDataSourceStatus.Failed && - !isCatalogCacheFetching(tablesLoadStatus)) - ) { - startLoadingTables({ - dataSourceName: selectedDataSetState.dataSource.title, - databaseName: externalDatabase.title, - dataSourceMDSId: selectedDataSetState.dataSource.id!, - }); - } else if (databaseCache.status === CachedDataSourceStatus.Updated) { - setCachedTables( - databaseCache.tables.map((table) => ({ - id: table.title, - title: table.title, - type: 'TABLE', - })) - ); - } - } - }, - [selectedDataSetState?.dataSource, tablesLoadStatus, startLoadingTables] - ); - - useEffect(() => { - const status = databasesLoadStatus.toLowerCase(); - if (selectedDataSetState?.dataSource && selectedDataSetState.dataSource.type === 'EXTERNAL') { - const dataSourceCache = CatalogCacheManager.getOrCreateDataSource( - selectedDataSetState.dataSource.title, - selectedDataSetState.dataSource.id! - ); - if (status === DirectQueryLoadingStatus.SUCCESS) { - setCachedDatabases( - dataSourceCache.databases.map((db) => ({ - id: db.title, - title: db.title, - type: 'DATABASE', - })) - ); - } else if ( - status === DirectQueryLoadingStatus.CANCELED || - status === DirectQueryLoadingStatus.FAILED - ) { - notifications.toasts.addWarning('Error loading databases'); - } - } - }, [databasesLoadStatus, selectedDataSetState?.dataSource, notifications.toasts]); - - useEffect(() => { - if ( - selectedDataSetState?.dataSource && - selectedDataSetState.dataSource?.type === 'EXTERNAL' && - selectedDataSetState.database - ) { - const tablesStatus = tablesLoadStatus.toLowerCase(); - let databaseCache: CachedDatabase; - try { - databaseCache = CatalogCacheManager.getDatabase( - selectedDataSetState.dataSource.title, - selectedDataSetState.database.title!, - selectedDataSetState.dataSource.id! - ); - } catch (error) { - return; - } - if (tablesStatus === DirectQueryLoadingStatus.SUCCESS) { - setCachedTables( - databaseCache.tables.map((table) => ({ - id: table.id, - title: table.title, - type: 'TABLE', - })) - ); - } else if ( - tablesStatus === DirectQueryLoadingStatus.CANCELED || - tablesStatus === DirectQueryLoadingStatus.FAILED - ) { - notifications.toasts.addWarning('Error loading tables'); - } - } - }, [ - tablesLoadStatus, - selectedDataSetState?.dataSource, - selectedDataSetState?.database, - notifications.toasts, - ]); - - const handleSelectedDataSource = useCallback( - async (source: DataSource) => { - if (source) { - setIsLoading(true); - try { - const indices = await fetchIndices(searchService, source.id!); - const updatedSource = { - ...source, - indices: indices.map((indexName: string) => ({ - id: indexName, - title: indexName, - dataSource: { - id: source.id, - title: source.title, - type: source.type, - }, - })), - }; - - setDataSources((prevDataSources) => - prevDataSources.map((ds) => (ds.id === source.id ? updatedSource : ds)) - ); - - setSelectedDataSetState((prevState) => ({ - ...prevState!, - dataSource: updatedSource, - })); - } finally { - setIsLoading(false); - } - } - }, - [searchService] - ); - - const handleSelectedObject = useCallback( - async (object: DataStructure) => { - if (object) { - setIsLoading(true); - const fields = await indexPatternsService.getFieldsForWildcard({ - pattern: object.title, - dataSourceId: object.parent?.id, - }); - - const timeFields = fields.filter((field: any) => field.type === 'date'); - const timeFieldName = timeFields?.length > 0 ? timeFields[0].title : undefined; - setSelectedDataSetState((prevState) => ({ - ...prevState!, - id: `${object.parent ? object.parent.id : ''}.${object.id}`, - title: object.title, - fields, - timeFields, - timeFieldName, - dataSource: object.parent, - type: 'TEMPORARY', - })); - setIsLoading(false); - } - }, - [indexPatternsService] - ); - - const indexPatternsLabel = i18n.translate('data.query.dataSetNavigator.indexPatternsName', { - defaultMessage: 'Index patterns', - }); - const indicesLabel = i18n.translate('data.query.dataSetNavigator.indicesName', { - defaultMessage: 'Indexes', - }); - const S3DataSourcesLabel = i18n.translate('data.query.dataSetNavigator.S3DataSourcesLabel', { - defaultMessage: 'S3', - }); - - const createRefreshButton = useCallback( - () => ( - - ), - [onRefresh, databasesLoadStatus, tablesLoadStatus] - ); - - const createLoadingSpinner = () => ( - - - - ); - - const panels = useMemo( - () => [ - { - id: 0, - items: [ - ...(CatalogCacheManager.getRecentDataSets().length > 0 - ? [ - { - title: 'Recently Used', - panel: 8, - }, - ] - : []), - { - title: indexPatternsLabel, - panel: 1, - }, - { - title: indicesLabel, - panel: 2, - }, - ...(isExternalDataSourcesEnabled - ? [ - { - title: S3DataSourcesLabel, - panel: 4, - onClick: () => {}, - }, - ] - : []), - ], - }, - { - id: 1, - title: indexPatternsLabel, - items: indexPatterns.map((indexPattern) => ({ - title: indexPattern.title, - onClick: () => handleSelectedDataSet(indexPattern), - })), - content: indexPatterns.length === 0 && createLoadingSpinner(), - }, - { - id: 2, - title: indicesLabel, - items: dataSources.map((dataSource) => ({ - title: dataSource.title, - panel: 3, - onClick: () => handleSelectedDataSource(dataSource), - })), - content: isLoading && createLoadingSpinner(), - }, - { - id: 3, - title: selectedDataSetState?.dataSource?.title ?? indicesLabel, - items: - selectedDataSetState?.dataSource?.indices?.map((object) => ({ - title: object.title, - panel: 7, - onClick: () => handleSelectedObject(object), - })) ?? [], - content: isLoading && !selectedDataSetState?.dataSource?.indices && createLoadingSpinner(), - }, - { - id: 4, - title: ( -
- {S3DataSourcesLabel} - {CatalogCacheManager.getExternalDataSourcesCache().status === - CachedDataSourceStatus.Updated && createRefreshButton()} -
- ), - items: externalDataSources.map((dataSource) => ({ - title: dataSource.title, - onClick: async () => await handleSelectExternalDataSource(dataSource), - panel: 5, - })), - content: isCatalogCacheFetching(dataSourcesLoadStatus) && createLoadingSpinner(), - }, - { - id: 5, - title: selectedDataSetState?.dataSource?.title ?? 'Databases', - items: cachedDatabases.map((db) => ({ - title: db.title, - onClick: () => { - setSelectedDataSetState((prevState) => ({ - ...prevState!, - database: db, - })); - handleSelectExternalDatabase(db); - }, - panel: 6, - })), - content: isCatalogCacheFetching(databasesLoadStatus) && createLoadingSpinner(), - }, - { - id: 6, - title: selectedDataSetState?.database?.title ?? 'Tables', - items: cachedTables.map((table) => ({ - title: table.title, - onClick: () => { - const tableObject: Dataset = { - ...selectedDataSetState!, - // TODO: potential error case where we do not append the MDS ID for ID - id: `${selectedDataSetState?.dataSource?.meta?.name}.${selectedDataSetState?.database?.title}.${table.title}`, - title: `${selectedDataSetState?.dataSource?.meta?.name}.${selectedDataSetState?.database?.title}.${table.title}`, - ...(selectedDataSetState?.dataSource && { - dataSource: { - id: selectedDataSetState?.dataSource!.id, - title: selectedDataSetState?.dataSource!.title, - type: selectedDataSetState?.dataSource!.type, - }, - }), - type: 'TEMPORARY_ASYNC', - }; - handleSelectedDataSet(tableObject); - }, - })), - content: isCatalogCacheFetching(tablesLoadStatus) && createLoadingSpinner(), - }, - { - id: 7, - title: selectedDataSetState?.title, - content: - !selectedDataSetState || !selectedDataSetState?.title ? ( - createLoadingSpinner() - ) : ( - - - 0 - ? selectedDataSetState.timeFields.map((field: any) => ({ - value: field.value, - text: field.title, - })) - : []), - { value: 'no-time-filter', text: "I don't want to use a time filter" }, - ]} - onChange={(event) => { - setSelectedDataSetState((prevState) => ({ - ...prevState!, - timeFieldName: - event.target.value !== 'no-time-filter' ? event.target.value : undefined, - })); - }} - aria-label="Select a date field" - /> - - handleSelectedDataSet()}> - Select - - - ), - }, - { - id: 8, - title: 'Recently Used', - items: CatalogCacheManager.getRecentDataSets().map((ds) => ({ - title: ds.title, - onClick: () => { - setSelectedDataSetState({ - id: ds.id ?? ds.title, - title: ds.title, - dataSource: ds.dataSource, - database: undefined, - timeFieldName: ds.timeFieldName, - type: ds.type, - }); - handleSelectedDataSet(); - }, - })), - }, - ], - [ - indexPatternsLabel, - indicesLabel, - isExternalDataSourcesEnabled, - S3DataSourcesLabel, - indexPatterns, - dataSources, - isLoading, - selectedDataSetState, - createRefreshButton, - externalDataSources, - dataSourcesLoadStatus, - cachedDatabases, - databasesLoadStatus, - cachedTables, - tablesLoadStatus, - handleSelectedDataSet, - handleSelectedDataSource, - handleSelectedObject, - handleSelectExternalDataSource, - handleSelectExternalDatabase, - ] - ); - - const dataSetTitle = - selectedDataSetState && - selectedDataSetState?.dataSource && - selectedDataSetState?.dataSource.title - ? `${selectedDataSetState.dataSource?.title}::${selectedDataSetState?.title}` - : selectedDataSetState?.title; - - return ( - - - - {dataSetTitle ?? 'Select data set'} - - - } - isOpen={isOpen} - closePopover={closePopover} - anchorPosition="downLeft" - display="block" - panelPaddingSize="none" - > - - - ); -}; - -// eslint-disable-next-line import/no-default-export -export default React.memo(DataSetNavigator); diff --git a/src/plugins/data/public/ui/dataset_navigator/index.tsx b/src/plugins/data/public/ui/dataset_navigator/index.tsx deleted file mode 100644 index e98e52c8421f..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator'; -export { createDataSetNavigator } from './create_dataset_navigator'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts deleted file mode 100644 index 0526cfd51212..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_intercept.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { HttpFetchOptionsWithPath, IHttpInterceptController } from 'opensearch-dashboards/public'; -import { SECURITY_DASHBOARDS_LOGOUT_URL } from '../constants'; -import { CatalogCacheManager } from './cache_manager'; - -export function catalogRequestIntercept(): any { - return ( - fetchOptions: Readonly, - _controller: IHttpInterceptController - ) => { - if (fetchOptions.path.includes(SECURITY_DASHBOARDS_LOGOUT_URL)) { - // Clears all user catalog cache details - CatalogCacheManager.clearDataSourceCache(); - CatalogCacheManager.clearAccelerationsCache(); - CatalogCacheManager.clearExternalDataSourcesCache(); - CatalogCacheManager.clearRecentDataSetsCache(); - } - }; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx deleted file mode 100644 index f5a6aa8aece0..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx +++ /dev/null @@ -1,482 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useRef, useState } from 'react'; -import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; -import { ASYNC_QUERY, SPARK } from '../constants'; -import { - AsyncPollingResult, - CachedColumn, - CachedDataSourceStatus, - CachedTable, - LoadCacheType, - StartLoadingParams, - DirectQueryLoadingStatus, - DirectQueryRequest, -} from '../types'; -import { SIMPLE_DATA_SOURCE_TYPES, SimpleDataSource } from '../../../../../common'; -import { addBackticksIfNeeded, combineSchemaAndDatarows, formatError } from '../utils/shared'; -import { usePolling } from '../utils/use_polling'; -import { CatalogCacheManager } from './cache_manager'; -import { fetchExternalDataSources } from '../utils'; -import { getUiService } from '../../../../services'; - -export const updateDatabasesToCache = ( - dataSourceName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - const cachedDataSource = CatalogCacheManager.getOrCreateDataSource( - dataSourceName, - dataSourceMDSId - ); - - const currentTime = new Date().toUTCString(); - - if (!pollingResult) { - CatalogCacheManager.addOrUpdateDataSource( - { - ...cachedDataSource, - databases: [], - lastUpdated: currentTime, - status: CachedDataSourceStatus.Failed, - ...(dataSourceMDSId && { dataSourceMDSId }), - }, - dataSourceMDSId - ); - return; - } - - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newDatabases = combinedData.map((row: any) => ({ - name: row.namespace, - tables: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - })); - - CatalogCacheManager.addOrUpdateDataSource( - { - ...cachedDataSource, - databases: newDatabases, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - ...(dataSourceMDSId && { dataSourceMDSId }), - }, - dataSourceMDSId - ); -}; - -export const updateTablesToCache = ( - dataSourceName: string, - databaseName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - try { - const cachedDatabase = CatalogCacheManager.getDatabase( - dataSourceName, - databaseName, - dataSourceMDSId - ); - const currentTime = new Date().toUTCString(); - - if (!pollingResult) { - CatalogCacheManager.updateDatabase( - dataSourceName, - { - ...cachedDatabase, - tables: [], - lastUpdated: currentTime, - status: CachedDataSourceStatus.Failed, - }, - dataSourceMDSId - ); - return; - } - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - const newTables = combinedData - .filter((row: any) => !SPARK.HIVE_TABLE_REGEX.test(row.information)) - .map((row: any) => ({ - name: row.tableName, - })); - - CatalogCacheManager.updateDatabase( - dataSourceName, - { - ...cachedDatabase, - tables: newTables, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - }, - dataSourceMDSId - ); - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } -}; - -export const updateAccelerationsToCache = ( - dataSourceName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - const currentTime = new Date().toUTCString(); - - if (!pollingResult) { - CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ - name: dataSourceName, - accelerations: [], - lastUpdated: currentTime, - status: CachedDataSourceStatus.Failed, - ...(dataSourceMDSId && { dataSourceMDSId }), - }); - return; - } - - const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows); - - const newAccelerations: any[] = combinedData.map((row: any) => ({ - flintIndexName: row.flint_index_name, - type: row.kind === 'mv' ? 'materialized' : row.kind, - database: row.database, - table: row.table, - indexName: row.index_name, - autoRefresh: row.auto_refresh, - status: row.status, - })); - - CatalogCacheManager.addOrUpdateAccelerationsByDataSource({ - name: dataSourceName, - accelerations: newAccelerations, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - ...(dataSourceMDSId && { dataSourceMDSId }), - }); -}; - -export const updateTableColumnsToCache = ( - dataSourceName: string, - databaseName: string, - tableName: string, - pollingResult: AsyncPollingResult, - dataSourceMDSId?: string -) => { - try { - if (!pollingResult) { - return; - } - const cachedDatabase = CatalogCacheManager.getDatabase( - dataSourceName, - databaseName, - dataSourceMDSId - ); - const currentTime = new Date().toUTCString(); - - const combinedData: Array<{ col_name: string; data_type: string }> = combineSchemaAndDatarows( - pollingResult.schema, - pollingResult.datarows - ); - - const tableColumns: CachedColumn[] = []; - for (const row of combinedData) { - if (row.col_name === SPARK.PARTITION_INFO) { - break; - } - tableColumns.push({ - fieldName: row.col_name, - dataType: row.data_type, - }); - } - - const newTables: CachedTable[] = cachedDatabase.tables.map((ts) => - ts.name === tableName ? { ...ts, columns: tableColumns } : { ...ts } - ); - - if (cachedDatabase.status === CachedDataSourceStatus.Updated) { - CatalogCacheManager.updateDatabase( - dataSourceName, - { - ...cachedDatabase, - tables: newTables, - lastUpdated: currentTime, - status: CachedDataSourceStatus.Updated, - }, - dataSourceMDSId - ); - } - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } -}; - -export const updateToCache = ( - pollResults: any, - loadCacheType: LoadCacheType, - dataSourceName: string, - databaseName?: string, - tableName?: string, - dataSourceMDSId?: string -) => { - switch (loadCacheType) { - case 'databases': - updateDatabasesToCache(dataSourceName, pollResults, dataSourceMDSId); - break; - case 'tables': - updateTablesToCache(dataSourceName, databaseName!, pollResults, dataSourceMDSId); - break; - case 'accelerations': - updateAccelerationsToCache(dataSourceName, pollResults, dataSourceMDSId); - break; - case 'tableColumns': - updateTableColumnsToCache( - dataSourceName, - databaseName!, - tableName!, - pollResults, - dataSourceMDSId - ); - default: - break; - } -}; - -export const createLoadQuery = ( - loadCacheType: LoadCacheType, - dataSourceName: string, - databaseName?: string, - tableName?: string -) => { - let query; - switch (loadCacheType) { - case 'databases': - query = `SHOW SCHEMAS IN ${addBackticksIfNeeded(dataSourceName)}`; - break; - case 'tables': - query = `SHOW TABLE EXTENDED IN ${addBackticksIfNeeded( - dataSourceName - )}.${addBackticksIfNeeded(databaseName!)} LIKE '*'`; - break; - case 'accelerations': - query = `SHOW FLINT INDEX in ${addBackticksIfNeeded(dataSourceName)}`; - break; - case 'tableColumns': - query = `DESC ${addBackticksIfNeeded(dataSourceName)}.${addBackticksIfNeeded( - databaseName! - )}.${addBackticksIfNeeded(tableName!)}`; - break; - default: - query = ''; - break; - } - return query; -}; - -export const useLoadToCache = ( - loadCacheType: LoadCacheType, - http: HttpStart, - notifications: NotificationsStart -) => { - const [currentDataSourceName, setCurrentDataSourceName] = useState(''); - const [currentDatabaseName, setCurrentDatabaseName] = useState(''); - const [currentTableName, setCurrentTableName] = useState(''); - const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.INITIAL - ); - const dataSourceMDSClientId = useRef(''); - - const { - data: pollingResult, - loading: _pollingLoading, - error: pollingError, - startPolling, - stopPolling: stopLoading, - } = usePolling((params) => { - return http.fetch(`../../api/enhancements/datasource/jobs`, { - query: { - id: dataSourceMDSClientId.current, - queryId: params.queryId, - }, - }); - }, ASYNC_QUERY.POLLING_INTERVAL); - - const onLoadingFailed = () => { - setLoadStatus(DirectQueryLoadingStatus.FAILED); - updateToCache( - null, - loadCacheType, - currentDataSourceName, - currentDatabaseName, - currentTableName, - dataSourceMDSClientId.current - ); - }; - - const startLoading = async ({ - dataSourceName, - dataSourceMDSId, - databaseName, - tableName, - }: StartLoadingParams) => { - const uiService = getUiService(); - setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); - setCurrentDataSourceName(dataSourceName); - setCurrentDatabaseName(databaseName); - setCurrentTableName(tableName); - dataSourceMDSClientId.current = dataSourceMDSId || ''; - - let requestPayload: DirectQueryRequest = { - lang: 'sql', - query: createLoadQuery(loadCacheType, dataSourceName, databaseName, tableName), - datasource: dataSourceName, - }; - - const sessionId = uiService.Settings.getUserQuerySessionId(dataSourceName); - if (sessionId) { - requestPayload = { ...requestPayload, sessionId }; - } - await http - .post(`../../api/enhancements/datasource/jobs`, { - body: JSON.stringify(requestPayload), - query: { - id: dataSourceMDSClientId.current, - }, - }) - .then((result) => { - uiService.Settings.setUserQuerySessionIdByObj(dataSourceName, result); - if (result.queryId) { - startPolling({ - queryId: result.queryId, - }); - } else { - // eslint-disable-next-line no-console - console.error('No query id found in response'); - onLoadingFailed(); - } - }) - .catch((e) => { - onLoadingFailed(); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - e.body?.message - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - // eslint-disable-next-line no-console - console.error(e); - }); - }; - - useEffect(() => { - // cancel direct query - if (!pollingResult) return; - const { status: anyCaseStatus, datarows, error } = pollingResult; - const status = anyCaseStatus?.toLowerCase(); - - if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { - setLoadStatus(status); - stopLoading(); - updateToCache( - pollingResult, - loadCacheType, - currentDataSourceName, - currentDatabaseName, - currentTableName, - dataSourceMDSClientId.current - ); - } else if (status === DirectQueryLoadingStatus.FAILED) { - onLoadingFailed(); - stopLoading(); - - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - error - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - } else { - setLoadStatus(status); - } - - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [pollingResult, pollingError]); - - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadDatabasesToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache( - 'databases', - http, - notifications - ); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadTablesToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache('tables', http, notifications); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadTableColumnsToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache( - 'tableColumns', - http, - notifications - ); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadAccelerationsToCache = (http: HttpStart, notifications: NotificationsStart) => { - const { loadStatus, startLoading, stopLoading } = useLoadToCache( - 'accelerations', - http, - notifications - ); - return { loadStatus, startLoading, stopLoading }; -}; - -export const useLoadExternalDataSourcesToCache = ( - http: HttpStart, - notifications: NotificationsStart -) => { - const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.INITIAL - ); - - const loadExternalDataSources = async ( - connectedClusters: SimpleDataSource[] - ): Promise => { - setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); - CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty); - - try { - const externalDataSources = await fetchExternalDataSources(http, connectedClusters); - CatalogCacheManager.updateExternalDataSources(externalDataSources); - setLoadStatus(DirectQueryLoadingStatus.SUCCESS); - CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated); - return externalDataSources.map((dataSource) => ({ - id: dataSource.dataSourceRef, - name: dataSource.name, - type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL, - })); - } catch (error) { - setLoadStatus(DirectQueryLoadingStatus.FAILED); - CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed); - notifications.toasts.addError(error, { - title: 'Failed to load external datasources', - }); - } - return []; - }; - - return { loadStatus, loadExternalDataSources }; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts deleted file mode 100644 index 6844bbd67078..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_manager.ts +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { ASYNC_QUERY, DATASET } from '../constants'; -import { - AccelerationsCacheData, - CachedAccelerationByDataSource, - CachedDataSource, - CachedDataSourceStatus, - CachedDatabase, - DataSourceCacheData, - ExternalDataSource, - ExternalDataSourcesCacheData, - RecentDataSetOptionsCacheData, -} from '../types'; -import { Dataset, DataStructure } from '../../../../../common'; - -/** - * Manages caching for catalog data including data sources and accelerations. - */ -export class CatalogCacheManager { - // TODO: make this an advanced setting - private static readonly maxRecentDataSet = 4; - - /** - * Saves data source cache to local storage. - * @param {DataSourceCacheData} cacheData - The data source cache data to save. - */ - static saveDataSourceCache(cacheData: DataSourceCacheData): void { - sessionStorage.setItem(ASYNC_QUERY.CATALOG_CACHE.KEY, JSON.stringify(cacheData)); - } - - /** - * Retrieves data source cache from local storage. - * @returns {DataSourceCacheData} The retrieved data source cache. - */ - static getDataSourceCache(): DataSourceCacheData { - const catalogData = sessionStorage.getItem(ASYNC_QUERY.CATALOG_CACHE.KEY); - - if (catalogData) { - return JSON.parse(catalogData); - } else { - const defaultCacheObject = { version: ASYNC_QUERY.CATALOG_CACHE.VERSION, dataSources: [] }; - this.saveDataSourceCache(defaultCacheObject); - return defaultCacheObject; - } - } - - /** - * Saves accelerations cache to local storage. - * @param {AccelerationsCacheData} cacheData - The accelerations cache data to save. - */ - static saveAccelerationsCache(cacheData: AccelerationsCacheData): void { - sessionStorage.setItem(ASYNC_QUERY.ACCELERATIONS_CACHE, JSON.stringify(cacheData)); - } - - /** - * Retrieves accelerations cache from local storage. - * @returns {AccelerationsCacheData} The retrieved accelerations cache. - */ - static getAccelerationsCache(): AccelerationsCacheData { - const accelerationCacheData = sessionStorage.getItem(ASYNC_QUERY.ACCELERATIONS_CACHE); - - if (accelerationCacheData) { - return JSON.parse(accelerationCacheData); - } else { - const defaultCacheObject = { - version: ASYNC_QUERY.CATALOG_CACHE.VERSION, - dataSources: [], - }; - this.saveAccelerationsCache(defaultCacheObject); - return defaultCacheObject; - } - } - - /** - * Adds or updates a data source in the accelerations cache. - * @param {CachedAccelerationByDataSource} dataSource - The data source to add or update. - */ - static addOrUpdateAccelerationsByDataSource( - dataSource: CachedAccelerationByDataSource, - dataSourceMDSId?: string - ): void { - let index = -1; - const accCacheData = this.getAccelerationsCache(); - if (dataSourceMDSId) { - index = accCacheData.dataSources.findIndex( - (ds: CachedAccelerationByDataSource) => - ds.name === dataSource.name && ds.dataSourceMDSId === dataSourceMDSId - ); - } else { - index = accCacheData.dataSources.findIndex( - (ds: CachedAccelerationByDataSource) => ds.name === dataSource.name - ); - } - if (index !== -1) { - accCacheData.dataSources[index] = dataSource; - } else { - accCacheData.dataSources.push(dataSource); - } - this.saveAccelerationsCache(accCacheData); - } - - /** - * Retrieves accelerations cache from local storage by the datasource name. - * @param {string} dataSourceName - The name of the data source. - * @returns {CachedAccelerationByDataSource} The retrieved accelerations by datasource in cache. - * @throws {Error} If the data source is not found. - */ - static getOrCreateAccelerationsByDataSource( - dataSourceName: string, - dataSourceMDSId?: string - ): CachedAccelerationByDataSource { - const accCacheData = this.getAccelerationsCache(); - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = accCacheData.dataSources.find( - (ds) => ds.name === dataSourceName && ds.dataSourceMDSId === dataSourceMDSId - ); - } else { - cachedDataSource = accCacheData.dataSources.find((ds) => ds.name === dataSourceName); - } - if (cachedDataSource) return cachedDataSource; - else { - let defaultDataSourceObject: CachedAccelerationByDataSource = { - name: dataSourceName, - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - accelerations: [], - }; - - if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { - defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; - } - this.addOrUpdateAccelerationsByDataSource(defaultDataSourceObject, dataSourceMDSId); - return defaultDataSourceObject; - } - } - - /** - * Adds or updates a data source in the cache. - * @param {CachedDataSource} dataSource - The data source to add or update. - */ - static addOrUpdateDataSource(dataSource: CachedDataSource, dataSourceMDSId?: string): void { - const cacheData = this.getDataSourceCache(); - const index = cacheData.dataSources.findIndex( - (ds) => - ds.name === dataSource.name && (!dataSourceMDSId || ds.dataSourceMDSId === dataSourceMDSId) - ); - cacheData.dataSources.splice(index, 1, dataSource); - this.saveDataSourceCache(cacheData); - } - - /** - * Retrieves or creates a data source with the specified name. - * @param {string} dataSourceName - The name of the data source. - * @returns {CachedDataSource} The retrieved or created data source. - */ - static getOrCreateDataSource(dataSourceName: string, dataSourceMDSId?: string): CachedDataSource { - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName - ); - } else { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.name === dataSourceName - ); - } - if (cachedDataSource) { - return cachedDataSource; - } else { - let defaultDataSourceObject: CachedDataSource = { - name: dataSourceName, - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - databases: [], - }; - if (dataSourceMDSId !== '' && dataSourceMDSId !== undefined) { - defaultDataSourceObject = { ...defaultDataSourceObject, dataSourceMDSId }; - } - this.addOrUpdateDataSource(defaultDataSourceObject, dataSourceMDSId); - return defaultDataSourceObject; - } - } - - /** - * Retrieves a database from the cache. - * @param {string} dataSourceName - The name of the data source containing the database. - * @param {string} databaseName - The name of the database. - * @returns {CachedDatabase} The retrieved database. - * @throws {Error} If the data source or database is not found. - */ - static getDatabase( - dataSourceName: string, - databaseName: string, - dataSourceMDSId?: string - ): CachedDatabase { - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName - ); - } else { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.name === dataSourceName - ); - } - if (!cachedDataSource) { - throw new Error('DataSource not found exception: ' + dataSourceName); - } - - const cachedDatabase = cachedDataSource.databases.find((db) => db.name === databaseName); - if (!cachedDatabase) { - throw new Error('Database not found exception: ' + databaseName); - } - - return cachedDatabase; - } - - /** - * Retrieves a table from the cache. - * @param {string} dataSourceName - The name of the data source containing the database. - * @param {string} databaseName - The name of the database. - * @param {string} tableName - The name of the database. - * @returns {Cachedtable} The retrieved database. - * @throws {Error} If the data source, database or table is not found. - */ - static getTable( - dataSourceName: string, - databaseName: string, - tableName: string, - dataSourceMDSId?: string - ): DataStructure { - const cachedDatabase = this.getDatabase(dataSourceName, databaseName, dataSourceMDSId); - - const cachedTable = cachedDatabase.tables!.find((table) => table.name === tableName); - if (!cachedTable) { - throw new Error('Table not found exception: ' + tableName); - } - - // TODO: Potentional source of error, id should be MDS_ID::connectionName.defaultDb.table1 - return { id: cachedTable.name, title: cachedTable.name, type: 'TABLE', ...cachedTable }; - } - - /** - * Updates a database in the cache. - * @param {string} dataSourceName - The name of the data source containing the database. - * @param {CachedDatabase} database - The database to be updated. - * @throws {Error} If the data source or database is not found. - */ - static updateDatabase( - dataSourceName: string, - database: CachedDatabase, - dataSourceMDSId?: string - ): void { - let cachedDataSource; - if (dataSourceMDSId) { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.dataSourceMDSId === dataSourceMDSId && ds.name === dataSourceName - ); - } else { - cachedDataSource = this.getDataSourceCache().dataSources.find( - (ds) => ds.name === dataSourceName - ); - } - - if (!cachedDataSource) { - throw new Error('DataSource not found exception: ' + dataSourceName); - } - - const index = cachedDataSource.databases.findIndex((db) => db.name === database.name); - if (index !== -1) { - cachedDataSource.databases[index] = database; - this.addOrUpdateDataSource(cachedDataSource, dataSourceMDSId); - } else { - throw new Error('Database not found exception: ' + database.name); - } - } - - /** - * Clears the data source cache from local storage. - */ - static clearDataSourceCache(): void { - sessionStorage.removeItem(ASYNC_QUERY.CATALOG_CACHE.KEY); - this.clearExternalDataSourcesCache(); - } - - /** - * Clears the accelerations cache from local storage. - */ - static clearAccelerationsCache(): void { - sessionStorage.removeItem(ASYNC_QUERY.ACCELERATIONS_CACHE); - } - - static saveExternalDataSourcesCache(cacheData: ExternalDataSourcesCacheData): void { - sessionStorage.setItem(ASYNC_QUERY.CATALOG_CACHE.KEY, JSON.stringify(cacheData)); - } - - static getExternalDataSourcesCache(): ExternalDataSourcesCacheData { - const externalDataSourcesData = sessionStorage.getItem(ASYNC_QUERY.CATALOG_CACHE.KEY); - - if (externalDataSourcesData) { - return JSON.parse(externalDataSourcesData); - } else { - const defaultCacheObject: ExternalDataSourcesCacheData = { - version: ASYNC_QUERY.CATALOG_CACHE.VERSION, - dataSources: [], - lastUpdated: '', - status: CachedDataSourceStatus.Empty, - }; - this.saveExternalDataSourcesCache(defaultCacheObject); - return defaultCacheObject; - } - } - - static updateExternalDataSources(externalDataSources: ExternalDataSource[]): void { - const currentTime = new Date().toUTCString(); - const cacheData = this.getExternalDataSourcesCache(); - cacheData.dataSources = externalDataSources; - cacheData.lastUpdated = currentTime; - cacheData.status = CachedDataSourceStatus.Updated; - this.saveExternalDataSourcesCache(cacheData); - } - - static getExternalDataSources(): ExternalDataSourcesCacheData { - return this.getExternalDataSourcesCache(); - } - - static clearExternalDataSourcesCache(): void { - sessionStorage.removeItem(ASYNC_QUERY.CATALOG_CACHE.KEY); - } - - static setExternalDataSourcesLoadingStatus(status: CachedDataSourceStatus): void { - const cacheData = this.getExternalDataSourcesCache(); - cacheData.status = status; - this.saveExternalDataSourcesCache(cacheData); - } - - static saveRecentDataSetsCache(cacheData: RecentDataSetOptionsCacheData): void { - sessionStorage.setItem(DATASET.OPTIONS_CACHE.KEY, JSON.stringify(cacheData)); - } - - static getRecentDataSetsCache(): RecentDataSetOptionsCacheData { - const recentDataSetOptionsData = sessionStorage.getItem(DATASET.OPTIONS_CACHE.KEY); - - if (recentDataSetOptionsData) { - return JSON.parse(recentDataSetOptionsData); - } else { - const defaultCacheObject: RecentDataSetOptionsCacheData = { - version: ASYNC_QUERY.CATALOG_CACHE.VERSION, - recentDataSets: [], - }; - this.saveRecentDataSetsCache(defaultCacheObject); - return defaultCacheObject; - } - } - - static addRecentDataSet(dataSet: Dataset): void { - const cacheData = this.getRecentDataSetsCache(); - - cacheData.recentDataSets = cacheData.recentDataSets.filter( - (option) => option.id !== dataSet.id - ); - - cacheData.recentDataSets.push(dataSet); - - if (cacheData.recentDataSets.length > this.maxRecentDataSet) { - cacheData.recentDataSets.shift(); - } - - this.saveRecentDataSetsCache(cacheData); - } - - static getRecentDataSets(): Dataset[] { - return this.getRecentDataSetsCache().recentDataSets; - } - - static clearRecentDataSetsCache(): void { - sessionStorage.removeItem(DATASET.OPTIONS_CACHE.KEY); - } -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx deleted file mode 100644 index 5449277b2bd8..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/index.tsx +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './cache_intercept'; -export * from './cache_loader'; -export * from './cache_manager'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts b/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts deleted file mode 100644 index eb970b791b3c..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/constants.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export const ASYNC_QUERY = { - SEARCH_STRATEGY: 'sqlasyncraw', - SESSION_ID: { - KEY: 'async-query-session-id', - }, - CATALOG_CACHE: { - KEY: 'async-query-catalog-cache', - VERSION: '1.0', - }, - ACCELERATIONS_CACHE: 'async-query-acclerations-cache', - POLLING_INTERVAL: 2000, -}; - -export const DATASET = { - OPTIONS_CACHE: { - KEY: 'recent_dataset_options_cache', - }, -}; - -export const DEFAULT_DATA_SOURCE = { - TYPE: 'DEFAULT_INDEX_PATTERNS', - NAME: 'Default cluster', - TITLE: 'Default Group', -}; - -export const ACCELERATION = { - DEFUALT_SKIPPING_INDEX_NAME: 'skipping', - TIME_INTERVAL: [ - { text: 'millisecond(s)', value: 'millisecond' }, - { text: 'second(s)', value: 'second' }, - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, - ], - REFRESH_TIME_INTERVAL: [ - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, - ], - ADD_FIELDS_TEXT: '(add fields here)', - INDEX_NAME_REGEX: /^[a-z0-9_]+$/, - S3_URL_REGEX: /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/, - INDEX_TYPES: [ - { label: 'Skipping Index', value: 'skipping' }, - { label: 'Covering Index', value: 'covering' }, - { label: 'Materialized View', value: 'materialized' }, - ], - INDEX_NAME_INFO: `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. - ##### Skipping Index - - For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. - - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. - ##### Covering Index - - 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. - - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. - ##### Materialized View Index - - 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. - - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. - ##### Note: - - All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. - `, - AGGREGRATION_FUNCTIONS: [ - { label: 'window.start' }, - { label: 'count' }, - { label: 'sum' }, - { label: 'avg' }, - { label: 'max' }, - { label: 'min' }, - ], -}; - -export const SKIPPING_INDEX = { - ACCELERATION_METHODS: [ - { value: 'PARTITION', text: 'Partition' }, - { value: 'VALUE_SET', text: 'Value Set' }, - { value: 'MIN_MAX', text: 'Min Max' }, - { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, - ], -}; - -export const SPARK = { - HIVE_TABLE_REGEX: /Provider:\s*hive/, - TIMESTAMP_DATATYPE: 'timestamp', - STRING_DATATYPE: 'string', - PARTITION_INFO: `# Partition Information`, -}; - -export const REGEX = { - SANITIZE_QUERY: /\s+/g, -}; - -export const DOCUMENTATION = { - ACC_INDEX_TYPE_URL: - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md', - ACC_CHECKPOINT_URL: - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options', -}; - -export const OBSERVABILITY = { - DEFAULT_CLUSTER: 'observability-default', - S3_DATA_SOURCE: 'observability-s3', -}; - -export const S3_DATA_SOURCE = { - GROUP_DISPLAY_NAME: 'Amazon S3', - GROUP_SPARK_DISPLAY_NAME: 'Spark', -}; - -export const SECURITY = { - DASHBOARDS_LOGOUT_URL: '/logout', -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx b/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx deleted file mode 100644 index 2bb3570376a7..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/types.tsx +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { EuiComboBoxOptionOption } from '@elastic/eui'; -import { Dataset, DataStructure } from '../../../../common'; - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SCHEDULED = 'scheduled', - CANCELED = 'canceled', - WAITING = 'waiting', - INITIAL = 'initial', -} - -export interface DirectQueryRequest { - query: string; - lang: string; - datasource: string; - sessionId?: string; -} - -export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; - -export interface PermissionsConfigurationProps { - roles: Role[]; - selectedRoles: Role[]; - setSelectedRoles: React.Dispatch>; - layout: 'horizontal' | 'vertical'; - hasSecurityAccess: boolean; -} - -export interface TableColumn { - name: string; - dataType: string; -} - -export interface Acceleration { - name: string; - status: AccelerationStatus; - type: string; - database: string; - table: string; - destination: string; - dateCreated: number; - dateUpdated: number; - index: string; - sql: string; -} - -export interface AssociatedObject { - tableName: string; - datasource: string; - id: string; - name: string; - database: string; - type: AssociatedObjectIndexType; - accelerations: CachedAcceleration[] | AssociatedObject; - columns?: CachedColumn[]; -} - -export type Role = EuiComboBoxOptionOption; - -export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; - -export interface S3GlueProperties { - 'glue.indexstore.opensearch.uri': string; - 'glue.indexstore.opensearch.region': string; -} - -export interface PrometheusProperties { - 'prometheus.uri': string; -} - -export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; - -export interface DatasourceDetails { - allowedRoles: string[]; - name: string; - connector: DatasourceType; - description: string; - properties: S3GlueProperties | PrometheusProperties; - status: DatasourceStatus; -} - -interface AsyncApiDataResponse { - status: string; - schema?: Array<{ name: string; type: string }>; - datarows?: any; - total?: number; - size?: number; - error?: string; -} - -export interface AsyncApiResponse { - data: { - ok: boolean; - resp: AsyncApiDataResponse; - }; -} - -export type PollingCallback = (statusObj: AsyncApiResponse) => void; - -export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; - -export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; - -export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; - -export enum CachedDataSourceStatus { - Updated = 'Updated', - Failed = 'Failed', - Empty = 'Empty', -} - -export interface CachedColumn { - fieldName: string; - dataType: string; -} - -export interface CachedTable { - id: any; - name: string; - columns?: CachedColumn[]; -} - -export interface CachedDatabase { - name: string; - tables: DataStructure[]; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; -} - -export interface CachedDataSource { - name: string; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; - databases: DataStructure[]; - dataSourceMDSId?: string; -} - -export interface DataSourceCacheData { - version: string; - dataSources: CachedDataSource[]; -} - -export interface CachedAcceleration { - flintIndexName: string; - type: AccelerationIndexType; - database: string; - table: string; - indexName: string; - autoRefresh: boolean; - status: string; -} - -export interface CachedAccelerationByDataSource { - name: string; - accelerations: CachedAcceleration[]; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; - dataSourceMDSId?: string; -} - -export interface AccelerationsCacheData { - version: string; - dataSources: CachedAccelerationByDataSource[]; -} - -export interface PollingSuccessResult { - schema: Array<{ name: string; type: string }>; - datarows: Array>; -} - -export type AsyncPollingResult = PollingSuccessResult | null; - -export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; - -export interface MaterializedViewColumn { - id: string; - functionName: AggregationFunctionType; - functionParam?: string; - fieldAlias?: string; -} - -export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; - -export interface SkippingIndexRowType { - id: string; - fieldName: string; - dataType: string; - accelerationMethod: SkippingIndexAccMethodType; -} - -export interface DataTableFieldsType { - id: string; - fieldName: string; - dataType: string; -} - -export interface RefreshIntervalType { - refreshWindow: number; - refreshInterval: string; -} - -export interface WatermarkDelayType { - delayWindow: number; - delayInterval: string; -} - -export interface GroupByTumbleType { - timeField: string; - tumbleWindow: number; - tumbleInterval: string; -} - -export interface MaterializedViewQueryType { - columnsValues: MaterializedViewColumn[]; - groupByTumbleValue: GroupByTumbleType; -} - -export interface FormErrorsType { - dataSourceError: string[]; - databaseError: string[]; - dataTableError: string[]; - skippingIndexError: string[]; - coveringIndexError: string[]; - materializedViewError: string[]; - indexNameError: string[]; - primaryShardsError: string[]; - replicaShardsError: string[]; - refreshIntervalError: string[]; - checkpointLocationError: string[]; - watermarkDelayError: string[]; -} - -export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; - -export interface CreateAccelerationForm { - dataSource: string; - database: string; - dataTable: string; - dataTableFields: DataTableFieldsType[]; - accelerationIndexType: AccelerationIndexType; - skippingIndexQueryData: SkippingIndexRowType[]; - coveringIndexQueryData: string[]; - materializedViewQueryData: MaterializedViewQueryType; - accelerationIndexName: string; - primaryShardsCount: number; - replicaShardsCount: number; - refreshType: AccelerationRefreshType; - checkpointLocation: string | undefined; - watermarkDelay: WatermarkDelayType; - refreshIntervalOptions: RefreshIntervalType; - formErrors: FormErrorsType; -} - -export interface LoadCachehookOutput { - loadStatus: DirectQueryLoadingStatus; - startLoading: (params: StartLoadingParams) => void; - stopLoading: () => void; -} - -export interface StartLoadingParams { - dataSourceName: string; - dataSourceMDSId?: string; - databaseName?: string; - tableName?: string; -} - -export interface RenderAccelerationFlyoutParams { - dataSourceName: string; - dataSourceMDSId?: string; - databaseName?: string; - tableName?: string; - handleRefresh?: () => void; -} - -export interface RenderAssociatedObjectsDetailsFlyoutParams { - tableDetail: AssociatedObject; - dataSourceName: string; - handleRefresh?: () => void; - dataSourceMDSId?: string; -} - -export interface RenderAccelerationDetailsFlyoutParams { - acceleration: CachedAcceleration; - dataSourceName: string; - handleRefresh?: () => void; - dataSourceMDSId?: string; -} - -export interface DataSetOption { - id?: string; - name: string; - dataSourceRef?: string; -} - -export interface RecentDataSetOptionsCacheData { - version: string; - recentDataSets: Dataset[]; -} - -export interface ExternalDataSource { - name: string; - status: string; - dataSource: string; -} - -export interface ExternalDataSourcesCacheData { - version: string; - dataSources: DataStructure[]; - lastUpdated: string; - status: CachedDataSourceStatus; -} - -interface DataSourceMeta { - // ref: string; // MDS ID - // dsName?: string; // flint datasource - id: string; - name: string; - type?: string; -} - -export interface DataSet { - id: string | undefined; // index pattern ID, index name, or flintdatasource.database.table - datasource?: DataSourceMeta; - meta?: { - timestampField: string; - mapping?: any; - }; - type?: 'dataSet' | 'temporary'; -} diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts deleted file mode 100644 index 697852fdd772..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_catalog_cache_status.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SCHEDULED = 'scheduled', - CANCELED = 'canceled', - WAITING = 'waiting', - INITIAL = 'initial', -} - -const catalogCacheFetchingStatus = [ - DirectQueryLoadingStatus.RUNNING, - DirectQueryLoadingStatus.WAITING, - DirectQueryLoadingStatus.SCHEDULED, -]; - -export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { - return statuses.some((status: DirectQueryLoadingStatus) => - catalogCacheFetchingStatus.includes(status) - ); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts deleted file mode 100644 index 3c62a2116b4d..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { SimpleDataSource } from '../../../../../common'; - -export const fetchDataSources = async (client: SavedObjectsClientContract) => { - const resp = await client.find({ - type: 'data-source', - perPage: 10000, - }); - const dataSources: SimpleDataSource[] = [{ id: '', name: 'Local Cluster', type: 'data-source' }]; - return dataSources.concat([ - ...(resp.savedObjects.map((savedObject) => ({ - id: savedObject.id, - name: savedObject.attributes.title, - type: 'data-source', - })) as SimpleDataSource[]), - ]); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts deleted file mode 100644 index 37a7d8bfad49..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { HttpStart } from 'opensearch-dashboards/public'; -import { CachedDataSourceStatus, DatasourceDetails, ExternalDataSource } from '../types'; -import { SimpleDataSource } from '../../../../../common'; - -export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => { - try { - await http.get('/api/enhancements/datasource/external'); - return true; - } catch (e) { - return false; - } -}; - -export const fetchExternalDataSources = async ( - http: HttpStart, - connectedClusters: SimpleDataSource[] -): Promise => { - let externalDataSources: ExternalDataSource[] = []; - - for (const cluster of connectedClusters) { - try { - const response = await http.fetch(`../../api/enhancements/datasource/external`, { - query: { - id: cluster.id, - }, - }); - - const clusterDataSources = response - .filter((dataSource: DatasourceDetails) => dataSource.connector === 'S3GLUE') - .map((dataSource: DatasourceDetails) => ({ - name: dataSource.name, - // status: dataSource.status, - dataSourceRef: cluster.id, - status: CachedDataSourceStatus.Empty, - })); - - externalDataSources = externalDataSources.concat(clusterDataSources); - } catch (error) { - // Ignore error and continue with the next cluster - } - } - - const flattenedResults = externalDataSources.flat(); - const uniqueResults = Array.from( - flattenedResults - .reduce((map, ds) => { - const key = `${ds.name}-${ds.status}`; - if (!map.has(key) || ds.dataSourceRef === '') { - map.set(key, ds); - } - return map; - }, new Map()) - .values() - ); - - return uniqueResults; -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts deleted file mode 100644 index 3f2cd230300e..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_index_patterns.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; -import { IIndexPattern } from '../.././../..'; -import { SIMPLE_DATA_SOURCE_TYPES, SIMPLE_DATA_SET_TYPES } from '../../../../../common'; - -export const fetchIndexPatterns = async (client: SavedObjectsClientContract, search: string) => { - const resp = await client.find({ - type: 'index-pattern', - fields: ['title', 'timeFieldName', 'references', 'fields'], - search: `${search}*`, - searchFields: ['title'], - perPage: 100, - }); - return resp.savedObjects.map((savedObject) => ({ - id: savedObject.id, - title: savedObject.attributes.title, - timeFieldName: savedObject.attributes.timeFieldName, - fields: savedObject.attributes.fields, - type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN, - ...(savedObject.references[0] - ? { - dataSourceRef: { - id: savedObject.references[0]?.id, - name: savedObject.references[0]?.name, - type: SIMPLE_DATA_SOURCE_TYPES.DEFAULT, - }, - } - : {}), - })); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts deleted file mode 100644 index ef10c72bc08c..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_indices.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { map } from 'rxjs/operators'; -import { ISearchStart } from '../../../../search'; - -export const fetchIndices = async (search: ISearchStart, dataSourceId?: string) => { - const buildSearchRequest = () => { - const request = { - params: { - ignoreUnavailable: true, - expand_wildcards: 'all', - index: '*', - body: { - size: 0, // no hits - aggs: { - indices: { - terms: { - field: '_index', - size: 100, - }, - }, - }, - }, - }, - dataSourceId, - }; - - return request; - }; - - const searchResponseToArray = (response: any) => { - const { rawResponse } = response; - return rawResponse.aggregations - ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key) - : []; - }; - - return search - .getDefaultSearchInterceptor() - .search(buildSearchRequest()) - .pipe(map(searchResponseToArray)) - .toPromise(); -}; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts deleted file mode 100644 index 79e55007654c..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export * from './fetch_catalog_cache_status'; -export * from './fetch_data_sources'; -export * from './fetch_external_data_sources'; -export * from './fetch_index_patterns'; -export * from './fetch_indices'; -export * from './shared'; -export * from './use_polling'; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts deleted file mode 100644 index 8077aa851051..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/shared.ts +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/** - * TODO making this method type-safe is nontrivial: if you just define - * `Nested = { [k: string]: Nested | T }` then you can't accumulate because `T` is not `Nested` - * There might be a way to define a recursive type that accumulates cleanly but it's probably not - * worth the effort. - */ - -export function get(obj: Record, path: string, defaultValue?: T): T { - return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; -} - -export function addBackticksIfNeeded(input: string): string { - if (input === undefined) { - return ''; - } - // Check if the string already has backticks - if (input.startsWith('`') && input.endsWith('`')) { - return input; // Return the string as it is - } else { - // Add backticks to the string - return '`' + input + '`'; - } -} - -export function combineSchemaAndDatarows( - schema: Array<{ name: string; type: string }>, - datarows: Array> -): object[] { - const combinedData: object[] = []; - - datarows.forEach((row) => { - const rowData: { [key: string]: string | number | boolean } = {}; - schema.forEach((field, index) => { - rowData[field.name] = row[index]; - }); - combinedData.push(rowData); - }); - - return combinedData; -} - -export const formatError = (name: string, message: string, details: string) => { - return { - name, - message, - body: { - attributes: { - error: { - caused_by: { - type: '', - reason: details, - }, - }, - }, - }, - }; -}; - -// TODO: relocate to a more appropriate location -// Client route -export const PPL_BASE = '/api/ppl'; -export const PPL_SEARCH = '/search'; -export const DSL_BASE = '/api/dsl'; -export const DSL_SEARCH = '/search'; -export const DSL_CAT = '/cat.indices'; -export const DSL_MAPPING = '/indices.getFieldMapping'; -export const DSL_SETTINGS = '/indices.getFieldSettings'; -export const OBSERVABILITY_BASE = '/api/observability'; -export const INTEGRATIONS_BASE = '/api/integrations'; -export const JOBS_BASE = '/query/jobs'; -export const DATACONNECTIONS_BASE = '/api/directquery/dataconnections'; -export const EDIT = '/edit'; -export const DATACONNECTIONS_UPDATE_STATUS = '/status'; -export const SECURITY_ROLES = '/api/v1/configuration/roles'; -export const EVENT_ANALYTICS = '/event_analytics'; -export const SAVED_OBJECTS = '/saved_objects'; -export const SAVED_QUERY = '/query'; -export const SAVED_VISUALIZATION = '/vis'; -export const CONSOLE_PROXY = '/api/console/proxy'; -export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; - -// Server route -export const PPL_ENDPOINT = '/_plugins/_ppl'; -export const SQL_ENDPOINT = '/_plugins/_sql'; -export const DSL_ENDPOINT = '/_plugins/_dsl'; -export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; -export const JOB_RESULT_ENDPOINT = '/result'; - -export const observabilityID = 'observability-logs'; -export const observabilityTitle = 'Observability'; -export const observabilityPluginOrder = 1500; - -export const observabilityApplicationsID = 'observability-applications'; -export const observabilityApplicationsTitle = 'Applications'; -export const observabilityApplicationsPluginOrder = 5090; - -export const observabilityLogsID = 'observability-logs'; -export const observabilityLogsTitle = 'Logs'; -export const observabilityLogsPluginOrder = 5091; - -export const observabilityMetricsID = 'observability-metrics'; -export const observabilityMetricsTitle = 'Metrics'; -export const observabilityMetricsPluginOrder = 5092; - -export const observabilityTracesID = 'observability-traces'; -export const observabilityTracesTitle = 'Traces'; -export const observabilityTracesPluginOrder = 5093; - -export const observabilityNotebookID = 'observability-notebooks'; -export const observabilityNotebookTitle = 'Notebooks'; -export const observabilityNotebookPluginOrder = 5094; - -export const observabilityPanelsID = 'observability-dashboards'; -export const observabilityPanelsTitle = 'Dashboards'; -export const observabilityPanelsPluginOrder = 5095; - -export const observabilityIntegrationsID = 'integrations'; -export const observabilityIntegrationsTitle = 'Integrations'; -export const observabilityIntegrationsPluginOrder = 9020; - -export const observabilityDataConnectionsID = 'datasources'; -export const observabilityDataConnectionsTitle = 'Data sources'; -export const observabilityDataConnectionsPluginOrder = 9030; - -export const queryWorkbenchPluginID = 'opensearch-query-workbench'; -export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; - -// Shared Constants -export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/'; -export const PPL_DOCUMENTATION_URL = - 'https://opensearch.org/docs/latest/search-plugins/sql/ppl/index'; -export const PPL_PATTERNS_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description'; -export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A'; -export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS'; -export const OTEL_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ss'; -export const SPAN_REGEX = /span/; - -export const PROMQL_METRIC_SUBTYPE = 'promqlmetric'; -export const OTEL_METRIC_SUBTYPE = 'openTelemetryMetric'; -export const PPL_METRIC_SUBTYPE = 'metric'; - -export const PPL_SPAN_REGEX = /by\s*span/i; -export const PPL_STATS_REGEX = /\|\s*stats/i; -export const PPL_INDEX_INSERT_POINT_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)(.*)/i; -export const PPL_INDEX_REGEX = /(search source|source|index)\s*=\s*([^|\s]+)/i; -export const PPL_WHERE_CLAUSE_REGEX = /\s*where\s+/i; -export const PPL_NEWLINE_REGEX = /[\n\r]+/g; -export const PPL_DESCRIBE_INDEX_REGEX = /(describe)\s+([^|\s]+)/i; - -// Observability plugin URI -const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; -const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; -export const OPENSEARCH_PANELS_API = { - OBJECT: `${BASE_OBSERVABILITY_URI}/object`, -}; -export const OPENSEARCH_DATACONNECTIONS_API = { - DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, -}; - -// Saved Objects -export const SAVED_OBJECT = '/object'; - -// Color Constants -export const PLOTLY_COLOR = [ - '#3CA1C7', - '#54B399', - '#DB748A', - '#F2BE4B', - '#68CCC2', - '#2A7866', - '#843769', - '#374FB8', - '#BD6F26', - '#4C636F', -]; - -export const LONG_CHART_COLOR = PLOTLY_COLOR[1]; - -export const pageStyles: CSS.Properties = { - float: 'left', - width: '100%', - maxWidth: '1130px', -}; - -export enum VIS_CHART_TYPES { - Bar = 'bar', - HorizontalBar = 'horizontal_bar', - Line = 'line', - Pie = 'pie', - HeatMap = 'heatmap', - Text = 'text', - Histogram = 'histogram', -} - -export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double']; - -export const ENABLED_VIS_TYPES = [ - VIS_CHART_TYPES.Bar, - VIS_CHART_TYPES.HorizontalBar, - VIS_CHART_TYPES.Line, - VIS_CHART_TYPES.Pie, - VIS_CHART_TYPES.HeatMap, - VIS_CHART_TYPES.Text, -]; - -// Live tail constants -export const LIVE_OPTIONS = [ - { - label: '5s', - startTime: 'now-5s', - delayTime: 5000, - }, - { - label: '10s', - startTime: 'now-10s', - delayTime: 10000, - }, - { - label: '30s', - startTime: 'now-30s', - delayTime: 30000, - }, - { - label: '1m', - startTime: 'now-1m', - delayTime: 60000, - }, - { - label: '5m', - startTime: 'now-5m', - delayTime: 60000 * 5, - }, - { - label: '15m', - startTime: 'now-15m', - delayTime: 60000 * 15, - }, - { - label: '30m', - startTime: 'now-30m', - delayTime: 60000 * 30, - }, - { - label: '1h', - startTime: 'now-1h', - delayTime: 60000 * 60, - }, - { - label: '2h', - startTime: 'now-2h', - delayTime: 60000 * 120, - }, -]; - -export const LIVE_END_TIME = 'now'; - -export interface DefaultChartStylesProps { - DefaultModeLine: string; - Interpolation: string; - LineWidth: number; - FillOpacity: number; - MarkerSize: number; - ShowLegend: string; - LegendPosition: string; - LabelAngle: number; - DefaultSortSectors: string; - DefaultModeScatter: string; -} - -export const DEFAULT_CHART_STYLES: DefaultChartStylesProps = { - DefaultModeLine: 'lines+markers', - Interpolation: 'spline', - LineWidth: 0, - FillOpacity: 100, - MarkerSize: 25, - ShowLegend: 'show', - LegendPosition: 'v', - LabelAngle: 0, - DefaultSortSectors: 'largest_to_smallest', - DefaultModeScatter: 'markers', -}; - -export const FILLOPACITY_DIV_FACTOR = 200; -export const SLIDER_MIN_VALUE = 0; -export const SLIDER_MAX_VALUE = 100; -export const SLIDER_STEP = 1; -export const THRESHOLD_LINE_WIDTH = 3; -export const THRESHOLD_LINE_OPACITY = 0.7; -export const MAX_BUCKET_LENGTH = 16; - -export enum BarOrientation { - horizontal = 'h', - vertical = 'v', -} - -export const PLOT_MARGIN = { - l: 30, - r: 5, - b: 30, - t: 50, - pad: 4, -}; - -export const WAITING_TIME_ON_USER_ACTIONS = 300; - -export const VISUALIZATION_ERROR = { - NO_DATA: 'No data found.', - INVALID_DATA: 'Invalid visualization data', - NO_SERIES: 'Add a field to start', - NO_METRIC: 'Invalid Metric MetaData', -}; - -export const S3_DATA_SOURCE_TYPE = 's3glue'; - -export const DIRECT_DUMMY_QUERY = 'select 1'; - -export const DEFAULT_START_TIME = 'now-15m'; -export const QUERY_ASSIST_START_TIME = 'now-40y'; -export const QUERY_ASSIST_END_TIME = 'now'; - -export const TIMESTAMP_DATETIME_TYPES = ['date', 'date_nanos']; diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts deleted file mode 100644 index 74fedd6cf110..000000000000 --- a/src/plugins/data/public/ui/dataset_navigator/lib/utils/use_polling.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useRef, useState } from 'react'; - -type FetchFunction = (params?: P) => Promise; - -export interface PollingConfigurations { - tabId: string; -} - -export class UsePolling { - public data: T | null = null; - public error: Error | null = null; - public loading: boolean = true; - private shouldPoll: boolean = false; - private intervalRef?: NodeJS.Timeout; - - constructor( - private fetchFunction: FetchFunction, - private interval: number = 5000, - private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, - private onPollingError?: (error: Error) => boolean, - private configurations?: PollingConfigurations - ) {} - - async fetchData(params?: P) { - this.loading = true; - try { - const result = await this.fetchFunction(params); - this.data = result; - this.loading = false; - - if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { - this.stopPolling(); - } - } catch (err) { - this.error = err as Error; - this.loading = false; - - if (this.onPollingError && this.onPollingError(this.error)) { - this.stopPolling(); - } - } - } - - startPolling(params?: P) { - this.shouldPoll = true; - if (!this.intervalRef) { - this.intervalRef = setInterval(() => { - if (this.shouldPoll) { - this.fetchData(params); - } - }, this.interval); - } - } - - stopPolling() { - this.shouldPoll = false; - if (this.intervalRef) { - clearInterval(this.intervalRef); - this.intervalRef = undefined; - } - } -} - -interface UsePollingReturn { - data: T | null; - loading: boolean; - error: Error | null; - startPolling: (params?: any) => void; - stopPolling: () => void; -} - -export function usePolling( - fetchFunction: FetchFunction, - interval: number = 5000, - onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, - onPollingError?: (error: Error) => boolean, - configurations?: PollingConfigurations -): UsePollingReturn { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [loading, setLoading] = useState(true); - const intervalRef = useRef(undefined); - const unmounted = useRef(false); - - const shouldPoll = useRef(false); - - const startPolling = (params?: P) => { - shouldPoll.current = true; - const intervalId = setInterval(() => { - if (shouldPoll.current) { - fetchData(params); - } - }, interval); - intervalRef.current = intervalId; - if (unmounted.current) { - clearInterval(intervalId); - } - }; - - const stopPolling = () => { - shouldPoll.current = false; - clearInterval(intervalRef.current); - }; - - const fetchData = async (params?: P) => { - try { - const result = await fetchFunction(params); - setData(result); - // Check the success condition and stop polling if it's met - if (onPollingSuccess && onPollingSuccess(result, configurations)) { - stopPolling(); - } - } catch (err: unknown) { - setError(err as Error); - - // Check the error condition and stop polling if it's met - if (onPollingError && onPollingError(err as Error)) { - stopPolling(); - } - } finally { - setLoading(false); - } - }; - - useEffect(() => { - return () => { - unmounted.current = true; - }; - }, []); - - return { data, loading, error, startPolling, stopPolling }; -} diff --git a/src/plugins/data/public/ui/dataset_selector/_dataset_explorer.scss b/src/plugins/data/public/ui/dataset_selector/_dataset_explorer.scss new file mode 100644 index 000000000000..0011db04fa29 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/_dataset_explorer.scss @@ -0,0 +1,28 @@ +.datasetExplorer { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 240px)) minmax(300px, 1fr); + height: 600px; + overflow-x: auto; + border: $euiBorderThin; + + &__column { + display: grid; + grid-template-rows: auto 1fr; + border-right: $euiBorderThin; + + &--empty, + &--leaf { + border-right: none; + } + } + + &__list { + flex-grow: 1; + overflow-y: auto; + } + + &__columnTitle { + padding: $euiSizeS; + border-bottom: $euiBorderThin; + } +} diff --git a/src/plugins/data/public/ui/dataset_selector/_dataset_selector.scss b/src/plugins/data/public/ui/dataset_selector/_dataset_selector.scss new file mode 100644 index 000000000000..9ec6ca531cf8 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/_dataset_selector.scss @@ -0,0 +1,14 @@ +.datasetSelector { + &__icon { + margin-right: 4px; + } + + &__advancedButton { + width: 100%; + } + + &__selectable { + width: 365px; + padding: $euiSizeXS; + } +} diff --git a/src/plugins/data/public/ui/dataset_selector/_index.scss b/src/plugins/data/public/ui/dataset_selector/_index.scss new file mode 100644 index 000000000000..b1d4dbe34c68 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/_index.scss @@ -0,0 +1,2 @@ +@import "./dataset_explorer"; +@import "./dataset_selector"; diff --git a/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx b/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx new file mode 100644 index 000000000000..a319c9a376bd --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/advanced_selector.tsx @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState } from 'react'; +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { + BaseDataset, + DATA_STRUCTURE_META_TYPES, + Dataset, + DataStructure, + DEFAULT_DATA, +} from '../../../common'; +import { DatasetExplorer } from './dataset_explorer'; +import { Configurator } from './configurator'; +import { getQueryService } from '../../services'; + +export const AdvancedSelector = ({ + savedObjects, + onSelect, + onCancel, +}: { + savedObjects: SavedObjectsClientContract; + onSelect: (dataset: Dataset) => void; + onCancel: () => void; +}) => { + const queryString = getQueryService().queryString; + + const [path, setPath] = useState([ + { + ...DEFAULT_DATA.STRUCTURES.ROOT, + columnHeader: 'Select data', + hasNext: true, + children: queryString + .getDatasetService() + .getTypes() + .map((type) => { + return { + id: type!.id, + title: type!.title, + type: type!.id, + meta: { + ...type!.meta, + type: DATA_STRUCTURE_META_TYPES.TYPE, + }, + } as DataStructure; + }), + }, + ]); + const [selectedDataset, setSelectedDataset] = useState(); + + return selectedDataset ? ( + setSelectedDataset(undefined)} + /> + ) : ( + setSelectedDataset(dataset)} + onCancel={onCancel} + /> + ); +}; diff --git a/src/plugins/data/public/ui/dataset_selector/configurator.tsx b/src/plugins/data/public/ui/dataset_selector/configurator.tsx new file mode 100644 index 000000000000..32e48691aa87 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/configurator.tsx @@ -0,0 +1,161 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useState } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiFieldText, + EuiForm, + EuiFormRow, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSelect, + EuiText, +} from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import { FormattedMessage } from '@osd/i18n/react'; +import { BaseDataset, Dataset, DatasetField } from '../../../common'; +import { getQueryService, getIndexPatterns } from '../../services'; + +export const Configurator = ({ + baseDataset, + onConfirm, + onCancel, + onPrevious, +}: { + baseDataset: BaseDataset; + onConfirm: (dataset: Dataset) => void; + onCancel: () => void; + onPrevious: () => void; +}) => { + const queryService = getQueryService(); + const queryString = queryService.queryString; + const indexPatternsService = getIndexPatterns(); + const type = queryString.getDatasetService().getType(baseDataset.type); + const languages = type?.supportedLanguages(baseDataset) || []; + + const [dataset, setDataset] = useState(baseDataset); + const [timeFields, setTimeFields] = useState(); + const [timeField, setTimeField] = useState(dataset.timeFieldName); + const [language, setLanguage] = useState(languages[0]); + + useEffect(() => { + const fetchFields = async () => { + const datasetFields = await queryString + .getDatasetService() + .getType(baseDataset.type) + ?.fetchFields(baseDataset); + + const dateFields = datasetFields?.filter((field) => field.type === 'date'); + setTimeFields(dateFields || []); + }; + + fetchFields(); + }, [baseDataset, indexPatternsService, queryString]); + + return ( + <> + + +

+ +

+ +

+ +

+ + + + + + + + + {timeFields && timeFields.length > 0 && ( + + ({ + text: field.displayName || field.name, + value: field.name, + })), + { text: '-----', value: '', disabled: true }, + { text: 'No time field', value: undefined }, + ]} + value={timeField} + onChange={(e) => { + const value = e.target.value === 'undefined' ? undefined : e.target.value; + setTimeField(value); + setDataset({ ...dataset, timeFieldName: value }); + }} + /> + + )} + + ({ + text: languageId, + value: languageId, + }))} + value={language} + onChange={(e) => setLanguage(e.target.value)} + /> + + + + + + + + + + + onConfirm(dataset)} fill> + + + + + ); +}; diff --git a/src/plugins/data/public/ui/dataset_selector/dataset_explorer.tsx b/src/plugins/data/public/ui/dataset_selector/dataset_explorer.tsx new file mode 100644 index 000000000000..dd35ce8065b2 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/dataset_explorer.tsx @@ -0,0 +1,228 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiIcon, + EuiLink, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSelectable, + EuiText, + EuiTitle, + EuiToolTip, +} from '@elastic/eui'; +import { FormattedMessage } from '@osd/i18n/react'; +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { BaseDataset, DATA_STRUCTURE_META_TYPES, DataStructure } from '../../../common'; +import { QueryStringContract } from '../../query'; + +export const DatasetExplorer = ({ + savedObjects, + queryString, + path, + setPath, + onNext, + onCancel, +}: { + savedObjects: SavedObjectsClientContract; + queryString: QueryStringContract; + path: DataStructure[]; + setPath: (path: DataStructure[]) => void; + onNext: (dataset: BaseDataset) => void; + onCancel: () => void; +}) => { + const [explorerDataset, setExplorerDataset] = useState(undefined); + const [loading, setLoading] = useState(false); + + const selectDataStructure = async (item: DataStructure, newPath: DataStructure[]) => { + const lastPathItem = newPath[newPath.length - 1]; + const nextPath = [...newPath, item]; + + const typeConfig = queryString.getDatasetService().getType(nextPath[1].id); + if (!typeConfig) return; + + if (!lastPathItem.hasNext) { + const dataset = typeConfig!.toDataset(nextPath); + setExplorerDataset(dataset as BaseDataset); + return; + } + + setLoading(true); + const nextDataStructure = await typeConfig.fetch(savedObjects, nextPath); + setLoading(false); + + setPath([...newPath, nextDataStructure]); + }; + + const columnCount = path[path.length - 1]?.hasNext ? path.length + 1 : path.length; + + return ( + <> + + +

+ +

+ +

+ + + Manage data sources + +

+ + + + +
+ {path.map((current, index) => { + const isLast = index === path.length - 1; + const isFinal = isLast && !current.hasNext; + return ( +
+ +

{current.columnHeader}

+ + ({ + label: child.parent ? `${child.parent.title}::${child.title}` : child.title, + value: child.id, + prepend: child.meta?.type === DATA_STRUCTURE_META_TYPES.TYPE && + child.meta?.icon && , + append: appendIcon(child), + checked: isChecked(child, index, path, explorerDataset), + }))} + onChange={(options) => { + const selected = options.find((option) => option.checked); + if (selected) { + const item = current.children?.find((child) => child.id === selected.value); + if (item) { + selectDataStructure(item, path.slice(0, index + 1)); + } + } + }} + singleSelection + {...(isFinal && { + searchProps: { + compressed: true, + }, + searchable: true, + })} + className="datasetExplorer__selectable" + > + {(list, search) => ( + <> + {isFinal && search} + {list} + + )} + +
+ ); + })} + {!!path[path.length - 1]?.hasNext && } +
+ + + + + + onNext(explorerDataset!)} + iconType="arrowRight" + iconSide="right" + fill + > + + + + + ); +}; + +const EmptyColumn = () => ( +
+); + +const LoadingEmptyColumn = ({ isLoading }: { isLoading: boolean }) => + isLoading ? ( +
+ +

...

+ + + {(list) => <>{list}} + +
+ ) : ( + + ); +const appendIcon = (item: DataStructure) => { + if (item.meta?.type === DATA_STRUCTURE_META_TYPES.FEATURE) { + if (item.meta?.icon && item.meta?.tooltip) { + return ( + + + + ); + } else if (item.meta?.icon) { + return ; + } + } + + if (item.meta?.type === DATA_STRUCTURE_META_TYPES.TYPE) { + return ( + + + + ); + } + + return null; +}; + +const isChecked = ( + child: DataStructure, + index: number, + path: DataStructure[], + explorerDataset?: BaseDataset +) => { + if (index === path.length - 1) { + // For the last level, check against the selectedDataSet + return child.id === explorerDataset?.id ? 'on' : undefined; + } + // For other levels, check against the next item in the path + return child.id === path[index + 1]?.id ? 'on' : undefined; +}; diff --git a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx new file mode 100644 index 000000000000..01441f736c79 --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx @@ -0,0 +1,198 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useMemo, useState, useCallback } from 'react'; +import { + EuiButtonEmpty, + EuiIcon, + EuiPopover, + EuiPopoverTitle, + EuiSelectable, + EuiSelectableOption, + EuiToolTip, +} from '@elastic/eui'; +import { FormattedMessage } from '@osd/i18n/react'; +import { toMountPoint } from '../../../../opensearch_dashboards_react/public'; +import { Dataset, DEFAULT_DATA } from '../../../common'; +import { IDataPluginServices } from '../../types'; +import { AdvancedSelector } from './advanced_selector'; +import { getQueryService } from '../../services'; + +interface DatasetSelectorProps { + selectedDataset?: Dataset; + setSelectedDataset: (dataset: Dataset) => void; + services: IDataPluginServices; +} + +export const DatasetSelector = ({ + selectedDataset, + setSelectedDataset, + services, +}: DatasetSelectorProps) => { + const [isOpen, setIsOpen] = useState(false); + const [datasets, setDatasets] = useState([]); + const { overlays, savedObjects } = services; + + const datasetService = getQueryService().queryString.getDatasetService(); + + const datasetIcon = + datasetService.getType(selectedDataset?.type || '')?.meta.icon.type || 'database'; + + const fetchDatasets = useCallback(async () => { + const typeConfig = datasetService.getType(selectedDataset?.type || ''); + if (!typeConfig || typeConfig.id !== DEFAULT_DATA.SET_TYPES.INDEX_PATTERN) { + return; + } + + const fetchedIndexPatternDataStructures = await typeConfig.fetch(savedObjects.client, []); + + const fetchedDatasets = + fetchedIndexPatternDataStructures.children?.map((pattern) => + typeConfig.toDataset([pattern]) + ) ?? []; + setDatasets(fetchedDatasets); + + // If no dataset is selected, select the first one + if (!selectedDataset && fetchedDatasets.length > 0) { + setSelectedDataset(fetchedDatasets[0]); + } + }, [datasetService, savedObjects.client, selectedDataset, setSelectedDataset]); + + useEffect(() => { + fetchDatasets(); + }, [fetchDatasets]); + + const togglePopover = useCallback(async () => { + if (!isOpen) { + await fetchDatasets(); + } + setIsOpen(!isOpen); + }, [isOpen, fetchDatasets]); + + const closePopover = useCallback(() => setIsOpen(false), []); + + const options = useMemo(() => { + const newOptions: EuiSelectableOption[] = [ + { + label: 'Index patterns', + isGroupLabel: true, + }, + ]; + + datasets.forEach(({ id, title, type, dataSource }) => { + const label = dataSource ? `${dataSource.title}::${title}` : title; + newOptions.push({ + label, + checked: id === selectedDataset?.id ? 'on' : undefined, + key: id, + prepend: , + }); + }); + + return newOptions; + }, [datasets, selectedDataset?.id, datasetService]); + + const handleOptionChange = useCallback( + (newOptions: EuiSelectableOption[]) => { + const selectedOption = newOptions.find((option) => option.checked === 'on'); + if (selectedOption) { + const foundDataset = datasets.find((dataset) => dataset.id === selectedOption.key); + if (foundDataset) { + closePopover(); + setSelectedDataset(foundDataset); + } + } + }, + [datasets, setSelectedDataset, closePopover] + ); + + const datasetTitle = useMemo(() => { + if (!selectedDataset) { + return 'Select data'; + } + + if (selectedDataset.dataSource) { + return `${selectedDataset.dataSource.title}::${selectedDataset.title}`; + } + + return selectedDataset.title; + }, [selectedDataset]); + + return ( + + + + {datasetTitle} + + + } + isOpen={isOpen} + closePopover={closePopover} + anchorPosition="downLeft" + display="block" + panelPaddingSize="none" + > + + { + closePopover(); + const overlay = overlays?.openModal( + toMountPoint( + { + overlay?.close(); + if (dataset) { + setSelectedDataset(dataset); + } + }} + onCancel={() => overlay?.close()} + /> + ) + ); + }} + > + + + + + {(list, search) => ( + <> + {search} + {list} + + )} + + + ); +}; diff --git a/src/plugins/data/public/ui/dataset_selector/index.tsx b/src/plugins/data/public/ui/dataset_selector/index.tsx new file mode 100644 index 000000000000..c5c2835fe6ef --- /dev/null +++ b/src/plugins/data/public/ui/dataset_selector/index.tsx @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useState } from 'react'; +import React from 'react'; +import { Dataset, Query, TimeRange } from '../../../common'; +import { DatasetSelector } from './dataset_selector'; +import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; +import { IDataPluginServices } from '../../types'; + +interface ConnectedDatasetSelectorProps { + onSubmit: ((query: Query, dateRange?: TimeRange | undefined) => void) | undefined; +} + +const ConnectedDatasetSelector = ({ onSubmit }: ConnectedDatasetSelectorProps) => { + const { services } = useOpenSearchDashboards(); + const queryString = services.data.query.queryString; + const initialDataset = queryString.getQuery().dataset || queryString.getDefaultQuery().dataset; + const [selectedDataset, setSelectedDataset] = useState(initialDataset); + + useEffect(() => { + const subscription = queryString.getUpdates$().subscribe((query) => { + setSelectedDataset(query.dataset); + }); + + return () => subscription.unsubscribe(); + }, [queryString]); + + const handleDatasetChange = (dataset?: Dataset) => { + setSelectedDataset(dataset); + if (dataset) { + const query = queryString.getInitialQueryByDataset(dataset); + queryString.setQuery(query); + onSubmit!(queryString.getQuery()); + } + }; + + return ( + + ); +}; + +export { ConnectedDatasetSelector as DatasetSelector }; diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index 400887e51d57..c1e29b57e86a 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -47,6 +47,10 @@ export { QueryEditorExtensionDependencies, QueryEditorExtensionConfig, } from './query_editor'; -export { SearchBar, SearchBarProps, StatefulSearchBarProps } from './search_bar'; +export { + SearchBar, + SearchBarProps, + StatefulSearchBarProps, + useQueryStringManager, +} from './search_bar'; export { SuggestionsComponent } from './typeahead'; -export { DataSetNavigator } from './dataset_navigator'; diff --git a/src/plugins/data/public/ui/mocks.ts b/src/plugins/data/public/ui/mocks.ts index dc70e5cac1d4..bbb3a118b9a9 100644 --- a/src/plugins/data/public/ui/mocks.ts +++ b/src/plugins/data/public/ui/mocks.ts @@ -38,7 +38,6 @@ function createStartContract( const queryEnhancements = new Map(); return { IndexPatternSelect: jest.fn(), - DataSetNavigator: jest.fn(), // Add the missing property SearchBar: jest.fn(), SuggestionsComponent: jest.fn(), // Add the missing property Settings: new SettingsMock( diff --git a/src/plugins/data/public/ui/query_editor/_query_editor.scss b/src/plugins/data/public/ui/query_editor/_query_editor.scss index 88ba526abcd5..82c71a199b59 100644 --- a/src/plugins/data/public/ui/query_editor/_query_editor.scss +++ b/src/plugins/data/public/ui/query_editor/_query_editor.scss @@ -102,7 +102,7 @@ } } -.osdQueryEditor__dataSetNavigatorWrapper { +.osdQueryEditor__datasetSelectorWrapper { :first-child { border-bottom: $euiBorderThin !important; } @@ -155,6 +155,11 @@ align-items: center; padding: $euiSizeXS; + > * { + flex: 0 1 auto; + min-width: 0; + } + .osdQueryEditor__collapseBtn { padding-right: $euiSizeXS; border-right: $euiBorderThin; diff --git a/src/plugins/data/public/ui/query_editor/language_selector.tsx b/src/plugins/data/public/ui/query_editor/language_selector.tsx index 455540d28df2..acfa03c34d3f 100644 --- a/src/plugins/data/public/ui/query_editor/language_selector.tsx +++ b/src/plugins/data/public/ui/query_editor/language_selector.tsx @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import React, { useState, useEffect } from 'react'; import { PopoverAnchorPosition, EuiContextMenuPanel, @@ -10,78 +11,72 @@ import { EuiButtonEmpty, EuiContextMenuItem, } from '@elastic/eui'; -import { i18n } from '@osd/i18n'; -import React, { useState } from 'react'; -import { getUiService } from '../../services'; +import { getQueryService, getUiService } from '../../services'; +import { LanguageConfig } from '../../query'; +import { Query } from '../..'; export interface QueryLanguageSelectorProps { - language: string; + query: Query; onSelectLanguage: (newLanguage: string) => void; anchorPosition?: PopoverAnchorPosition; appName?: string; } -const mapExternalLanguageToOptions = (language: string) => { +const mapExternalLanguageToOptions = (language: LanguageConfig) => { return { - label: language, - value: language, + label: language.title, + value: language.id, }; }; export const QueryLanguageSelector = (props: QueryLanguageSelectorProps) => { const [isPopoverOpen, setPopover] = useState(false); + const [currentLanguage, setCurrentLanguage] = useState(props.query.language); + + const uiService = getUiService(); + const queryString = getQueryService().queryString; + const languageService = queryString.getLanguageService(); + + useEffect(() => { + const subscription = queryString.getUpdates$().subscribe((query: Query) => { + if (query.language !== currentLanguage) { + setCurrentLanguage(query.language); + } + }); + + return () => { + subscription.unsubscribe(); + }; + }, [queryString, currentLanguage, props]); const onButtonClick = () => { setPopover(!isPopoverOpen); }; - const dqlLabel = i18n.translate('data.query.queryEditor.dqlLanguageName', { - defaultMessage: 'DQL', - }); - const luceneLabel = i18n.translate('data.query.queryEditor.luceneLanguageName', { - defaultMessage: 'Lucene', - }); - - const languageOptions = [ - { - label: dqlLabel, - value: 'kuery', - }, - { - label: luceneLabel, - value: 'lucene', - }, - ]; + const languageOptions: Array<{ label: string; value: string }> = []; - const uiService = getUiService(); - - const queryEnhancements = uiService.Settings.getAllQueryEnhancements(); - queryEnhancements.forEach((enhancement) => { + languageService.getLanguages().forEach((language) => { if ( - (enhancement.supportedAppNames && - props.appName && - !enhancement.supportedAppNames.includes(props.appName)) || - uiService.Settings.getUserQueryLanguageBlocklist().includes( - enhancement.language.toLowerCase() - ) + (language && props.appName && !language.supportedAppNames.includes(props.appName)) || + uiService.Settings.getUserQueryLanguageBlocklist().includes(language?.id) ) return; - languageOptions.unshift(mapExternalLanguageToOptions(enhancement.language)); + languageOptions.unshift(mapExternalLanguageToOptions(language!)); }); const selectedLanguage = { label: (languageOptions.find( - (option) => (option.value as string).toLowerCase() === props.language.toLowerCase() + (option) => (option.value as string).toLowerCase() === currentLanguage.toLowerCase() )?.label as string) ?? languageOptions[0].label, }; const handleLanguageChange = (newLanguage: string) => { + setCurrentLanguage(newLanguage); props.onSelectLanguage(newLanguage); - uiService.Settings.setUserQueryLanguage(newLanguage); }; - uiService.Settings.setUserQueryLanguage(props.language); + uiService.Settings.setUserQueryLanguage(currentLanguage); const languageOptionsMenu = languageOptions .sort((a, b) => { @@ -102,6 +97,7 @@ export const QueryLanguageSelector = (props: QueryLanguageSelectorProps) => { ); }); + return ( ; settings: Settings; disableAutoFocus?: boolean; screenTitle?: string; @@ -44,12 +42,10 @@ export interface QueryEditorProps { onChange?: (query: Query, dateRange?: TimeRange) => void; onChangeQueryEditorFocus?: (isFocused: boolean) => void; onSubmit?: (query: Query, dateRange?: TimeRange) => void; - getQueryStringInitialValue?: (language: string) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; isInvalid?: boolean; - queryLanguage?: string; headerClassName?: string; bannerClassName?: string; footerClassName?: string; @@ -70,18 +66,6 @@ interface State { lineCount: number | undefined; } -const KEY_CODES = { - LEFT: 37, - UP: 38, - RIGHT: 39, - DOWN: 40, - ENTER: 13, - ESC: 27, - TAB: 9, - HOME: 36, - END: 35, -}; - // Needed for React.lazy // TODO: MQL export this and let people extended this // eslint-disable-next-line import/no-default-export @@ -98,7 +82,7 @@ export default class QueryEditorUI extends Component { public inputRef: monaco.editor.IStandaloneCodeEditor | null = null; - private queryService = getQueryService(); + private queryString = getQueryService().queryString; private persistedLog: PersistedLog | undefined; private abortController?: AbortController; @@ -108,9 +92,6 @@ export default class QueryEditorUI extends Component { private extensionMap = this.props.settings?.getQueryEditorExtensionMap(); private getQueryString = () => { - if (!this.props.query.query) { - return this.props.getQueryStringInitialValue?.(this.props.query.language) ?? ''; - } return toUser(this.props.query.query); }; @@ -123,7 +104,7 @@ export default class QueryEditorUI extends Component { !( this.headerRef.current && this.bannerRef.current && - this.props.queryLanguage && + this.props.query.language && this.extensionMap && Object.keys(this.extensionMap).length > 0 ) @@ -132,7 +113,7 @@ export default class QueryEditorUI extends Component { } return ( { this.persistedLog.add(query.query); } - this.props.onSubmit({ query: fromUser(query.query), language: query.language }); + this.props.onSubmit({ + query: fromUser(query.query), + language: query.language, + dataset: query.dataset, + }); } }; private onChange = (query: Query, dateRange?: TimeRange) => { if (this.props.onChange) { - this.props.onChange({ query: fromUser(query.query), language: query.language }, dateRange); + this.props.onChange( + { query: fromUser(query.query), language: query.language, dataset: query.dataset }, + dateRange + ); } }; @@ -165,7 +153,11 @@ export default class QueryEditorUI extends Component { index: null, }); - this.onChange({ query: value, language: this.props.query.language }); + this.onChange({ + query: value, + language: this.props.query.language, + dataset: this.props.query.dataset, + }); }; private onInputChange = (value: string) => { @@ -185,21 +177,18 @@ export default class QueryEditorUI extends Component { }; // TODO: MQL consider moving language select language of setting search source here - private onSelectLanguage = (language: string) => { + private onSelectLanguage = (languageId: string) => { // Send telemetry info every time the user opts in or out of kuery // As a result it is important this function only ever gets called in the // UI component's change handler. this.services.http.post('/api/opensearch-dashboards/dql_opt_in_stats', { - body: JSON.stringify({ opt_in: language === 'kuery' }), + body: JSON.stringify({ opt_in: languageId === 'kuery' }), }); - const newQuery = { - query: this.props.getQueryStringInitialValue?.(language) ?? '', - language, - }; + const languageConfig = this.queryString.getLanguageService().getLanguage(languageId); + const newQuery = this.queryString.getInitialQueryByLanguage(languageId); - const enhancement = this.props.settings.getQueryEnhancements(newQuery.language); - const fields = enhancement?.fields; + const fields = languageConfig?.fields; const newSettings: DataSettings = { userQueryLanguage: newQuery.language, userQueryString: newQuery.query, @@ -207,7 +196,7 @@ export default class QueryEditorUI extends Component { }; this.props.settings?.updateSettings(newSettings); - const dateRangeEnhancement = enhancement?.searchBar?.dateRange; + const dateRangeEnhancement = languageConfig?.searchBar?.dateRange; const dateRange = dateRangeEnhancement ? { from: dateRangeEnhancement.initialFrom!, @@ -239,6 +228,19 @@ export default class QueryEditorUI extends Component { } public componentDidUpdate(prevProps: Props) { + const prevQuery = prevProps.query; + + if (!isEqual(this.props.query.dataset, prevQuery.dataset)) { + if (this.inputRef) { + const newQuery = this.queryString.getInitialQuery(); + const newQueryString = newQuery.query; + if (this.inputRef.getValue() !== newQueryString) { + this.inputRef.setValue(newQueryString); + this.onSubmit(newQuery); + } + } + } + const parsedQuery = fromUser(toUser(this.props.query.query)); if (!isEqual(this.props.query.query, parsedQuery)) { this.onChange({ ...this.props.query, query: parsedQuery }); @@ -258,9 +260,10 @@ export default class QueryEditorUI extends Component { }; private fetchIndexPattern = async () => { - const dataSetTitle = this.queryService.dataSetManager.getDataSet()?.title; - if (!dataSetTitle) return undefined; - return getIndexPatterns().getByTitle(dataSetTitle); + const dataset = this.queryString.getQuery().dataset; + if (!dataset) return undefined; + const indexPattern = await getIndexPatterns().get(dataset.id); + return indexPattern; }; provideCompletionItems = async ( @@ -309,7 +312,7 @@ export default class QueryEditorUI extends Component { const languageSelector = ( { footerItems: { start: [ `${this.state.lineCount} ${this.state.lineCount === 1 ? 'line' : 'lines'}`, - this.props.dataSet?.timeFieldName || '', + this.props.query.dataset?.timeFieldName || '', ], }, provideCompletionItems: this.provideCompletionItems, @@ -343,7 +346,6 @@ export default class QueryEditorUI extends Component { onChange: (value: string) => { // Replace new lines with an empty string to prevent multi-line input this.onQueryStringChange(value.replace(/[\r\n]+/gm, '')); - this.setState({ lineCount: undefined }); }, editorDidMount: (editor: monaco.editor.IStandaloneCodeEditor) => { @@ -393,7 +395,7 @@ export default class QueryEditorUI extends Component { onClick={() => this.setState({ isCollapsed: !this.state.isCollapsed })} isCollapsed={!this.state.isCollapsed} /> -
+
{this.state.isCollapsed ? languageEditor.TopBar.Collapsed() diff --git a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx index 4621944ecce3..da3ba70b3837 100644 --- a/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx +++ b/src/plugins/data/public/ui/query_editor/query_editor_top_row.tsx @@ -13,7 +13,6 @@ import { prettyDuration, } from '@elastic/eui'; import classNames from 'classnames'; -import { isEqual } from 'lodash'; import React, { useState } from 'react'; import { createPortal } from 'react-dom'; import { IDataPluginServices, IIndexPattern, Query, TimeHistoryContract, TimeRange } from '../..'; @@ -26,14 +25,12 @@ import { getQueryLog, PersistedLog } from '../../query'; import { Settings } from '../types'; import { NoDataPopover } from './no_data_popover'; import QueryEditorUI from './query_editor'; -import { useDataSetManager } from '../search_bar/lib/use_dataset_manager'; const QueryEditor = withOpenSearchDashboards(QueryEditorUI); // @internal export interface QueryEditorTopRowProps { query?: Query; - dataSetContainerRef?: React.RefCallback; settings?: Settings; onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void; onChange: (payload: { dateRange: TimeRange; query?: Query }) => void; @@ -71,31 +68,17 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { storage, appName, data: { - query: { dataSetManager: dataSetManager }, + query: { queryString }, }, } = opensearchDashboards.services; - const { dataSet } = useDataSetManager({ dataSetManager: dataSetManager! }); const queryLanguage = props.query && props.query.language; - const queryUiEnhancement = - (queryLanguage && - props.settings && - props.settings.getQueryEnhancements(queryLanguage)?.searchBar) || - null; - const parsedQuery = - !queryUiEnhancement || isValidQuery(props.query) - ? props.query! - : { query: getQueryStringInitialValue(queryLanguage!), language: queryLanguage! }; - if (!isEqual(parsedQuery?.query, props.query?.query)) { - onQueryChange(parsedQuery); - onSubmit({ query: parsedQuery, dateRange: getDateRange() }); - } const persistedLog: PersistedLog | undefined = React.useMemo( () => queryLanguage && uiSettings && storage && appName ? getQueryLog(uiSettings!, storage, appName, queryLanguage) : undefined, - [appName, queryLanguage, uiSettings, storage] + [queryLanguage, uiSettings, storage, appName] ); function onClickSubmitButton(event: React.MouseEvent) { @@ -108,17 +91,20 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { function getDateRange() { const defaultTimeSetting = uiSettings!.get(UI_SETTINGS.TIMEPICKER_TIME_DEFAULTS); + const languageConfig = queryString.getLanguageService().getLanguage(queryLanguage!); return { from: props.dateRangeFrom || - queryUiEnhancement?.dateRange?.initialFrom || + languageConfig?.searchBar?.dateRange?.initialFrom || defaultTimeSetting.from, - to: props.dateRangeTo || queryUiEnhancement?.dateRange?.initialTo || defaultTimeSetting.to, + to: + props.dateRangeTo || + languageConfig?.searchBar?.dateRange?.initialTo || + defaultTimeSetting.to, }; } function onQueryChange(query: Query, dateRange?: TimeRange) { - if (queryUiEnhancement && !isValidQuery(query)) return; props.onChange({ query, dateRange: dateRange ?? getDateRange(), @@ -191,40 +177,22 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { return valueAsMoment.toISOString(); } - function isValidQuery(query: Query | undefined) { - if (query && query.query) return true; - } - - function getQueryStringInitialValue(language: string) { - const { settings } = props; - const input = settings?.getQueryEnhancements(language)?.searchBar?.queryStringInput - ?.initialValue; - - if (!input) return ''; - - return input.replace('', dataSet?.title ?? dataSet?.title ?? ''); - } - function renderQueryEditor() { if (!shouldRenderQueryEditor()) return; return ( @@ -248,9 +216,10 @@ export default function QueryEditorTopRow(props: QueryEditorTopRowProps) { } function shouldRenderDatePicker(): boolean { + const languageConfig = queryString.getLanguageService().getLanguage(queryLanguage!); return Boolean( - (props.showDatePicker && (queryUiEnhancement?.showDatePicker ?? true)) ?? - (props.showAutoRefreshOnly && (queryUiEnhancement?.showAutoRefreshOnly ?? true)) + (props.showDatePicker && (languageConfig?.searchBar?.showDatePicker ?? true)) ?? + (props.showAutoRefreshOnly && (languageConfig?.searchBar?.showAutoRefreshOnly ?? true)) ); } diff --git a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx index 675f6cdc5791..a310d6e67c21 100644 --- a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx @@ -154,8 +154,7 @@ export function createSearchBar({ filterManager: data.query.filterManager, }); const { query } = useQueryStringManager({ - query: props.query, - queryStringManager: data.query.queryString, + queryString: data.query.queryString, }); const { timeRange, refreshInterval } = useTimefilter({ diff --git a/src/plugins/data/public/ui/search_bar/index.tsx b/src/plugins/data/public/ui/search_bar/index.tsx index 41cc1c2a992b..6fffe20199e2 100644 --- a/src/plugins/data/public/ui/search_bar/index.tsx +++ b/src/plugins/data/public/ui/search_bar/index.tsx @@ -45,3 +45,4 @@ const WrappedSearchBar = (props: SearchBarProps) => ( export const SearchBar = injectI18n(withOpenSearchDashboards(WrappedSearchBar)); export { StatefulSearchBarProps } from './create_search_bar'; export type { SearchBarProps, SearchBarOwnProps } from './search_bar'; +export { useQueryStringManager } from './lib/use_query_string_manager'; diff --git a/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts b/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts deleted file mode 100644 index 9fb96cdead3c..000000000000 --- a/src/plugins/data/public/ui/search_bar/lib/use_dataset_manager.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useState, useEffect } from 'react'; -import { Subscription } from 'rxjs'; -import { Dataset } from '../../../../../data/common'; -import { DataSetContract } from '../../../query'; - -interface UseDataSetManagerProps { - dataSet?: Dataset; - dataSetManager: DataSetContract; -} - -export const useDataSetManager = (props: UseDataSetManagerProps) => { - const [dataSet, setDataSet] = useState( - props.dataSet || props.dataSetManager.getDataSet() - ); - - useEffect(() => { - const subscriptions = new Subscription(); - - subscriptions.add( - props.dataSetManager.getUpdates$().subscribe({ - next: () => { - const newDataSet = props.dataSetManager.getDataSet(); - setDataSet(newDataSet); - }, - }) - ); - - return () => { - subscriptions.unsubscribe(); - }; - }, [dataSet, props.dataSet, props.dataSetManager]); - - return { dataSet }; -}; diff --git a/src/plugins/data/public/ui/search_bar/lib/use_query_string_manager.ts b/src/plugins/data/public/ui/search_bar/lib/use_query_string_manager.ts index b88bd282d442..8f9d49f80fef 100644 --- a/src/plugins/data/public/ui/search_bar/lib/use_query_string_manager.ts +++ b/src/plugins/data/public/ui/search_bar/lib/use_query_string_manager.ts @@ -28,35 +28,50 @@ * under the License. */ -import { useState, useEffect } from 'react'; +import { useState, useEffect, useCallback } from 'react'; import { Subscription } from 'rxjs'; import { Query } from '../../..'; import { QueryStringContract } from '../../../query/query_string'; interface UseQueryStringProps { query?: Query; - queryStringManager: QueryStringContract; + queryString: QueryStringContract; } export const useQueryStringManager = (props: UseQueryStringProps) => { // Filters should be either what's passed in the initial state or the current state of the filter manager - const [query, setQuery] = useState(props.query || props.queryStringManager.getQuery()); + const [query, setQuery] = useState(() => props.query || props.queryString.getQuery()); + useEffect(() => { const subscriptions = new Subscription(); - subscriptions.add( - props.queryStringManager.getUpdates$().subscribe({ + props.queryString.getUpdates$().subscribe({ next: () => { - const newQuery = props.queryStringManager.getQuery(); - setQuery(newQuery); + setQuery((prevQuery) => { + const newQuery = props.queryString.getQuery(); + // Only update if the query has actually changed + return JSON.stringify(prevQuery) !== JSON.stringify(newQuery) ? newQuery : prevQuery; + }); }, }) ); - return () => { subscriptions.unsubscribe(); }; - }, [props.queryStringManager]); + }, [props.queryString]); + + // Use callback to memoize the function + const updateQuery = useCallback( + (newQueryPartial: Partial) => { + const updatedQuery = { ...query, ...newQueryPartial }; + props.queryString.setQuery(updatedQuery); + setQuery(updatedQuery); + }, + [query, props.queryString] + ); - return { query }; + return { + query, + updateQuery, + }; }; diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index fd8ff3dc21c1..666a54946f57 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -119,9 +119,8 @@ class SearchBarUI extends Component { }; private services = this.props.opensearchDashboards.services; - private dataSetService = this.services.data.query.dataSetManager; - private queryStringService = this.services.data.query.queryString; - private savedQueryService = this.services.data.query.savedQueries; + private queryService = this.services.data.query; + private savedQueryService = this.queryService.savedQueries; public filterBarRef: Element | null = null; public filterBarWrapperRef: Element | null = null; private useNewHeader = Boolean(this.services.uiSettings.get(UI_SETTINGS.NEW_HOME_PAGE)); @@ -136,6 +135,7 @@ class SearchBarUI extends Component { nextQuery = { query: nextProps.query.query, language: nextProps.query.language, + dataset: nextProps.query.dataset, }; } else if ( nextProps.query && @@ -145,6 +145,17 @@ class SearchBarUI extends Component { nextQuery = { query: '', language: nextProps.query.language, + dataset: nextProps.query.dataset, + }; + } else if ( + nextProps.query && + prevState.query && + nextProps.query.dataset !== prevState.query.dataset + ) { + nextQuery = { + query: nextProps.query.query, + language: nextProps.query.language, + dataset: nextProps.query.dataset, }; } @@ -238,8 +249,8 @@ class SearchBarUI extends Component { (!this.useNewHeader || this.props.filters.length > 0) && this.props.indexPatterns && compact(this.props.indexPatterns).length > 0 && - (this.props.settings?.getQueryEnhancements(this.state.query?.language!)?.searchBar - ?.showFilterBar ?? + (this.queryService.queryString.getLanguageService().getLanguage(this.state.query?.language!) + ?.searchBar?.showFilterBar ?? true) ); } @@ -374,10 +385,10 @@ class SearchBarUI extends Component { } } ); - const dataSet = this.dataSetService.getDataSet(); - if (dataSet && queryAndDateRange.query) { - this.queryStringService.addToQueryHistory( - dataSet, + const dataset = this.queryService.queryString.getQuery().dataset; + if (dataset && queryAndDateRange.query) { + this.queryService.queryString.addToQueryHistory( + dataset, queryAndDateRange.query, queryAndDateRange.dateRange ); @@ -512,7 +523,6 @@ class SearchBarUI extends Component { queryEditor = ( , private readonly queryEditorExtensionMap: Record ) { this.isEnabled = true; @@ -58,14 +58,6 @@ export class Settings { return true; } - getAllQueryEnhancements() { - return this.queryEnhancements; - } - - getQueryEnhancements(language: string) { - return this.queryEnhancements.get(language); - } - getQueryEditorExtensionMap() { return this.queryEditorExtensionMap; } @@ -86,18 +78,12 @@ export class Settings { return this.storage.get('userQueryLanguage') || 'kuery'; } - setUserQueryLanguage(language: string) { - if (language !== this.getUserQueryLanguage()) { + setUserQueryLanguage(languageId: string) { + if (languageId !== this.getUserQueryLanguage()) { this.search.df.clear(); } - this.storage.set('userQueryLanguage', language); - const queryEnhancement = this.queryEnhancements.get(language); - this.search.__enhance({ - searchInterceptor: queryEnhancement - ? queryEnhancement.search - : this.search.getDefaultSearchInterceptor(), - }); - this.setUiOverridesByUserQueryLanguage(language); + this.storage.set('userQueryLanguage', languageId); + this.setUiOverridesByUserQueryLanguage(languageId); return true; } @@ -126,10 +112,10 @@ export class Settings { return true; } - setUiOverridesByUserQueryLanguage(language: string) { - const queryEnhancement = this.queryEnhancements.get(language); - if (queryEnhancement) { - const { fields = {}, showDocLinks } = queryEnhancement; + setUiOverridesByUserQueryLanguage(languageId: string) { + const language = this.query.queryString.getLanguageService().getLanguage(languageId); + if (language) { + const { fields = {}, showDocLinks } = language; this.setUiOverrides({ fields, showDocLinks }); } else { this.setUiOverrides({ fields: undefined, showDocLinks: undefined }); @@ -170,17 +156,11 @@ export class Settings { interface Deps { config: ConfigSchema['enhancements']; search: ISearchStart; + query: IQueryStart; storage: DataStorage; - queryEnhancements: Map; queryEditorExtensionMap: Record; } -export function createSettings({ - config, - search, - storage, - queryEnhancements, - queryEditorExtensionMap, -}: Deps) { - return new Settings(config, search, storage, queryEnhancements, queryEditorExtensionMap); +export function createSettings({ config, search, query, storage, queryEditorExtensionMap }: Deps) { + return new Settings(config, search, query, storage, queryEditorExtensionMap); } diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts index 268e274c199d..1d13a13c6040 100644 --- a/src/plugins/data/public/ui/types.ts +++ b/src/plugins/data/public/ui/types.ts @@ -4,8 +4,6 @@ */ import { Observable } from 'rxjs'; -import { SearchInterceptor } from '../search'; -import { DataSetNavigatorProps } from './dataset_navigator'; import { IndexPatternSelectProps } from './index_pattern_select'; import { StatefulSearchBarProps } from './search_bar'; import { QueryEditorExtensionConfig } from './query_editor/query_editor_extensions'; @@ -14,41 +12,7 @@ import { SuggestionsComponentProps } from './typeahead/suggestions_component'; export * from './settings'; -export interface QueryEnhancement { - // TODO: MQL do want to default have supported all data_sources? - // or should data connect have a record of query enhancements that are supported - language: string; - search: SearchInterceptor; - // Leave blank to support all data sources - // supportedDataSourceTypes?: Record; - searchBar?: { - showDataSetsSelector?: boolean; - showDataSourcesSelector?: boolean; - showQueryInput?: boolean; - showFilterBar?: boolean; - showDatePicker?: boolean; - showAutoRefreshOnly?: boolean; - queryStringInput?: { - // will replace '' with the data source name - initialValue?: string; - }; - dateRange?: { - initialFrom?: string; - initialTo?: string; - }; - }; - fields?: { - filterable?: boolean; - visualizable?: boolean; - }; - showDocLinks?: boolean; - // List of supported app names that this enhancement should be enabled for, - // if not provided it will be enabled for all apps - supportedAppNames?: string[]; -} - export interface UiEnhancements { - query?: QueryEnhancement; queryEditorExtension?: QueryEditorExtensionConfig; } @@ -65,10 +29,6 @@ export interface IUiSetup { */ export interface IUiStart { IndexPatternSelect: React.ComponentType; - /** - * @experimental - Subject to change - */ - DataSetNavigator: React.ComponentType; SearchBar: React.ComponentType; SuggestionsComponent: React.ComponentType; /** diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts index 1802e3b79320..faea0b738571 100644 --- a/src/plugins/data/public/ui/ui_service.ts +++ b/src/plugins/data/public/ui/ui_service.ts @@ -7,13 +7,12 @@ import { BehaviorSubject } from 'rxjs'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/public'; import { ConfigSchema } from '../../config'; import { DataPublicPluginStart } from '../types'; -import { createDataSetNavigator } from './dataset_navigator'; import { createIndexPatternSelect } from './index_pattern_select'; import { QueryEditorExtensionConfig } from './query_editor'; import { createSearchBar } from './search_bar/create_search_bar'; import { createSettings } from './settings'; import { SuggestionsComponent } from './typeahead'; -import { IUiSetup, IUiStart, QueryEnhancement, UiEnhancements } from './types'; +import { IUiSetup, IUiStart, UiEnhancements } from './types'; import { DataStorage } from '../../common'; /** @internal */ @@ -28,7 +27,6 @@ export interface UiServiceStartDependencies { export class UiService implements Plugin { enhancementsConfig: ConfigSchema['enhancements']; - private queryEnhancements: Map = new Map(); private queryEditorExtensionMap: Record = {}; private dataSetContainer$ = new BehaviorSubject(null); @@ -42,9 +40,6 @@ export class UiService implements Plugin { return { __enhance: (enhancements?: UiEnhancements) => { if (!enhancements) return; - if (enhancements.query && enhancements.query.language) { - this.queryEnhancements.set(enhancements.query.language, enhancements.query); - } if (enhancements.queryEditorExtension) { this.queryEditorExtensionMap[enhancements.queryEditorExtension.id] = enhancements.queryEditorExtension; @@ -57,8 +52,8 @@ export class UiService implements Plugin { const Settings = createSettings({ config: this.enhancementsConfig, search: dataServices.search, + query: dataServices.query, storage, - queryEnhancements: this.queryEnhancements, queryEditorExtensionMap: this.queryEditorExtensionMap, }); @@ -76,11 +71,6 @@ export class UiService implements Plugin { return { IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), - DataSetNavigator: createDataSetNavigator( - core.savedObjects.client, - core.http, - dataServices.query.dataSetManager - ), SearchBar, SuggestionsComponent, Settings, diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index deb277087fcf..98ef84e20008 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -78,7 +78,6 @@ import { IDataFrame, IDataFrameResponse, createDataFrameCache, - dataFrameToSpec, } from '../../common'; type StrategyMap = Record>; @@ -215,29 +214,7 @@ export class SearchService implements Plugin { const dfService: DataFrameService = { get: () => this.dfCache.get(), set: async (dataFrame: IDataFrame) => { - if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) { - scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false); - } - if ( - dataFrame.meta && - dataFrame.meta.queryConfig && - 'dataSource' in dataFrame.meta.queryConfig - ) { - const dataSource = await scopedIndexPatterns.findDataSourceByTitle( - dataFrame.meta.queryConfig.dataSource - ); - dataFrame.meta.queryConfig.dataSourceId = dataSource?.id; - } this.dfCache.set(dataFrame); - const dataSetName = `${dataFrame.meta?.queryConfig?.dataSourceId ?? ''}.${ - dataFrame.name - }`; - const existingIndexPattern = await scopedIndexPatterns.get(dataSetName, true); - const dataSet = await scopedIndexPatterns.create( - dataFrameToSpec(dataFrame, existingIndexPattern?.id ?? dataSetName), - !existingIndexPattern?.id - ); - scopedIndexPatterns.saveToCache(dataSetName, dataSet); }, clear: () => { if (this.dfCache.get() === undefined) return; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 616be16e9f56..e6547af63d0d 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -4,7 +4,7 @@ */ import React, { FC, useCallback, useEffect, useRef, useState } from 'react'; -import { EuiPageSideBar, EuiPortal, EuiSplitPanel } from '@elastic/eui'; +import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DataSource, DataSourceGroup, DataSourceSelectable } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; @@ -30,8 +30,6 @@ export const Sidebar: FC = ({ children }) => { }, } = useOpenSearchDashboards(); - const { DataSetNavigator } = ui; - useEffect(() => { const subscriptions = ui.Settings.getEnabledQueryEnhancementsUpdated$().subscribe( (enabledQueryEnhancements) => { @@ -144,15 +142,6 @@ export const Sidebar: FC = ({ children }) => { borderRadius="none" color="transparent" > - {isEnhancementsEnabled && ( - { - containerRef.current = node; - }} - > - - - )} {!isEnhancementsEnabled && ( ; } const initialState: MetadataState = {}; @@ -20,12 +19,16 @@ export const getPreloadedState = async ({ embeddable, scopedHistory, data, + uiSettings, }: DataExplorerServices): Promise => { const { originatingApp } = embeddable .getStateTransfer(scopedHistory) .getIncomingEditorState({ keysToRemoveAfterFetch: ['id', 'input'] }) || {}; - const defaultIndexPattern = await data.indexPatterns.getDefault(); + const isQueryEnhancementEnabled = uiSettings.get(QUERY_ENHANCEMENT_ENABLED_SETTING); + const defaultIndexPattern = isQueryEnhancementEnabled + ? undefined + : await data.indexPatterns.getDefault(); const preloadedState: MetadataState = { ...initialState, originatingApp, @@ -39,12 +42,9 @@ export const slice = createSlice({ name: 'metadata', initialState, reducers: { - setIndexPattern: (state, action: PayloadAction) => { + setIndexPattern: (state, action: PayloadAction) => { state.indexPattern = action.payload; }, - setDataSet: (state, action: PayloadAction>) => { - state.dataSet = action.payload; - }, setOriginatingApp: (state, action: PayloadAction) => { state.originatingApp = action.payload; }, @@ -58,4 +58,4 @@ export const slice = createSlice({ }); export const { reducer } = slice; -export const { setIndexPattern, setDataSet, setOriginatingApp, setView, setState } = slice.actions; +export const { setIndexPattern, setOriginatingApp, setView, setState } = slice.actions; diff --git a/src/plugins/data_explorer/public/utils/state_management/redux_persistence.ts b/src/plugins/data_explorer/public/utils/state_management/redux_persistence.ts index 81517f3e9f4f..7eb715433f01 100644 --- a/src/plugins/data_explorer/public/utils/state_management/redux_persistence.ts +++ b/src/plugins/data_explorer/public/utils/state_management/redux_persistence.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { Dataset, DEFAULT_DATA } from '../../../../data/common'; +import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../components/constants'; import { DataExplorerServices } from '../../types'; import { getPreloadedState } from './preload'; import { RootState } from './store'; @@ -10,12 +12,49 @@ import { RootState } from './store'; export const loadReduxState = async (services: DataExplorerServices) => { try { const serializedState = services.osdUrlStateStorage.get('_a'); - if (serializedState !== null) return serializedState; + if (serializedState !== null) { + const isQueryEnhancementEnabled = services.uiSettings.get(QUERY_ENHANCEMENT_ENABLED_SETTING); + + // Migrate index pattern to query state + if (isQueryEnhancementEnabled && serializedState.metadata.indexPattern) { + const indexPattern = await services.data.indexPatterns.get( + serializedState.metadata.indexPattern + ); + + const dataset: Dataset = { + id: serializedState.metadata.indexPattern, + title: indexPattern.title, + type: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN, + }; + + if (indexPattern.dataSourceRef) { + const dataSource = await services.data.indexPatterns.getDataSource( + indexPattern.dataSourceRef.id + ); + + if (dataSource) { + dataset.dataSource = { + id: dataSource.id, + title: dataSource.attributes.title, + type: dataSource.attributes.dataSourceEngineType || '', + }; + } + } + services.data.query.queryString.setQuery({ + dataset, + }); + + delete serializedState.metadata.indexPattern; + } + + return serializedState; + } } catch (err) { // eslint-disable-next-line no-console console.error(err); } + // If state is not found, load the default state return await getPreloadedState(services); }; diff --git a/src/plugins/data_explorer/public/utils/state_management/store.ts b/src/plugins/data_explorer/public/utils/state_management/store.ts index 9d320de4b54b..daf0b3d7e369 100644 --- a/src/plugins/data_explorer/public/utils/state_management/store.ts +++ b/src/plugins/data_explorer/public/utils/state_management/store.ts @@ -116,4 +116,4 @@ export type RenderState = Omit; // Remaining state after export type Store = ReturnType; export type AppDispatch = Store['dispatch']; -export { MetadataState, setIndexPattern, setDataSet, setOriginatingApp } from './metadata_slice'; +export { MetadataState, setIndexPattern, setOriginatingApp } from './metadata_slice'; diff --git a/src/plugins/discover/public/application/utils/state_management/index.ts b/src/plugins/discover/public/application/utils/state_management/index.ts index e6df7e4774b8..989b2662f0d4 100644 --- a/src/plugins/discover/public/application/utils/state_management/index.ts +++ b/src/plugins/discover/public/application/utils/state_management/index.ts @@ -7,7 +7,6 @@ import { TypedUseSelectorHook } from 'react-redux'; import { RootState, setIndexPattern as updateIndexPattern, - setDataSet as updateDataSet, useTypedDispatch, useTypedSelector, } from '../../../../../data_explorer/public'; @@ -21,4 +20,4 @@ export interface DiscoverRootState extends RootState { export const useSelector: TypedUseSelectorHook = useTypedSelector; export const useDispatch = useTypedDispatch; -export { updateIndexPattern, updateDataSet }; +export { updateIndexPattern }; diff --git a/src/plugins/discover/public/application/view_components/canvas/index.tsx b/src/plugins/discover/public/application/view_components/canvas/index.tsx index 500c8ebdc80c..a44ac89c5d62 100644 --- a/src/plugins/discover/public/application/view_components/canvas/index.tsx +++ b/src/plugins/discover/public/application/view_components/canvas/index.tsx @@ -3,15 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { i18n } from '@osd/i18n'; import React, { useEffect, useState, useRef, useCallback } from 'react'; -import { - EuiButtonIcon, - EuiContextMenu, - EuiPanel, - EuiPopover, - EuiCompressedSwitch, -} from '@elastic/eui'; +import { EuiPanel } from '@elastic/eui'; import { TopNav } from './top_nav'; import { ViewProps } from '../../../../../data_explorer/public'; import { DiscoverTable } from './discover_table'; @@ -33,7 +26,6 @@ import { import { OpenSearchSearchHit } from '../../../application/doc_views/doc_views_types'; import { buildColumns } from '../../utils/columns'; import './discover_canvas.scss'; -import { getNewDiscoverSetting, setNewDiscoverSetting } from '../../components/utils/local_storage'; import { HeaderVariant } from '../../../../../../core/public'; // eslint-disable-next-line import/no-default-export diff --git a/src/plugins/discover/public/application/view_components/canvas/top_nav.tsx b/src/plugins/discover/public/application/view_components/canvas/top_nav.tsx index fa3ee3524994..34cf98e12496 100644 --- a/src/plugins/discover/public/application/view_components/canvas/top_nav.tsx +++ b/src/plugins/discover/public/application/view_components/canvas/top_nav.tsx @@ -20,7 +20,6 @@ import { useDiscoverContext } from '../context'; import { useDispatch, setSavedQuery, useSelector } from '../../utils/state_management'; import './discover_canvas.scss'; -import { useDataSetManager } from '../utils/use_dataset_manager'; import { TopNavMenuItemRenderType } from '../../../../../navigation/public'; export interface TopNavProps { @@ -72,22 +71,22 @@ export const TopNav = ({ opts, showSaveQuery, isEnhancementsEnabled }: TopNavPro useEffect(() => { let isMounted = true; - const initializeDataSet = async () => { + const initializeDataset = async () => { await data.indexPatterns.ensureDefaultIndexPattern(); const defaultIndexPattern = await data.indexPatterns.getDefault(); - const { dataSetManager } = data.query; - dataSetManager.initWithIndexPattern(defaultIndexPattern); - const defaultDataSet = dataSetManager.getDefaultDataSet(); + // TODO: ROCKY do we need this? + // const queryString = data.query.queryString; + // const defaultDataset = queryString.getDatasetService().getDefault(); if (!isMounted) return; setIndexPatterns(defaultIndexPattern ? [defaultIndexPattern] : undefined); - if (defaultDataSet) { - dataSetManager.setDataSet(defaultDataSet); - } + // if (defaultDataset) { + // datasetManager.setDataset(defaultDataset); + // } }; - initializeDataSet(); + initializeDataset(); return () => { isMounted = false; diff --git a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts index 88982650bbcb..4d837a632520 100644 --- a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts +++ b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts @@ -30,27 +30,16 @@ export const updateSearchSource = async ({ histogramConfigs, }: Props) => { const { uiSettings, data } = services; - const queryDataSet = data.query.dataSetManager.getDataSet(); + const queryDataset = data.query.queryString.getQuery().dataset; - let dataSet = - indexPattern.id === queryDataSet?.id - ? await data.indexPatterns.get(queryDataSet?.id!, true) + const dataset = + indexPattern.id === queryDataset?.id + ? await data.indexPatterns.get(queryDataset?.id!, true) : indexPattern; - const dataFrame = searchSource?.getDataFrame(); - if ( - searchSource && - dataFrame && - dataFrame.name && - dataFrame.name !== '' && - dataSet.title !== dataFrame.name - ) { - dataSet = data.indexPatterns.getByTitle(dataFrame.name, true) ?? dataSet; - searchSource.setField('index', dataSet); - } const sortForSearchSource = getSortForSearchSource( sort, - dataSet, + dataset, uiSettings.get(SORT_DEFAULT_ORDER_SETTING) ); const size = uiSettings.get(SAMPLE_SIZE_SETTING); @@ -61,18 +50,18 @@ export const updateSearchSource = async ({ // searchSource which applies time range const timeRangeSearchSource = await data.search.searchSource.create(); const { isDefault } = indexPatternUtils; - if (isDefault(dataSet)) { + if (isDefault(dataset)) { const timefilter = data.query.timefilter.timefilter; timeRangeSearchSource.setField('filter', () => { - return timefilter.createFilter(dataSet); + return timefilter.createFilter(dataset); }); } searchSourceInstance.setParent(timeRangeSearchSource); searchSourceInstance.setFields({ - index: dataSet, + index: dataset, sort: sortForSearchSource, size, query: data.query.queryString.getQuery() || null, diff --git a/src/plugins/discover/public/application/view_components/utils/use_dataset_manager.ts b/src/plugins/discover/public/application/view_components/utils/use_dataset_manager.ts deleted file mode 100644 index 3716b9360703..000000000000 --- a/src/plugins/discover/public/application/view_components/utils/use_dataset_manager.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useState, useEffect } from 'react'; -import { Subscription } from 'rxjs'; -import { DataSetManager } from '../../../../../data/public'; -import { Dataset } from '../../../../../data/common'; - -interface UseDataSetManagerProps { - dataSet?: Dataset; - dataSetManager: DataSetManager; -} - -export const useDataSetManager = (props: UseDataSetManagerProps) => { - const [dataSet, setDataSet] = useState( - props.dataSet || props.dataSetManager.getDataSet() - ); - - useEffect(() => { - const subscriptions = new Subscription(); - - subscriptions.add( - props.dataSetManager.getUpdates$().subscribe({ - next: () => { - const newDataSet = props.dataSetManager.getDataSet(); - setDataSet(newDataSet); - }, - }) - ); - - return () => { - subscriptions.unsubscribe(); - }; - }, [dataSet, props.dataSet, props.dataSetManager]); - - return { dataSet }; -}; diff --git a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts index 408e58b92aec..2f8bd6fbebf6 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_index_pattern.ts @@ -5,12 +5,10 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { i18n } from '@osd/i18n'; -import { DEFAULT_QUERY, Dataset, SavedObjectReference } from '../../../../../data/common'; -import { IndexPattern } from '../../../../../data/public'; +import { IndexPattern, useQueryStringManager } from '../../../../../data/public'; import { useSelector, updateIndexPattern } from '../../utils/state_management'; import { DiscoverViewServices } from '../../../build_services'; import { getIndexPatternId } from '../../helpers/get_index_pattern_id'; -import { useDataSetManager } from './use_dataset_manager'; import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../../../common'; /** @@ -29,7 +27,9 @@ import { QUERY_ENHANCEMENT_ENABLED_SETTING } from '../../../../common'; */ export const useIndexPattern = (services: DiscoverViewServices) => { const { data, toastNotifications, uiSettings, store } = services; - const { dataSet } = useDataSetManager({ dataSetManager: data.query.dataSetManager }); + const { query } = useQueryStringManager({ + queryString: data.query.queryString, + }); const indexPatternIdFromState = useSelector((state) => state.metadata.indexPattern); const [indexPattern, setIndexPattern] = useState(undefined); const isQueryEnhancementEnabled = useMemo( @@ -41,40 +41,12 @@ export const useIndexPattern = (services: DiscoverViewServices) => { data.indexPatterns, ]); - const createTempIndexPattern = useCallback( - async (dataSetFromState: Dataset) => { - try { - const tempIndexPattern = await data.indexPatterns.create( - { - id: `${dataSetFromState.dataSource?.id || ''}::${dataSetFromState.title}`, - title: dataSetFromState.title, - dataSourceRef: dataSetFromState.dataSource as SavedObjectReference, - type: dataSetFromState.type, - timeFieldName: dataSetFromState.timeFieldName, - }, - true - ); - data.indexPatterns.saveToCache(tempIndexPattern.id!, tempIndexPattern); - return tempIndexPattern; - } catch (error) { - return null; - } - }, - [data.indexPatterns] - ); - useEffect(() => { let isMounted = true; const handleIndexPattern = async () => { - if (isQueryEnhancementEnabled && dataSet) { - let pattern; - - if (dataSet.type === DEFAULT_QUERY.DATASET_TYPE) { - pattern = await fetchIndexPatternDetails(dataSet.id); - } else { - pattern = await createTempIndexPattern(dataSet); - } + if (isQueryEnhancementEnabled && query?.dataset) { + const pattern = await data.indexPatterns.get(query.dataset.id); if (isMounted && pattern) { setIndexPattern(pattern); @@ -82,7 +54,11 @@ export const useIndexPattern = (services: DiscoverViewServices) => { } else if (!isQueryEnhancementEnabled) { if (!indexPatternIdFromState) { const indexPatternList = await data.indexPatterns.getCache(); - const newId = getIndexPatternId('', indexPatternList, uiSettings.get('defaultIndex')); + const newId = getIndexPatternId( + '', + indexPatternList || [], + uiSettings.get('defaultIndex') + ); if (isMounted) { store!.dispatch(updateIndexPattern(newId)); handleIndexPattern(); @@ -113,14 +89,13 @@ export const useIndexPattern = (services: DiscoverViewServices) => { }; }, [ isQueryEnhancementEnabled, - dataSet, indexPatternIdFromState, fetchIndexPatternDetails, - createTempIndexPattern, data.indexPatterns, store, toastNotifications, uiSettings, + query?.dataset, ]); return indexPattern; diff --git a/src/plugins/discover/public/application/view_components/utils/use_search.ts b/src/plugins/discover/public/application/view_components/utils/use_search.ts index 7e8095c5af30..b6c13d4982f2 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_search.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_search.ts @@ -118,8 +118,8 @@ export const useSearch = (services: DiscoverViewServices) => { const refetch$ = useMemo(() => new Subject(), []); const fetch = useCallback(async () => { - let dataSet = indexPattern; - if (!dataSet) { + let dataset = indexPattern; + if (!dataset) { data$.next({ status: shouldSearchOnPageLoad() ? ResultStatus.LOADING : ResultStatus.UNINITIALIZED, }); @@ -136,18 +136,18 @@ export const useSearch = (services: DiscoverViewServices) => { // Abort any in-progress requests before fetching again if (fetchStateRef.current.abortController) fetchStateRef.current.abortController.abort(); fetchStateRef.current.abortController = new AbortController(); - const histogramConfigs = dataSet.timeFieldName - ? createHistogramConfigs(dataSet, interval || 'auto', data) + const histogramConfigs = dataset.timeFieldName + ? createHistogramConfigs(dataset, interval || 'auto', data) : undefined; const searchSource = await updateSearchSource({ - indexPattern: dataSet, + indexPattern: dataset, services, sort, searchSource: savedSearch?.searchSource, histogramConfigs, }); - dataSet = searchSource.getField('index'); + dataset = searchSource.getField('index'); try { // Only show loading indicator if we are fetching when the rows are empty @@ -183,7 +183,7 @@ export const useSearch = (services: DiscoverViewServices) => { let bucketInterval = {}; let chartData; for (const row of rows) { - const fields = Object.keys(dataSet!.flattenHit(row)); + const fields = Object.keys(dataset!.flattenHit(row)); for (const fieldName of fields) { fetchStateRef.current.fieldCounts[fieldName] = (fetchStateRef.current.fieldCounts[fieldName] || 0) + 1; @@ -251,8 +251,7 @@ export const useSearch = (services: DiscoverViewServices) => { timefilter.getFetch$(), timefilter.getTimeUpdate$(), timefilter.getAutoRefreshFetch$(), - data.query.queryString.getUpdates$(), - data.query.dataSetManager.getUpdates$() + data.query.queryString.getUpdates$() ).pipe(debounceTime(100)); const subscription = fetch$.subscribe(() => { @@ -282,7 +281,6 @@ export const useSearch = (services: DiscoverViewServices) => { fetch, core.fatalErrors, shouldSearchOnPageLoad, - data.query.dataSetManager, ]); // Get savedSearch if it exists diff --git a/src/plugins/opensearch_dashboards_utils/public/state_management/url/errors.ts b/src/plugins/opensearch_dashboards_utils/public/state_management/url/errors.ts index 87abbeff60b1..28fcbf5a6549 100644 --- a/src/plugins/opensearch_dashboards_utils/public/state_management/url/errors.ts +++ b/src/plugins/opensearch_dashboards_utils/public/state_management/url/errors.ts @@ -61,11 +61,13 @@ export const withNotifyOnErrors = (toasts: NotificationsStart['toasts']) => { return { onGetError: (error: Error) => { toasts.addError(error, { + id: 'opensearch_dashboards_utils.state_management.url.restoreUrlError', title: restoreUrlErrorTitle, }); }, onSetError: (error: Error) => { toasts.addError(error, { + id: 'opensearch_dashboards_utils.state_management.url.saveStateInUrlError', title: saveStateInUrlErrorTitle, }); }, diff --git a/src/plugins/query_enhancements/common/utils.ts b/src/plugins/query_enhancements/common/utils.ts index df300e92a413..208256bd4dd4 100644 --- a/src/plugins/query_enhancements/common/utils.ts +++ b/src/plugins/query_enhancements/common/utils.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IDataFrame } from 'src/plugins/data/common'; +import { IDataFrame, Query } from 'src/plugins/data/common'; import { Observable, Subscription, from, throwError, timer } from 'rxjs'; import { catchError, concatMap, last, takeWhile, tap } from 'rxjs/operators'; import { FetchDataFrameContext, FetchFunction } from './types'; @@ -133,13 +133,9 @@ export const handleDataFrameError = (response: any) => { } }; -export const fetchDataFrame = ( - context: FetchDataFrameContext, - queryString: string, - df: IDataFrame -) => { +export const fetchDataFrame = (context: FetchDataFrameContext, query: Query, df: IDataFrame) => { const { http, path, signal } = context; - const body = JSON.stringify({ query: { qs: queryString, format: 'jdbc' }, df }); + const body = JSON.stringify({ query: { ...query, format: 'jdbc' }, df }); return from( http.fetch({ method: 'POST', diff --git a/src/plugins/query_enhancements/public/assets/s3_mark.svg b/src/plugins/query_enhancements/public/assets/s3_mark.svg new file mode 100644 index 000000000000..5b0c3a35aaad --- /dev/null +++ b/src/plugins/query_enhancements/public/assets/s3_mark.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx b/src/plugins/query_enhancements/public/datasets/index.ts similarity index 59% rename from src/plugins/data/public/ui/dataset_navigator/lib/index.tsx rename to src/plugins/query_enhancements/public/datasets/index.ts index 3e55b949c98d..ddd7bf74995c 100644 --- a/src/plugins/data/public/ui/dataset_navigator/lib/index.tsx +++ b/src/plugins/query_enhancements/public/datasets/index.ts @@ -3,5 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './catalog_cache'; -export * from './utils'; +export * from './s3_handler'; diff --git a/src/plugins/query_enhancements/public/datasets/s3_handler.ts b/src/plugins/query_enhancements/public/datasets/s3_handler.ts new file mode 100644 index 000000000000..ce18662146bc --- /dev/null +++ b/src/plugins/query_enhancements/public/datasets/s3_handler.ts @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { DataStructure, Dataset, DatasetField } from 'src/plugins/data/common'; +import { DatasetTypeConfig } from 'src/plugins/data/public'; + +const S3_ICON = 'visTable'; +const S3_ID = 'S3'; + +export const s3TypeConfig: DatasetTypeConfig = { + id: S3_ID, + title: S3_ID, + meta: { + icon: { type: S3_ICON }, + tooltip: 'S3 Data Source', + }, + + toDataset: (path: DataStructure[]): Dataset => { + const s3 = path[path.length - 1]; + const dataSource = path.find((ds) => ds.type === S3_ID); + + return { + id: s3.id, + title: s3.title, + type: S3_ID, + dataSource: dataSource + ? { + id: dataSource.id, + title: dataSource.title, + type: dataSource.type, + } + : undefined, + }; + }, + + fetch: async ( + savedObjects: SavedObjectsClientContract, + path: DataStructure[] + ): Promise => { + const dataStructure = path[path.length - 1]; + switch (dataStructure.type) { + case S3_ID: + return { + ...dataStructure, + columnHeader: 'Connections', + hasNext: true, + children: [ + { + id: `${dataStructure.id}::mys3`, + title: 'mys3', + type: 'CONNECTION', + }, + ], + }; + case 'CONNECTION': + return { + ...dataStructure, + columnHeader: 'Databases', + hasNext: true, + children: [ + { + id: `${dataStructure.id}.defaultDb`, + title: 'defaultDb', + type: 'DATABASE', + }, + ], + }; + case 'DATABASE': + return { + ...dataStructure, + columnHeader: 'Tables', + hasNext: false, + children: [ + { + id: `${dataStructure.id}.table1`, + title: 'table1', + type: 'TABLE', + }, + { + id: `${dataStructure.id}.table2`, + title: 'table2', + type: 'TABLE', + }, + ], + }; + default: + const s3DataSources = await fetchS3DataSources(savedObjects); + return { + ...dataStructure, + columnHeader: 'S3 Data Sources', + hasNext: false, + children: s3DataSources, + }; + } + }, + + fetchFields: async (dataset: Dataset): Promise => { + // This is a placeholder. You'll need to implement the actual logic to fetch S3 fields. + // For now, we'll return an empty array. + return []; + }, + + supportedLanguages: (): string[] => { + return ['sql']; // Assuming S3 only supports SQL queries + }, +}; + +const fetchS3DataSources = async (client: SavedObjectsClientContract): Promise => { + return []; +}; diff --git a/src/plugins/query_enhancements/public/plugin.tsx b/src/plugins/query_enhancements/public/plugin.tsx index ccc00822c1a1..d09189eda3b9 100644 --- a/src/plugins/query_enhancements/public/plugin.tsx +++ b/src/plugins/query_enhancements/public/plugin.tsx @@ -16,7 +16,8 @@ import { QueryEnhancementsPluginStart, QueryEnhancementsPluginStartDependencies, } from './types'; -import { UI_SETTINGS } from '../common'; +import { LanguageConfig, Query } from '../../data/public'; +import { s3TypeConfig } from './datasets'; export class QueryEnhancementsPlugin implements @@ -38,19 +39,7 @@ export class QueryEnhancementsPlugin core: CoreSetup, { data }: QueryEnhancementsPluginSetupDependencies ): QueryEnhancementsPluginSetup { - core.uiSettings.getUpdate$().subscribe(({ key, newValue }) => { - if (key === UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED) { - if (newValue) { - core.uiSettings.set(UI_SETTINGS.STATE_STORE_IN_SESSION_STORAGE, true); - } - } - if (key === UI_SETTINGS.STATE_STORE_IN_SESSION_STORAGE) { - if (!newValue) { - core.uiSettings.set(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED, false); - } - } - }); - + const { queryString } = data.query; const pplSearchInterceptor = new PPLSearchInterceptor({ toasts: core.notifications.toasts, http: core.http, @@ -67,52 +56,50 @@ export class QueryEnhancementsPlugin usageCollector: data.search.usageCollector, }); - data.__enhance({ - ui: { - query: { - language: 'PPL', - search: pplSearchInterceptor, - searchBar: { - queryStringInput: { initialValue: 'source=' }, - dateRange: { - initialFrom: moment().subtract(2, 'days').toISOString(), - initialTo: moment().add(2, 'days').toISOString(), - }, - showFilterBar: false, - showDataSetsSelector: true, - showDataSourcesSelector: true, - }, - fields: { - filterable: false, - visualizable: false, - }, - showDocLinks: false, - supportedAppNames: ['discover'], + // Register PPL language + const pplLanguageConfig: LanguageConfig = { + id: 'PPL', + title: 'PPL', + search: pplSearchInterceptor, + getQueryString: (query: Query) => { + return `source = ${query.dataset?.title}`; + }, + searchBar: { + dateRange: { + initialFrom: moment().subtract(2, 'days').toISOString(), + initialTo: moment().add(2, 'days').toISOString(), }, + showFilterBar: false, }, - }); + fields: { + filterable: false, + visualizable: false, + }, + showDocLinks: false, + supportedAppNames: ['discover'], + }; + queryString.getLanguageService().registerLanguage(pplLanguageConfig); - data.__enhance({ - ui: { - query: { - language: 'SQL', - search: sqlSearchInterceptor, - searchBar: { - showDatePicker: false, - showFilterBar: false, - showDataSetsSelector: true, - showDataSourcesSelector: true, - queryStringInput: { initialValue: 'SELECT * FROM LIMIT 10' }, - }, - fields: { - filterable: false, - visualizable: false, - }, - showDocLinks: false, - supportedAppNames: ['discover'], - }, + // Register SQL language + const sqlLanguageConfig: LanguageConfig = { + id: 'SQL', + title: 'SQL', + search: sqlSearchInterceptor, + getQueryString: (query: Query) => { + return `SELECT * FROM ${queryString.getQuery().dataset?.title} LIMIT 10`; }, - }); + searchBar: { + showDatePicker: false, + showFilterBar: false, + }, + fields: { + filterable: false, + visualizable: false, + }, + showDocLinks: false, + supportedAppNames: ['discover'], + }; + queryString.getLanguageService().registerLanguage(sqlLanguageConfig); data.__enhance({ ui: { @@ -120,6 +107,8 @@ export class QueryEnhancementsPlugin }, }); + queryString.getDatasetService().registerType(s3TypeConfig); + return {}; } diff --git a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx index 5e39f64b3ff8..85f951fc7515 100644 --- a/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx +++ b/src/plugins/query_enhancements/public/query_assist/components/query_assist_bar.tsx @@ -26,6 +26,7 @@ interface QueryAssistInputProps { export const QueryAssistBar: React.FC = (props) => { const { services } = useOpenSearchDashboards(); + const queryString = services.data.query.queryString; const inputRef = useRef(null); const storage = getStorage(); const persistedLog: PersistedLog = useMemo( @@ -35,18 +36,18 @@ export const QueryAssistBar: React.FC = (props) => { const { generateQuery, loading } = useGenerateQuery(); const [callOutType, setCallOutType] = useState(); const dismissCallout = () => setCallOutType(undefined); - const [selectedDataSet, setSelectedDataSet] = useState( - services.data.query.dataSetManager.getDataSet() + const [selectedDataset, setSelectedDataset] = useState( + queryString.getQuery().dataset ); - const selectedIndex = selectedDataSet?.title; + const selectedIndex = selectedDataset?.title; const previousQuestionRef = useRef(); useEffect(() => { - const subscription = services.data.query.dataSetManager.getUpdates$().subscribe((dataSet) => { - setSelectedDataSet(dataSet); + const subscription = queryString.getUpdates$().subscribe((query) => { + setSelectedDataset(query.dataset); }); return () => subscription.unsubscribe(); - }, [services.data.query.dataSetManager]); + }, [queryString]); const onSubmit = async (e: SyntheticEvent) => { e.preventDefault(); @@ -65,7 +66,7 @@ export const QueryAssistBar: React.FC = (props) => { question: inputRef.current.value, index: selectedIndex, language: props.dependencies.language, - dataSourceId: selectedDataSet?.dataSourceRef?.id, + dataSourceId: selectedDataset?.dataSource?.id, }; const { response, error } = await generateQuery(params); if (error) { @@ -78,6 +79,7 @@ export const QueryAssistBar: React.FC = (props) => { services.data.query.queryString.setQuery({ query: response.query, language: params.language, + dataset: selectedDataset, }); if (response.timeRange) services.data.query.timefilter.timefilter.setTime(response.timeRange); setCallOutType('query_generated'); diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.test.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.test.tsx index ec3304e987fc..3afb986b259f 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.test.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.test.tsx @@ -6,12 +6,9 @@ import { firstValueFrom } from '@osd/std'; import { act, render, screen } from '@testing-library/react'; import React from 'react'; -import { of } from 'rxjs'; import { coreMock } from '../../../../../core/public/mocks'; -import { Dataset } from '../../../../data/common'; import { QueryEditorExtensionDependencies } from '../../../../data/public'; import { dataPluginMock } from '../../../../data/public/mocks'; -import { DataSetContract } from '../../../../data/public/query'; import { ConfigSchema } from '../../../common/config'; import { createQueryAssistExtension } from './create_extension'; @@ -24,18 +21,6 @@ const coreSetupMock = coreMock.createSetup({ }); const httpMock = coreSetupMock.http; const dataMock = dataPluginMock.createSetupContract(); -const dataSetMock = (dataMock.query.dataSetManager as unknown) as jest.Mocked; - -const mockDataset = { - id: 'mock-data-set-id', - title: 'mock-title', - dataSource: { - id: 'mock-data-source-id', - }, -} as Dataset; - -dataSetMock.getDataSet.mockReturnValue(mockDataset); -dataSetMock.getUpdates$.mockReturnValue(of(mockDataset)); jest.mock('../components', () => ({ QueryAssistBar: jest.fn(() =>
QueryAssistBar
), diff --git a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx index b427824bd581..e1c8d56908df 100644 --- a/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx +++ b/src/plugins/query_enhancements/public/query_assist/utils/create_extension.tsx @@ -25,15 +25,15 @@ const getAvailableLanguages$ = ( http: HttpSetup, data: DataPublicPluginSetup ) => - data.query.dataSetManager.getUpdates$().pipe( - startWith(data.query.dataSetManager.getDataSet()), + data.query.queryString.getUpdates$().pipe( + startWith(data.query.queryString.getQuery()), distinctUntilChanged(), - switchMap(async (dataset) => { + switchMap(async (query) => { // currently query assist tool relies on opensearch API to get index // mappings, external data source types (e.g. s3) are not supported - if (dataset?.dataSource?.type !== DEFAULT_QUERY.ENGINE_TYPE) return []; + if (query.dataset?.dataSource?.type !== DEFAULT_QUERY.DATASET.DATASOURCE.TYPE) return []; - const dataSourceId = dataset?.dataSource?.id; + const dataSourceId = query.dataset?.dataSource?.id; const cached = availableLanguagesByDataSource.get(dataSourceId); if (cached !== undefined) return cached; const languages = await http diff --git a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts index 1f28df9af998..3a5edd67ca7f 100644 --- a/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts @@ -5,16 +5,14 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; -import { catchError, concatMap } from 'rxjs/operators'; +import { catchError } from 'rxjs/operators'; import { DataFrameAggConfig, - getAggConfig, getRawDataFrame, - getRawQueryString, formatTimePickerDate, getUniqueValuesForRawAggs, updateDataFrameMeta, - getRawAggs, + Query, } from '../../../data/common'; import { DataPublicPluginStart, @@ -73,17 +71,17 @@ export class PPLSearchInterceptor extends SearchInterceptor { }; const getAggQsFn = ({ - qs, + query, aggConfig, timeField, timeFilter, }: { - qs: string; + query: Query; aggConfig: DataFrameAggConfig; timeField: any; timeFilter: string; }) => { - return removeKeyword(`${qs} ${getAggString(timeField, aggConfig)} ${timeFilter}`); + return removeKeyword(`${query.query} ${getAggString(timeField, aggConfig)} ${timeFilter}`); }; const getAggString = (timeField: any, aggsConfig?: DataFrameAggConfig) => { @@ -138,77 +136,24 @@ export class PPLSearchInterceptor extends SearchInterceptor { }; const dataFrame = getRawDataFrame(searchRequest); - - let queryString = dataFrame.meta?.queryConfig?.qs ?? getRawQueryString(searchRequest) ?? ''; - - dataFrame.meta = { - ...dataFrame.meta, - aggConfig: { - ...dataFrame.meta.aggConfig, - ...(getRawAggs(searchRequest) && - this.aggsService.types.get.bind(this) && - getAggConfig(searchRequest, {}, this.aggsService.types.get.bind(this))), - }, - queryConfig: { - ...dataFrame.meta.queryConfig, - ...(this.queryService.dataSetManager.getDataSet() && { - dataSourceId: this.queryService.dataSetManager.getDataSet()?.dataSourceRef?.id, - dataSourceName: this.queryService.dataSetManager.getDataSet()?.dataSourceRef?.name, - timeFieldName: this.queryService.dataSetManager.getDataSet()?.timeFieldName, - }), - }, - }; - - if (!dataFrame.schema) { - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( - concatMap((response) => { - const df = response.body; - if (df.error) { - const jsError = new Error(df.error.response); - return throwError(jsError); - } - const timeField = dataFrame.meta?.queryConfig?.timeFieldName; - const aggConfig = dataFrame.meta?.aggConfig; - if (timeField && aggConfig) { - const timeFilter = getTimeFilter(timeField); - const newQuery = insertTimeFilter(queryString, timeFilter); - updateDataFrameMeta({ - dataFrame: df, - qs: newQuery, - aggConfig, - timeField, - timeFilter, - getAggQsFn: getAggQsFn.bind(this), - }); - return fetchDataFrame(dfContext, newQuery, df); - } - return fetchDataFrame(dfContext, queryString, df); - }), - catchError((error) => { - return throwError(error); - }) - ); - } - - if (dataFrame.schema) { - const timeField = dataFrame.meta?.queryConfig?.timeFieldName; - const aggConfig = dataFrame.meta?.aggConfig; - if (timeField && aggConfig) { - const timeFilter = getTimeFilter(timeField); - const newQuery = insertTimeFilter(queryString, timeFilter); - updateDataFrameMeta({ - dataFrame, - qs: newQuery, - aggConfig: dataFrame.meta?.aggConfig, - timeField, - timeFilter, - getAggQsFn: getAggQsFn.bind(this), - }); - queryString += timeFilter; - } + const query = this.queryService.queryString.getQuery(); + const timeField = query.dataset?.timeFieldName; + const aggConfig = dataFrame?.meta?.aggConfig; + if (timeField && aggConfig) { + const timeFilter = getTimeFilter(timeField); + const newQuery = insertTimeFilter(query.query as string, timeFilter); + updateDataFrameMeta({ + dataFrame, + query: { ...query, query: newQuery }, + aggConfig: dataFrame?.meta?.aggConfig, + timeField, + timeFilter, + getAggQsFn: getAggQsFn.bind(this), + }); + query.query += timeFilter; } - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( + return fetchDataFrame(dfContext, query, dataFrame).pipe( catchError((error) => { return throwError(error); }) diff --git a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts index a73a1cee8206..3aba5cca7fad 100644 --- a/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts +++ b/src/plugins/query_enhancements/public/search/sql_search_interceptor.ts @@ -7,7 +7,7 @@ import { trimEnd } from 'lodash'; import { Observable, throwError } from 'rxjs'; import { i18n } from '@osd/i18n'; import { concatMap, map } from 'rxjs/operators'; -import { DATA_FRAME_TYPES, getRawDataFrame, getRawQueryString } from '../../../data/common'; +import { DATA_FRAME_TYPES, getRawDataFrame } from '../../../data/common'; import { DataPublicPluginStart, IOpenSearchDashboardsSearchRequest, @@ -55,34 +55,7 @@ export class SQLSearchInterceptor extends SearchInterceptor { const dataFrame = getRawDataFrame(searchRequest); - const queryString = dataFrame.meta?.queryConfig?.qs ?? getRawQueryString(searchRequest) ?? ''; - - dataFrame.meta = { - ...dataFrame.meta, - queryConfig: { - ...dataFrame.meta.queryConfig, - ...(this.queryService.dataSetManager.getDataset() && { - dataSourceId: this.queryService.dataSetManager.getDataset()?.dataSource?.id, - dataSourceName: this.queryService.dataSetManager.getDataset()?.dataSource?.title, - timeFieldName: this.queryService.dataSetManager.getDataset()?.timeFieldName, - }), - }, - }; - - if (!dataFrame.schema) { - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( - concatMap((response) => { - const df = response.body; - if (df.error) { - const jsError = new Error(df.error.response); - return throwError(jsError); - } - return fetchDataFrame(dfContext, queryString, df); - }) - ); - } - - return fetchDataFrame(dfContext, queryString, dataFrame); + return fetchDataFrame(dfContext, this.queryService.queryString.getQuery(), dataFrame); } protected runSearchAsync( @@ -99,22 +72,19 @@ export class SQLSearchInterceptor extends SearchInterceptor { }; const dataFrame = getRawDataFrame(searchRequest); - if (!dataFrame) { - return throwError(this.handleSearchError('DataFrame is not defined', request, signal!)); - } + const query = this.queryService.queryString.getQuery(); - const queryString = getRawQueryString(searchRequest) ?? ''; - const dataSourceRef = this.queryService.dataSetManager.getDataset() + const dataSourceRef = query.dataset ? { - dataSourceId: this.queryService.dataSetManager.getDataset()?.dataSource?.id, - dataSourceName: this.queryService.dataSetManager.getDataset()?.dataSource?.title, + dataSourceId: query.dataset.dataSource?.id, + dataSourceName: query.dataset.dataSource?.title, } : {}; dataFrame.meta = { - ...dataFrame.meta, + ...dataFrame?.meta, queryConfig: { - ...dataFrame.meta.queryConfig, + ...dataFrame?.meta.queryConfig, ...dataSourceRef, }, sessionId: dataSourceRef @@ -155,7 +125,7 @@ export class SQLSearchInterceptor extends SearchInterceptor { defaultMessage: 'Starting query job...', }), }); - return fetchDataFrame(dfContext, queryString, dataFrame).pipe( + return fetchDataFrame(dfContext, query, dataFrame).pipe( concatMap((jobResponse) => { const df = jobResponse.body; if (dataSourceRef?.dataSourceName && df?.meta?.sessionId) { @@ -182,8 +152,8 @@ export class SQLSearchInterceptor extends SearchInterceptor { } public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) { - const dataSet = this.queryService.dataSetManager.getDataset(); - if (dataSet?.type === 'TEMPORARY_ASYNC') { + const dataset = this.queryService.queryString.getQuery().dataset; + if (dataset?.type === 'S3') { return this.runSearchAsync(request, options.abortSignal, SEARCH_STRATEGY.SQL_ASYNC); } return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.SQL); diff --git a/src/plugins/query_enhancements/server/routes/index.ts b/src/plugins/query_enhancements/server/routes/index.ts index 3c23db3c87b9..a17c4f2294bd 100644 --- a/src/plugins/query_enhancements/server/routes/index.ts +++ b/src/plugins/query_enhancements/server/routes/index.ts @@ -32,7 +32,9 @@ function defineRoute( validate: { body: schema.object({ query: schema.object({ - qs: schema.string(), + query: schema.string(), + language: schema.string(), + dataset: schema.nullable(schema.object({}, { unknowns: 'allow' })), format: schema.string(), }), df: schema.nullable(schema.object({}, { unknowns: 'allow' })), diff --git a/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts b/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts index f0b01cd51da4..f426b2d9980f 100644 --- a/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/ppl_search_strategy.ts @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { first } from 'rxjs/operators'; import { SharedGlobalConfig, Logger, ILegacyClusterClient } from 'opensearch-dashboards/server'; import { Observable } from 'rxjs'; import { ISearchStrategy, getDefaultSearchParams, SearchUsage } from '../../../data/server'; @@ -13,6 +12,7 @@ import { IDataFrameResponse, IDataFrameWithAggs, IOpenSearchDashboardsSearchRequest, + Query, createDataFrame, } from '../../../data/common'; import { getFields } from '../../common/utils'; @@ -45,8 +45,6 @@ export const pplSearchStrategyProvider = ( const source = pipeMap.get('source'); - const searchQuery = query; - const filters = pipeMap.get('where'); const stats = pipeMap.get('stats'); @@ -55,15 +53,12 @@ export const pplSearchStrategyProvider = ( : undefined; return { - map: pipeMap, - search: searchQuery, aggs: aggsQuery, }; }; return { search: async (context, request: any, options) => { - const config = await config$.pipe(first()).toPromise(); const uiSettingsClient = await context.core.uiSettings.client; const { dataFrameHydrationStrategy, ...defaultParams } = await getDefaultSearchParams( @@ -71,14 +66,9 @@ export const pplSearchStrategyProvider = ( ); try { - const requestParams = parseRequest(request.body.query.qs); - const source = requestParams?.map.get('source'); - const { schema, meta } = request.body.df; - - request.body.query = - !schema || dataFrameHydrationStrategy === 'perQuery' - ? `source=${source} | head` - : requestParams.search; + const query: Query = request.body.query; + const { df } = request.body; + const rawResponse: any = await pplFacet.describeQuery(context, request); if (!rawResponse.success) { @@ -90,9 +80,9 @@ export const pplSearchStrategyProvider = ( } const dataFrame = createDataFrame({ - name: source, - schema: schema ?? rawResponse.data.schema, - meta, + name: query.dataset?.id, + schema: rawResponse.data.schema, + meta: df?.meta, fields: getFields(rawResponse), }); @@ -100,11 +90,10 @@ export const pplSearchStrategyProvider = ( if (usage) usage.trackSuccess(rawResponse.took); - if (dataFrame.meta?.aggsQs) { - for (const [key, aggQueryString] of Object.entries(dataFrame.meta.aggsQs)) { + if (dataFrame?.meta?.aggsQs) { + for (const [key, aggQueryString] of Object.entries(dataFrame?.meta?.aggsQs)) { const aggRequest = parseRequest(aggQueryString as string); - const query = aggRequest.aggs; - request.body.query = query; + request.body.query = aggRequest.aggs; const rawAggs: any = await pplFacet.describeQuery(context, request); (dataFrame as IDataFrameWithAggs).aggs = {}; (dataFrame as IDataFrameWithAggs).aggs[key] = rawAggs.data.datarows?.map((hit: any) => { diff --git a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts index 3c3802eae261..c32159e5b968 100644 --- a/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_async_search_strategy.ts @@ -12,6 +12,7 @@ import { IDataFrameResponse, IOpenSearchDashboardsSearchRequest, PartialDataFrame, + Query, createDataFrame, } from '../../../data/common'; import { Facet } from '../utils'; @@ -38,12 +39,13 @@ export const sqlAsyncSearchStrategyProvider = ( return { search: async (context, request: any, options) => { try { + const query: Query = request?.body?.query; // Create job: this should return a queryId and sessionId - if (request?.body?.query?.qs) { + if (query) { const df = request.body?.df; request.body = { - query: request.body.query.qs, - datasource: df?.meta?.queryConfig?.dataSourceName, + query: query.query, + datasource: query.dataset?.dataSource?.title, lang: SEARCH_STRATEGY.SQL, sessionId: df?.meta?.sessionId, }; @@ -60,13 +62,13 @@ export const sqlAsyncSearchStrategyProvider = ( const sessionId = rawResponse.data?.sessionId; const partial: PartialDataFrame = { - ...request.body.df, + ...df, fields: rawResponse?.data?.schema || [], }; const dataFrame = createDataFrame(partial); dataFrame.meta = { - ...dataFrame.meta, - query: request.body.query, + ...dataFrame?.meta, + query: query.query, queryId, sessionId, }; @@ -93,7 +95,7 @@ export const sqlAsyncSearchStrategyProvider = ( dataFrame.size = asyncResponse?.data?.datarows?.length || 0; dataFrame.meta = { - ...dataFrame.meta, + ...dataFrame?.meta, status, queryId, error: status === 'FAILED' && asyncResponse.data?.error, diff --git a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts index b2f6af9ca144..914391471baa 100644 --- a/src/plugins/query_enhancements/server/search/sql_search_strategy.ts +++ b/src/plugins/query_enhancements/server/search/sql_search_strategy.ts @@ -12,6 +12,7 @@ import { IDataFrameResponse, IOpenSearchDashboardsSearchRequest, PartialDataFrame, + Query, createDataFrame, } from '../../../data/common'; import { Facet } from '../utils'; @@ -27,7 +28,6 @@ export const sqlSearchStrategyProvider = ( return { search: async (context, request: any, _options) => { try { - request.body.query = request.body.query.qs; const rawResponse: any = await sqlFacet.describeQuery(context, request); if (!rawResponse.success) { diff --git a/src/plugins/query_enhancements/server/utils/facet.ts b/src/plugins/query_enhancements/server/utils/facet.ts index e930ce612d9b..287e8b6e86ec 100644 --- a/src/plugins/query_enhancements/server/utils/facet.ts +++ b/src/plugins/query_enhancements/server/utils/facet.ts @@ -6,6 +6,7 @@ import { Logger } from 'opensearch-dashboards/server'; import { FacetResponse, IPPLEventsDataSource, IPPLVisualizationDataSource } from '../types'; import { shimSchemaRow, shimStats } from '.'; +import { Query } from '../../../data/common'; export interface FacetProps { client: any; @@ -36,12 +37,13 @@ export class Facet { endpoint: string ): Promise => { try { - const { format, df, dataSourceId, ...query } = request.body; + const query: Query = request.body.query; + const { format, df } = request.body; const params = { body: { ...query }, ...(format !== 'jdbc' && { format }), }; - const clientId = dataSourceId ?? df?.meta?.queryConfig?.dataSourceId; + const clientId = query.dataset?.dataSource?.id ?? df?.meta?.queryConfig?.dataSourceId; const client = clientId ? context.dataSource.opensearch.legacy.getClient(clientId).callAPI : this.defaultClient.asScoped(request).callAsCurrentUser; diff --git a/yarn.lock b/yarn.lock index e3cacd0f1623..a8d9af23c63e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15986,7 +15986,7 @@ string-similarity@^4.0.1: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -16021,15 +16021,6 @@ string-width@^3.0.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - string-width@^5.0.1, string-width@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" @@ -16108,7 +16099,7 @@ stringify-entities@^3.0.1: character-entities-legacy "^1.0.0" xtend "^4.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -16150,13 +16141,6 @@ strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -18300,7 +18284,7 @@ workerpool@6.2.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -18326,15 +18310,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"