[discover] update interfaces and selector (#7835)

* dataset handler and move manager Signed-off-by: Kawika Avilla <[email protected]> * Adds new Data selector Signed-off-by: Ashwin P Chandran <[email protected]> fixes loading state Signed-off-by: Ashwin P Chandran <[email protected]> * start wiring into the selector Signed-off-by: Kawika Avilla <[email protected]> no more white screen Signed-off-by: Kawika Avilla <[email protected]> updating handler Signed-off-by: Kawika Avilla <[email protected]> rendering data Signed-off-by: Kawika Avilla <[email protected]> data structures working ok Signed-off-by: Kawika Avilla <[email protected]> need to add datasets Signed-off-by: Kawika Avilla <[email protected]> clean up Signed-off-by: Kawika Avilla <[email protected]> deleted unneeded types Signed-off-by: Kawika Avilla <[email protected]> * still working on the data structure and field Signed-off-by: Kawika Avilla <[email protected]> * still not working. the leaf logic isnt right i believe Signed-off-by: Kawika Avilla <[email protected]> * indices Signed-off-by: Kawika Avilla <[email protected]> * pushing datasets Signed-off-by: Kawika Avilla <[email protected]> * fix index pattern Signed-off-by: Kawika Avilla <[email protected]> * fixes with column Signed-off-by: Kawika Avilla <[email protected]> * working creation step Signed-off-by: Kawika Avilla <[email protected]> * get dataset from state in use index pattern Signed-off-by: Kawika Avilla <[email protected]> * dataset selector working Signed-off-by: Kawika Avilla <[email protected]> * update ppl interceptor Signed-off-by: Kawika Avilla <[email protected]> * add dataset service Signed-off-by: Kawika Avilla <[email protected]> * language service Signed-off-by: Kawika Avilla <[email protected]> * wired up but the dataset is off Signed-off-by: Kawika Avilla <[email protected]> * Address review comments Signed-off-by: Kawika Avilla <[email protected]> * ppl query Signed-off-by: Kawika Avilla <[email protected]> * dql and lucene working again Signed-off-by: Kawika Avilla <[email protected]> * fix issue that would deselect dataset Signed-off-by: Kawika Avilla <[email protected]> * language switches and sets the query correctly Signed-off-by: Kawika Avilla <[email protected]> * upating query Signed-off-by: Kawika Avilla <[email protected]> * fix the styling a little Signed-off-by: Kawika Avilla <[email protected]> * little bit width Signed-off-by: Kawika Avilla <[email protected]> * submit on update Signed-off-by: Kawika Avilla <[email protected]> * fix sql Signed-off-by: Kawika Avilla <[email protected]> * ppl working better Signed-off-by: Kawika Avilla <[email protected]> * need to still fix the aggregations Signed-off-by: Kawika Avilla <[email protected]> * gotta fix the aggs and still need to update dql lucene dataset Signed-off-by: Kawika Avilla <[email protected]> * use patch Signed-off-by: Kawika Avilla <[email protected]> * Metadata slice Signed-off-by: Kawika Avilla <[email protected]> * index pattern updates Signed-off-by: Kawika Avilla <[email protected]> * types and datasource prepend Signed-off-by: Kawika Avilla <[email protected]> --------- Signed-off-by: Kawika Avilla <[email protected]> Signed-off-by: Ashwin P Chandran <[email protected]> Co-authored-by: Ashwin P Chandran <[email protected]>
opensearch-project · Aug 26, 2024 · 5e6e818 · 5e6e818
1 parent 21604e8
commit 5e6e818
Show file tree

Hide file tree

Showing 115 changed files with 2,171 additions and 3,915 deletions.
diff --git a/package.json b/package.json
@@ -65,7 +65,7 @@
     "start": "scripts/use_node scripts/opensearch_dashboards --dev",
     "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST",
     "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security",
-    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true --uiSettings.overrides['state:storeInSessionStorage']=true",
+    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true",
     "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev",
     "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev",
     "lint": "yarn run lint:es && yarn run lint:style",

diff --git a/packages/osd-stylelint-config/config/global_selectors.json b/packages/osd-stylelint-config/config/global_selectors.json
@@ -27,8 +27,7 @@
       "src/plugins/discover/public/application/view_components/canvas/discover_canvas.scss",
       "src/plugins/discover/public/application/components/sidebar/discover_sidebar.scss",
       "src/plugins/data/public/ui/query_string_input/_query_bar.scss",
-      "src/plugins/data/public/ui/query_editor/_query_editor.scss",
-      "src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss"
+      "src/plugins/data/public/ui/query_editor/_query_editor.scss"
     ]
   }
 }
diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts
@@ -457,14 +457,14 @@ export class SavedObjectsClient {
    *   { id: 'foo', type: 'index-pattern' }
    * ])
    */
-  public bulkGet = (objects: Array<{ id: string; type: string }> = []) => {
+  public bulkGet = <T = unknown>(objects: Array<{ id: string; type: string }> = []) => {
     const filteredObjects = objects.map((obj) => pick(obj, ['id', 'type']));
     return this.performBulkGet(filteredObjects).then((resp) => {
       resp.saved_objects = resp.saved_objects.map((d) => this.createSavedObject(d));
       return renameKeys<
         PromiseType<ReturnType<SavedObjectsApi['bulkGet']>>,
-        SavedObjectsBatchResponse
-      >({ saved_objects: 'savedObjects' }, resp) as SavedObjectsBatchResponse;
+        SavedObjectsBatchResponse<T>
+      >({ saved_objects: 'savedObjects' }, resp) as SavedObjectsBatchResponse<T>;
     });
   };
 

diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts
@@ -9,6 +9,8 @@
  * GitHub history for details.
  */
 
+import { DATA_STRUCTURE_META_TYPES, DataStructure } from './types';
+
 /*
  * Licensed to Elasticsearch B.V. under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -28,10 +30,41 @@
  * under the License.
  */
 
+export const DEFAULT_DATA = {
+  STRUCTURES: {
+    ROOT: {
+      id: 'ROOT',
+      title: 'Data',
+      type: 'ROOT',
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.FEATURE,
+        icon: 'folderOpen',
+        tooltip: 'Root Data Structure',
+      },
+    } as DataStructure,
+  },
+
+  SET_TYPES: {
+    INDEX_PATTERN: 'INDEX_PATTERN',
+    INDEX: 'INDEXES',
+  },
+
+  SOURCE_TYPES: {
+    OPENSEARCH: 'OPENSEARCH',
+    LEGACY: 'LEGACY',
+  },
+};
+
+export const DEFAULT_QUERY_LANGUAGE = 'kuery';
+
 export const DEFAULT_QUERY = {
-  LANGUAGE: 'kuery',
-  DATASET_TYPE: 'INDEX_PATTERN',
-  ENGINE_TYPE: 'OPENSEARCH',
+  LANGUAGE: DEFAULT_QUERY_LANGUAGE,
+  DATASET: {
+    TYPE: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN,
+    DATASOURCE: {
+      TYPE: DEFAULT_DATA.SOURCE_TYPES.OPENSEARCH,
+    },
+  },
 };
 
 export const UI_SETTINGS = {

diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts
@@ -18,7 +18,7 @@ import {
 import { IFieldType } from './fields';
 import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns';
 import { IOpenSearchDashboardsSearchRequest } from '../search';
-import { GetAggTypeFn, GetDataFrameAggQsFn, TimeRange } from '../types';
+import { GetAggTypeFn, GetDataFrameAggQsFn, Query, TimeRange } from '../types';
 
 /**
  * Returns the raw data frame from the search request.
@@ -380,25 +380,25 @@ export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFr
  */
 export const updateDataFrameMeta = ({
   dataFrame,
-  qs,
+  query,
   aggConfig,
   timeField,
   timeFilter,
   getAggQsFn,
 }: {
   dataFrame: IDataFrame;
-  qs: string;
+  query: Query;
   aggConfig: DataFrameAggConfig;
   timeField: any;
   timeFilter: string;
   getAggQsFn: GetDataFrameAggQsFn;
 }) => {
   dataFrame.meta = {
-    ...dataFrame.meta,
+    ...dataFrame?.meta,
     aggs: aggConfig,
     aggsQs: {
       [aggConfig.id]: getAggQsFn({
-        qs,
+        query,
         aggConfig,
         timeField,
         timeFilter,
@@ -411,7 +411,7 @@ export const updateDataFrameMeta = ({
     for (const [key, subAgg] of Object.entries(subAggs)) {
       const subAggConfig: Record<string, any> = { [key]: subAgg };
       dataFrame.meta.aggsQs[subAgg.id] = getAggQsFn({
-        qs,
+        query,
         aggConfig: subAggConfig as DataFrameAggConfig,
         timeField,
         timeFilter,

diff --git a/src/plugins/data/common/datasets/_structure_cache.ts b/src/plugins/data/common/datasets/_structure_cache.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { DataStructure, CachedDataStructure } from './types';
+
+export interface DataStructureCache {
+  get: (id: string) => CachedDataStructure | undefined;
+  set: (id: string, value: CachedDataStructure) => CachedDataStructure;
+  clear: (id: string) => void;
+  clearAll: () => void;
+}
+
+export function createDataStructureCache(): DataStructureCache {
+  const cache: Record<string, CachedDataStructure> = {};
+
+  const dataStructureCache: DataStructureCache = {
+    get: (id: string) => {
+      return cache[id];
+    },
+    set: (id: string, value: CachedDataStructure) => {
+      cache[id] = value;
+      return value;
+    },
+    clear: (id: string) => {
+      delete cache[id];
+    },
+    // TODO: call this on log out
+    clearAll: () => {
+      Object.keys(cache).forEach((key) => delete cache[key]);
+    },
+  };
+
+  return dataStructureCache;
+}
+
+export function toCachedDataStructure(dataStructure: DataStructure): CachedDataStructure {
+  return {
+    id: dataStructure.id,
+    title: dataStructure.title,
+    type: dataStructure.type,
+    parent: dataStructure.parent?.id || '',
+    children: dataStructure.children?.map((child) => child.id) || [],
+  };
+}
diff --git a/src/plugins/data/common/datasets/types.ts b/src/plugins/data/common/datasets/types.ts
@@ -3,6 +3,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { EuiIconProps } from '@elastic/eui';
+export * from './_structure_cache';
+
 /**
  * Describes a data source with its properties.
  */
@@ -116,42 +119,55 @@ export interface DataStructure {
   parent?: DataStructure;
   /** Optional array of child data structures */
   children?: DataStructure[];
+  hasNext?: boolean;
+  columnHeader?: string;
   /** Optional metadata for the data structure */
   meta?: DataStructureMeta;
 }
 
-/**
- * Metadata for a data structure, used for additional properties like icons or tooltips.
- */
-export interface DataStructureMeta {
-  type: DATA_STRUCTURE_META_TYPES;
-}
-
 /**
  * DataStructureMeta types
  */
 export enum DATA_STRUCTURE_META_TYPES {
   FEATURE = 'FEATURE',
+  TYPE = 'TYPE',
   CUSTOM = 'CUSTOM',
 }
 
 /**
  * Metadata for a data structure, used for additional properties like icons or tooltips.
  */
-export interface DataStructureFeatureMeta extends DataStructureMeta {
+export interface DataStructureFeatureMeta {
   type: DATA_STRUCTURE_META_TYPES.FEATURE;
-  icon: string;
+  icon?: string;
+  tooltip?: string;
+}
+
+/**
+ * Metadata for dataset type
+ */
+export interface DataStructureDataTypeMeta {
+  type: DATA_STRUCTURE_META_TYPES.TYPE;
+  icon: EuiIconProps;
   tooltip: string;
 }
 
 /**
  * Metadata for a data structure with CUSTOM type, allowing any additional fields.
  */
-export interface DataStructureCustomMeta extends DataStructureMeta {
+export interface DataStructureCustomMeta {
   type: DATA_STRUCTURE_META_TYPES.CUSTOM;
   [key: string]: any;
 }
 
+/**
+ * Union type for DataStructureMeta
+ */
+export type DataStructureMeta =
+  | DataStructureFeatureMeta
+  | DataStructureDataTypeMeta
+  | DataStructureCustomMeta;
+
 /**
  * Represents a cached version of DataStructure with string references instead of object references.
  *
@@ -168,13 +184,24 @@ export interface DataStructureCustomMeta extends DataStructureMeta {
  *   ]
  * };
  */
-export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'children' | 'meta'> {
+export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'children'> {
   /** ID of the parent data structure */
   parent: string;
   /** Array of child data structure IDs */
   children: string[];
 }
 
+export interface BaseDataset {
+  /** Unique identifier for the dataset, for non-index pattern based datasets, we will append the data source ID if present */
+  id: string;
+  /** Human-readable name of the dataset that is used to query */
+  title: string;
+  /** The type of the dataset, registered by other classes */
+  type: string;
+  /** Optional reference to the data source */
+  dataSource?: DataSource;
+}
+
 /**
  * Defines the structure of a dataset, including its type and reference to a data source.
  * NOTE: For non-index pattern datasets we will append the data source ID to the front of
@@ -189,10 +216,10 @@ export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'chi
  *   type: "INDEX_PATTERN",
  *   timeFieldName: "@timestamp",
  *   dataSource: {
- *     id: "main-cluster",
- *     title: "Main OpenSearch Cluster",
- *     type: "OPENSEARCH"
- *   },
+ *     id: "2e1b1b80-9c4d-11ee-8c90-0242ac120001",
+ *     title: "Cluster1",
+ *     type: "OpenSearch"
+ *   }
  * };
  *
  * @example
@@ -209,15 +236,16 @@ export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'chi
  *   },
  * };
  */
-export interface Dataset {
-  /** Unique identifier for the dataset, for non-index pattern based datasets, we will append the data source ID if present */
-  id: string;
-  /** Human-readable name of the dataset that is used to query */
-  title: string;
-  /** The type of the dataset, registered by other classes */
-  type: string;
+export interface Dataset extends BaseDataset {
   /** Optional name of the field used for time-based operations */
   timeFieldName?: string;
-  /** Optional reference to the data source */
-  dataSource?: DataSource;
+  /** Optional language to default to from the language selector */
+  language?: string;
+}
+
+export interface DatasetField {
+  name: string;
+  type: string;
+  displayName?: string;
+  // TODO:  osdFieldType?
 }
diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts
@@ -122,6 +122,7 @@ export class IndexPatternsService {
 
     this.savedObjectsCache = await Promise.all(
       this.savedObjectsCache.map(async (obj) => {
+        // TODO: This behaviour will cause the index pattern title to be resolved differently depending on how its fetched since the get method in this service will not append the datasource title
         if (obj.type === 'index-pattern') {
           const result = { ...obj };
           result.attributes.title = await getIndexPatternTitle(

diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts
@@ -320,16 +320,9 @@ export class SearchSource {
    * @return {undefined|IDataFrame}
    */
   async createDataFrame(searchRequest: SearchRequest) {
-    const rawQueryString = this.getRawQueryStringFromRequest(searchRequest);
     const dataFrame = createDataFrame({
       name: searchRequest.index.title || searchRequest.index,
       fields: [],
-      ...(rawQueryString && {
-        meta: {
-          queryConfig: { qs: rawQueryString },
-          ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }),
-        },
-      }),
     });
     await this.setDataFrame(dataFrame);
     return this.getDataFrame();

diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts
@@ -29,6 +29,7 @@
  */
 
 import { DataFrameAggConfig, IDataFrame } from './data_frames';
+import { Query } from './query';
 import { BucketAggType, MetricAggType } from './search';
 
 export * from './query/types';
@@ -50,12 +51,12 @@ export * from './datasets/types';
 export type GetConfigFn = <T = any>(key: string, defaultOverride?: T) => T;
 export type GetDataFrameFn = () => IDataFrame | undefined;
 export type GetDataFrameAggQsFn = ({
-  qs,
+  query,
   aggConfig,
   timeField,
   timeFilter,
 }: {
-  qs: string;
+  query: Query;
   aggConfig: DataFrameAggConfig;
   timeField: any;
   timeFilter: any;

diff --git a/src/plugins/data/public/antlr/opensearch_sql/code_completion.ts b/src/plugins/data/public/antlr/opensearch_sql/code_completion.ts
@@ -47,7 +47,7 @@ export const getSuggestions = async ({
   services,
 }: QuerySuggestionGetFnArgs): Promise<QuerySuggestion[]> => {
   const { api } = services.uiSettings;
-  const dataSetManager = services.data.query.dataSetManager;
+  const queryString = services.data.query.queryString;
   const { lineNumber, column } = position || {};
   const suggestions = getOpenSearchSqlAutoCompleteSuggestions(query, {
     line: lineNumber || selectionStart,
@@ -60,7 +60,7 @@ export const getSuggestions = async ({
     // Fetch columns and values
     if (suggestions.suggestColumns?.tables?.length) {
       const tableNames = suggestions.suggestColumns.tables.map((table) => table.name);
-      const schemas = await fetchTableSchemas(tableNames, api, dataSetManager);
+      const schemas = await fetchTableSchemas(tableNames, api, queryString);
 
       (schemas as IDataFrameResponse[]).forEach((schema: IDataFrameResponse) => {
         if ('body' in schema && schema.body && 'fields' in schema.body) {