Initial updates to new interfaces Discover expects

Signed-off-by: Kawika Avilla <[email protected]>

src/plugins/dashboard/server/saved_objects/migrate_match_all_query.ts

@@ -30,7 +30,7 @@
 
 import { SavedObjectMigrationFn } from 'opensearch-dashboards/server';
 import { get } from 'lodash';
-import { DEFAULT_QUERY_LANGUAGE } from '../../../data/common';
+import { DEFAULT_QUERY } from '../../../data/common';
 
 /**
  * This migration script is related to:
@@ -61,7 +61,7 @@ export const migrateMatchAllQuery: SavedObjectMigrationFn<any, any> = (doc) => {
               ...searchSource,
               query: {
                 query: '',
-                language: DEFAULT_QUERY_LANGUAGE,
+                language: DEFAULT_QUERY.LANGUAGE,
               },
             }),
           },

src/plugins/data/common/constants.ts

@@ -28,7 +28,11 @@
  * under the License.
  */
 
-export const DEFAULT_QUERY_LANGUAGE = 'kuery';
+export const DEFAULT_QUERY = {
+  LANGUAGE: 'kuery',
+  DATASET_TYPE: 'INDEX_PATTERN',
+  ENGINE_TYPE: 'OPENSEARCH',
+};
 
 export const UI_SETTINGS = {
   META_FIELDS: 'metaFields',

src/plugins/data/common/datasets/types.ts

@@ -0,0 +1,144 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Describes a data source with its properties.
+ */
+export interface DataSource {
+  /** Optional unique identifier for the data source, if MDS enabled this is the ID used to fetch the data source */
+  id?: string;
+  /** Human-readable name of the data source */
+  title: string;
+  /** The engine type of the data source */
+  type: string;
+  /** Optional metadata for the data source */
+  meta?: DataSourceMeta;
+}
+
+/**
+ * Metadata for a data source, generic to allow for additional fields.
+ */
+export interface DataSourceMeta {
+  /** Optional name used for specific purposes like async query sources */
+  name?: string;
+  /** Optional session ID for faster responses when utilizing async query sources */
+  sessionId?: string;
+}
+
+/**
+ * Represents the hierarchical structure of data within a data source.
+ *
+ * @example
+ * Example of an OpenSearch cluster with indices
+ *
+ * const openSearchCluster: DataStructure = {
+ *   id: "b18e5f58-cf71-11ee-ad92-2468ce360004",
+ *   title: "Production Cluster",
+ *   type: "OPENSEARCH",
+ *   children: [
+ *     {
+ *       id: "b18e5f58-cf71-11ee-ad92-2468ce360004::logs-2023.05",
+ *       title: "logs-2023.05",
+ *       type: "INDEX",
+ *       parent: { id: "b18e5f58-cf71-11ee-ad92-2468ce360004", title: "Production Cluster", type: "OPENSEARCH" }
+ *     },
+ *     {
+ *       id: "b18e5f58-cf71-11ee-ad92-2468ce360004::logs-2023.06",
+ *       title: "logs-2023.06",
+ *       type: "INDEX",
+ *       parent: { id: "b18e5f58-cf71-11ee-ad92-2468ce360004", title: "Production Cluster", type: "OPENSEARCH" }
+ *     }
+ *   ]
+ * };
+ *
+ * Example of an S3 data source with a database and tables:
+ *
+ * const s3DataSource: DataStructure = {
+ *   id: "7d5c3e1c-ae5f-11ee-9c91-1357bd240003",
+ *   title: "mys3",
+ *   type: "S3,
+ *   children: [
+ *     {
+ *       id: "mys3.defaultDb",
+ *       title: "defaultDb",
+ *       type: "DATABASE",
+ *       parent: { id: "myS3", title: "My S3 Bucket", type: "S3" },
+ *       children: [
+ *         {
+ *           id: "mys3.defaultDb.table1",
+ *           title: "table1",
+ *           type: "TABLE",
+ *           parent: { id: "sales-db", title: "Sales Database", type: "DATABASE" }
+ *         },
+ *         {
+ *           id: "mys3.defaultDb.table2",
+ *           title: "table2",
+ *           type: "TABLE",
+ *           parent: { id: "sales-db", title: "Sales Database", type: "DATABASE" }
+ *         }
+ *       ]
+ *     }
+ *   ]
+ * };
+ */
+export interface DataStructure {
+  /** Unique identifier for the data structure. */
+  id: string;
+  /** Human-readable name of the data structure */
+  title: string;
+  /** The type of the data structure, registered by other classes */
+  type: string;
+  /** Optional reference to the parent data structure */
+  parent?: DataStructure;
+  /** Optional array of child data structures */
+  children?: DataStructure[];
+}
+
+/**
+ * Defines the structure of a dataset, including its type and reference to a data source.
+ * NOTE: For non-index pattern datasets we will append the data source ID to the front of
+ * the title of the dataset to ensure we do not have any conflicts. Many clusters could
+ * have similar titles and the data plugin assumes unique data set IDs.
+ *
+ * @example
+ * Example of a Dataset for an OpenSearch index
+ * const logsIndexDataset: Dataset = {
+ *   id: "2e1b1b80-9c4d-11ee-8c90-0242ac120001",
+ *   title: "logs-*",
+ *   type: "INDEX_PATTERN",
+ *   timeFieldName: "@timestamp",
+ *   dataSource: {
+ *     id: "main-cluster",
+ *     title: "Main OpenSearch Cluster",
+ *     type: "DEFAULT"
+ *   },
+ * };
+ *
+ * @example
+ * Example of a Dataset for an S3 table
+ * const ordersTableDataset: Dataset = {
+ *   id: "7d5c3e1c-ae5f-11ee-9c91-1357bd240003::mys3.defaultDb.table1",
+ *   title: "mys3.defaultDb.table1",
+ *   type: "S3",
+ *   timeFieldName: "order_date",
+ *   dataSource: {
+ *     id: "7d5c3e1c-ae5f-11ee-9c91-1357bd240003",
+ *     title: "My S3 Connect",
+ *     type: "EXAMPLE_DATASOURCE"
+ *   },
+ * };
+ */
+export interface Dataset {
+  /** Unique identifier for the dataset, for non-index pattern based datasets, we will append the data source ID if present */
+  id: string;
+  /** Human-readable name of the dataset that is used to query */
+  title: string;
+  /** The type of the dataset, registered by other classes */
+  type: string;
+  /** Optional name of the field used for time-based operations */
+  timeFieldName?: string;
+  /** Optional reference to the data source */
+  dataSource?: DataSource;
+}

src/plugins/data/common/index.ts

@@ -31,7 +31,7 @@
 export * from './constants';
 export * from './opensearch_query';
 export * from './data_frames';
-export * from './data_sets';
+export * from './datasets';
 export * from './field_formats';
 export * from './field_mapping';
 export * from './index_patterns';

src/plugins/data/common/query/types.ts

@@ -9,6 +9,8 @@
  * GitHub history for details.
  */
 
+import { Dataset } from '../types';
+
 /*
  * Licensed to Elasticsearch B.V. under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -34,4 +36,5 @@ export * from './timefilter/types';
 export type Query = {
   query: string | { [key: string]: any };
   language: string;
+  dataset?: Dataset;
 };

src/plugins/data/common/types.ts

@@ -35,7 +35,7 @@ export * from './query/types';
 export * from './osd_field_types/types';
 export * from './index_patterns/types';
 export * from './data_frames/types';
-export * from './data_sets/types';
+export * from './datasets/types';
 
 /**
  * If a service is being shared on both the client and the server, and

src/plugins/data/public/query/dataset_manager/dataset_manager.test.ts

@@ -5,8 +5,9 @@
 
 import { DataSetManager } from './dataset_manager';
 import { coreMock } from '../../../../../core/public/mocks';
-import { SimpleDataSet } from '../../../common';
-describe('DataSetManager', () => {
+import { Dataset } from '../../../common';
+
+describe('DatasetManager', () => {
   let service: DataSetManager;
 
   beforeEach(() => {
@@ -17,14 +18,18 @@ describe('DataSetManager', () => {
 
   test('getUpdates$ is a cold emits only after dataset changes', () => {
     const obs$ = service.getUpdates$();
-    const emittedValues: SimpleDataSet[] = [];
+    const emittedValues: Dataset[] = [];
     obs$.subscribe((v) => {
       emittedValues.push(v!);
     });
     expect(emittedValues).toHaveLength(0);
     expect(emittedValues[0]).toEqual(undefined);
 
-    const newDataSet: SimpleDataSet = { id: 'test_dataset', title: 'Test Dataset' };
+    const newDataSet: Dataset = {
+      id: 'test_dataset',
+      title: 'Test Dataset',
+      type: 'INDEX_PATTERN',
+    };
     service.setDataSet(newDataSet);
     expect(emittedValues).toHaveLength(1);
     expect(emittedValues[0]).toEqual(newDataSet);

src/plugins/data/public/query/dataset_manager/dataset_manager.ts

@@ -6,22 +6,16 @@
 import { BehaviorSubject } from 'rxjs';
 import { CoreStart } from 'opensearch-dashboards/public';
 import { skip } from 'rxjs/operators';
-import {
-  IndexPattern,
-  SIMPLE_DATA_SET_TYPES,
-  SimpleDataSet,
-  SimpleDataSource,
-  UI_SETTINGS,
-} from '../../../common';
+import { DEFAULT_QUERY, Dataset, DataSource, IndexPattern, UI_SETTINGS } from '../../../common';
 import { IndexPatternsContract } from '../../index_patterns';
 
 export class DataSetManager {
-  private dataSet$: BehaviorSubject<SimpleDataSet | undefined>;
+  private dataSet$: BehaviorSubject<Dataset | undefined>;
   private indexPatterns?: IndexPatternsContract;
-  private defaultDataSet?: SimpleDataSet;
+  private defaultDataSet?: Dataset;
 
   constructor(private readonly uiSettings: CoreStart['uiSettings']) {
-    this.dataSet$ = new BehaviorSubject<SimpleDataSet | undefined>(undefined);
+    this.dataSet$ = new BehaviorSubject<Dataset | undefined>(undefined);
   }
 
   public init = async (indexPatterns: IndexPatternsContract) => {
@@ -39,16 +33,15 @@ export class DataSetManager {
     this.defaultDataSet = {
       id: indexPattern.id,
       title: indexPattern.title,
-      type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN,
+      type: DEFAULT_QUERY.DATASET_TYPE,
       timeFieldName: indexPattern.timeFieldName,
-      fields: indexPattern.fields,
       ...(indexPattern.dataSourceRef
         ? {
-            dataSourceRef: {
+            dataSource: {
               id: indexPattern.dataSourceRef?.id,
-              name: indexPattern.dataSourceRef?.name,
+              title: indexPattern.dataSourceRef?.name,
               type: indexPattern.dataSourceRef?.type,
-            } as SimpleDataSource,
+            } as DataSource,
           }
         : {}),
     };
@@ -66,23 +59,16 @@ export class DataSetManager {
    * Updates the query.
    * @param {Query} query
    */
-  public setDataSet = (dataSet: SimpleDataSet | undefined) => {
+  public setDataSet = (dataSet: Dataset | undefined) => {
     if (!this.uiSettings.get(UI_SETTINGS.QUERY_ENHANCEMENTS_ENABLED)) return;
-
-    // if (dataSet) {
-    //   const { fields, ...dataSetWithoutFields } = dataSet;
-    //   this.dataSet$.next(dataSetWithoutFields);
-    // } else {
-    //   this.dataSet$.next(undefined);
-    // }
     this.dataSet$.next(dataSet);
   };
 
   public getDefaultDataSet = () => {
     return this.defaultDataSet;
   };
 
-  public fetchDefaultDataSet = async (): Promise<SimpleDataSet | undefined> => {
+  public fetchDefaultDataSet = async (): Promise<Dataset | undefined> => {
     const defaultIndexPatternId = this.uiSettings.get('defaultIndex');
     if (!defaultIndexPatternId) {
       return undefined;
@@ -96,15 +82,15 @@ export class DataSetManager {
     return {
       id: indexPattern.id,
       title: indexPattern.title,
-      type: SIMPLE_DATA_SET_TYPES.INDEX_PATTERN,
+      type: DEFAULT_QUERY.DATASET_TYPE,
       timeFieldName: indexPattern.timeFieldName,
       ...(indexPattern.dataSourceRef
         ? {
-            dataSourceRef: {
+            dataSource: {
               id: indexPattern.dataSourceRef?.id,
-              name: indexPattern.dataSourceRef?.name,
+              title: indexPattern.dataSourceRef?.name,
               type: indexPattern.dataSourceRef?.type,
-            } as SimpleDataSource,
+            } as DataSource,
           }
         : {}),
     };

src/plugins/data/public/query/state_sync/types.ts

@@ -28,7 +28,7 @@
  * under the License.
  */
 
-import { Filter, RefreshInterval, TimeRange, Query, SimpleDataSet } from '../../../common';
+import { Filter, RefreshInterval, TimeRange, Query, Dataset } from '../../../common';
 
 /**
  * All query state service state
@@ -38,7 +38,6 @@ export interface QueryState {
   refreshInterval?: RefreshInterval;
   filters?: Filter[];
   query?: Query;
-  dataSet?: SimpleDataSet;
 }
 
 type QueryStateChangePartial = {