[8.x] [ML][ES|QL] Adds query guardrails and technical preview badge t…

…o ES|QL data visualizer (#200325) (#200677) # Backport This will backport the following commits from `main` to `8.x`: - [[ML][ES|QL] Adds query guardrails and technical preview badge to ES|QL data visualizer (#200325)](#200325)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Quynh Nguyen (Quinn) <[email protected]>
elastic · Nov 19, 2024 · 14d8575 · 14d8575
1 parent cffa72b
commit 14d8575
Show file tree

Hide file tree

Showing 16 changed files with 155 additions and 105 deletions.
diff --git a/...visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/...visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React from 'react';
+import React, { useEffect } from 'react';
 import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
 import { useExpandedRowCss } from './use_expanded_row_css';
 import { GeoPointContentWithMap } from './geo_point_content_with_map';
@@ -34,6 +34,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
   totalDocuments,
   timeFieldName,
   typeAccessor = 'type',
+  onVisibilityChange,
 }: {
   item: FieldVisConfig;
   dataView: DataView | undefined;
@@ -46,6 +47,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
    */
   onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
   timeFieldName?: string;
+  onVisibilityChange?: (visible: boolean, item: FieldVisConfig) => void;
 }) => {
   const config = { ...item, stats: { ...item.stats, totalDocuments } };
   const { loading, existsInDocs, fieldName } = config;
@@ -98,6 +100,14 @@ export const IndexBasedDataVisualizerExpandedRow = ({
     }
   }
 
+  useEffect(() => {
+    onVisibilityChange?.(true, item);
+
+    return () => {
+      onVisibilityChange?.(false, item);
+    };
+  }, [item, onVisibilityChange]);
+
   return (
     <div css={dvExpandedRow} data-test-subj={`dataVisualizerFieldExpandedRow-${fieldName}`}>
       {loading === true ? <LoadingIndicator /> : getCardContent()}

diff --git a/...sualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/...sualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx
@@ -63,6 +63,7 @@ interface DataVisualizerTableProps<T extends object> {
   overallStatsRunning: boolean;
   renderFieldName?: FieldStatisticTableEmbeddableProps['renderFieldName'];
   error?: Error | string;
+  isEsql?: boolean;
 }
 
 const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
@@ -78,6 +79,7 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
   overallStatsRunning,
   renderFieldName,
   error,
+  isEsql = false,
 }: DataVisualizerTableProps<T>) => {
   const { euiTheme } = useEuiTheme();
 
@@ -87,7 +89,8 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
   const { onTableChange, pagination, sorting } = useTableSettings<T>(
     items,
     pageState,
-    updatePageState
+    updatePageState,
+    isEsql
   );
   const [showDistributions, setShowDistributions] = useState<boolean>(showPreviewByDefault ?? true);
   const [dimensions, setDimensions] = useState(calculateTableColumnsDimensions());

diff --git a/...ns/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts b/...ns/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts
@@ -27,7 +27,8 @@ interface UseTableSettingsReturnValue<T extends object> {
 export function useTableSettings<TypeOfItem extends object>(
   items: TypeOfItem[],
   pageState: DataVisualizerTableState,
-  updatePageState: (update: DataVisualizerTableState) => void
+  updatePageState: (update: DataVisualizerTableState) => void,
+  isEsql: boolean = false
 ): UseTableSettingsReturnValue<TypeOfItem> {
   const { pageIndex, pageSize, sortField, sortDirection } = pageState;
 
@@ -50,9 +51,9 @@ export function useTableSettings<TypeOfItem extends object>(
       pageIndex,
       pageSize,
       totalItemCount: items.length,
-      pageSizeOptions: PAGE_SIZE_OPTIONS,
+      pageSizeOptions: isEsql ? [10, 25] : PAGE_SIZE_OPTIONS,
     }),
-    [items, pageIndex, pageSize]
+    [items, pageIndex, pageSize, isEsql]
   );
 
   const sorting = useMemo(

diff --git a/...ndex_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx b/...ndex_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx
@@ -332,6 +332,7 @@ export const IndexDataVisualizerESQL: FC<IndexDataVisualizerESQLProps> = (dataVi
 
               <EuiProgress value={combinedProgress} max={100} size="xs" />
               <DataVisualizerTable
+                isEsql={true}
                 items={configs}
                 pageState={dataVisualizerListState}
                 updatePageState={setDataVisualizerListState}

diff --git a/...izer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx b/...izer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx
@@ -27,14 +27,6 @@ const options = [
       values: { limit: '10,000' },
     }),
   },
-  {
-    'data-test-subj': 'dvESQLLimitSize-100000',
-    value: '100000',
-    text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
-      defaultMessage: '{limit} rows',
-      values: { limit: '100,000' },
-    }),
-  },
 ];
 
 export const ESQLDefaultLimitSizeSelect = ({

diff --git a/...gins/data_visualizer/public/application/index_data_visualizer/constants/esql_constants.ts b/...gins/data_visualizer/public/application/index_data_visualizer/constants/esql_constants.ts
@@ -5,4 +5,4 @@
  * 2.0.
  */
 
-export const DEFAULT_ESQL_LIMIT = 10000;
+export const DEFAULT_ESQL_LIMIT = 5000;
diff --git a/...n/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx b/...n/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx
@@ -70,6 +70,7 @@ const EmbeddableESQLFieldStatsTableWrapper = React.memo(
 
     return (
       <DataVisualizerTable<FieldVisConfig>
+        isEsql={true}
         items={configs}
         pageState={dataVisualizerListState}
         updatePageState={onTableChange}

diff --git a/...zer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx b/...zer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx
@@ -53,14 +53,14 @@ const defaultSearchQuery = {
 };
 
 const FALLBACK_ESQL_QUERY: ESQLQuery = { esql: '' };
-const DEFAULT_LIMIT_SIZE = '10000';
+const DEFAULT_LIMIT_SIZE = '5000';
 const defaults = getDefaultPageState();
 
 export const getDefaultESQLDataVisualizerListState = (
   overrides?: Partial<ESQLDataVisualizerIndexBasedAppState>
 ): Required<ESQLDataVisualizerIndexBasedAppState> => ({
   pageIndex: 0,
-  pageSize: 25,
+  pageSize: 10,
   sortField: 'fieldName',
   sortDirection: 'asc',
   visibleFieldTypes: [],
@@ -70,7 +70,7 @@ export const getDefaultESQLDataVisualizerListState = (
   searchQuery: defaultSearchQuery,
   searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY,
   filters: [],
-  showDistributions: true,
+  showDistributions: false,
   showAllFields: false,
   showEmptyFields: false,
   probability: null,
@@ -229,6 +229,21 @@ export const useESQLDataVisualizerData = (
           } as QueryDslQueryContainer;
         }
       }
+
+      // Ensure that we don't query frozen data
+      if (filter.bool === undefined) {
+        filter.bool = Object.create(null);
+      }
+
+      if (filter.bool && filter.bool.must_not === undefined) {
+        filter.bool.must_not = [];
+      }
+
+      if (filter.bool && Array.isArray(filter?.bool?.must_not)) {
+        filter.bool.must_not!.push({
+          term: { _tier: 'data_frozen' },
+        });
+      }
       return {
         id: input.id,
         earliest,
@@ -332,9 +347,25 @@ export const useESQLDataVisualizerData = (
 
   const visibleFieldTypes =
     dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes;
+  const [expandedRows, setExpandedRows] = useState<string[]>([]);
+
+  const onVisibilityChange = useCallback((visible: boolean, item: FieldVisConfig) => {
+    if (visible) {
+      setExpandedRows((prev) => [...prev, item.fieldName]);
+    } else {
+      setExpandedRows((prev) => prev.filter((fieldName) => fieldName !== item.fieldName));
+    }
+  }, []);
+
+  const hasExpandedRows = useMemo(() => expandedRows.length > 0, [expandedRows]);
 
   useEffect(
     function updateFieldStatFieldsToFetch() {
+      if (dataVisualizerListState?.showDistributions === false && !hasExpandedRows) {
+        setFieldStatFieldsToFetch(undefined);
+        return;
+      }
+
       const { sortField, sortDirection } = dataVisualizerListState;
 
       // Otherwise, sort the list of fields by the initial sort field and sort direction
@@ -376,6 +407,8 @@ export const useESQLDataVisualizerData = (
       dataVisualizerListState.sortDirection,
       nonMetricConfigs,
       metricConfigs,
+      dataVisualizerListState?.showDistributions,
+      hasExpandedRows,
     ]
   );
 
@@ -618,14 +651,15 @@ export const useESQLDataVisualizerData = (
               typeAccessor="secondaryType"
               timeFieldName={timeFieldName}
               onAddFilter={input.onAddFilter}
+              onVisibilityChange={onVisibilityChange}
             />
           );
         }
         return map;
       }, {} as ItemIdToExpandedRowMap);
     },
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    [currentDataView, totalCount, query.esql, timeFieldName]
+    [currentDataView, totalCount, query.esql, timeFieldName, onVisibilityChange]
   );
 
   const combinedProgress = useMemo(

diff --git a/...alizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts b/...alizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts
@@ -313,7 +313,6 @@ export const useESQLOverallStatsData = (
           { strategy: ESQL_ASYNC_SEARCH_STRATEGY }
         )) as ESQLResponse | undefined;
         setQueryHistoryStatus(false);
-
         const columnInfo = columnsResp?.rawResponse
           ? columnsResp.rawResponse.all_columns ?? columnsResp.rawResponse.columns
           : [];

diff --git a/...lugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/...lugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx
@@ -16,7 +16,6 @@ import { i18n } from '@kbn/i18n';
 import { Storage } from '@kbn/kibana-utils-plugin/public';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
-import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme';
 import { StorageContextProvider } from '@kbn/ml-local-storage';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { getNestedProperty } from '@kbn/ml-nested-property';
@@ -49,8 +48,6 @@ import { DATA_VISUALIZER_INDEX_VIEWER } from './constants/index_data_visualizer_
 import { INDEX_DATA_VISUALIZER_NAME } from '../common/constants';
 import { DV_STORAGE_KEYS } from './types/storage';
 
-const XXL_BREAKPOINT = 1400;
-
 const localStorage = new Storage(window.localStorage);
 
 export interface DataVisualizerStateContextProviderProps {
@@ -341,29 +338,20 @@ export const IndexDataVisualizer: FC<Props> = ({
 
   return (
     <KibanaRenderContextProvider {...startServices}>
-      <KibanaThemeProvider
-        theme={coreStart.theme}
-        modify={{
-          breakpoint: {
-            xxl: XXL_BREAKPOINT,
-          },
-        }}
-      >
-        <KibanaContextProvider services={{ ...services }}>
-          <StorageContextProvider storage={localStorage} storageKeys={DV_STORAGE_KEYS}>
-            <DatePickerContextProvider {...datePickerDeps}>
-              {!esql ? (
-                <DataVisualizerStateContextProvider
-                  IndexDataVisualizerComponent={IndexDataVisualizerView}
-                  getAdditionalLinks={getAdditionalLinks}
-                />
-              ) : (
-                <DataVisualizerESQLStateContextProvider />
-              )}
-            </DatePickerContextProvider>
-          </StorageContextProvider>
-        </KibanaContextProvider>
-      </KibanaThemeProvider>
+      <KibanaContextProvider services={{ ...services }}>
+        <StorageContextProvider storage={localStorage} storageKeys={DV_STORAGE_KEYS}>
+          <DatePickerContextProvider {...datePickerDeps}>
+            {!esql ? (
+              <DataVisualizerStateContextProvider
+                IndexDataVisualizerComponent={IndexDataVisualizerView}
+                getAdditionalLinks={getAdditionalLinks}
+              />
+            ) : (
+              <DataVisualizerESQLStateContextProvider />
+            )}
+          </DatePickerContextProvider>
+        </StorageContextProvider>
+      </KibanaContextProvider>
     </KibanaRenderContextProvider>
   );
 };
diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json
@@ -76,7 +76,6 @@
     "@kbn/ml-time-buckets",
     "@kbn/aiops-log-rate-analysis",
     "@kbn/react-kibana-context-render",
-    "@kbn/react-kibana-context-theme",
     "@kbn/presentation-publishing",
     "@kbn/shared-ux-utility",
     "@kbn/search-types",

diff --git a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx
@@ -18,6 +18,7 @@ import {
   EuiLink,
   EuiSpacer,
   EuiText,
+  EuiBetaBadge,
   EuiTextAlign,
 } from '@elastic/eui';
 
@@ -64,7 +65,6 @@ export const DatavisualizerSelector: FC = () => {
     },
   } = useMlKibana();
   const isEsqlEnabled = useMemo(() => uiSettings.get(ENABLE_ESQL), [uiSettings]);
-
   const helpLink = docLinks.links.ml.guide;
   const navigateToPath = useNavigateToPath();
 
@@ -172,6 +172,19 @@ export const DatavisualizerSelector: FC = () => {
                       <FormattedMessage
                         id="xpack.ml.datavisualizer.selector.selectESQLTitle"
                         defaultMessage="Visualize data using ES|QL"
+                      />{' '}
+                      <EuiBetaBadge
+                        label=""
+                        iconType="beaker"
+                        size="m"
+                        color="hollow"
+                        tooltipContent={
+                          <FormattedMessage
+                            id="xpack.ml.datavisualizer.selector.esqlTechnicalPreviewBadge.titleMsg"
+                            defaultMessage="ES|QL data visualizer is in technical preview."
+                          />
+                        }
+                        tooltipPosition={'right'}
                       />
                     </>
                   </EuiTextAlign>

diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx
@@ -16,7 +16,7 @@ import type {
   GetAdditionalLinksParams,
 } from '@kbn/data-visualizer-plugin/public';
 import { useTimefilter } from '@kbn/ml-date-picker';
-import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { EuiFlexGroup, EuiFlexItem, useEuiTheme } from '@elastic/eui';
 import useMountedState from 'react-use/lib/useMountedState';
 import { useMlApi, useMlKibana, useMlLocator } from '../../contexts/kibana';
 import { HelpMenu } from '../../components/help_menu';
@@ -26,6 +26,7 @@ import { mlNodesAvailable, getMlNodeCount } from '../../ml_nodes_check/check_ml_
 import { checkPermission } from '../../capabilities/check_capabilities';
 import { MlPageHeader } from '../../components/page_header';
 import { useEnabledFeatures } from '../../contexts/ml';
+import { TechnicalPreviewBadge } from '../../components/technical_preview_badge';
 export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false }) => {
   useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false });
   const {
@@ -188,6 +189,7 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
     // eslint-disable-next-line react-hooks/exhaustive-deps
     [mlLocator, mlFeaturesDisabled]
   );
+  const { euiTheme } = useEuiTheme();
   return IndexDataVisualizer ? (
     <Fragment>
       {IndexDataVisualizer !== null ? (
@@ -203,6 +205,9 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
                   <EuiFlexItem grow={false}>
                     <FormattedMessage id="xpack.ml.datavisualizer" defaultMessage="(ES|QL)" />
                   </EuiFlexItem>
+                  <EuiFlexItem grow={false} css={{ marginTop: euiTheme.size.xs }}>
+                    <TechnicalPreviewBadge />
+                  </EuiFlexItem>
                 </>
               ) : null}
             </EuiFlexGroup>
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,4 +5,4 @@ @@
      * 2.0.
      */
-    export const DEFAULT_ESQL_LIMIT = 10000;
+    export const DEFAULT_ESQL_LIMIT = 5000;