[ML][ES|QL] Adds query guardrails and technical preview badge to ES|Q…

…L data visualizer (#200325) ## Summary Adds extra guardrails to the searches, plus the technical preview badge, to the ES|QL data visualizer. \ The extra safety guards applied are: - Lowers the limit of # of rows sampled to 5000 - Changes the pagination to 10 by default, and only allows 10 to 25 rows visible concurrently - Hides distribution charts by default - Always have frozen tier excluded in the queries <img width="1164" alt="Screenshot 2024-11-16 at 18 26 20" src="https://github.com/user-attachments/assets/a558fdfa-189a-4ba9-b038-8d14df4594be"> https://github.com/user-attachments/assets/488f44dd-4a3c-48c7-8416-5091b614edfd ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_node:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) ### Identify risks Does this PR introduce any risks? For example, consider risks like hard to test bugs, performance regression, potential of data loss. Describe the risk, its severity, and mitigation for each identified risk. Invite stakeholders and evaluate how to proceed before merging. - [ ] [See some risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) - [ ] ...
elastic · Nov 19, 2024 · 94aaed3 · 94aaed3
1 parent 45056f3
commit 94aaed3
Show file tree

Hide file tree

Showing 16 changed files with 155 additions and 105 deletions.
diff --git a/...visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx b/...visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React from 'react';
+import React, { useEffect } from 'react';
 import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
 import { useExpandedRowCss } from './use_expanded_row_css';
 import { GeoPointContentWithMap } from './geo_point_content_with_map';
@@ -34,6 +34,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
   totalDocuments,
   timeFieldName,
   typeAccessor = 'type',
+  onVisibilityChange,
 }: {
   item: FieldVisConfig;
   dataView: DataView | undefined;
@@ -46,6 +47,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
    */
   onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
   timeFieldName?: string;
+  onVisibilityChange?: (visible: boolean, item: FieldVisConfig) => void;
 }) => {
   const config = { ...item, stats: { ...item.stats, totalDocuments } };
   const { loading, existsInDocs, fieldName } = config;
@@ -98,6 +100,14 @@ export const IndexBasedDataVisualizerExpandedRow = ({
     }
   }
 
+  useEffect(() => {
+    onVisibilityChange?.(true, item);
+
+    return () => {
+      onVisibilityChange?.(false, item);
+    };
+  }, [item, onVisibilityChange]);
+
   return (
     <div css={dvExpandedRow} data-test-subj={`dataVisualizerFieldExpandedRow-${fieldName}`}>
       {loading === true ? <LoadingIndicator /> : getCardContent()}

diff --git a/...sualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx b/...sualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx
@@ -63,6 +63,7 @@ interface DataVisualizerTableProps<T extends object> {
   overallStatsRunning: boolean;
   renderFieldName?: FieldStatisticTableEmbeddableProps['renderFieldName'];
   error?: Error | string;
+  isEsql?: boolean;
 }
 
 const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
@@ -78,6 +79,7 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
   overallStatsRunning,
   renderFieldName,
   error,
+  isEsql = false,
 }: DataVisualizerTableProps<T>) => {
   const { euiTheme } = useEuiTheme();
 
@@ -87,7 +89,8 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
   const { onTableChange, pagination, sorting } = useTableSettings<T>(
     items,
     pageState,
-    updatePageState
+    updatePageState,
+    isEsql
   );
   const [showDistributions, setShowDistributions] = useState<boolean>(showPreviewByDefault ?? true);
   const [dimensions, setDimensions] = useState(calculateTableColumnsDimensions());

diff --git a/...ns/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts b/...ns/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts
@@ -27,7 +27,8 @@ interface UseTableSettingsReturnValue<T extends object> {
 export function useTableSettings<TypeOfItem extends object>(
   items: TypeOfItem[],
   pageState: DataVisualizerTableState,
-  updatePageState: (update: DataVisualizerTableState) => void
+  updatePageState: (update: DataVisualizerTableState) => void,
+  isEsql: boolean = false
 ): UseTableSettingsReturnValue<TypeOfItem> {
   const { pageIndex, pageSize, sortField, sortDirection } = pageState;
 
@@ -50,9 +51,9 @@ export function useTableSettings<TypeOfItem extends object>(
       pageIndex,
       pageSize,
       totalItemCount: items.length,
-      pageSizeOptions: PAGE_SIZE_OPTIONS,
+      pageSizeOptions: isEsql ? [10, 25] : PAGE_SIZE_OPTIONS,
     }),
-    [items, pageIndex, pageSize]
+    [items, pageIndex, pageSize, isEsql]
   );
 
   const sorting = useMemo(

diff --git a/...ndex_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx b/...ndex_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx
@@ -332,6 +332,7 @@ export const IndexDataVisualizerESQL: FC<IndexDataVisualizerESQLProps> = (dataVi
 
               <EuiProgress value={combinedProgress} max={100} size="xs" />
               <DataVisualizerTable
+                isEsql={true}
                 items={configs}
                 pageState={dataVisualizerListState}
                 updatePageState={setDataVisualizerListState}

diff --git a/...izer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx b/...izer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx
@@ -27,14 +27,6 @@ const options = [
       values: { limit: '10,000' },
     }),
   },
-  {
-    'data-test-subj': 'dvESQLLimitSize-100000',
-    value: '100000',
-    text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
-      defaultMessage: '{limit} rows',
-      values: { limit: '100,000' },
-    }),
-  },
 ];
 
 export const ESQLDefaultLimitSizeSelect = ({

diff --git a/...gins/data_visualizer/public/application/index_data_visualizer/constants/esql_constants.ts b/...gins/data_visualizer/public/application/index_data_visualizer/constants/esql_constants.ts
@@ -5,4 +5,4 @@
  * 2.0.
  */
 
-export const DEFAULT_ESQL_LIMIT = 10000;
+export const DEFAULT_ESQL_LIMIT = 5000;
diff --git a/...n/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx b/...n/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx
@@ -70,6 +70,7 @@ const EmbeddableESQLFieldStatsTableWrapper = React.memo(
 
     return (
       <DataVisualizerTable<FieldVisConfig>
+        isEsql={true}
         items={configs}
         pageState={dataVisualizerListState}
         updatePageState={onTableChange}

diff --git a/...zer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx b/...zer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx
@@ -53,14 +53,14 @@ const defaultSearchQuery = {
 };
 
 const FALLBACK_ESQL_QUERY: ESQLQuery = { esql: '' };
-const DEFAULT_LIMIT_SIZE = '10000';
+const DEFAULT_LIMIT_SIZE = '5000';
 const defaults = getDefaultPageState();
 
 export const getDefaultESQLDataVisualizerListState = (
   overrides?: Partial<ESQLDataVisualizerIndexBasedAppState>
 ): Required<ESQLDataVisualizerIndexBasedAppState> => ({
   pageIndex: 0,
-  pageSize: 25,
+  pageSize: 10,
   sortField: 'fieldName',
   sortDirection: 'asc',
   visibleFieldTypes: [],
@@ -70,7 +70,7 @@ export const getDefaultESQLDataVisualizerListState = (
   searchQuery: defaultSearchQuery,
   searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY,
   filters: [],
-  showDistributions: true,
+  showDistributions: false,
   showAllFields: false,
   showEmptyFields: false,
   probability: null,
@@ -229,6 +229,21 @@ export const useESQLDataVisualizerData = (
           } as QueryDslQueryContainer;
         }
       }
+
+      // Ensure that we don't query frozen data
+      if (filter.bool === undefined) {
+        filter.bool = Object.create(null);
+      }
+
+      if (filter.bool && filter.bool.must_not === undefined) {
+        filter.bool.must_not = [];
+      }
+
+      if (filter.bool && Array.isArray(filter?.bool?.must_not)) {
+        filter.bool.must_not!.push({
+          term: { _tier: 'data_frozen' },
+        });
+      }
       return {
         id: input.id,
         earliest,
@@ -332,9 +347,25 @@ export const useESQLDataVisualizerData = (
 
   const visibleFieldTypes =
     dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes;
+  const [expandedRows, setExpandedRows] = useState<string[]>([]);
+
+  const onVisibilityChange = useCallback((visible: boolean, item: FieldVisConfig) => {
+    if (visible) {
+      setExpandedRows((prev) => [...prev, item.fieldName]);
+    } else {
+      setExpandedRows((prev) => prev.filter((fieldName) => fieldName !== item.fieldName));
+    }
+  }, []);
+
+  const hasExpandedRows = useMemo(() => expandedRows.length > 0, [expandedRows]);
 
   useEffect(
     function updateFieldStatFieldsToFetch() {
+      if (dataVisualizerListState?.showDistributions === false && !hasExpandedRows) {
+        setFieldStatFieldsToFetch(undefined);
+        return;
+      }
+
       const { sortField, sortDirection } = dataVisualizerListState;
 
       // Otherwise, sort the list of fields by the initial sort field and sort direction
@@ -376,6 +407,8 @@ export const useESQLDataVisualizerData = (
       dataVisualizerListState.sortDirection,
       nonMetricConfigs,
       metricConfigs,
+      dataVisualizerListState?.showDistributions,
+      hasExpandedRows,
     ]
   );
 
@@ -618,14 +651,15 @@ export const useESQLDataVisualizerData = (
               typeAccessor="secondaryType"
               timeFieldName={timeFieldName}
               onAddFilter={input.onAddFilter}
+              onVisibilityChange={onVisibilityChange}
             />
           );
         }
         return map;
       }, {} as ItemIdToExpandedRowMap);
     },
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    [currentDataView, totalCount, query.esql, timeFieldName]
+    [currentDataView, totalCount, query.esql, timeFieldName, onVisibilityChange]
   );
 
   const combinedProgress = useMemo(

diff --git a/...alizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts b/...alizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts
@@ -313,7 +313,6 @@ export const useESQLOverallStatsData = (
           { strategy: ESQL_ASYNC_SEARCH_STRATEGY }
         )) as ESQLResponse | undefined;
         setQueryHistoryStatus(false);
-
         const columnInfo = columnsResp?.rawResponse
           ? columnsResp.rawResponse.all_columns ?? columnsResp.rawResponse.columns
           : [];

diff --git a/...lugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/...lugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx
@@ -16,7 +16,6 @@ import { i18n } from '@kbn/i18n';
 import { Storage } from '@kbn/kibana-utils-plugin/public';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
-import { KibanaThemeProvider } from '@kbn/react-kibana-context-theme';
 import { StorageContextProvider } from '@kbn/ml-local-storage';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { getNestedProperty } from '@kbn/ml-nested-property';
@@ -49,8 +48,6 @@ import { DATA_VISUALIZER_INDEX_VIEWER } from './constants/index_data_visualizer_
 import { INDEX_DATA_VISUALIZER_NAME } from '../common/constants';
 import { DV_STORAGE_KEYS } from './types/storage';
 
-const XXL_BREAKPOINT = 1400;
-
 const localStorage = new Storage(window.localStorage);
 
 export interface DataVisualizerStateContextProviderProps {
@@ -341,29 +338,20 @@ export const IndexDataVisualizer: FC<Props> = ({
 
   return (
     <KibanaRenderContextProvider {...startServices}>
-      <KibanaThemeProvider
-        theme={coreStart.theme}
-        modify={{
-          breakpoint: {
-            xxl: XXL_BREAKPOINT,
-          },
-        }}
-      >
-        <KibanaContextProvider services={{ ...services }}>
-          <StorageContextProvider storage={localStorage} storageKeys={DV_STORAGE_KEYS}>
-            <DatePickerContextProvider {...datePickerDeps}>
-              {!esql ? (
-                <DataVisualizerStateContextProvider
-                  IndexDataVisualizerComponent={IndexDataVisualizerView}
-                  getAdditionalLinks={getAdditionalLinks}
-                />
-              ) : (
-                <DataVisualizerESQLStateContextProvider />
-              )}
-            </DatePickerContextProvider>
-          </StorageContextProvider>
-        </KibanaContextProvider>
-      </KibanaThemeProvider>
+      <KibanaContextProvider services={{ ...services }}>
+        <StorageContextProvider storage={localStorage} storageKeys={DV_STORAGE_KEYS}>
+          <DatePickerContextProvider {...datePickerDeps}>
+            {!esql ? (
+              <DataVisualizerStateContextProvider
+                IndexDataVisualizerComponent={IndexDataVisualizerView}
+                getAdditionalLinks={getAdditionalLinks}
+              />
+            ) : (
+              <DataVisualizerESQLStateContextProvider />
+            )}
+          </DatePickerContextProvider>
+        </StorageContextProvider>
+      </KibanaContextProvider>
     </KibanaRenderContextProvider>
   );
 };
diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json
@@ -76,7 +76,6 @@
     "@kbn/ml-time-buckets",
     "@kbn/aiops-log-rate-analysis",
     "@kbn/react-kibana-context-render",
-    "@kbn/react-kibana-context-theme",
     "@kbn/presentation-publishing",
     "@kbn/shared-ux-utility",
     "@kbn/search-types",

diff --git a/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx b/x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx
@@ -18,6 +18,7 @@ import {
   EuiLink,
   EuiSpacer,
   EuiText,
+  EuiBetaBadge,
   EuiTextAlign,
 } from '@elastic/eui';
 
@@ -64,7 +65,6 @@ export const DatavisualizerSelector: FC = () => {
     },
   } = useMlKibana();
   const isEsqlEnabled = useMemo(() => uiSettings.get(ENABLE_ESQL), [uiSettings]);
-
   const helpLink = docLinks.links.ml.guide;
   const navigateToPath = useNavigateToPath();
 
@@ -172,6 +172,19 @@ export const DatavisualizerSelector: FC = () => {
                       <FormattedMessage
                         id="xpack.ml.datavisualizer.selector.selectESQLTitle"
                         defaultMessage="Visualize data using ES|QL"
+                      />{' '}
+                      <EuiBetaBadge
+                        label=""
+                        iconType="beaker"
+                        size="m"
+                        color="hollow"
+                        tooltipContent={
+                          <FormattedMessage
+                            id="xpack.ml.datavisualizer.selector.esqlTechnicalPreviewBadge.titleMsg"
+                            defaultMessage="ES|QL data visualizer is in technical preview."
+                          />
+                        }
+                        tooltipPosition={'right'}
                       />
                     </>
                   </EuiTextAlign>

diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx
@@ -16,7 +16,7 @@ import type {
   GetAdditionalLinksParams,
 } from '@kbn/data-visualizer-plugin/public';
 import { useTimefilter } from '@kbn/ml-date-picker';
-import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { EuiFlexGroup, EuiFlexItem, useEuiTheme } from '@elastic/eui';
 import useMountedState from 'react-use/lib/useMountedState';
 import { useMlApi, useMlKibana, useMlLocator } from '../../contexts/kibana';
 import { HelpMenu } from '../../components/help_menu';
@@ -26,6 +26,7 @@ import { mlNodesAvailable, getMlNodeCount } from '../../ml_nodes_check/check_ml_
 import { checkPermission } from '../../capabilities/check_capabilities';
 import { MlPageHeader } from '../../components/page_header';
 import { useEnabledFeatures } from '../../contexts/ml';
+import { TechnicalPreviewBadge } from '../../components/technical_preview_badge';
 export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false }) => {
   useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false });
   const {
@@ -188,6 +189,7 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
     // eslint-disable-next-line react-hooks/exhaustive-deps
     [mlLocator, mlFeaturesDisabled]
   );
+  const { euiTheme } = useEuiTheme();
   return IndexDataVisualizer ? (
     <Fragment>
       {IndexDataVisualizer !== null ? (
@@ -203,6 +205,9 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
                   <EuiFlexItem grow={false}>
                     <FormattedMessage id="xpack.ml.datavisualizer" defaultMessage="(ES|QL)" />
                   </EuiFlexItem>
+                  <EuiFlexItem grow={false} css={{ marginTop: euiTheme.size.xs }}>
+                    <TechnicalPreviewBadge />
+                  </EuiFlexItem>
                 </>
               ) : null}
             </EuiFlexGroup>
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,4 +5,4 @@ @@
      * 2.0.
      */
-    export const DEFAULT_ESQL_LIMIT = 10000;
+    export const DEFAULT_ESQL_LIMIT = 5000;