[ML][ES|QL] Adds query guardrails and technical preview badge to ES|QL data visualizer

x-pack/plugins/data_visualizer/public/application/common/components/expanded_row/index_based_expanded_row.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React from 'react';
+import React, { useEffect } from 'react';
 import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
 import { useExpandedRowCss } from './use_expanded_row_css';
 import { GeoPointContentWithMap } from './geo_point_content_with_map';
@@ -34,6 +34,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
   totalDocuments,
   timeFieldName,
   typeAccessor = 'type',
+  onVisibilityChange,
 }: {
   item: FieldVisConfig;
   dataView: DataView | undefined;
@@ -46,6 +47,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
    */
   onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
   timeFieldName?: string;
+  onVisibilityChange?: (visible: boolean, item: FieldVisConfig) => void;
 }) => {
   const config = { ...item, stats: { ...item.stats, totalDocuments } };
   const { loading, existsInDocs, fieldName } = config;
@@ -98,6 +100,14 @@ export const IndexBasedDataVisualizerExpandedRow = ({
     }
   }
 
+  useEffect(() => {
+    onVisibilityChange?.(true, item);
+
+    return () => {
+      onVisibilityChange?.(false, item);
+    };
+  }, [item, onVisibilityChange]);
+
   return (
     <div css={dvExpandedRow} data-test-subj={`dataVisualizerFieldExpandedRow-${fieldName}`}>
       {loading === true ? <LoadingIndicator /> : getCardContent()}

x-pack/plugins/data_visualizer/public/application/common/components/stats_table/data_visualizer_stats_table.tsx

@@ -63,6 +63,7 @@ interface DataVisualizerTableProps<T extends object> {
   overallStatsRunning: boolean;
   renderFieldName?: FieldStatisticTableEmbeddableProps['renderFieldName'];
   error?: Error | string;
+  isEsql?: boolean;
 }
 
 const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
@@ -78,6 +79,7 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
   overallStatsRunning,
   renderFieldName,
   error,
+  isEsql = false,
 }: DataVisualizerTableProps<T>) => {
   const { euiTheme } = useEuiTheme();
 
@@ -87,7 +89,8 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
   const { onTableChange, pagination, sorting } = useTableSettings<T>(
     items,
     pageState,
-    updatePageState
+    updatePageState,
+    isEsql
   );
   const [showDistributions, setShowDistributions] = useState<boolean>(showPreviewByDefault ?? true);
   const [dimensions, setDimensions] = useState(calculateTableColumnsDimensions());

x-pack/plugins/data_visualizer/public/application/common/components/stats_table/use_table_settings.ts

@@ -27,7 +27,8 @@ interface UseTableSettingsReturnValue<T extends object> {
 export function useTableSettings<TypeOfItem extends object>(
   items: TypeOfItem[],
   pageState: DataVisualizerTableState,
-  updatePageState: (update: DataVisualizerTableState) => void
+  updatePageState: (update: DataVisualizerTableState) => void,
+  isEsql: boolean = false
 ): UseTableSettingsReturnValue<TypeOfItem> {
   const { pageIndex, pageSize, sortField, sortDirection } = pageState;
 
@@ -50,9 +51,9 @@ export function useTableSettings<TypeOfItem extends object>(
       pageIndex,
       pageSize,
       totalItemCount: items.length,
-      pageSizeOptions: PAGE_SIZE_OPTIONS,
+      pageSizeOptions: isEsql ? [10, 25] : PAGE_SIZE_OPTIONS,
     }),
-    [items, pageIndex, pageSize]
+    [items, pageIndex, pageSize, isEsql]
   );
 
   const sorting = useMemo(

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx

@@ -332,6 +332,7 @@ export const IndexDataVisualizerESQL: FC<IndexDataVisualizerESQLProps> = (dataVi
 
               <EuiProgress value={combinedProgress} max={100} size="xs" />
               <DataVisualizerTable
+                isEsql={true}
                 items={configs}
                 pageState={dataVisualizerListState}
                 updatePageState={setDataVisualizerListState}

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/esql/limit_size.tsx

@@ -27,14 +27,6 @@ const options = [
       values: { limit: '10,000' },
     }),
   },
-  {
-    'data-test-subj': 'dvESQLLimitSize-100000',
-    value: '100000',
-    text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
-      defaultMessage: '{limit} rows',
-      values: { limit: '100,000' },
-    }),
-  },
 ];
 
 export const ESQLDefaultLimitSizeSelect = ({

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/constants/esql_constants.ts

@@ -5,4 +5,4 @@
  * 2.0.
  */
 
-export const DEFAULT_ESQL_LIMIT = 10000;
+export const DEFAULT_ESQL_LIMIT = 5000;

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/embeddable_esql_field_stats_table.tsx

@@ -70,6 +70,7 @@ const EmbeddableESQLFieldStatsTableWrapper = React.memo(
 
     return (
       <DataVisualizerTable<FieldVisConfig>
+        isEsql={true}
         items={configs}
         pageState={dataVisualizerListState}
         updatePageState={onTableChange}

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx

@@ -53,14 +53,14 @@ const defaultSearchQuery = {
 };
 
 const FALLBACK_ESQL_QUERY: ESQLQuery = { esql: '' };
-const DEFAULT_LIMIT_SIZE = '10000';
+const DEFAULT_LIMIT_SIZE = '5000';
 const defaults = getDefaultPageState();
 
 export const getDefaultESQLDataVisualizerListState = (
   overrides?: Partial<ESQLDataVisualizerIndexBasedAppState>
 ): Required<ESQLDataVisualizerIndexBasedAppState> => ({
   pageIndex: 0,
-  pageSize: 25,
+  pageSize: 10,
   sortField: 'fieldName',
   sortDirection: 'asc',
   visibleFieldTypes: [],
@@ -70,7 +70,7 @@ export const getDefaultESQLDataVisualizerListState = (
   searchQuery: defaultSearchQuery,
   searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY,
   filters: [],
-  showDistributions: true,
+  showDistributions: false,
   showAllFields: false,
   showEmptyFields: false,
   probability: null,
@@ -229,6 +229,21 @@ export const useESQLDataVisualizerData = (
           } as QueryDslQueryContainer;
         }
       }
+
+      // Ensure that we don't query frozen data
+      if (filter.bool === undefined) {
+        filter.bool = Object.create(null);
+      }
+
+      if (filter.bool && filter.bool.must_not === undefined) {
+        filter.bool.must_not = [];
+      }
+
+      if (filter.bool && Array.isArray(filter?.bool?.must_not)) {
+        filter.bool.must_not!.push({
+          term: { _tier: 'data_frozen' },
+        });
+      }
       return {
         id: input.id,
         earliest,
@@ -332,9 +347,25 @@ export const useESQLDataVisualizerData = (
 
   const visibleFieldTypes =
     dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes;
+  const [expandedRows, setExpandedRows] = useState<string[]>([]);
+
+  const onVisibilityChange = useCallback((visible: boolean, item: FieldVisConfig) => {
+    if (visible) {
+      setExpandedRows((prev) => [...prev, item.fieldName]);
+    } else {
+      setExpandedRows((prev) => prev.filter((fieldName) => fieldName !== item.fieldName));
+    }
+  }, []);
+
+  const hasExpandedRows = useMemo(() => expandedRows.length > 0, [expandedRows]);
 
   useEffect(
     function updateFieldStatFieldsToFetch() {
+      if (dataVisualizerListState?.showDistributions === false && !hasExpandedRows) {
+        setFieldStatFieldsToFetch(undefined);
+        return;
+      }
+
       const { sortField, sortDirection } = dataVisualizerListState;
 
       // Otherwise, sort the list of fields by the initial sort field and sort direction
@@ -376,6 +407,8 @@ export const useESQLDataVisualizerData = (
       dataVisualizerListState.sortDirection,
       nonMetricConfigs,
       metricConfigs,
+      dataVisualizerListState?.showDistributions,
+      hasExpandedRows,
     ]
   );
 
@@ -618,14 +651,15 @@ export const useESQLDataVisualizerData = (
               typeAccessor="secondaryType"
               timeFieldName={timeFieldName}
               onAddFilter={input.onAddFilter}
+              onVisibilityChange={onVisibilityChange}
             />
           );
         }
         return map;
       }, {} as ItemIdToExpandedRowMap);
     },
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    [currentDataView, totalCount, query.esql, timeFieldName]
+    [currentDataView, totalCount, query.esql, timeFieldName, onVisibilityChange]
   );
 
   const combinedProgress = useMemo(

x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts

@@ -313,7 +313,6 @@ export const useESQLOverallStatsData = (
           { strategy: ESQL_ASYNC_SEARCH_STRATEGY }
         )) as ESQLResponse | undefined;
         setQueryHistoryStatus(false);
-
         const columnInfo = columnsResp?.rawResponse
           ? columnsResp.rawResponse.all_columns ?? columnsResp.rawResponse.columns
           : [];

x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx

@@ -18,6 +18,7 @@ import {
   EuiLink,
   EuiSpacer,
   EuiText,
+  EuiBetaBadge,
   EuiTextAlign,
 } from '@elastic/eui';
 
@@ -64,7 +65,6 @@ export const DatavisualizerSelector: FC = () => {
     },
   } = useMlKibana();
   const isEsqlEnabled = useMemo(() => uiSettings.get(ENABLE_ESQL), [uiSettings]);
-
   const helpLink = docLinks.links.ml.guide;
   const navigateToPath = useNavigateToPath();
 
@@ -172,6 +172,19 @@ export const DatavisualizerSelector: FC = () => {
                       <FormattedMessage
                         id="xpack.ml.datavisualizer.selector.selectESQLTitle"
                         defaultMessage="Visualize data using ES|QL"
+                      />{' '}
+                      <EuiBetaBadge
+                        label=""
+                        iconType="beaker"
+                        size="m"
+                        color="hollow"
+                        tooltipContent={
+                          <FormattedMessage
+                            id="xpack.ml.datavisualizer.selector.esqlTechnicalPreviewBadge.titleMsg"
+                            defaultMessage="ES|QL data visualizer is in technical preview."
+                          />
+                        }
+                        tooltipPosition={'right'}
                       />
                     </>
                   </EuiTextAlign>

x-pack/plugins/ml/public/application/datavisualizer/index_based/index_data_visualizer.tsx

@@ -16,7 +16,7 @@ import type {
   GetAdditionalLinksParams,
 } from '@kbn/data-visualizer-plugin/public';
 import { useTimefilter } from '@kbn/ml-date-picker';
-import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { EuiFlexGroup, EuiFlexItem, useEuiTheme } from '@elastic/eui';
 import useMountedState from 'react-use/lib/useMountedState';
 import { useMlApi, useMlKibana, useMlLocator } from '../../contexts/kibana';
 import { HelpMenu } from '../../components/help_menu';
@@ -26,6 +26,7 @@ import { mlNodesAvailable, getMlNodeCount } from '../../ml_nodes_check/check_ml_
 import { checkPermission } from '../../capabilities/check_capabilities';
 import { MlPageHeader } from '../../components/page_header';
 import { useEnabledFeatures } from '../../contexts/ml';
+import { TechnicalPreviewBadge } from '../../components/technical_preview_badge';
 export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false }) => {
   useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false });
   const {
@@ -188,6 +189,7 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
     // eslint-disable-next-line react-hooks/exhaustive-deps
     [mlLocator, mlFeaturesDisabled]
   );
+  const { euiTheme } = useEuiTheme();
   return IndexDataVisualizer ? (
     <Fragment>
       {IndexDataVisualizer !== null ? (
@@ -203,6 +205,9 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
                   <EuiFlexItem grow={false}>
                     <FormattedMessage id="xpack.ml.datavisualizer" defaultMessage="(ES|QL)" />
                   </EuiFlexItem>
+                  <EuiFlexItem grow={false} css={{ marginTop: euiTheme.size.xs }}>
+                    <TechnicalPreviewBadge />
+                  </EuiFlexItem>
                 </>
               ) : null}
             </EuiFlexGroup>

x-pack/test/functional/apps/ml/data_visualizer/esql_data_visualizer.ts

@@ -39,7 +39,7 @@ const esqlFarequoteData = {
   sourceIndexOrSavedSearch: 'ft_farequote',
   expected: {
     hasDocCountChart: true,
-    initialLimitSize: '10,000 (100%)',
+    initialLimitSize: '5,000 (100%)',
     totalDocCountFormatted: '86,274',
     metricFields: [
       {
@@ -48,7 +48,7 @@ const esqlFarequoteData = {
         existsInDocs: true,
         aggregatable: true,
         loading: false,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         statsMaxDecimalPlaces: 3,
         topValuesCount: 11,
         viewableInLens: false,
@@ -61,7 +61,7 @@ const esqlFarequoteData = {
         existsInDocs: true,
         aggregatable: true,
         loading: false,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         exampleCount: 2,
         viewableInLens: false,
       },
@@ -72,7 +72,7 @@ const esqlFarequoteData = {
         aggregatable: false,
         loading: false,
         exampleCount: 1,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         viewableInLens: false,
       },
       {
@@ -82,7 +82,7 @@ const esqlFarequoteData = {
         aggregatable: true,
         loading: false,
         exampleCount: 1,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         viewableInLens: false,
       },
       {
@@ -92,7 +92,7 @@ const esqlFarequoteData = {
         aggregatable: true,
         loading: false,
         exampleCount: 10,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         viewableInLens: false,
       },
       {
@@ -102,7 +102,7 @@ const esqlFarequoteData = {
         aggregatable: false,
         loading: false,
         exampleCount: 1,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         viewableInLens: false,
       },
       {
@@ -112,7 +112,7 @@ const esqlFarequoteData = {
         aggregatable: true,
         loading: false,
         exampleCount: 1,
-        docCountFormatted: '86,274 (100%)',
+        docCountFormatted: '10,000 (100%)',
         viewableInLens: false,
       },
     ],
@@ -253,7 +253,7 @@ export default function ({ getPageObject, getService }: FtrProviderContext) {
 
     it(`${testData.suiteTitle} updates data when limit size changes`, async () => {
       if (testData.expected.initialLimitSize !== undefined) {
-        await ml.testExecution.logTestStep('shows analysis for 10,000 rows by default');
+        await ml.testExecution.logTestStep('shows analysis for 5,000 rows by default');
         for (const fieldRow of testData.expected.metricFields as Array<
           Required<MetricFieldVisConfig>
         >) {
@@ -268,8 +268,8 @@ export default function ({ getPageObject, getService }: FtrProviderContext) {
         }
       }
 
-      await ml.testExecution.logTestStep('sets limit size to Analyze all');
-      await ml.dataVisualizer.setLimitSize(100000);
+      await ml.testExecution.logTestStep('sets limit size to 10,000 rows');
+      await ml.dataVisualizer.setLimitSize(10000);
 
       await ml.testExecution.logTestStep('updates table with newly set limit size');
       for (const fieldRow of testData.expected.metricFields as Array<