Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML][ES|QL] Adds query guardrails and technical preview badge to ES|QL data visualizer #200325

Merged
merged 9 commits into from
Nov 19, 2024
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import React from 'react';
import React, { useEffect } from 'react';
import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
import { useExpandedRowCss } from './use_expanded_row_css';
import { GeoPointContentWithMap } from './geo_point_content_with_map';
Expand Down Expand Up @@ -34,6 +34,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
totalDocuments,
timeFieldName,
typeAccessor = 'type',
onVisibilityChange,
}: {
item: FieldVisConfig;
dataView: DataView | undefined;
Expand All @@ -46,6 +47,7 @@ export const IndexBasedDataVisualizerExpandedRow = ({
*/
onAddFilter?: (field: DataViewField | string, value: string, type: '+' | '-') => void;
timeFieldName?: string;
onVisibilityChange?: (visible: boolean, item: FieldVisConfig) => void;
}) => {
const config = { ...item, stats: { ...item.stats, totalDocuments } };
const { loading, existsInDocs, fieldName } = config;
Expand Down Expand Up @@ -98,6 +100,14 @@ export const IndexBasedDataVisualizerExpandedRow = ({
}
}

useEffect(() => {
onVisibilityChange?.(true, item);

return () => {
onVisibilityChange?.(false, item);
};
}, [item, onVisibilityChange]);

return (
<div css={dvExpandedRow} data-test-subj={`dataVisualizerFieldExpandedRow-${fieldName}`}>
{loading === true ? <LoadingIndicator /> : getCardContent()}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ interface DataVisualizerTableProps<T extends object> {
overallStatsRunning: boolean;
renderFieldName?: FieldStatisticTableEmbeddableProps['renderFieldName'];
error?: Error | string;
isEsql?: boolean;
}

const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
Expand All @@ -78,6 +79,7 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
overallStatsRunning,
renderFieldName,
error,
isEsql = false,
}: DataVisualizerTableProps<T>) => {
const { euiTheme } = useEuiTheme();

Expand All @@ -87,7 +89,8 @@ const UnmemoizedDataVisualizerTable = <T extends DataVisualizerTableItem>({
const { onTableChange, pagination, sorting } = useTableSettings<T>(
items,
pageState,
updatePageState
updatePageState,
isEsql
);
const [showDistributions, setShowDistributions] = useState<boolean>(showPreviewByDefault ?? true);
const [dimensions, setDimensions] = useState(calculateTableColumnsDimensions());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ interface UseTableSettingsReturnValue<T extends object> {
export function useTableSettings<TypeOfItem extends object>(
items: TypeOfItem[],
pageState: DataVisualizerTableState,
updatePageState: (update: DataVisualizerTableState) => void
updatePageState: (update: DataVisualizerTableState) => void,
isEsql: boolean = false
): UseTableSettingsReturnValue<TypeOfItem> {
const { pageIndex, pageSize, sortField, sortDirection } = pageState;

Expand All @@ -50,9 +51,9 @@ export function useTableSettings<TypeOfItem extends object>(
pageIndex,
pageSize,
totalItemCount: items.length,
pageSizeOptions: PAGE_SIZE_OPTIONS,
pageSizeOptions: isEsql ? [10, 25] : PAGE_SIZE_OPTIONS,
}),
[items, pageIndex, pageSize]
[items, pageIndex, pageSize, isEsql]
);

const sorting = useMemo(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ export const IndexDataVisualizerESQL: FC<IndexDataVisualizerESQLProps> = (dataVi

<EuiProgress value={combinedProgress} max={100} size="xs" />
<DataVisualizerTable
isEsql={true}
items={configs}
pageState={dataVisualizerListState}
updatePageState={setDataVisualizerListState}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,6 @@ const options = [
values: { limit: '10,000' },
}),
},
{
'data-test-subj': 'dvESQLLimitSize-100000',
value: '100000',
text: i18n.translate('xpack.dataVisualizer.searchPanel.esql.limitSizeOptionLabel', {
defaultMessage: '{limit} rows',
values: { limit: '100,000' },
}),
},
];

export const ESQLDefaultLimitSizeSelect = ({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
* 2.0.
*/

export const DEFAULT_ESQL_LIMIT = 10000;
export const DEFAULT_ESQL_LIMIT = 5000;
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ const EmbeddableESQLFieldStatsTableWrapper = React.memo(

return (
<DataVisualizerTable<FieldVisConfig>
isEsql={true}
items={configs}
pageState={dataVisualizerListState}
updatePageState={onTableChange}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,14 @@ const defaultSearchQuery = {
};

const FALLBACK_ESQL_QUERY: ESQLQuery = { esql: '' };
const DEFAULT_LIMIT_SIZE = '10000';
const DEFAULT_LIMIT_SIZE = '5000';
const defaults = getDefaultPageState();

export const getDefaultESQLDataVisualizerListState = (
overrides?: Partial<ESQLDataVisualizerIndexBasedAppState>
): Required<ESQLDataVisualizerIndexBasedAppState> => ({
pageIndex: 0,
pageSize: 25,
pageSize: 10,
sortField: 'fieldName',
sortDirection: 'asc',
visibleFieldTypes: [],
Expand All @@ -70,7 +70,7 @@ export const getDefaultESQLDataVisualizerListState = (
searchQuery: defaultSearchQuery,
searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY,
filters: [],
showDistributions: true,
showDistributions: false,
showAllFields: false,
showEmptyFields: false,
probability: null,
Expand Down Expand Up @@ -229,6 +229,24 @@ export const useESQLDataVisualizerData = (
} as QueryDslQueryContainer;
}
}

// Ensure that we don't query frozen data
if (filter.bool && Array.isArray(filter.bool?.must_not)) {
filter.bool.must_not.push({
term: {
_tier: 'data_frozen',
},
});
} else {
filter.bool = {
must_not: [
{
term: { _tier: 'data_frozen' },
},
],
};
}
Copy link
Member

@jgowdyelastic jgowdyelastic Nov 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If a bool object already exists containing a must but not must_not this will hit the else clause where a new bool object is created, wiping out the exiting must.

I think a series of ifs might be safer. something like

if (filter.bool === undefined) {
  filter.bool = Object.create(null);
}

if (filter.bool.must_not === undefined) {
  filter.bool.must_not = [];
}

filter.bool.must_not.push({
  term: { _tier: 'data_frozen' },
  })

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated here 83ff92c (#200325)


return {
id: input.id,
earliest,
Expand Down Expand Up @@ -332,9 +350,25 @@ export const useESQLDataVisualizerData = (

const visibleFieldTypes =
dataVisualizerListState.visibleFieldTypes ?? restorableDefaults.visibleFieldTypes;
const [expandedRows, setExpandedRows] = useState<string[]>([]);

const onVisibilityChange = useCallback((visible: boolean, item: FieldVisConfig) => {
if (visible) {
setExpandedRows((prev) => [...prev, item.fieldName]);
} else {
setExpandedRows((prev) => prev.filter((fieldName) => fieldName !== item.fieldName));
}
}, []);

const hasExpandedRows = useMemo(() => expandedRows.length > 0, [expandedRows]);

useEffect(
function updateFieldStatFieldsToFetch() {
if (dataVisualizerListState?.showDistributions === false && !hasExpandedRows) {
setFieldStatFieldsToFetch(undefined);
return;
}

const { sortField, sortDirection } = dataVisualizerListState;

// Otherwise, sort the list of fields by the initial sort field and sort direction
Expand Down Expand Up @@ -376,6 +410,8 @@ export const useESQLDataVisualizerData = (
dataVisualizerListState.sortDirection,
nonMetricConfigs,
metricConfigs,
dataVisualizerListState?.showDistributions,
hasExpandedRows,
]
);

Expand Down Expand Up @@ -618,14 +654,15 @@ export const useESQLDataVisualizerData = (
typeAccessor="secondaryType"
timeFieldName={timeFieldName}
onAddFilter={input.onAddFilter}
onVisibilityChange={onVisibilityChange}
/>
);
}
return map;
}, {} as ItemIdToExpandedRowMap);
},
// eslint-disable-next-line react-hooks/exhaustive-deps
[currentDataView, totalCount, query.esql, timeFieldName]
[currentDataView, totalCount, query.esql, timeFieldName, onVisibilityChange]
);

const combinedProgress = useMemo(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,6 @@ export const useESQLOverallStatsData = (
{ strategy: ESQL_ASYNC_SEARCH_STRATEGY }
)) as ESQLResponse | undefined;
setQueryHistoryStatus(false);

const columnInfo = columnsResp?.rawResponse
? columnsResp.rawResponse.all_columns ?? columnsResp.rawResponse.columns
: [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
EuiLink,
EuiSpacer,
EuiText,
EuiBetaBadge,
EuiTextAlign,
} from '@elastic/eui';

Expand Down Expand Up @@ -64,7 +65,6 @@ export const DatavisualizerSelector: FC = () => {
},
} = useMlKibana();
const isEsqlEnabled = useMemo(() => uiSettings.get(ENABLE_ESQL), [uiSettings]);

const helpLink = docLinks.links.ml.guide;
const navigateToPath = useNavigateToPath();

Expand Down Expand Up @@ -172,6 +172,19 @@ export const DatavisualizerSelector: FC = () => {
<FormattedMessage
id="xpack.ml.datavisualizer.selector.selectESQLTitle"
defaultMessage="Visualize data using ES|QL"
/>{' '}
<EuiBetaBadge
label=""
iconType="beaker"
size="m"
color="hollow"
tooltipContent={
<FormattedMessage
id="xpack.ml.datavisualizer.selector.esqlTechnicalPreviewBadge.titleMsg"
defaultMessage="ES|QL data visualizer is in technical preview."
/>
}
tooltipPosition={'right'}
/>
</>
</EuiTextAlign>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import type {
GetAdditionalLinksParams,
} from '@kbn/data-visualizer-plugin/public';
import { useTimefilter } from '@kbn/ml-date-picker';
import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import { EuiFlexGroup, EuiFlexItem, useEuiTheme } from '@elastic/eui';
import useMountedState from 'react-use/lib/useMountedState';
import { useMlApi, useMlKibana, useMlLocator } from '../../contexts/kibana';
import { HelpMenu } from '../../components/help_menu';
Expand All @@ -26,6 +26,7 @@ import { mlNodesAvailable, getMlNodeCount } from '../../ml_nodes_check/check_ml_
import { checkPermission } from '../../capabilities/check_capabilities';
import { MlPageHeader } from '../../components/page_header';
import { useEnabledFeatures } from '../../contexts/ml';
import { TechnicalPreviewBadge } from '../../components/technical_preview_badge';
export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false }) => {
useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false });
const {
Expand Down Expand Up @@ -188,6 +189,7 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
// eslint-disable-next-line react-hooks/exhaustive-deps
[mlLocator, mlFeaturesDisabled]
);
const { euiTheme } = useEuiTheme();
return IndexDataVisualizer ? (
<Fragment>
{IndexDataVisualizer !== null ? (
Expand All @@ -203,6 +205,9 @@ export const IndexDataVisualizerPage: FC<{ esql: boolean }> = ({ esql = false })
<EuiFlexItem grow={false}>
<FormattedMessage id="xpack.ml.datavisualizer" defaultMessage="(ES|QL)" />
</EuiFlexItem>
<EuiFlexItem grow={false} css={{ marginTop: euiTheme.size.xs }}>
<TechnicalPreviewBadge />
</EuiFlexItem>
</>
) : null}
</EuiFlexGroup>
Expand Down