From f611a41a8abf751b7c909686cafba6a17546ac5e Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 12 Oct 2023 20:03:17 -0400 Subject: [PATCH] [8.11] [ML] Add functional tests for Data Drift view (#167911) (#168756) # Backport This will backport the following commits from `main` to `8.11`: - [[ML] Add functional tests for Data Drift view (#167911)](https://github.com/elastic/kibana/pull/167911) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Quynh Nguyen (Quinn) <43350163+qn895@users.noreply.github.com> --- .../data_drift/data_drift_overview_table.tsx | 8 +- .../data_drift/data_drift_page.tsx | 6 +- .../data_drift/data_drift_utils.ts | 5 +- .../document_count_with_dual_brush.tsx | 4 +- .../data_drift_index_patterns_editor.tsx | 8 +- .../data_drift/data_view_editor.tsx | 11 +- .../data_drift/index_patterns_picker.tsx | 1 + .../model_management/expanded_row.tsx | 4 +- .../apps/ml/data_visualizer/data_drift.ts | 282 +++++++++++++----- .../model_management/model_list.ts | 72 ++++- .../test/functional/services/ml/data_drift.ts | 140 ++++++++- .../services/ml/trained_models_table.ts | 78 ++++- 12 files changed, 525 insertions(+), 94 deletions(-) diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_overview_table.tsx b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_overview_table.tsx index 1c03e18f52e8e..57d2f28af4c18 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_overview_table.tsx +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_overview_table.tsx @@ -112,6 +112,9 @@ export const DataDriftOverviewTable = ({ return ( toggleDetails(item)} aria-label={itemIdToExpandedRowMapValues[item.featureName] ? COLLAPSE_ROW : EXPAND_ROW} iconType={itemIdToExpandedRowMapValues[item.featureName] ? 'arrowDown' : 'arrowRight'} @@ -149,7 +152,10 @@ export const DataDriftOverviewTable = ({ 'data-test-subj': 'mlDataDriftOverviewTableDriftDetected', sortable: true, textOnly: true, - render: (driftDetected: boolean) => { + render: (driftDetected: boolean, item) => { + // @ts-expect-error currently ES two_sided does return string NaN, will be fixed + // NaN happens when the distributions are non overlapping. This means there is a drift. + if (item.similarityTestPValue === 'NaN') return dataComparisonYesLabel; return {driftDetected ? dataComparisonYesLabel : dataComparisonNoLabel}; }, }, diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx index 86ddff5c25e3f..2411c9096be70 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_page.tsx @@ -94,7 +94,11 @@ export const PageHeader: FC = () => { return ( {dataView.getName()}} + pageTitle={ +
+ {dataView.getName()} +
+ } rightSideItems={[ {hasValidTimeField ? ( diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_utils.ts b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_utils.ts index bb9b5cbc5a99b..43c0afee07e3e 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_utils.ts @@ -9,8 +9,11 @@ * formatSignificanceLevel * @param significanceLevel */ -export const formatSignificanceLevel = (significanceLevel: number) => { +export const formatSignificanceLevel = (significanceLevel: number | 'NaN') => { + // NaN happens when the distributions are non overlapping. This means there is a drift, and the p-value would be astronomically small. + if (significanceLevel === 'NaN') return '< 0.000001'; if (typeof significanceLevel !== 'number' || isNaN(significanceLevel)) return ''; + if (significanceLevel < 1e-6) { return '< 0.000001'; } else if (significanceLevel < 0.01) { diff --git a/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx b/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx index 63d717cec0393..0d83879c37486 100644 --- a/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx +++ b/x-pack/plugins/data_visualizer/public/application/data_drift/document_count_with_dual_brush.tsx @@ -141,7 +141,7 @@ export const DocumentCountWithDualBrush: FC = ({ timeRangeEarliest === undefined || timeRangeLatest === undefined ) { - return totalCount !== undefined ? : null; + return totalCount !== undefined ? : null; } const chartPoints: LogRateHistogramItem[] = Object.entries(documentCountStats.buckets).map( @@ -166,7 +166,7 @@ export const DocumentCountWithDualBrush: FC = ({ data-test-subj={getDataTestSubject('dataDriftTotalDocCountHeader', id)} > - + diff --git a/x-pack/plugins/ml/public/application/datavisualizer/data_drift/data_drift_index_patterns_editor.tsx b/x-pack/plugins/ml/public/application/datavisualizer/data_drift/data_drift_index_patterns_editor.tsx index 76cf041426758..5e388d72af92c 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/data_drift/data_drift_index_patterns_editor.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/data_drift/data_drift_index_patterns_editor.tsx @@ -246,7 +246,7 @@ export function DataDriftIndexPatternsEditor({ children: ( @@ -183,6 +186,12 @@ export function DataViewEditor({ columns={columns} pagination={pagination} onChange={onTableChange} + data-test-subject={`mlDataDriftIndexPatternTable-${id ?? ''}`} + rowProps={(item) => { + return { + 'data-test-subj': `mlDataDriftIndexPatternTableRow row-${id}`, + }; + }} /> diff --git a/x-pack/plugins/ml/public/application/datavisualizer/data_drift/index_patterns_picker.tsx b/x-pack/plugins/ml/public/application/datavisualizer/data_drift/index_patterns_picker.tsx index 1f414d8224578..18f3dcea8ae70 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/data_drift/index_patterns_picker.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/data_drift/index_patterns_picker.tsx @@ -88,6 +88,7 @@ export const DataDriftIndexOrSearchRedirect: FC = () => { iconType="plusInCircleFilled" onClick={() => navigateToPath(createPath(ML_PAGES.DATA_DRIFT_CUSTOM))} disabled={!canEditDataView} + data-test-subj={'dataDriftCreateDataViewButton'} > = ({ item }) => { : []), { id: 'models_map', - 'data-test-subj': 'mlTrainedModelsMap', + 'data-test-subj': 'mlTrainedModelMap', name: ( = ({ item }) => { /> ), content: ( -
+
{ - await elasticChart.setNewChartUiDebugFlag(true); - - await ml.testExecution.logTestStep( - `${testData.suiteTitle} loads the saved search selection page` - ); - await ml.navigation.navigateToDataDrift(); - - await ml.testExecution.logTestStep( - `${testData.suiteTitle} loads the data drift index or saved search select page` - ); - await ml.jobSourceSelection.selectSourceForDataDrift(testData.sourceIndexOrSavedSearch); - }); + async function assertDataDriftPageContent(testData: TestData) { + await PageObjects.header.waitUntilLoadingHasFinished(); - it(`${testData.suiteTitle} displays index details`, async () => { - await ml.testExecution.logTestStep(`${testData.suiteTitle} displays the time range step`); - await ml.dataDrift.assertTimeRangeSelectorSectionExists(); + await ml.testExecution.logTestStep(`${testData.suiteTitle} displays the time range step`); + await ml.dataDrift.assertTimeRangeSelectorSectionExists(); - await ml.testExecution.logTestStep(`${testData.suiteTitle} loads data for full time range`); - await ml.dataDrift.clickUseFullDataButton(); + await ml.testExecution.logTestStep(`${testData.suiteTitle} loads data for full time range`); + await ml.dataDrift.clickUseFullDataButton(); - await ml.dataDrift.setRandomSamplingOption('Reference', 'dvRandomSamplerOptionOff'); - await ml.dataDrift.setRandomSamplingOption('Comparison', 'dvRandomSamplerOptionOff'); + await ml.dataDrift.setRandomSamplingOption('Reference', 'dvRandomSamplerOptionOff'); + await ml.dataDrift.setRandomSamplingOption('Comparison', 'dvRandomSamplerOptionOff'); - await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.header.waitUntilLoadingHasFinished(); - await ml.testExecution.logTestStep( - `${testData.suiteTitle} displays elements in the doc count panel correctly` - ); - await ml.dataDrift.assertPrimarySearchBarExists(); - await ml.dataDrift.assertReferenceDocCountContent(); - await ml.dataDrift.assertComparisonDocCountContent(); + await ml.testExecution.logTestStep( + `${testData.suiteTitle} displays elements in the doc count panel correctly` + ); + await ml.dataDrift.assertPrimarySearchBarExists(); + await ml.dataDrift.assertReferenceDocCountContent(); + await ml.dataDrift.assertComparisonDocCountContent(); - await ml.testExecution.logTestStep( - `${testData.suiteTitle} displays elements in the page correctly` - ); - await ml.dataDrift.assertNoWindowParametersEmptyPromptExists(); + await ml.testExecution.logTestStep( + `${testData.suiteTitle} displays elements on the page correctly` + ); + await ml.dataDrift.assertNoWindowParametersEmptyPromptExists(); + if (testData.chartClickCoordinates) { await ml.testExecution.logTestStep('clicks the document count chart to start analysis'); await ml.dataDrift.clickDocumentCountChart( 'dataDriftDocCountChart-Reference', testData.chartClickCoordinates ); - await ml.dataDrift.runAnalysis(); - }); + } + await ml.dataDrift.runAnalysis(); } describe('data drift', async function () { - for (const testData of [farequoteDataViewTestDataWithQuery]) { - describe(`with '${testData.sourceIndexOrSavedSearch}'`, function () { - before(async () => { - await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); - - await ml.testResources.createIndexPatternIfNeeded( - testData.sourceIndexOrSavedSearch, - '@timestamp' - ); - - await ml.testResources.setKibanaTimeZoneToUTC(); - - if (testData.dataGenerator === 'kibana_sample_data_logs') { - await PageObjects.security.login('elastic', 'changeme', { - expectSuccess: true, - }); - - await PageObjects.common.navigateToUrl('home', '/tutorial_directory/sampleData', { - useActualUrl: true, - }); - await PageObjects.header.waitUntilLoadingHasFinished(); - await PageObjects.home.addSampleDataSet('logs'); - await PageObjects.header.waitUntilLoadingHasFinished(); - } else { - await ml.securityUI.loginAsMlPowerUser(); - } - }); - - after(async () => { - await elasticChart.setNewChartUiDebugFlag(false); - await ml.testResources.deleteIndexPatternByTitle(testData.sourceIndexOrSavedSearch); - await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); - }); - - it(`${testData.suiteTitle} loads the ml page`, async () => { - // Start navigation from the base of the ML app. - await ml.navigation.navigateToMl(); - await elasticChart.setNewChartUiDebugFlag(true); - }); - - runTests(testData); + before(async () => { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ihp_outlier'); + await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier'); + + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.createSavedSearchFarequoteFilterAndKueryIfNeeded(); + + await ml.testResources.setKibanaTimeZoneToUTC(); + await ml.securityUI.loginAsMlPowerUser(); + }); + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/ml/ihp_outlier'); + await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); + await Promise.all([ + ml.testResources.deleteIndexPatternByTitle('ft_fare*'), + ml.testResources.deleteIndexPatternByTitle('ft_fare*,ft_fareq*'), + ml.testResources.deleteIndexPatternByTitle('ft_farequote'), + ml.testResources.deleteIndexPatternByTitle('ft_ihp_outlier'), + ]); + }); + + describe('with ft_farequote_filter_and_kuery from index selection page', async function () { + after(async () => { + await elasticChart.setNewChartUiDebugFlag(false); }); - } + + it(`${farequoteKQLFiltersSearchTestData.suiteTitle} loads the ml page`, async () => { + // Start navigation from the base of the ML app. + await ml.navigation.navigateToMl(); + await elasticChart.setNewChartUiDebugFlag(true); + }); + + it(`${farequoteKQLFiltersSearchTestData.suiteTitle} loads the source data in data drift`, async () => { + await ml.testExecution.logTestStep( + `${farequoteKQLFiltersSearchTestData.suiteTitle} loads the data drift index or saved search select page` + ); + await ml.navigation.navigateToDataDrift(); + + await ml.testExecution.logTestStep( + `${farequoteKQLFiltersSearchTestData.suiteTitle} loads the data drift view` + ); + await ml.jobSourceSelection.selectSourceForDataDrift( + farequoteKQLFiltersSearchTestData.sourceIndexOrSavedSearch + ); + await assertDataDriftPageContent(farequoteKQLFiltersSearchTestData); + + if (farequoteKQLFiltersSearchTestData.dataViewName !== undefined) { + await ml.dataDrift.assertDataViewTitle(farequoteKQLFiltersSearchTestData.dataViewName); + } + + await ml.dataDrift.assertTotalDocumentCount( + 'Reference', + farequoteKQLFiltersSearchTestData.totalDocCount + ); + await ml.dataDrift.assertTotalDocumentCount( + 'Comparison', + farequoteKQLFiltersSearchTestData.totalDocCount + ); + }); + }); + + describe(dataViewCreationTestData.suiteTitle, function () { + beforeEach(`${dataViewCreationTestData.suiteTitle} loads the ml page`, async () => { + // Start navigation from the base of the ML app. + await ml.navigation.navigateToMl(); + await elasticChart.setNewChartUiDebugFlag(true); + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} loads the saved search selection page` + ); + await ml.navigation.navigateToDataDrift(); + }); + + it(`${dataViewCreationTestData.suiteTitle} allows analyzing data drift without saving`, async () => { + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} creates new data view` + ); + await ml.dataDrift.navigateToCreateNewDataViewPage(); + await ml.dataDrift.assertIndexPatternNotEmptyFormErrorExists('reference'); + await ml.dataDrift.assertIndexPatternNotEmptyFormErrorExists('comparison'); + await ml.dataDrift.assertAnalyzeWithoutSavingButtonState(true); + await ml.dataDrift.assertAnalyzeDataDriftButtonState(true); + + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} sets index patterns` + ); + await ml.dataDrift.setIndexPatternInput('reference', 'ft_fare*'); + await ml.dataDrift.setIndexPatternInput('comparison', 'ft_fareq*'); + + await ml.dataDrift.selectTimeField(dataViewCreationTestData.dateTimeField); + + await ml.dataDrift.assertAnalyzeWithoutSavingButtonState(false); + await ml.dataDrift.assertAnalyzeDataDriftButtonState(false); + + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} redirects to data drift page` + ); + await ml.dataDrift.clickAnalyzeWithoutSavingButton(); + await assertDataDriftPageContent(dataViewCreationTestData); + await ml.dataDrift.assertDataViewTitle('ft_fare*,ft_fareq*'); + await ml.dataDrift.assertTotalDocumentCount( + 'Reference', + dataViewCreationTestData.totalDocCount + ); + await ml.dataDrift.assertTotalDocumentCount( + 'Comparison', + dataViewCreationTestData.totalDocCount + ); + }); + + it(`${dataViewCreationTestData.suiteTitle} hides analyze data drift without saving option if patterns are same`, async () => { + await ml.dataDrift.navigateToCreateNewDataViewPage(); + await ml.dataDrift.assertAnalyzeWithoutSavingButtonState(true); + await ml.dataDrift.assertAnalyzeDataDriftButtonState(true); + + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} sets index patterns` + ); + await ml.dataDrift.setIndexPatternInput('reference', 'ft_fare*'); + await ml.dataDrift.setIndexPatternInput('comparison', 'ft_fare*'); + + await ml.dataDrift.assertAnalyzeWithoutSavingButtonMissing(); + await ml.dataDrift.assertAnalyzeDataDriftButtonState(false); + + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} redirects to data drift page` + ); + await ml.dataDrift.clickAnalyzeDataDrift(); + await assertDataDriftPageContent(dataViewCreationTestData); + + await ml.testExecution.logTestStep( + `${dataViewCreationTestData.suiteTitle} does not create new data view, and uses available one with matching index pattern` + ); + await ml.dataDrift.assertDataViewTitle('ft_farequote'); + await ml.dataDrift.assertTotalDocumentCount( + 'Reference', + dataViewCreationTestData.totalDocCount + ); + await ml.dataDrift.assertTotalDocumentCount( + 'Comparison', + dataViewCreationTestData.totalDocCount + ); + }); + + it(`${nonTimeSeriesTestData.suiteTitle} loads non-time series data`, async () => { + await ml.jobSourceSelection.selectSourceForDataDrift( + nonTimeSeriesTestData.sourceIndexOrSavedSearch + ); + await ml.dataDrift.runAnalysis(); + }); + }); }); } diff --git a/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts b/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts index 67f3728eab9f9..9ae541c834714 100644 --- a/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts +++ b/x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts @@ -67,6 +67,25 @@ export default function ({ getService }: FtrProviderContext) { }, }; + const modelWithPipelineAndDestIndex = { + modelId: 'dfa_regression_model_n_1', + description: '', + modelTypes: ['regression', 'tree_ensemble'], + }; + const modelWithPipelineAndDestIndexExpectedValues = { + dataViewTitle: `user-index_${modelWithPipelineAndDestIndex.modelId}`, + index: `user-index_${modelWithPipelineAndDestIndex.modelId}`, + name: `ml-inference-${modelWithPipelineAndDestIndex.modelId}`, + description: '', + inferenceConfig: { + regression: { + results_field: 'predicted_value', + num_top_feature_importance_values: 0, + }, + }, + fieldMap: {}, + }; + before(async () => { for (const model of trainedModels) { await ml.api.importTrainedModel(model.id, model.name); @@ -77,17 +96,32 @@ export default function ({ getService }: FtrProviderContext) { // Make sure the .ml-stats index is created in advance, see https://github.com/elastic/elasticsearch/issues/65846 await ml.api.assureMlStatsIndexExists(); + + // Create ingest pipeline and destination index that's tied to model + await ml.api.createIngestPipeline(modelWithPipelineAndDestIndex.modelId); + await ml.api.createIndex(modelWithPipelineAndDestIndexExpectedValues.index, undefined, { + index: { default_pipeline: `pipeline_${modelWithPipelineAndDestIndex.modelId}` }, + }); }); after(async () => { await ml.api.stopAllTrainedModelDeploymentsES(); await ml.api.deleteAllTrainedModelsES(); + + await ml.api.cleanMlIndices(); + await ml.api.deleteIndices(modelWithPipelineAndDestIndexExpectedValues.index); + await ml.api.deleteIngestPipeline(modelWithoutPipelineDataExpectedValues.name, false); await ml.api.deleteIngestPipeline( modelWithoutPipelineDataExpectedValues.duplicateName, false ); - await ml.api.cleanMlIndices(); + + // Need to delete index before ingest pipeline, else it will give error + await ml.api.deleteIngestPipeline(modelWithPipelineAndDestIndex.modelId); + await ml.testResources.deleteIndexPatternByTitle( + modelWithPipelineAndDestIndexExpectedValues.dataViewTitle + ); }); describe('for ML user with read-only access', () => { @@ -387,7 +421,43 @@ export default function ({ getService }: FtrProviderContext) { ); }); + it('navigates to data drift', async () => { + await ml.testExecution.logTestStep('should show the model map in the expanded row'); + await ml.trainedModelsTable.ensureRowIsExpanded(modelWithPipelineAndDestIndex.modelId); + await ml.trainedModelsTable.assertModelsMapTabContent(); + + await ml.testExecution.logTestStep( + 'should navigate to data drift index pattern creation page' + ); + + await ml.trainedModelsTable.assertAnalyzeDataDriftActionButtonEnabled( + modelWithPipelineAndDestIndex.modelId, + true + ); + await ml.trainedModelsTable.clickAnalyzeDataDriftActionButton( + modelWithPipelineAndDestIndex.modelId + ); + + await ml.testExecution.logTestStep(`sets index pattern for reference data set`); + await ml.dataDrift.setIndexPatternInput( + 'reference', + `${modelWithPipelineAndDestIndexExpectedValues.index}*` + ); + await ml.dataDrift.assertIndexPatternInput( + 'comparison', + modelWithPipelineAndDestIndexExpectedValues.index + ); + + await ml.testExecution.logTestStep(`redirects to data drift page`); + await ml.trainedModelsTable.clickAnalyzeDataDriftWithoutSaving(); + await ml.navigation.navigateToTrainedModels(); + }); + describe('with imported models', function () { + before(async () => { + await ml.navigation.navigateToTrainedModels(); + }); + for (const model of trainedModels) { it(`renders expanded row content correctly for imported tiny model ${model.id} without pipelines`, async () => { await ml.trainedModelsTable.ensureRowIsExpanded(model.id); diff --git a/x-pack/test/functional/services/ml/data_drift.ts b/x-pack/test/functional/services/ml/data_drift.ts index 2e0eec6f0e10e..b077caafd0bee 100644 --- a/x-pack/test/functional/services/ml/data_drift.ts +++ b/x-pack/test/functional/services/ml/data_drift.ts @@ -8,6 +8,8 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; +type SubjectId = 'reference' | 'comparison'; + export function MachineLearningDataDriftProvider({ getService, getPageObjects, @@ -17,6 +19,7 @@ export function MachineLearningDataDriftProvider({ const PageObjects = getPageObjects(['discover', 'header']); const elasticChart = getService('elasticChart'); const browser = getService('browser'); + const comboBox = getService('comboBox'); type RandomSamplerOption = | 'dvRandomSamplerOptionOnAutomatic' @@ -29,11 +32,27 @@ export function MachineLearningDataDriftProvider({ return `${testSubject}-${id}`; }, + async assertDataViewTitle(expectedTitle: string) { + const selector = 'mlDataDriftPageDataViewTitle'; + await testSubjects.existOrFail(selector); + await retry.tryForTime(5000, async () => { + const title = await testSubjects.getVisibleText(selector); + expect(title).to.eql( + expectedTitle, + `Expected data drift page's data view title to be '${expectedTitle}' (got '${title}')` + ); + }); + }, + async assertTimeRangeSelectorSectionExists() { await testSubjects.existOrFail('dataComparisonTimeRangeSelectorSection'); }, - async assertTotalDocumentCount(selector: string, expectedFormattedTotalDocCount: string) { + async assertTotalDocumentCount( + id: 'Reference' | 'Comparison', + expectedFormattedTotalDocCount: string + ) { + const selector = `dataVisualizerTotalDocCount-${id}`; await retry.tryForTime(5000, async () => { const docCount = await testSubjects.getVisibleText(selector); expect(docCount).to.eql( @@ -206,9 +225,126 @@ export function MachineLearningDataDriftProvider({ async runAnalysis() { await retry.tryForTime(5000, async () => { await testSubjects.click(`aiopsRerunAnalysisButton`); - // As part of the interface for the histogram brushes, the button to clear the selection should be present await this.assertDataDriftTableExists(); }); }, + + async navigateToCreateNewDataViewPage() { + await retry.tryForTime(5000, async () => { + await testSubjects.click(`dataDriftCreateDataViewButton`); + await testSubjects.existOrFail(`mlPageDataDriftCustomIndexPatterns`); + }); + }, + + async assertIndexPatternNotEmptyFormErrorExists(id: SubjectId) { + const subj = `mlDataDriftIndexPatternFormRow-${id ?? ''}`; + await retry.tryForTime(5000, async () => { + await testSubjects.existOrFail(subj); + const row = await testSubjects.find(subj); + const errorElements = await row.findAllByClassName('euiFormErrorText'); + expect(await errorElements[0].getVisibleText()).eql('Index pattern must not be empty.'); + }); + }, + + async assertIndexPatternInput(id: SubjectId, expectedText: string) { + const inputSelector = `mlDataDriftIndexPatternTitleInput-${id}`; + + await retry.tryForTime(5000, async () => { + const input = await testSubjects.find(inputSelector); + const text = await input.getAttribute('value'); + expect(text).eql( + expectedText, + `Expected ${inputSelector} to have text ${expectedText} (got ${text})` + ); + }); + }, + + async setIndexPatternInput(id: SubjectId, pattern: string) { + const inputSelector = `mlDataDriftIndexPatternTitleInput-${id}`; + + // The input for index pattern automatically appends "*" at the end of the string + // So here we just omit that * at the end to avoid double characters + + await retry.tryForTime(10 * 1000, async () => { + const hasWildCard = pattern.endsWith('*'); + const trimmedPattern = hasWildCard ? pattern.substring(0, pattern.length - 1) : pattern; + + const input = await testSubjects.find(inputSelector); + await input.clearValue(); + + await testSubjects.setValue(inputSelector, trimmedPattern, { + clearWithKeyboard: true, + typeCharByChar: true, + }); + + if (!hasWildCard) { + // If original pattern does not have wildcard, make to delete the wildcard + await input.focus(); + await browser.pressKeys(browser.keys.DELETE); + } + + await this.assertIndexPatternInput(id, pattern); + }); + }, + + async assertAnalyzeWithoutSavingButtonMissing() { + await retry.tryForTime(5000, async () => { + await testSubjects.missingOrFail('analyzeDataDriftWithoutSavingButton'); + }); + }, + + async assertAnalyzeWithoutSavingButtonState(disabled = true) { + await retry.tryForTime(5000, async () => { + const isDisabled = !(await testSubjects.isEnabled('analyzeDataDriftWithoutSavingButton')); + expect(isDisabled).to.equal( + disabled, + `Expect analyze without saving button disabled state to be ${disabled} (got ${isDisabled})` + ); + }); + }, + + async assertAnalyzeDataDriftButtonState(disabled = true) { + await retry.tryForTime(5000, async () => { + const isDisabled = !(await testSubjects.isEnabled('analyzeDataDriftButton')); + expect(isDisabled).to.equal( + disabled, + `Expect analyze data drift button disabled state to be ${disabled} (got ${isDisabled})` + ); + }); + }, + + async clickAnalyzeWithoutSavingButton() { + await retry.tryForTime(5000, async () => { + await testSubjects.existOrFail('analyzeDataDriftWithoutSavingButton'); + await testSubjects.click('analyzeDataDriftWithoutSavingButton'); + await testSubjects.existOrFail(`mlPageDataDriftCustomIndexPatterns`); + }); + }, + + async clickAnalyzeDataDrift() { + await retry.tryForTime(5000, async () => { + await testSubjects.existOrFail('analyzeDataDriftButton'); + await testSubjects.click('analyzeDataDriftButton'); + await testSubjects.existOrFail(`mlPageDataDriftCustomIndexPatterns`); + }); + }, + + async assertDataDriftTimestampField(expectedIdentifier: string) { + await retry.tryForTime(2000, async () => { + const comboBoxSelectedOptions = await comboBox.getComboBoxSelectedOptions( + 'mlDataDriftTimestampField > comboBoxInput' + ); + expect(comboBoxSelectedOptions).to.eql( + expectedIdentifier === '' ? [] : [expectedIdentifier], + `Expected type field to be '${expectedIdentifier}' (got '${comboBoxSelectedOptions}')` + ); + }); + }, + + async selectTimeField(timeFieldName: string) { + await comboBox.set('mlDataDriftTimestampField', timeFieldName); + + await this.assertDataDriftTimestampField(timeFieldName); + }, }; } diff --git a/x-pack/test/functional/services/ml/trained_models_table.ts b/x-pack/test/functional/services/ml/trained_models_table.ts index 6616d61a78f42..878d1a0d8d1d1 100644 --- a/x-pack/test/functional/services/ml/trained_models_table.ts +++ b/x-pack/test/functional/services/ml/trained_models_table.ts @@ -226,6 +226,71 @@ export function TrainedModelsTableProvider( ); } + public async assertModelAnalyzeDataDriftButtonExists(modelId: string, expectedValue: boolean) { + const actionsExists = await testSubjects.exists( + this.rowSelector(modelId, 'mlModelsAnalyzeDataDriftAction') + ); + + expect(actionsExists).to.eql( + expectedValue, + `Expected row analyze data drift action button for trained model '${modelId}' to be ${ + expectedValue ? 'visible' : 'hidden' + } (got ${actionsExists ? 'visible' : 'hidden'})` + ); + } + + public async assertAnalyzeDataDriftActionButtonEnabled( + modelId: string, + expectedValue: boolean + ) { + const actionsButtonExists = await this.doesModelCollapsedActionsButtonExist(modelId); + + let isEnabled = null; + await retry.tryForTime(5 * 1000, async () => { + if (actionsButtonExists) { + await this.toggleActionsContextMenu(modelId, true); + const panelElement = await find.byCssSelector('.euiContextMenuPanel'); + const actionButton = await panelElement.findByTestSubject('mlModelsTableRowDeleteAction'); + isEnabled = await actionButton.isEnabled(); + // escape popover + await browser.pressKeys(browser.keys.ESCAPE); + } else { + await this.assertModelDeleteActionButtonExists(modelId, true); + isEnabled = await testSubjects.isEnabled( + this.rowSelector(modelId, 'mlModelsAnalyzeDataDriftAction') + ); + } + + expect(isEnabled).to.eql( + expectedValue, + `Expected row analyze data drift action button for trained model '${modelId}' to be '${ + expectedValue ? 'enabled' : 'disabled' + }' (got '${isEnabled ? 'enabled' : 'disabled'}')` + ); + }); + } + + public async clickAnalyzeDataDriftActionButton(modelId: string) { + await retry.tryForTime(30 * 1000, async () => { + const actionsButtonExists = await this.doesModelCollapsedActionsButtonExist(modelId); + if (actionsButtonExists) { + await this.toggleActionsContextMenu(modelId, true); + const panelElement = await find.byCssSelector('.euiContextMenuPanel'); + const actionButton = await panelElement.findByTestSubject( + 'mlModelsAnalyzeDataDriftAction' + ); + await actionButton.click(); + // escape popover + await browser.pressKeys(browser.keys.ESCAPE); + } else { + await this.assertModelDeleteActionButtonExists(modelId, true); + await testSubjects.click(this.rowSelector(modelId, 'mlModelsAnalyzeDataDriftAction')); + } + + await testSubjects.existOrFail('mlPageDataDriftCustomIndexPatterns'); + }); + } + public async assertModelTestButtonExists(modelId: string, expectedValue: boolean) { const actionExists = await testSubjects.exists( this.rowSelector(modelId, 'mlModelsTableRowTestAction') @@ -480,7 +545,7 @@ export function TrainedModelsTableProvider( } public async assertTabContent( - type: 'details' | 'stats' | 'inferenceConfig' | 'pipelines', + type: 'details' | 'stats' | 'inferenceConfig' | 'pipelines' | 'map', expectVisible = true ) { const tabTestSubj = `mlTrainedModel${upperFirst(type)}`; @@ -500,6 +565,10 @@ export function TrainedModelsTableProvider( await this.assertTabContent('details', expectVisible); } + public async assertModelsMapTabContent(expectVisible = true) { + await this.assertTabContent('map', expectVisible); + } + public async assertInferenceConfigTabContent(expectVisible = true) { await this.assertTabContent('inferenceConfig', expectVisible); } @@ -526,5 +595,12 @@ export function TrainedModelsTableProvider( } } } + + public async clickAnalyzeDataDriftWithoutSaving() { + await retry.tryForTime(5 * 1000, async () => { + await testSubjects.clickWhenNotDisabled('analyzeDataDriftWithoutSavingButton'); + await testSubjects.existOrFail('mlDataDriftTable'); + }); + } })(); }