Skip to content

Commit

Permalink
[8.12] [ML] Enable Trained models functional tests (#173517) (#173914)
Browse files Browse the repository at this point in the history 
# Backport

This will backport the following commits from `main` to `8.12`:
- [[ML] Enable Trained models functional tests
(#173517)](#173517)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Dima
Arnautov","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-12-22T11:23:50Z","message":"[ML]
Enable Trained models functional tests (#173517)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/168899\r\nCloses
https://github.com/elastic/kibana/issues/168492\r\nCloses
https://github.com/elastic/kibana/issues/156243\r\n\r\nEnables Trained
models functional tests \r\n\r\n### Checklist\r\n\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed\r\n(https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4664)","sha":"b5cd85c21fa60c121c30219666a89135c9a95124","branchLabelMapping":{"^v8.13.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":[":ml","test_ui_functional","release_note:skip","Team:ML","v8.12.0","v8.13.0"],"number":173517,"url":"https://github.com/elastic/kibana/pull/173517","mergeCommit":{"message":"[ML]
Enable Trained models functional tests (#173517)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/168899\r\nCloses
https://github.com/elastic/kibana/issues/168492\r\nCloses
https://github.com/elastic/kibana/issues/156243\r\n\r\nEnables Trained
models functional tests \r\n\r\n### Checklist\r\n\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed\r\n(https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4664)","sha":"b5cd85c21fa60c121c30219666a89135c9a95124"}},"sourceBranch":"main","suggestedTargetBranches":["8.12"],"targetPullRequestStates":[{"branch":"8.12","label":"v8.12.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.13.0","labelRegex":"^v8.13.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/173517","number":173517,"mergeCommit":{"message":"[ML]
Enable Trained models functional tests (#173517)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/168899\r\nCloses
https://github.com/elastic/kibana/issues/168492\r\nCloses
https://github.com/elastic/kibana/issues/156243\r\n\r\nEnables Trained
models functional tests \r\n\r\n### Checklist\r\n\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed\r\n(https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4664)","sha":"b5cd85c21fa60c121c30219666a89135c9a95124"}}]}]
BACKPORT-->

Co-authored-by: Dima Arnautov <[email protected]>
  • Loading branch information
@kibanamachine @darnautov
kibanamachine and darnautov authored Dec 22, 2023
1 parent a2630f0 commit 193a858
Show file tree
Hide file tree
Showing 4 changed files with 189 additions and 133 deletions.
3 changes: 3 additions & 0 deletions x-pack/plugins/ml/public/application/model_management/deployment_setup.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ export const DeploymentSetup: FC<DeploymentSetupProps> = ({
id,
label: id,
value,
'data-test-subj': `mlModelsStartDeploymentModalThreadsPerAllocation_${id}`,
};
}),
[maxSingleMlNodeProcessors]
Expand Down Expand Up @@ -215,6 +216,7 @@ export const DeploymentSetup: FC<DeploymentSetupProps> = ({
defaultMessage: 'low',
}
),
'data-test-subj': 'mlModelsStartDeploymentModalLowPriority',
},
{
id: 'normal',
Expand All @@ -225,6 +227,7 @@ export const DeploymentSetup: FC<DeploymentSetupProps> = ({
defaultMessage: 'normal',
}
),
'data-test-subj': 'mlModelsStartDeploymentModalNormalPriority',
},
]}
data-test-subj={'mlModelsStartDeploymentModalPriority'}
Expand Down
255 changes: 134 additions & 121 deletions x-pack/test/functional/apps/ml/short_tests/model_management/model_list.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@ export default function ({ getService }: FtrProviderContext) {
id: model.name,
}));

// FLAKY: https://github.com/elastic/kibana/issues/165084
describe.skip('trained models', function () {
describe('trained models', function () {
// 'Created at' will be different on each run,
// so we will just assert that the value is in the expected timestamp format.
const builtInModelData = {
Expand Down Expand Up @@ -112,12 +111,6 @@ export default function ({ getService }: FtrProviderContext) {
await ml.api.cleanMlIndices();
await ml.api.deleteIndices(modelWithPipelineAndDestIndexExpectedValues.index);

await ml.api.deleteIngestPipeline(modelWithoutPipelineDataExpectedValues.name, false);
await ml.api.deleteIngestPipeline(
modelWithoutPipelineDataExpectedValues.duplicateName,
false
);

// Need to delete index before ingest pipeline, else it will give error
await ml.api.deleteIngestPipeline(modelWithPipelineAndDestIndex.modelId);
await ml.testResources.deleteDataViewByTitle(
Expand Down Expand Up @@ -188,122 +181,141 @@ export default function ({ getService }: FtrProviderContext) {
await ml.trainedModelsTable.assertPipelinesTabContent(false);
});

it('deploys the trained model with default values', async () => {
await ml.testExecution.logTestStep('should display the trained model in the table');
await ml.trainedModelsTable.filterWithSearchString(modelWithoutPipelineData.modelId, 1);
await ml.testExecution.logTestStep(
'should show collapsed actions menu for the model in the table'
);
await ml.trainedModelsTable.assertModelCollapsedActionsButtonExists(
modelWithoutPipelineData.modelId,
true
);
await ml.testExecution.logTestStep('should show deploy action for the model in the table');
await ml.trainedModelsTable.assertModelDeployActionButtonEnabled(
modelWithoutPipelineData.modelId,
true
);
await ml.testExecution.logTestStep('should open the deploy model flyout');
await ml.trainedModelsTable.clickDeployAction(modelWithoutPipelineData.modelId);
await ml.testExecution.logTestStep('should complete the deploy model Details step');
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutDetails({
name: modelWithoutPipelineDataExpectedValues.name,
description: modelWithoutPipelineDataExpectedValues.description,
// If no metadata is provided, the target field will default to empty string
targetField: '',
// FLAKY: https://github.com/elastic/kibana/issues/165084
describe.skip('DFA model deployment', () => {
after(async () => {
await ml.api.deleteIngestPipeline(modelWithoutPipelineDataExpectedValues.name, false);
await ml.api.deleteIngestPipeline(
modelWithoutPipelineDataExpectedValues.duplicateName,
false
);
});
await ml.testExecution.logTestStep('should complete the deploy model Pipeline Config step');
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutPipelineConfig({
inferenceConfig: modelWithoutPipelineDataExpectedValues.inferenceConfig,
fieldMap: modelWithoutPipelineDataExpectedValues.fieldMap,
});
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline On Failure step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutOnFailure(
getDefaultOnFailureConfiguration()
);
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline Create pipeline step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutCreateStep({
description: modelWithoutPipelineDataExpectedValues.description,
processors: [
{
inference: {
model_id: modelWithoutPipelineData.modelId,
ignore_failure: false,
inference_config: modelWithoutPipelineDataExpectedValues.inferenceConfig,
on_failure: getDefaultOnFailureConfiguration(),
},
},
],
});
});

it('deploys the trained model with custom values', async () => {
await ml.testExecution.logTestStep('should display the trained model in the table');
await ml.trainedModelsTable.filterWithSearchString(modelWithoutPipelineData.modelId, 1);
await ml.testExecution.logTestStep(
'should not show collapsed actions menu for the model in the table'
);
await ml.trainedModelsTable.assertModelCollapsedActionsButtonExists(
modelWithoutPipelineData.modelId,
true
);
await ml.testExecution.logTestStep('should show deploy action for the model in the table');
await ml.trainedModelsTable.assertModelDeployActionButtonExists(
modelWithoutPipelineData.modelId,
false
);
await ml.testExecution.logTestStep('should open the deploy model flyout');
await ml.trainedModelsTable.clickDeployAction(modelWithoutPipelineData.modelId);
await ml.testExecution.logTestStep('should complete the deploy model Details step');
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutDetails(
{
name: modelWithoutPipelineDataExpectedValues.duplicateName,
description: modelWithoutPipelineDataExpectedValues.duplicateDescription,
targetField: 'myTargetField',
},
true
);
await ml.testExecution.logTestStep('should complete the deploy model Pipeline Config step');
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutPipelineConfig(
{
it.skip('deploys the trained model with default values', async () => {
await ml.testExecution.logTestStep('should display the trained model in the table');
await ml.trainedModelsTable.filterWithSearchString(modelWithoutPipelineData.modelId, 1);
await ml.testExecution.logTestStep(
'should show collapsed actions menu for the model in the table'
);
await ml.trainedModelsTable.assertModelCollapsedActionsButtonExists(
modelWithoutPipelineData.modelId,
true
);
await ml.testExecution.logTestStep(
'should show deploy action for the model in the table'
);
await ml.trainedModelsTable.assertModelDeployActionButtonEnabled(
modelWithoutPipelineData.modelId,
true
);
await ml.testExecution.logTestStep('should open the deploy model flyout');
await ml.trainedModelsTable.clickDeployAction(modelWithoutPipelineData.modelId);
await ml.testExecution.logTestStep('should complete the deploy model Details step');
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutDetails({
name: modelWithoutPipelineDataExpectedValues.name,
description: modelWithoutPipelineDataExpectedValues.description,
// If no metadata is provided, the target field will default to empty string
targetField: '',
});
await ml.testExecution.logTestStep(
'should complete the deploy model Pipeline Config step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutPipelineConfig({
inferenceConfig: modelWithoutPipelineDataExpectedValues.inferenceConfig,
editedInferenceConfig: modelWithoutPipelineDataExpectedValues.editedInferenceConfig,
fieldMap: modelWithoutPipelineDataExpectedValues.fieldMap,
editedFieldMap: modelWithoutPipelineDataExpectedValues.editedFieldMap,
},
true
);
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline On Failure step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutOnFailure(
getDefaultOnFailureConfiguration(),
true
);
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline Create pipeline step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutCreateStep({
description: modelWithoutPipelineDataExpectedValues.duplicateDescription,
processors: [
{
inference: {
field_map: {
incoming_field: 'old_field',
});
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline On Failure step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutOnFailure(
getDefaultOnFailureConfiguration()
);
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline Create pipeline step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutCreateStep({
description: modelWithoutPipelineDataExpectedValues.description,
processors: [
{
inference: {
model_id: modelWithoutPipelineData.modelId,
ignore_failure: false,
inference_config: modelWithoutPipelineDataExpectedValues.inferenceConfig,
on_failure: getDefaultOnFailureConfiguration(),
},
ignore_failure: true,
if: "ctx?.network?.name == 'Guest'",
model_id: modelWithoutPipelineData.modelId,
inference_config: modelWithoutPipelineDataExpectedValues.inferenceConfigDuplicate,
tag: 'tag',
target_field: 'myTargetField',
},
],
});
});

it.skip('deploys the trained model with custom values', async () => {
await ml.testExecution.logTestStep('should display the trained model in the table');
await ml.trainedModelsTable.filterWithSearchString(modelWithoutPipelineData.modelId, 1);
await ml.testExecution.logTestStep(
'should not show collapsed actions menu for the model in the table'
);
await ml.trainedModelsTable.assertModelCollapsedActionsButtonExists(
modelWithoutPipelineData.modelId,
true
);
await ml.testExecution.logTestStep(
'should show deploy action for the model in the table'
);
await ml.trainedModelsTable.assertModelDeployActionButtonExists(
modelWithoutPipelineData.modelId,
false
);
await ml.testExecution.logTestStep('should open the deploy model flyout');
await ml.trainedModelsTable.clickDeployAction(modelWithoutPipelineData.modelId);
await ml.testExecution.logTestStep('should complete the deploy model Details step');
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutDetails(
{
name: modelWithoutPipelineDataExpectedValues.duplicateName,
description: modelWithoutPipelineDataExpectedValues.duplicateDescription,
targetField: 'myTargetField',
},
true
);
await ml.testExecution.logTestStep(
'should complete the deploy model Pipeline Config step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutPipelineConfig(
{
inferenceConfig: modelWithoutPipelineDataExpectedValues.inferenceConfig,
editedInferenceConfig: modelWithoutPipelineDataExpectedValues.editedInferenceConfig,
fieldMap: modelWithoutPipelineDataExpectedValues.fieldMap,
editedFieldMap: modelWithoutPipelineDataExpectedValues.editedFieldMap,
},
],
true
);
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline On Failure step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutOnFailure(
getDefaultOnFailureConfiguration(),
true
);
await ml.testExecution.logTestStep(
'should complete the deploy model pipeline Create pipeline step'
);
await ml.deployDFAModelFlyout.completeTrainedModelsInferenceFlyoutCreateStep({
description: modelWithoutPipelineDataExpectedValues.duplicateDescription,
processors: [
{
inference: {
field_map: {
incoming_field: 'old_field',
},
ignore_failure: true,
if: "ctx?.network?.name == 'Guest'",
model_id: modelWithoutPipelineData.modelId,
inference_config: modelWithoutPipelineDataExpectedValues.inferenceConfigDuplicate,
tag: 'tag',
target_field: 'myTargetField',
},
},
],
});
});
});

Expand Down Expand Up @@ -418,7 +430,7 @@ export default function ({ getService }: FtrProviderContext) {
);
});

it('navigates to data drift', async () => {
it.skip('navigates to data drift', async () => {
await ml.testExecution.logTestStep('should show the model map in the expanded row');
await ml.trainedModelsTable.ensureRowIsExpanded(modelWithPipelineAndDestIndex.modelId);
await ml.trainedModelsTable.assertModelsMapTabContent();
Expand Down Expand Up @@ -450,8 +462,7 @@ export default function ({ getService }: FtrProviderContext) {
await ml.navigation.navigateToTrainedModels();
});

// FLAKY: https://github.com/elastic/kibana/issues/168899
describe.skip('with imported models', function () {
describe('with imported models', function () {
before(async () => {
await ml.navigation.navigateToTrainedModels();
});
Expand All @@ -475,8 +486,10 @@ export default function ({ getService }: FtrProviderContext) {
});

it(`stops deployment of the imported model ${model.id}`, async () => {
// Wait for the model to be deployed before stopping it.
await ml.testExecution.logTestStep('should have a Deployed state');
await ml.trainedModelsTable.assertModelState(model.id, 'Deployed');
await ml.trainedModelsTable.stopDeployment(model.id);
await ml.trainedModelsTable.assertModelDeleteActionButtonEnabled(model.id, true);
});

it(`deletes the imported model ${model.id}`, async () => {
Expand Down
14 changes: 10 additions & 4 deletions x-pack/test/functional/services/ml/common_ui.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,15 +363,21 @@ export function MachineLearningCommonUIProvider({
});
},

async selectButtonGroupValue(inputTestSubj: string, value: string) {
async selectButtonGroupValue(inputTestSubj: string, value: string, valueTestSubj?: string) {
await retry.tryForTime(5000, async () => {
// The input element can not be clicked directly.
// Instead, we need to click the corresponding label

const fieldSetElement = await testSubjects.find(inputTestSubj);
let labelElement: WebElementWrapper;

const labelElement = await fieldSetElement.findByCssSelector(`label[title="${value}"]`);
await labelElement.click();
if (valueTestSubj) {
await testSubjects.click(valueTestSubj);
labelElement = await testSubjects.find(valueTestSubj);
} else {
const fieldSetElement = await testSubjects.find(inputTestSubj);
labelElement = await fieldSetElement.findByCssSelector(`label[title="${value}"]`);
await labelElement.click();
}

const labelClasses = await labelElement.getAttribute('class');
expect(labelClasses).to.contain(
Expand Down
Loading

0 comments on commit 193a858

Please sign in to comment.