From babee9a0f9d13501d31c8bcf8e6e48498417a5c5 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 12 Nov 2024 10:02:26 +0000 Subject: [PATCH] [ML] File upload adding deployment initialization step (#198446) Fixes https://github.com/elastic/kibana/issues/196696 When adding a semantic text field, we now have an additional step in the file uploading process which calls inference for the selected inference endpoint. The response of the inference call is ignored and a poll is started to check to see of the model has been deployed by check to see if `num_allocations > 0` Any errors returned from the inference call will stop the upload, unless they are timeout errors which are ignored. https://github.com/user-attachments/assets/382ce565-3b4b-47a3-a081-d79c15aa462f --- .../file_data_visualizer_view.js | 3 - .../import_progress/import_progress.tsx | 136 +++++++++++++----- .../components/import_view/auto_deploy.ts | 76 ++++++++++ .../components/import_view/import.ts | 57 ++++++-- .../components/import_view/import_view.js | 26 +++- .../file_data_visualizer.tsx | 1 - .../plugins/data_visualizer/server/routes.ts | 50 ++++++- 7 files changed, 291 insertions(+), 58 deletions(-) create mode 100644 x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js index e19b2cbceda33..676830e94a280 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/file_data_visualizer_view/file_data_visualizer_view.js @@ -360,9 +360,6 @@ export class FileDataVisualizerView extends Component { fileName={fileName} fileContents={fileContents} data={data} - dataViewsContract={this.props.dataViewsContract} - dataStart={this.props.dataStart} - fileUpload={this.props.fileUpload} getAdditionalLinks={this.props.getAdditionalLinks} resultLinks={this.props.resultLinks} capabilities={this.props.capabilities} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx index 53b92ab1f6414..5a6db4f544fd9 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_progress/import_progress.tsx @@ -31,6 +31,8 @@ export interface Statuses { createDataView: boolean; createPipeline: boolean; permissionCheckStatus: IMPORT_STATUS; + initializeDeployment: boolean; + initializeDeploymentStatus: IMPORT_STATUS; } export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { @@ -45,6 +47,8 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { uploadStatus, createDataView, createPipeline, + initializeDeployment, + initializeDeploymentStatus, } = statuses; let statusInfo = null; @@ -58,27 +62,37 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { ) { completedStep = 0; } + + if ( + readStatus === IMPORT_STATUS.COMPLETE && + initializeDeployment === true && + initializeDeploymentStatus === IMPORT_STATUS.INCOMPLETE + ) { + completedStep = 1; + } + if ( readStatus === IMPORT_STATUS.COMPLETE && + (initializeDeployment === false || initializeDeploymentStatus === IMPORT_STATUS.COMPLETE) && indexCreatedStatus === IMPORT_STATUS.INCOMPLETE && ingestPipelineCreatedStatus === IMPORT_STATUS.INCOMPLETE ) { - completedStep = 1; + completedStep = 2; } if (indexCreatedStatus === IMPORT_STATUS.COMPLETE) { - completedStep = 2; + completedStep = 3; } if ( ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE || (createPipeline === false && indexCreatedStatus === IMPORT_STATUS.COMPLETE) ) { - completedStep = 3; + completedStep = 4; } if (uploadStatus === IMPORT_STATUS.COMPLETE) { - completedStep = 4; + completedStep = 5; } if (dataViewCreatedStatus === IMPORT_STATUS.COMPLETE) { - completedStep = 5; + completedStep = 6; } let processFileTitle = i18n.translate( @@ -87,6 +101,12 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { defaultMessage: 'Process file', } ); + let initializeDeploymentTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.initializeDeploymentTitle', + { + defaultMessage: 'Initialize model deployment', + } + ); let createIndexTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.createIndexTitle', { @@ -146,13 +166,43 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => {

); } - if (completedStep >= 1) { + if (initializeDeployment) { + if (completedStep >= 1) { + processFileTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.fileProcessedTitle', + { + defaultMessage: 'File processed', + } + ); + initializeDeploymentTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.initializingDeploymentTitle', + { + defaultMessage: 'Initializing model deployment', + } + ); + statusInfo = ( +

+ +

+ ); + } + } + if (completedStep >= 2) { processFileTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.fileProcessedTitle', { defaultMessage: 'File processed', } ); + initializeDeploymentTitle = i18n.translate( + 'xpack.dataVisualizer.file.importProgress.deploymentInitializedTitle', + { + defaultMessage: 'Model deployed', + } + ); createIndexTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.creatingIndexTitle', { @@ -162,7 +212,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { statusInfo = createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus; } - if (completedStep >= 2) { + if (completedStep >= 3) { createIndexTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.indexCreatedTitle', { @@ -178,7 +228,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { statusInfo = createPipeline === true ? creatingIndexAndIngestPipelineStatus : creatingIndexStatus; } - if (completedStep >= 3) { + if (completedStep >= 4) { createIngestPipelineTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.ingestPipelineCreatedTitle', { @@ -193,7 +243,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { ); statusInfo = ; } - if (completedStep >= 4) { + if (completedStep >= 5) { uploadingDataTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.dataUploadedTitle', { @@ -219,7 +269,7 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { statusInfo = null; } } - if (completedStep >= 5) { + if (completedStep >= 6) { createDataViewTitle = i18n.translate( 'xpack.dataVisualizer.file.importProgress.dataViewCreatedTitle', { @@ -239,44 +289,58 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { : 'selected') as EuiStepStatus, onClick: () => {}, }, - { - title: createIndexTitle, - status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first - ? indexCreatedStatus + ]; + + if (initializeDeployment === true) { + steps.push({ + title: initializeDeploymentTitle, + status: (initializeDeploymentStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first + ? initializeDeploymentStatus : completedStep === 1 // Then show selected/incomplete states ? 'selected' : 'incomplete') as EuiStepStatus, onClick: () => {}, - }, - { - title: uploadingDataTitle, - status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first - ? uploadStatus - : completedStep === 3 // Then show selected/incomplete states - ? 'selected' - : 'incomplete') as EuiStepStatus, - onClick: () => {}, - }, - ]; + }); + } + + steps.push({ + title: createIndexTitle, + status: (indexCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first + ? indexCreatedStatus + : completedStep === 2 // Then show selected/incomplete states + ? 'selected' + : 'incomplete') as EuiStepStatus, + onClick: () => {}, + }); if (createPipeline === true) { - steps.splice(2, 0, { + steps.push({ title: createIngestPipelineTitle, status: (ingestPipelineCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first ? ingestPipelineCreatedStatus - : completedStep === 2 // Then show selected/incomplete states + : completedStep === 3 // Then show selected/incomplete states ? 'selected' : 'incomplete') as EuiStepStatus, onClick: () => {}, }); } + steps.push({ + title: uploadingDataTitle, + status: (uploadStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first + ? uploadStatus + : completedStep === 4 // Then show selected/incomplete states + ? 'selected' + : 'incomplete') as EuiStepStatus, + onClick: () => {}, + }); + if (createDataView === true) { steps.push({ title: createDataViewTitle, status: (dataViewCreatedStatus !== IMPORT_STATUS.INCOMPLETE // Show failure/completed states first ? dataViewCreatedStatus - : completedStep === 4 // Then show selected/incomplete states + : completedStep === 5 // Then show selected/incomplete states ? 'selected' : 'incomplete') as EuiStepStatus, onClick: () => {}, @@ -284,21 +348,21 @@ export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { } return ( - + <> {statusInfo && ( - + <> {statusInfo} - + )} - + ); }; const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => { return ( - + <>

= ({ progress }) => { />

{progress < 100 && ( - + <> - + )} - + ); }; diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts new file mode 100644 index 0000000000000..a402d203585d2 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/auto_deploy.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InferenceInferenceEndpointInfo } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { HttpSetup } from '@kbn/core/public'; + +const POLL_INTERVAL = 5; // seconds + +export class AutoDeploy { + private inferError: Error | null = null; + constructor(private readonly http: HttpSetup, private readonly inferenceId: string) {} + + public async deploy() { + this.inferError = null; + if (await this.isDeployed()) { + return; + } + + this.infer().catch((e) => { + // ignore timeout errors + // The deployment may take a long time + // we'll know when it's ready from polling the inference endpoints + // looking for num_allocations + const status = e.response?.status; + if (status === 408 || status === 504 || status === 502) { + return; + } + this.inferError = e; + }); + await this.pollIsDeployed(); + } + + private async infer() { + return this.http.fetch( + `/internal/data_visualizer/inference/${this.inferenceId}`, + { + method: 'POST', + version: '1', + body: JSON.stringify({ input: '' }), + } + ); + } + + private async isDeployed() { + const inferenceEndpoints = await this.http.fetch( + '/internal/data_visualizer/inference_endpoints', + { + method: 'GET', + version: '1', + } + ); + return inferenceEndpoints.some((endpoint) => { + return ( + endpoint.inference_id === this.inferenceId && endpoint.service_settings.num_allocations > 0 + ); + }); + } + + private async pollIsDeployed() { + while (true) { + if (this.inferError !== null) { + throw this.inferError; + } + const isDeployed = await this.isDeployed(); + if (isDeployed) { + // break out of the loop once we have a successful deployment + return; + } + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL * 1000)); + } + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts index 8611e1ccd2cab..a44cb4fb890fe 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts @@ -12,13 +12,17 @@ import type { } from '@kbn/file-upload-plugin/common/types'; import type { FileUploadStartApi } from '@kbn/file-upload-plugin/public/api'; import { i18n } from '@kbn/i18n'; +import type { HttpSetup } from '@kbn/core/public'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { IMPORT_STATUS } from '../import_progress/import_progress'; +import { AutoDeploy } from './auto_deploy'; interface Props { data: ArrayBuffer; results: FindFileStructureResponse; dataViewsContract: DataViewsServicePublic; fileUpload: FileUploadStartApi; + http: HttpSetup; } interface Config { @@ -33,7 +37,7 @@ interface Config { } export async function importData(props: Props, config: Config, setState: (state: unknown) => void) { - const { data, results, dataViewsContract, fileUpload } = props; + const { data, results, dataViewsContract, fileUpload, http } = props; const { index, dataView, @@ -76,14 +80,6 @@ export async function importData(props: Props, config: Config, setState: (state: return; } - setState({ - importing: true, - imported: false, - reading: true, - initialized: true, - permissionCheckStatus: IMPORT_STATUS.COMPLETE, - }); - let success = true; let settings = {}; @@ -122,7 +118,15 @@ export async function importData(props: Props, config: Config, setState: (state: errors.push(`${parseError} ${error.message}`); } + const inferenceId = getInferenceId(mappings); + setState({ + importing: true, + imported: false, + reading: true, + initialized: true, + permissionCheckStatus: IMPORT_STATUS.COMPLETE, + initializeDeployment: inferenceId !== null, parseJSONStatus: getSuccess(success), }); @@ -147,6 +151,32 @@ export async function importData(props: Props, config: Config, setState: (state: return; } + if (inferenceId) { + // Initialize deployment + const autoDeploy = new AutoDeploy(http, inferenceId); + + try { + await autoDeploy.deploy(); + setState({ + initializeDeploymentStatus: IMPORT_STATUS.COMPLETE, + }); + } catch (error) { + success = false; + const deployError = i18n.translate('xpack.dataVisualizer.file.importView.deployModelError', { + defaultMessage: 'Error deploying trained model:', + }); + errors.push(`${deployError} ${error.message}`); + setState({ + initializeDeploymentStatus: IMPORT_STATUS.FAILED, + errors, + }); + } + } + + if (success === false) { + return; + } + const initializeImportResp = await importer.initializeImport(index, settings, mappings, pipeline); const timeFieldName = importer.getTimeField(); @@ -245,3 +275,12 @@ async function createKibanaDataView( function getSuccess(success: boolean) { return success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED; } + +function getInferenceId(mappings: MappingTypeMapping) { + for (const value of Object.values(mappings.properties ?? {})) { + if (value.type === 'semantic_text') { + return value.inference_id; + } + } + return null; +} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js index 8481ed76e5654..3f30062270060 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js @@ -21,6 +21,7 @@ import { } from '@elastic/eui'; import { debounce } from 'lodash'; +import { context } from '@kbn/kibana-react-plugin/public'; import { ResultsLinks } from '../../../common/components/results_links'; import { FilebeatConfigFlyout } from '../../../common/components/filebeat_config_flyout'; import { ImportProgress, IMPORT_STATUS } from '../import_progress'; @@ -76,14 +77,17 @@ const DEFAULT_STATE = { combinedFields: [], importer: undefined, createPipeline: true, + initializeDeployment: false, + initializeDeploymentStatus: IMPORT_STATUS.INCOMPLETE, }; export class ImportView extends Component { + static contextType = context; + constructor(props) { super(props); this.state = getDefaultState(DEFAULT_STATE, this.props.results, this.props.capabilities); - this.dataViewsContract = props.dataViewsContract; } componentDidMount() { @@ -98,7 +102,12 @@ export class ImportView extends Component { }; clickImport = () => { - const { data, results, dataViewsContract, fileUpload } = this.props; + const { data, results } = this.props; + const { + data: { dataViews: dataViewsContract }, + fileUpload, + http, + } = this.context.services; const { index, dataView, @@ -110,12 +119,13 @@ export class ImportView extends Component { } = this.state; const createPipeline = pipelineString !== ''; + this.setState({ createPipeline, }); importData( - { data, results, dataViewsContract, fileUpload }, + { data, results, dataViewsContract, fileUpload, http }, { index, dataView, @@ -150,7 +160,7 @@ export class ImportView extends Component { return; } - const exists = await this.props.fileUpload.checkIndexExists(index); + const exists = await this.context.services.fileUpload.checkIndexExists(index); const indexNameError = exists ? ( )} diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx index 01d9d2c37194f..6289a73e2a664 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/file_data_visualizer.tsx @@ -44,7 +44,6 @@ export const FileDataVisualizer: FC = ({ getAdditionalLinks, resultLinks , logger: Logger) const filteredInferenceEndpoints = endpoints.filter((endpoint) => { return ( - (endpoint.task_type === 'sparse_embedding' || - endpoint.task_type === 'text_embedding') && - endpoint.service_settings.num_allocations >= 0 + endpoint.task_type === 'sparse_embedding' || endpoint.task_type === 'text_embedding' ); }); @@ -108,4 +106,50 @@ export function routes(coreSetup: CoreSetup, logger: Logger) } } ); + + /** + * @apiGroup DataVisualizer + * + * @api {get} /internal/data_visualizer/inference/{inferenceId} Runs inference on a given inference endpoint with the provided input + * @apiName inference + * @apiDescription Runs inference on a given inference endpoint with the provided input. + */ + router.versioned + .post({ + path: '/internal/data_visualizer/inference/{inferenceId}', + access: 'internal', + options: { + tags: ['access:fileUpload:analyzeFile'], + }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: schema.object({ + inferenceId: schema.string(), + }), + body: schema.object({ + input: schema.string(), + }), + }, + }, + }, + async (context, request, response) => { + try { + const inferenceId = request.params.inferenceId; + const input = request.body.input; + const esClient = (await context.core).elasticsearch.client; + const body = await esClient.asCurrentUser.inference.inference({ + inference_id: inferenceId, + input, + }); + + return response.ok({ body }); + } catch (e) { + return response.customError(wrapError(e)); + } + } + ); }