Skip to content

Commit

Permalink
[GenAI][Integrations] Add telemetry to the integration assistant (#18…
Browse files Browse the repository at this point in the history 
…6654)

## Summary

Adds telemetry to the Integration assistant.

**Staging dashboard**:
https://telemetry-v2-staging.elastic.dev/s/securitysolution/app/r/s/nJaND

### Event definitions

```
'upload_integration_zip_complete' 
{
  integrationName?: string;
  errorMessage?: string;
}
```

```
'integration_assistant_open' 
{
  sessionId: string; // Unique identifier to represent the current generation session
}
```

```
'integration_assistant_step_complete' 
{
  sessionId: string;
  step: number; // the step number (1-4)
  stepName: string; 
  durationMs: number; // Time spent in the current step
  sessionElapsedTime: number; // Total time spent in the current generation session
}
```

```
'integration_assistant_generation_complete' 
{
  sessionId: string;
  sampleRows: number;  // number of rows in the logs sample (1-10)
  durationMs: number;
  actionTypeId: string; // connector type (bedrock, gen-ai, gemini)
  model: string;  //  LLM used (claude 3 opus, ...)
  provider: string; // API provider (only for OpenAI)
  errorMessage?: string;
}
```

```
'integration_assistant_complete' 
{
  sessionId: string;
  durationMs: number;
  integrationName: string;
  integrationDescription: string;
  dataStreamName: string;
  inputType: string;
  actionTypeId: string; // connector type (bedrock, gen-ai, gemini)
  model: string;  // LLM used (claude 3 opus, claude 3.5 sonnet ...)
  provider: string; // API provider (only for OpenAI)
  errorMessage?: string;
}
```

---------

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
@semd @kibanamachine
semd and kibanamachine authored Jun 27, 2024
1 parent 1e4dba7 commit 2ed3609
Show file tree
Hide file tree
Showing 29 changed files with 677 additions and 78 deletions.
4 changes: 2 additions & 2 deletions x-pack/plugins/integration_assistant/public/common/hooks/use_kibana.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@
*/

import { useKibana as _useKibana } from '@kbn/kibana-react-plugin/public';
import type { CreateIntegrationServices } from '../../components/create_integration/types';
import type { Services } from '../../services';

export const useKibana = () => _useKibana<CreateIntegrationServices>();
export const useKibana = () => _useKibana<Services>();
10 changes: 7 additions & 3 deletions ...plugins/integration_assistant/public/components/create_integration/create_integration.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,25 @@ import React from 'react';
import { Switch } from 'react-router-dom';
import { Route } from '@kbn/shared-ux-router';
import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
import type { CreateIntegrationServices } from './types';
import type { Services } from '../../services';
import { TelemetryContextProvider } from './telemetry';
import { CreateIntegrationLanding } from './create_integration_landing';
import { CreateIntegrationUpload } from './create_integration_upload';
import { CreateIntegrationAssistant } from './create_integration_assistant';
import { Page, PagePath } from '../../common/constants';
import { useRoutesAuthorization } from '../../common/hooks/use_authorization';

interface CreateIntegrationProps {
services: CreateIntegrationServices;
services: Services;
}
export const CreateIntegration = React.memo<CreateIntegrationProps>(({ services }) => (
<KibanaContextProvider services={services}>
<CreateIntegrationRouter />
<TelemetryContextProvider>
<CreateIntegrationRouter />
</TelemetryContextProvider>
</KibanaContextProvider>
));

CreateIntegration.displayName = 'CreateIntegration';

const CreateIntegrationRouter = React.memo(() => {
Expand Down
19 changes: 12 additions & 7 deletions ...mponents/create_integration/create_integration_assistant/create_integration_assistant.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import React, { useReducer, useMemo, useCallback } from 'react';
import React, { useReducer, useMemo, useCallback, useEffect } from 'react';
import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template';
import { Header } from './header';
import { Footer } from './footer';
Expand All @@ -15,17 +15,23 @@ import { DataStreamStep, isDataStreamStepReady } from './steps/data_stream_step'
import { ReviewStep, isReviewStepReady } from './steps/review_step';
import { DeployStep } from './steps/deploy_step';
import { reducer, initialState, ActionsProvider, type Actions } from './state';
import { useTelemetry } from '../telemetry';

export const CreateIntegrationAssistant = React.memo(() => {
const [state, dispatch] = useReducer(reducer, initialState);

const telemetry = useTelemetry();
useEffect(() => {
telemetry.reportAssistantOpen();
}, [telemetry]);

const actions = useMemo<Actions>(
() => ({
setStep: (payload) => {
dispatch({ type: 'SET_STEP', payload });
},
setConnectorId: (payload) => {
dispatch({ type: 'SET_CONNECTOR_ID', payload });
setConnector: (payload) => {
dispatch({ type: 'SET_CONNECTOR', payload });
},
setIntegrationSettings: (payload) => {
dispatch({ type: 'SET_INTEGRATION_SETTINGS', payload });
Expand Down Expand Up @@ -60,19 +66,18 @@ export const CreateIntegrationAssistant = React.memo(() => {
<KibanaPageTemplate>
<Header currentStep={state.step} isGenerating={state.isGenerating} />
<KibanaPageTemplate.Section grow paddingSize="l">
{state.step === 1 && <ConnectorStep connectorId={state.connectorId} />}
{state.step === 1 && <ConnectorStep connector={state.connector} />}
{state.step === 2 && <IntegrationStep integrationSettings={state.integrationSettings} />}
{state.step === 3 && (
<DataStreamStep
integrationSettings={state.integrationSettings}
connectorId={state.connectorId}
connector={state.connector}
isGenerating={state.isGenerating}
/>
)}
{state.step === 4 && (
<ReviewStep
integrationSettings={state.integrationSettings}
connectorId={state.connectorId}
isGenerating={state.isGenerating}
result={state.result}
/>
Expand All @@ -81,7 +86,7 @@ export const CreateIntegrationAssistant = React.memo(() => {
<DeployStep
integrationSettings={state.integrationSettings}
result={state.result}
connectorId={state.connectorId}
connector={state.connector}
/>
)}
</KibanaPageTemplate.Section>
Expand Down
5 changes: 4 additions & 1 deletion ...stant/public/components/create_integration/create_integration_assistant/footer/footer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { EuiLoadingSpinner } from '@elastic/eui';
import React, { useCallback, useMemo } from 'react';
import { ButtonsFooter } from '../../../../common/components/buttons_footer';
import { useNavigate, Page } from '../../../../common/hooks/use_navigate';
import { useTelemetry } from '../../telemetry';
import { useActions, type State } from '../state';
import * as i18n from './translations';

Expand All @@ -35,6 +36,7 @@ interface FooterProps {

export const Footer = React.memo<FooterProps>(
({ currentStep, onGenerate, isGenerating, isNextStepEnabled = false }) => {
const telemetry = useTelemetry();
const { setStep } = useActions();
const navigate = useNavigate();

Expand All @@ -47,12 +49,13 @@ export const Footer = React.memo<FooterProps>(
}, [currentStep, navigate, setStep]);

const onNext = useCallback(() => {
telemetry.reportAssistantStepComplete({ step: currentStep });
if (currentStep === 3) {
onGenerate();
} else {
setStep(currentStep + 1);
}
}, [currentStep, onGenerate, setStep]);
}, [currentStep, onGenerate, setStep, telemetry]);

const nextButtonText = useMemo(() => {
if (currentStep === 3) {
Expand Down
12 changes: 6 additions & 6 deletions ...tion_assistant/public/components/create_integration/create_integration_assistant/state.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import type { AIConnector, IntegrationSettings } from './types';

export interface State {
step: number;
connectorId?: AIConnector['id'];
connector?: AIConnector;
integrationSettings?: IntegrationSettings;
isGenerating: boolean;
result?: {
Expand All @@ -21,15 +21,15 @@ export interface State {

export const initialState: State = {
step: 1,
connectorId: undefined,
connector: undefined,
integrationSettings: undefined,
isGenerating: false,
result: undefined,
};

type Action =
| { type: 'SET_STEP'; payload: State['step'] }
| { type: 'SET_CONNECTOR_ID'; payload: State['connectorId'] }
| { type: 'SET_CONNECTOR'; payload: State['connector'] }
| { type: 'SET_INTEGRATION_SETTINGS'; payload: State['integrationSettings'] }
| { type: 'SET_IS_GENERATING'; payload: State['isGenerating'] }
| { type: 'SET_GENERATED_RESULT'; payload: State['result'] };
Expand All @@ -43,8 +43,8 @@ export const reducer = (state: State, action: Action): State => {
isGenerating: false,
...(action.payload < state.step && { result: undefined }), // reset the result when we go back
};
case 'SET_CONNECTOR_ID':
return { ...state, connectorId: action.payload };
case 'SET_CONNECTOR':
return { ...state, connector: action.payload };
case 'SET_INTEGRATION_SETTINGS':
return { ...state, integrationSettings: action.payload };
case 'SET_IS_GENERATING':
Expand All @@ -58,7 +58,7 @@ export const reducer = (state: State, action: Action): State => {

export interface Actions {
setStep: (payload: State['step']) => void;
setConnectorId: (payload: State['connectorId']) => void;
setConnector: (payload: State['connector']) => void;
setIntegrationSettings: (payload: State['integrationSettings']) => void;
setIsGenerating: (payload: State['isGenerating']) => void;
setResult: (payload: State['result']) => void;
Expand Down
4 changes: 2 additions & 2 deletions ...eate_integration/create_integration_assistant/steps/connector_step/connector_selector.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,15 @@ export const ConnectorSelector = React.memo<ConnectorSelectorProps>(
const {
triggersActionsUi: { actionTypeRegistry },
} = useKibana().services;
const { setConnectorId } = useActions();
const { setConnector } = useActions();
const rowCss = useRowCss();
return (
<>
{connectors.map((connector) => (
<EuiFlexItem key={connector.id}>
<EuiPanel
key={connector.id}
onClick={() => setConnectorId(connector.id)}
onClick={() => setConnector(connector)}
hasShadow={false}
hasBorder
paddingSize="l"
Expand Down
12 changes: 6 additions & 6 deletions ...s/create_integration/create_integration_assistant/steps/connector_step/connector_step.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ import * as i18n from './translations';
const AllowedActionTypeIds = ['.bedrock'];

interface ConnectorStepProps {
connectorId: string | undefined;
connector: AIConnector | undefined;
}
export const ConnectorStep = React.memo<ConnectorStepProps>(({ connectorId }) => {
export const ConnectorStep = React.memo<ConnectorStepProps>(({ connector }) => {
const { http, notifications } = useKibana().services;
const { setConnectorId } = useActions();
const { setConnector } = useActions();
const [connectors, setConnectors] = useState<AIConnector[]>();
const {
isLoading,
Expand All @@ -48,10 +48,10 @@ export const ConnectorStep = React.memo<ConnectorStepProps>(({ connectorId }) =>
setConnectors(filteredAiConnectors);
if (filteredAiConnectors && filteredAiConnectors.length === 1) {
// pre-select the connector if there is only one
setConnectorId(filteredAiConnectors[0].id);
setConnector(filteredAiConnectors[0]);
}
}
}, [aiConnectors, setConnectorId]);
}, [aiConnectors, setConnector]);

const onConnectorSaved = useCallback(() => refetchConnectors(), [refetchConnectors]);

Expand All @@ -71,7 +71,7 @@ export const ConnectorStep = React.memo<ConnectorStepProps>(({ connectorId }) =>
<>
{hasConnectors ? (
<EuiFlexGroup alignItems="stretch" direction="column" gutterSize="s">
<ConnectorSelector connectors={connectors} selectedConnectorId={connectorId} />
<ConnectorSelector connectors={connectors} selectedConnectorId={connector?.id} />
</EuiFlexGroup>
) : (
<AuthorizationWrapper canCreateConnectors>
Expand Down
2 changes: 1 addition & 1 deletion ...nts/create_integration/create_integration_assistant/steps/connector_step/is_step_ready.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@

import type { State } from '../../state';

export const isConnectorStepReady = ({ connectorId }: State) => connectorId != null;
export const isConnectorStepReady = ({ connector }: State) => connector != null;
6 changes: 3 additions & 3 deletions ...eate_integration/create_integration_assistant/steps/data_stream_step/data_stream_step.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ const getNameFromTitle = (title: string) => title.toLowerCase().replaceAll(/[^a-

interface DataStreamStepProps {
integrationSettings: State['integrationSettings'];
connectorId: State['connectorId'];
connector: State['connector'];
isGenerating: State['isGenerating'];
}
export const DataStreamStep = React.memo<DataStreamStepProps>(
({ integrationSettings, connectorId, isGenerating }) => {
({ integrationSettings, connector, isGenerating }) => {
const { setIntegrationSettings, setIsGenerating, setStep, setResult } = useActions();
const { isLoading: isLoadingPackageNames, packageNames } = useLoadPackageNames(); // this is used to avoid duplicate names

Expand Down Expand Up @@ -217,7 +217,7 @@ export const DataStreamStep = React.memo<DataStreamStepProps>(
{isGenerating && (
<GenerationModal
integrationSettings={integrationSettings}
connectorId={connectorId}
connector={connector}
onComplete={onGenerationCompleted}
onClose={onGenerationClosed}
/>
Expand Down
56 changes: 45 additions & 11 deletions ...eate_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
import { useKibana } from '../../../../../common/hooks/use_kibana';
import type { State } from '../../state';
import * as i18n from './translations';
import { useTelemetry } from '../../../telemetry';

export type OnComplete = (result: State['result']) => void;

Expand All @@ -49,22 +50,29 @@ const progressText: Record<ProgressItem, string> = {

interface UseGenerationProps {
integrationSettings: State['integrationSettings'];
connectorId: State['connectorId'];
connector: State['connector'];
onComplete: OnComplete;
}
export const useGeneration = ({
integrationSettings,
connectorId,
connector,
onComplete,
}: UseGenerationProps) => {
const { reportGenerationComplete } = useTelemetry();
const { http, notifications } = useKibana().services;
const [progress, setProgress] = useState<ProgressItem>();
const [error, setError] = useState<null | string>(null);

useEffect(() => {
if (http == null || integrationSettings == null || notifications?.toasts == null) {
if (
http == null ||
connector == null ||
integrationSettings == null ||
notifications?.toasts == null
) {
return;
}
const generationStartedAt = Date.now();
const abortController = new AbortController();
const deps = { http, abortSignal: abortController.signal };

Expand All @@ -74,7 +82,7 @@ export const useGeneration = ({
packageName: integrationSettings.name ?? '',
dataStreamName: integrationSettings.dataStreamName ?? '',
rawSamples: integrationSettings.logsSampleParsed ?? [],
connectorId: connectorId ?? '',
connectorId: connector.id,
};

setProgress('ecs');
Expand All @@ -100,18 +108,44 @@ export const useGeneration = ({
setProgress('related');
const relatedGraphResult = await runRelatedGraph(relatedRequest, deps);
if (abortController.signal.aborted) return;
if (!isEmpty(relatedGraphResult?.results)) {
onComplete(relatedGraphResult.results);

if (isEmpty(relatedGraphResult?.results)) {
throw new Error('Results not found in response');
}

reportGenerationComplete({
connector,
integrationSettings,
durationMs: Date.now() - generationStartedAt,
});

onComplete(relatedGraphResult.results);
} catch (e) {
if (abortController.signal.aborted) return;
setError(`Error: ${e.body.message}`);
const errorMessage = e.body?.message ?? e.message;

reportGenerationComplete({
connector,
integrationSettings,
durationMs: Date.now() - generationStartedAt,
error: errorMessage,
});

setError(`Error: ${errorMessage}`);
}
})();
return () => {
abortController.abort();
};
}, [onComplete, setProgress, connectorId, http, integrationSettings, notifications?.toasts]);
}, [
onComplete,
setProgress,
connector,
http,
integrationSettings,
reportGenerationComplete,
notifications?.toasts,
]);

return {
progress,
Expand All @@ -135,16 +169,16 @@ const useModalCss = () => {

interface GenerationModalProps {
integrationSettings: State['integrationSettings'];
connectorId: State['connectorId'];
connector: State['connector'];
onComplete: OnComplete;
onClose: () => void;
}
export const GenerationModal = React.memo<GenerationModalProps>(
({ integrationSettings, connectorId, onComplete, onClose }) => {
({ integrationSettings, connector, onComplete, onClose }) => {
const { headerCss, bodyCss } = useModalCss();
const { progress, error } = useGeneration({
integrationSettings,
connectorId,
connector,
onComplete,
});

Expand Down
Loading

0 comments on commit 2ed3609

Please sign in to comment.