[8.x] [Automatic Import] Reject CEF logs in Auto Import until it is s…

…upported (#201792) (#202444) # Backport This will backport the following commits from `main` to `8.x`: - [[Automatic Import] Reject CEF logs in Auto Import until it is supported (#201792)](#201792)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Bharat Pasupula <[email protected]>
elastic · Dec 2, 2024 · a1cb4c2 · a1cb4c2
1 parent 9728176
commit a1cb4c2
Show file tree

Hide file tree

Showing 13 changed files with 258 additions and 6 deletions.
diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenerationErrorCode } from '../constants';
+import { isGenerationErrorBody } from './generation_error';
+import type { GenerationErrorBody } from './generation_error';
+
+describe('isGenerationErrorBody', () => {
+  it('should return true for a valid GenerationErrorBody object', () => {
+    const validErrorBody: GenerationErrorBody = {
+      message: 'An error occurred',
+      attributes: {
+        errorCode: GenerationErrorCode.CEF_ERROR,
+        underlyingMessages: ['Error message 1', 'Error message 2'],
+      },
+    };
+
+    expect(isGenerationErrorBody(validErrorBody)).toBe(true);
+  });
+
+  it('should return false for an object without a message', () => {
+    const invalidErrorBody = {
+      attributes: {
+        errorCode: 'ERROR_CODE',
+        underlyingMessages: ['Error message 1', 'Error message 2'],
+      },
+    };
+
+    expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+  });
+
+  it('should return false for an object without attributes', () => {
+    const invalidErrorBody = {
+      message: 'An error occurred',
+    };
+
+    expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+  });
+
+  it('should return false for an object with invalid attributes', () => {
+    const invalidErrorBody = {
+      message: 'An error occurred',
+      attributes: {
+        errorCode: 123, // errorCode should be a string
+        underlyingMessages: 'Error message', // underlyingMessages should be an array
+      },
+    };
+
+    expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+  });
+
+  it('should return false for a non-object value', () => {
+    expect(isGenerationErrorBody(null)).toBe(false);
+    expect(isGenerationErrorBody(undefined)).toBe(false);
+    expect(isGenerationErrorBody('string')).toBe(false);
+    expect(isGenerationErrorBody(123)).toBe(false);
+    expect(isGenerationErrorBody(true)).toBe(false);
+  });
+});
diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.ts
@@ -13,6 +13,12 @@ export interface GenerationErrorBody {
   attributes: GenerationErrorAttributes;
 }
 
+export interface ErrorMessageWithLink {
+  link: string;
+  linkText: string;
+  errorMessage: string;
+}
+
 export function isGenerationErrorBody(obj: unknown | undefined): obj is GenerationErrorBody {
   return (
     typeof obj === 'object' &&
@@ -27,7 +33,8 @@ export function isGenerationErrorBody(obj: unknown | undefined): obj is Generati
 
 export interface GenerationErrorAttributes {
   errorCode: GenerationErrorCode;
-  underlyingMessages: string[] | undefined;
+  underlyingMessages?: string[] | undefined;
+  errorMessageWithLink?: ErrorMessageWithLink | undefined;
 }
 
 export function isGenerationErrorAttributes(obj: unknown): obj is GenerationErrorAttributes {

diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts
@@ -84,6 +84,7 @@ export const SamplesFormatName = z.enum([
   'structured',
   'unstructured',
   'unsupported',
+  'cef',
 ]);
 export type SamplesFormatNameEnum = typeof SamplesFormatName.enum;
 export const SamplesFormatNameEnum = SamplesFormatName.enum;

diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml
@@ -64,6 +64,7 @@ components:
         - structured
         - unstructured
         - unsupported
+        - cef
 
     SamplesFormat:
       type: object

diff --git a/x-pack/plugins/integration_assistant/common/constants.ts b/x-pack/plugins/integration_assistant/common/constants.ts
@@ -35,6 +35,7 @@ export enum GenerationErrorCode {
   RECURSION_LIMIT_ANALYZE_LOGS = 'recursion-limit-analyze-logs',
   UNSUPPORTED_LOG_SAMPLES_FORMAT = 'unsupported-log-samples-format',
   UNPARSEABLE_CSV_DATA = 'unparseable-csv-data',
+  CEF_ERROR = 'cef-not-supported',
 }
 
 // Size limits

diff --git a/..._integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx b/..._integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { render } from '@testing-library/react';
+import { ErrorMessage, isErrorMessageWithLink, MessageLink } from './error_with_link';
+
+describe('isErrorMessageWithLink', () => {
+  it('should return true when error is an ErrorMessageWithLink', () => {
+    const error = {
+      link: 'http://example.com',
+      errorMessage: 'An error occurred',
+      linkText: 'decode_cef',
+    };
+    expect(isErrorMessageWithLink(error)).toBe(true);
+  });
+
+  it('should return false when error is a string', () => {
+    const error = 'An error occurred';
+    expect(isErrorMessageWithLink(error)).toBe(false);
+  });
+
+  describe('MessageLink', () => {
+    it('should render link with correct href and text', () => {
+      const { getByText } = render(<MessageLink link="http://example.com" linkText="decode_cef" />);
+      const linkElement = getByText('decode_cef');
+      expect(linkElement).toBeInTheDocument();
+      expect(linkElement).toHaveAttribute('href', 'http://example.com');
+    });
+  });
+
+  describe('ErrorMessage', () => {
+    it('should render error message when error is a string', () => {
+      const error = 'An error occurred';
+      const { getByText } = render(<ErrorMessage error={error} />);
+      expect(getByText('An error occurred')).toBeInTheDocument();
+    });
+  });
+});
diff --git a/...reate_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx b/...reate_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { EuiLink } from '@elastic/eui';
+import type { ErrorMessageWithLink } from '../../../../../../common/api/generation_error';
+
+interface ErrorMessageProps {
+  error: string | null | ErrorMessageWithLink;
+}
+
+interface MessageLinkProps {
+  link: string;
+  linkText: string;
+}
+
+export const isErrorMessageWithLink = (
+  error: string | ErrorMessageWithLink | null
+): error is ErrorMessageWithLink => {
+  return (
+    (error as ErrorMessageWithLink).link !== undefined &&
+    (error as ErrorMessageWithLink).linkText !== undefined &&
+    (error as ErrorMessageWithLink).errorMessage !== undefined
+  );
+};
+
+export const MessageLink = React.memo<MessageLinkProps>(({ link, linkText }) => {
+  return (
+    <EuiLink href={link} target="_blank">
+      {linkText}
+    </EuiLink>
+  );
+});
+MessageLink.displayName = 'MessageLink';
+
+export const ErrorMessage = React.memo<ErrorMessageProps>(({ error }) => {
+  return (
+    <>
+      {isErrorMessageWithLink(error) ? (
+        <FormattedMessage
+          id="xpack.integrationAssistant.createIntegration.generateErrorWithLink"
+          defaultMessage="{errorMessage} {link}"
+          values={{
+            errorMessage: error.errorMessage,
+            link: <MessageLink link={error.link} linkText={error.linkText} />,
+          }}
+        />
+      ) : typeof error === 'string' ? (
+        <>{error}</>
+      ) : null}
+    </>
+  );
+});
+ErrorMessage.displayName = 'ErrorMessage';
diff --git a/...eate_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx b/...eate_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx
@@ -28,6 +28,7 @@ import * as i18n from './translations';
 
 import type { OnComplete, ProgressItem } from './use_generation';
 import { ProgressOrder, useGeneration } from './use_generation';
+import { ErrorMessage } from './error_with_link';
 
 const progressText: Record<ProgressItem, string> = {
   analyzeLogs: i18n.PROGRESS_ANALYZE_LOGS,
@@ -87,7 +88,7 @@ export const GenerationModal = React.memo<GenerationModalProps>(
                       iconType="alert"
                       data-test-subj="generationErrorCallout"
                     >
-                      {error}
+                      <ErrorMessage error={error} />
                     </EuiCallOut>
                   </EuiFlexItem>
                 ) : (

diff --git a/...ts/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts b/...ts/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
@@ -188,6 +188,13 @@ export const RETRY = i18n.translate('xpack.integrationAssistant.step.dataStream.
   defaultMessage: 'Retry',
 });
 
+export const DECODE_CEF_LINK = i18n.translate(
+  'xpack.integrationAssistant.errors.cefFormat.decodeLink',
+  {
+    defaultMessage: 'CEF format not supported yet. Instead please use CEF Integration:',
+  }
+);
+
 export const GENERATION_ERROR_TRANSLATION: Record<
   GenerationErrorCode,
   string | ((attributes: GenerationErrorAttributes) => string)
@@ -211,6 +218,11 @@ export const GENERATION_ERROR_TRANSLATION: Record<
       defaultMessage: 'Unsupported log format in the samples.',
     }
   ),
+  [GenerationErrorCode.CEF_ERROR]: i18n.translate('xpack.integrationAssistant.errors.cefError', {
+    // This is a default error message if the linking does not work.
+    defaultMessage:
+      'CEF format detected. Please decode the CEF logs into JSON format using filebeat decode_cef processor.',
+  }),
   [GenerationErrorCode.UNPARSEABLE_CSV_DATA]: (attributes) => {
     if (
       attributes.underlyingMessages !== undefined &&

diff --git a/...create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx b/...create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx
@@ -28,6 +28,8 @@ import type { State } from '../../state';
 import * as i18n from './translations';
 import { useTelemetry } from '../../../telemetry';
 import type { AIConnector, IntegrationSettings } from '../../types';
+import type { ErrorMessageWithLink } from '../../../../../../common/api/generation_error';
+import { GenerationErrorCode } from '../../../../../../common/constants';
 
 export type OnComplete = (result: State['result']) => void;
 export const ProgressOrder = ['analyzeLogs', 'ecs', 'categorization', 'related'] as const;
@@ -48,12 +50,23 @@ interface RunGenerationProps {
 
 // If the result is classified as a generation error, produce an error message
 // as defined in the i18n file. Otherwise, return undefined.
-function generationErrorMessage(body: unknown | undefined): string | undefined {
+function generationErrorMessage(
+  body: unknown | undefined
+): string | ErrorMessageWithLink | undefined {
   if (!isGenerationErrorBody(body)) {
     return;
   }
 
   const errorCode = body.attributes.errorCode;
+  if (errorCode === GenerationErrorCode.CEF_ERROR) {
+    if (body.attributes.errorMessageWithLink !== undefined) {
+      return {
+        link: body.attributes.errorMessageWithLink.link,
+        errorMessage: i18n.DECODE_CEF_LINK,
+        linkText: body.attributes.errorMessageWithLink.linkText,
+      };
+    }
+  }
   const translation = i18n.GENERATION_ERROR_TRANSLATION[errorCode];
   return typeof translation === 'function' ? translation(body.attributes) : translation;
 }
@@ -72,7 +85,7 @@ export const useGeneration = ({
   const { reportGenerationComplete } = useTelemetry();
   const { http, notifications } = useKibana().services;
   const [progress, setProgress] = useState<ProgressItem>();
-  const [error, setError] = useState<null | string>(null);
+  const [error, setError] = useState<null | string | ErrorMessageWithLink>(null);
   const [isRequesting, setIsRequesting] = useState<boolean>(true);
 
   useEffect(() => {

diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts
@@ -23,6 +23,7 @@ Follow these steps to do this:
      b. If there is no csv header then set "header: false" and try to find good names for the columns in the "columns" array by looking into the values of data in those columns. For each column, if you are unable to find good name candidate for it then output an empty string, like in the example.
   * 'structured': If the log samples have structured message body with key-value pairs then classify it as "name: structured". Look for a flat list of key-value pairs, often separated by some delimiters. Consider variations in formatting, such as quotes around values ("key=value", key="value"), special characters in keys or values, or escape sequences.
   * 'unstructured': If the log samples have unstructured body like a free-form text then classify it as "name: unstructured".
+  * 'cef': If the log samples have Common Event Format (CEF) then classify it as "name: cef".
   * 'unsupported': If you cannot put the format into any of the above categories then classify it with "name: unsupported".
 2. Header: for structured and unstructured format:
   - if the samples have any or all of priority, timestamp, loglevel, hostname, ipAddress, messageId in the beginning information then set "header: true".

diff --git a/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts b/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaResponseFactory } from '@kbn/core/server';
+import {
+  GenerationErrorAttributes,
+  GenerationErrorBody,
+} from '../../../common/api/generation_error';
+import { ErrorThatHandlesItsOwnResponse } from './types';
+import { GenerationErrorCode } from '../../../common/constants';
+
+export class CefError extends Error implements ErrorThatHandlesItsOwnResponse {
+  private readonly errorCode: GenerationErrorCode = GenerationErrorCode.CEF_ERROR;
+  attributes: GenerationErrorAttributes;
+
+  constructor(message: string) {
+    super(message);
+    this.attributes = {
+      errorCode: this.errorCode,
+      errorMessageWithLink: {
+        linkText: 'cef-integration',
+        link: 'https://www.elastic.co/docs/current/integrations/cef',
+        errorMessage: '', // Will be set using translation in the UI.
+      },
+    };
+  }
+
+  public sendResponse(res: KibanaResponseFactory) {
+    const body: GenerationErrorBody = {
+      message: this.errorCode,
+      attributes: this.attributes,
+    };
+    return res.customError({
+      statusCode: 501,
+      body,
+    });
+  }
+}
diff --git a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts
@@ -19,6 +19,7 @@ import { withAvailability } from './with_availability';
 import { isErrorThatHandlesItsOwnResponse, UnsupportedLogFormatError } from '../lib/errors';
 import { handleCustomErrors } from './routes_util';
 import { GenerationErrorCode } from '../../common/constants';
+import { CefError } from '../lib/errors/cef_error';
 
 export function registerAnalyzeLogsRoutes(
   router: IRouter<IntegrationAssistantRouteHandlerContext>
@@ -102,9 +103,16 @@ export function registerAnalyzeLogsRoutes(
             .withConfig({ runName: 'Log Format' })
             .invoke(logFormatParameters, options);
           const graphLogFormat = graphResults.results.samplesFormat.name;
-          if (graphLogFormat === 'unsupported') {
-            throw new UnsupportedLogFormatError(GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT);
+
+          switch (graphLogFormat) {
+            case 'unsupported':
+              throw new UnsupportedLogFormatError(
+                GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT
+              );
+            case 'cef':
+              throw new CefError(GenerationErrorCode.CEF_ERROR);
           }
+
           return res.ok({ body: AnalyzeLogsResponse.parse(graphResults) });
         } catch (err) {
           try {