[Automatic Import] Reject CEF logs in Auto Import until it is support…

…ed (#201792) ## Release Note Restrict and Reject CEF logs in Automatic Import and redirect to CEF integration instead. ## Summary Currently Automatic Import does not handle CEF logs properly and gives wierd errors. This PR identifies the CEF logs and sends an error popup to alternatively go for CEF integration instead. <img width="1229" alt="image" src="https://github.com/user-attachments/assets/59037dd4-323a-476a-9747-950fbc6e384d"> ## Testing Tested this with different types of CEF logs ``` <14>Nov 22 16:19:13 ABQ-ZTA-VRNS-3 CEF:0|Varonis Inc.|DatAdvantage|8.6.51|6000|Folder permissions added|3|rt=Nov 22 2024 16:19:09 cat=Alert cs2=Permissions granted to Global Access Groups cs2Label=RuleName cn1=132 cn1Label=RuleID end=Nov 22 2024 16:19:05 duser=zta.local\\Dani Lulli (ADMIN) dhost=10.100.20.12 filePath=E:\\Share\\Share\\Finance fname=Finance act=Folder permissions added dvchost= outcome=Success msg=Read & Execute permissions for This folder, subfolders and files (not inherited) was added to Everyone on E:\\Share\\Share\\Finance cs3= cs3Label=AttachmentName cs4= cs4Label=ClientAccessType deviceCustomDate1= fileType= cs1= cs1Label=MailRecipient suser= cs5= cs5Label=MailboxAccessType cnt= cs6=Read & Execute cs6Label=ChangedPermissions oldFilePermission=None filePermission=Read & Execute dpriv=Everyone start= <14>Nov 22 16:44:31 ABQ-ZTA-VRNS-3 CEF:0|Varonis Inc.|DatAdvantage|8.6.51|1|File opened|2|rt=Nov 22 2024 16:44:31 cat=Alert cs2=Dani Test - access of credentials cs2Label=RuleName cn1=184 cn1Label=RuleID end=Nov 22 2024 16:34:33 duser=zta.local\\Dani Lulli (ADMIN) dhost=10.100.20.12 filePath=E:\\Share\\Share\\B4\\Project mgmt\\U3 projects11.txt:Zone.Identifier fname=U3 projects11.txt:Zone.Identifier act=File opened dvchost= outcome=Success msg= cs3= cs3Label=AttachmentName cs4= cs4Label=ClientAccessType deviceCustomDate1= fileType= cs1= cs1Label=MailRecipient suser= cs5= cs5Label=MailboxAccessType cnt= cs6=None cs6Label=ChangedPermissions oldFilePermission=None filePermission=None dpriv= start= ``` ``` CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=89.160.20.156 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=89.160.20.156 spt=33876 dst=89.160.20.156 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4 ``` ``` <163>Apr 1 05:14:15 192.0.2.1 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|msg=rfc3164 Apr 1 05:14:15 192.0.2.1 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|msg=rfc3164 <164>1 2021-04-01T05:14:15.000003-05:00 192.0.2.1 rfc5424App 8710 - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|msg=rfc5424 2021-04-01T05:14:15.000003-05:00 192.0.2.1 rfc5424App 8710 - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|msg=rfc5424 <165>1 2021-04-01T05:14:15.000003Z 192.0.2.1 rfc5424App 8710 - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|msg=rfc5424 2021-04-01T05:14:15.000003Z 192.0.2.1 rfc5424App 8710 - - CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|msg=rfc5424 ``` ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Elastic Machine <[email protected]>
elastic · Dec 2, 2024 · f6fa94f · f6fa94f
1 parent 76c2f31
commit f6fa94f
Show file tree

Hide file tree

Showing 13 changed files with 258 additions and 6 deletions.
diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenerationErrorCode } from '../constants';
+import { isGenerationErrorBody } from './generation_error';
+import type { GenerationErrorBody } from './generation_error';
+
+describe('isGenerationErrorBody', () => {
+  it('should return true for a valid GenerationErrorBody object', () => {
+    const validErrorBody: GenerationErrorBody = {
+      message: 'An error occurred',
+      attributes: {
+        errorCode: GenerationErrorCode.CEF_ERROR,
+        underlyingMessages: ['Error message 1', 'Error message 2'],
+      },
+    };
+
+    expect(isGenerationErrorBody(validErrorBody)).toBe(true);
+  });
+
+  it('should return false for an object without a message', () => {
+    const invalidErrorBody = {
+      attributes: {
+        errorCode: 'ERROR_CODE',
+        underlyingMessages: ['Error message 1', 'Error message 2'],
+      },
+    };
+
+    expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+  });
+
+  it('should return false for an object without attributes', () => {
+    const invalidErrorBody = {
+      message: 'An error occurred',
+    };
+
+    expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+  });
+
+  it('should return false for an object with invalid attributes', () => {
+    const invalidErrorBody = {
+      message: 'An error occurred',
+      attributes: {
+        errorCode: 123, // errorCode should be a string
+        underlyingMessages: 'Error message', // underlyingMessages should be an array
+      },
+    };
+
+    expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+  });
+
+  it('should return false for a non-object value', () => {
+    expect(isGenerationErrorBody(null)).toBe(false);
+    expect(isGenerationErrorBody(undefined)).toBe(false);
+    expect(isGenerationErrorBody('string')).toBe(false);
+    expect(isGenerationErrorBody(123)).toBe(false);
+    expect(isGenerationErrorBody(true)).toBe(false);
+  });
+});
diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.ts
@@ -13,6 +13,12 @@ export interface GenerationErrorBody {
   attributes: GenerationErrorAttributes;
 }
 
+export interface ErrorMessageWithLink {
+  link: string;
+  linkText: string;
+  errorMessage: string;
+}
+
 export function isGenerationErrorBody(obj: unknown | undefined): obj is GenerationErrorBody {
   return (
     typeof obj === 'object' &&
@@ -27,7 +33,8 @@ export function isGenerationErrorBody(obj: unknown | undefined): obj is Generati
 
 export interface GenerationErrorAttributes {
   errorCode: GenerationErrorCode;
-  underlyingMessages: string[] | undefined;
+  underlyingMessages?: string[] | undefined;
+  errorMessageWithLink?: ErrorMessageWithLink | undefined;
 }
 
 export function isGenerationErrorAttributes(obj: unknown): obj is GenerationErrorAttributes {

diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts
@@ -84,6 +84,7 @@ export const SamplesFormatName = z.enum([
   'structured',
   'unstructured',
   'unsupported',
+  'cef',
 ]);
 export type SamplesFormatNameEnum = typeof SamplesFormatName.enum;
 export const SamplesFormatNameEnum = SamplesFormatName.enum;

diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml
@@ -64,6 +64,7 @@ components:
         - structured
         - unstructured
         - unsupported
+        - cef
 
     SamplesFormat:
       type: object

diff --git a/x-pack/plugins/integration_assistant/common/constants.ts b/x-pack/plugins/integration_assistant/common/constants.ts
@@ -35,6 +35,7 @@ export enum GenerationErrorCode {
   RECURSION_LIMIT_ANALYZE_LOGS = 'recursion-limit-analyze-logs',
   UNSUPPORTED_LOG_SAMPLES_FORMAT = 'unsupported-log-samples-format',
   UNPARSEABLE_CSV_DATA = 'unparseable-csv-data',
+  CEF_ERROR = 'cef-not-supported',
 }
 
 // Size limits

diff --git a/..._integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx b/..._integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { render } from '@testing-library/react';
+import { ErrorMessage, isErrorMessageWithLink, MessageLink } from './error_with_link';
+
+describe('isErrorMessageWithLink', () => {
+  it('should return true when error is an ErrorMessageWithLink', () => {
+    const error = {
+      link: 'http://example.com',
+      errorMessage: 'An error occurred',
+      linkText: 'decode_cef',
+    };
+    expect(isErrorMessageWithLink(error)).toBe(true);
+  });
+
+  it('should return false when error is a string', () => {
+    const error = 'An error occurred';
+    expect(isErrorMessageWithLink(error)).toBe(false);
+  });
+
+  describe('MessageLink', () => {
+    it('should render link with correct href and text', () => {
+      const { getByText } = render(<MessageLink link="http://example.com" linkText="decode_cef" />);
+      const linkElement = getByText('decode_cef');
+      expect(linkElement).toBeInTheDocument();
+      expect(linkElement).toHaveAttribute('href', 'http://example.com');
+    });
+  });
+
+  describe('ErrorMessage', () => {
+    it('should render error message when error is a string', () => {
+      const error = 'An error occurred';
+      const { getByText } = render(<ErrorMessage error={error} />);
+      expect(getByText('An error occurred')).toBeInTheDocument();
+    });
+  });
+});
diff --git a/...reate_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx b/...reate_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { EuiLink } from '@elastic/eui';
+import type { ErrorMessageWithLink } from '../../../../../../common/api/generation_error';
+
+interface ErrorMessageProps {
+  error: string | null | ErrorMessageWithLink;
+}
+
+interface MessageLinkProps {
+  link: string;
+  linkText: string;
+}
+
+export const isErrorMessageWithLink = (
+  error: string | ErrorMessageWithLink | null
+): error is ErrorMessageWithLink => {
+  return (
+    (error as ErrorMessageWithLink).link !== undefined &&
+    (error as ErrorMessageWithLink).linkText !== undefined &&
+    (error as ErrorMessageWithLink).errorMessage !== undefined
+  );
+};
+
+export const MessageLink = React.memo<MessageLinkProps>(({ link, linkText }) => {
+  return (
+    <EuiLink href={link} target="_blank">
+      {linkText}
+    </EuiLink>
+  );
+});
+MessageLink.displayName = 'MessageLink';
+
+export const ErrorMessage = React.memo<ErrorMessageProps>(({ error }) => {
+  return (
+    <>
+      {isErrorMessageWithLink(error) ? (
+        <FormattedMessage
+          id="xpack.integrationAssistant.createIntegration.generateErrorWithLink"
+          defaultMessage="{errorMessage} {link}"
+          values={{
+            errorMessage: error.errorMessage,
+            link: <MessageLink link={error.link} linkText={error.linkText} />,
+          }}
+        />
+      ) : typeof error === 'string' ? (
+        <>{error}</>
+      ) : null}
+    </>
+  );
+});
+ErrorMessage.displayName = 'ErrorMessage';
diff --git a/...eate_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx b/...eate_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx
@@ -28,6 +28,7 @@ import * as i18n from './translations';
 
 import type { OnComplete, ProgressItem } from './use_generation';
 import { ProgressOrder, useGeneration } from './use_generation';
+import { ErrorMessage } from './error_with_link';
 
 const progressText: Record<ProgressItem, string> = {
   analyzeLogs: i18n.PROGRESS_ANALYZE_LOGS,
@@ -87,7 +88,7 @@ export const GenerationModal = React.memo<GenerationModalProps>(
                       iconType="alert"
                       data-test-subj="generationErrorCallout"
                     >
-                      {error}
+                      <ErrorMessage error={error} />
                     </EuiCallOut>
                   </EuiFlexItem>
                 ) : (

diff --git a/...ts/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts b/...ts/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
@@ -188,6 +188,13 @@ export const RETRY = i18n.translate('xpack.integrationAssistant.step.dataStream.
   defaultMessage: 'Retry',
 });
 
+export const DECODE_CEF_LINK = i18n.translate(
+  'xpack.integrationAssistant.errors.cefFormat.decodeLink',
+  {
+    defaultMessage: 'CEF format not supported yet. Instead please use CEF Integration:',
+  }
+);
+
 export const GENERATION_ERROR_TRANSLATION: Record<
   GenerationErrorCode,
   string | ((attributes: GenerationErrorAttributes) => string)
@@ -211,6 +218,11 @@ export const GENERATION_ERROR_TRANSLATION: Record<
       defaultMessage: 'Unsupported log format in the samples.',
     }
   ),
+  [GenerationErrorCode.CEF_ERROR]: i18n.translate('xpack.integrationAssistant.errors.cefError', {
+    // This is a default error message if the linking does not work.
+    defaultMessage:
+      'CEF format detected. Please decode the CEF logs into JSON format using filebeat decode_cef processor.',
+  }),
   [GenerationErrorCode.UNPARSEABLE_CSV_DATA]: (attributes) => {
     if (
       attributes.underlyingMessages !== undefined &&

diff --git a/...create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx b/...create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx
@@ -28,6 +28,8 @@ import type { State } from '../../state';
 import * as i18n from './translations';
 import { useTelemetry } from '../../../telemetry';
 import type { AIConnector, IntegrationSettings } from '../../types';
+import type { ErrorMessageWithLink } from '../../../../../../common/api/generation_error';
+import { GenerationErrorCode } from '../../../../../../common/constants';
 
 export type OnComplete = (result: State['result']) => void;
 export const ProgressOrder = ['analyzeLogs', 'ecs', 'categorization', 'related'] as const;
@@ -48,12 +50,23 @@ interface RunGenerationProps {
 
 // If the result is classified as a generation error, produce an error message
 // as defined in the i18n file. Otherwise, return undefined.
-function generationErrorMessage(body: unknown | undefined): string | undefined {
+function generationErrorMessage(
+  body: unknown | undefined
+): string | ErrorMessageWithLink | undefined {
   if (!isGenerationErrorBody(body)) {
     return;
   }
 
   const errorCode = body.attributes.errorCode;
+  if (errorCode === GenerationErrorCode.CEF_ERROR) {
+    if (body.attributes.errorMessageWithLink !== undefined) {
+      return {
+        link: body.attributes.errorMessageWithLink.link,
+        errorMessage: i18n.DECODE_CEF_LINK,
+        linkText: body.attributes.errorMessageWithLink.linkText,
+      };
+    }
+  }
   const translation = i18n.GENERATION_ERROR_TRANSLATION[errorCode];
   return typeof translation === 'function' ? translation(body.attributes) : translation;
 }
@@ -72,7 +85,7 @@ export const useGeneration = ({
   const { reportGenerationComplete } = useTelemetry();
   const { http, notifications } = useKibana().services;
   const [progress, setProgress] = useState<ProgressItem>();
-  const [error, setError] = useState<null | string>(null);
+  const [error, setError] = useState<null | string | ErrorMessageWithLink>(null);
   const [isRequesting, setIsRequesting] = useState<boolean>(true);
 
   useEffect(() => {

diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts
@@ -23,6 +23,7 @@ Follow these steps to do this:
      b. If there is no csv header then set "header: false" and try to find good names for the columns in the "columns" array by looking into the values of data in those columns. For each column, if you are unable to find good name candidate for it then output an empty string, like in the example.
   * 'structured': If the log samples have structured message body with key-value pairs then classify it as "name: structured". Look for a flat list of key-value pairs, often separated by some delimiters. Consider variations in formatting, such as quotes around values ("key=value", key="value"), special characters in keys or values, or escape sequences.
   * 'unstructured': If the log samples have unstructured body like a free-form text then classify it as "name: unstructured".
+  * 'cef': If the log samples have Common Event Format (CEF) then classify it as "name: cef".
   * 'unsupported': If you cannot put the format into any of the above categories then classify it with "name: unsupported".
 2. Header: for structured and unstructured format:
   - if the samples have any or all of priority, timestamp, loglevel, hostname, ipAddress, messageId in the beginning information then set "header: true".

diff --git a/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts b/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaResponseFactory } from '@kbn/core/server';
+import {
+  GenerationErrorAttributes,
+  GenerationErrorBody,
+} from '../../../common/api/generation_error';
+import { ErrorThatHandlesItsOwnResponse } from './types';
+import { GenerationErrorCode } from '../../../common/constants';
+
+export class CefError extends Error implements ErrorThatHandlesItsOwnResponse {
+  private readonly errorCode: GenerationErrorCode = GenerationErrorCode.CEF_ERROR;
+  attributes: GenerationErrorAttributes;
+
+  constructor(message: string) {
+    super(message);
+    this.attributes = {
+      errorCode: this.errorCode,
+      errorMessageWithLink: {
+        linkText: 'cef-integration',
+        link: 'https://www.elastic.co/docs/current/integrations/cef',
+        errorMessage: '', // Will be set using translation in the UI.
+      },
+    };
+  }
+
+  public sendResponse(res: KibanaResponseFactory) {
+    const body: GenerationErrorBody = {
+      message: this.errorCode,
+      attributes: this.attributes,
+    };
+    return res.customError({
+      statusCode: 501,
+      body,
+    });
+  }
+}
diff --git a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts
@@ -19,6 +19,7 @@ import { withAvailability } from './with_availability';
 import { isErrorThatHandlesItsOwnResponse, UnsupportedLogFormatError } from '../lib/errors';
 import { handleCustomErrors } from './routes_util';
 import { GenerationErrorCode } from '../../common/constants';
+import { CefError } from '../lib/errors/cef_error';
 
 export function registerAnalyzeLogsRoutes(
   router: IRouter<IntegrationAssistantRouteHandlerContext>
@@ -102,9 +103,16 @@ export function registerAnalyzeLogsRoutes(
             .withConfig({ runName: 'Log Format' })
             .invoke(logFormatParameters, options);
           const graphLogFormat = graphResults.results.samplesFormat.name;
-          if (graphLogFormat === 'unsupported') {
-            throw new UnsupportedLogFormatError(GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT);
+
+          switch (graphLogFormat) {
+            case 'unsupported':
+              throw new UnsupportedLogFormatError(
+                GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT
+              );
+            case 'cef':
+              throw new CefError(GenerationErrorCode.CEF_ERROR);
           }
+
           return res.ok({ body: AnalyzeLogsResponse.parse(graphResults) });
         } catch (err) {
           try {