From a1cb4c22e82e107313fa8a9c587a9de47a280689 Mon Sep 17 00:00:00 2001
From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Date: Tue, 3 Dec 2024 00:44:53 +1100
Subject: [PATCH] [8.x] [Automatic Import] Reject CEF logs in Auto Import until
it is supported (#201792) (#202444)
# Backport
This will backport the following commits from `main` to `8.x`:
- [[Automatic Import] Reject CEF logs in Auto Import until it is
supported (#201792)](https://github.com/elastic/kibana/pull/201792)
### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)
Co-authored-by: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>
---
.../common/api/generation_error.test.ts | 63 +++++++++++++++++++
.../common/api/generation_error.ts | 9 ++-
.../common/api/model/common_attributes.gen.ts | 1 +
.../api/model/common_attributes.schema.yaml | 1 +
.../integration_assistant/common/constants.ts | 1 +
.../data_stream_step/error_with_link.test.tsx | 43 +++++++++++++
.../data_stream_step/error_with_link.tsx | 59 +++++++++++++++++
.../data_stream_step/generation_modal.tsx | 3 +-
.../steps/data_stream_step/translations.ts | 12 ++++
.../steps/data_stream_step/use_generation.tsx | 17 ++++-
.../graphs/log_type_detection/prompts.ts | 1 +
.../server/lib/errors/cef_error.ts | 42 +++++++++++++
.../server/routes/analyze_logs_routes.ts | 12 +++-
13 files changed, 258 insertions(+), 6 deletions(-)
create mode 100644 x-pack/plugins/integration_assistant/common/api/generation_error.test.ts
create mode 100644 x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx
create mode 100644 x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx
create mode 100644 x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts
diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts
new file mode 100644
index 0000000000000..b18414610664c
--- /dev/null
+++ b/x-pack/plugins/integration_assistant/common/api/generation_error.test.ts
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenerationErrorCode } from '../constants';
+import { isGenerationErrorBody } from './generation_error';
+import type { GenerationErrorBody } from './generation_error';
+
+describe('isGenerationErrorBody', () => {
+ it('should return true for a valid GenerationErrorBody object', () => {
+ const validErrorBody: GenerationErrorBody = {
+ message: 'An error occurred',
+ attributes: {
+ errorCode: GenerationErrorCode.CEF_ERROR,
+ underlyingMessages: ['Error message 1', 'Error message 2'],
+ },
+ };
+
+ expect(isGenerationErrorBody(validErrorBody)).toBe(true);
+ });
+
+ it('should return false for an object without a message', () => {
+ const invalidErrorBody = {
+ attributes: {
+ errorCode: 'ERROR_CODE',
+ underlyingMessages: ['Error message 1', 'Error message 2'],
+ },
+ };
+
+ expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+ });
+
+ it('should return false for an object without attributes', () => {
+ const invalidErrorBody = {
+ message: 'An error occurred',
+ };
+
+ expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+ });
+
+ it('should return false for an object with invalid attributes', () => {
+ const invalidErrorBody = {
+ message: 'An error occurred',
+ attributes: {
+ errorCode: 123, // errorCode should be a string
+ underlyingMessages: 'Error message', // underlyingMessages should be an array
+ },
+ };
+
+ expect(isGenerationErrorBody(invalidErrorBody)).toBe(false);
+ });
+
+ it('should return false for a non-object value', () => {
+ expect(isGenerationErrorBody(null)).toBe(false);
+ expect(isGenerationErrorBody(undefined)).toBe(false);
+ expect(isGenerationErrorBody('string')).toBe(false);
+ expect(isGenerationErrorBody(123)).toBe(false);
+ expect(isGenerationErrorBody(true)).toBe(false);
+ });
+});
diff --git a/x-pack/plugins/integration_assistant/common/api/generation_error.ts b/x-pack/plugins/integration_assistant/common/api/generation_error.ts
index e96ad8bff9b59..03f01e96bee53 100644
--- a/x-pack/plugins/integration_assistant/common/api/generation_error.ts
+++ b/x-pack/plugins/integration_assistant/common/api/generation_error.ts
@@ -13,6 +13,12 @@ export interface GenerationErrorBody {
attributes: GenerationErrorAttributes;
}
+export interface ErrorMessageWithLink {
+ link: string;
+ linkText: string;
+ errorMessage: string;
+}
+
export function isGenerationErrorBody(obj: unknown | undefined): obj is GenerationErrorBody {
return (
typeof obj === 'object' &&
@@ -27,7 +33,8 @@ export function isGenerationErrorBody(obj: unknown | undefined): obj is Generati
export interface GenerationErrorAttributes {
errorCode: GenerationErrorCode;
- underlyingMessages: string[] | undefined;
+ underlyingMessages?: string[] | undefined;
+ errorMessageWithLink?: ErrorMessageWithLink | undefined;
}
export function isGenerationErrorAttributes(obj: unknown): obj is GenerationErrorAttributes {
diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts
index 803f9b8a6c3af..3b8dac7af22ca 100644
--- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts
+++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.gen.ts
@@ -84,6 +84,7 @@ export const SamplesFormatName = z.enum([
'structured',
'unstructured',
'unsupported',
+ 'cef',
]);
export type SamplesFormatNameEnum = typeof SamplesFormatName.enum;
export const SamplesFormatNameEnum = SamplesFormatName.enum;
diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml
index 900b6e362a754..23ad137d8d83a 100644
--- a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml
+++ b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml
@@ -64,6 +64,7 @@ components:
- structured
- unstructured
- unsupported
+ - cef
SamplesFormat:
type: object
diff --git a/x-pack/plugins/integration_assistant/common/constants.ts b/x-pack/plugins/integration_assistant/common/constants.ts
index 4d791341e34f9..3026101ebf54d 100644
--- a/x-pack/plugins/integration_assistant/common/constants.ts
+++ b/x-pack/plugins/integration_assistant/common/constants.ts
@@ -35,6 +35,7 @@ export enum GenerationErrorCode {
RECURSION_LIMIT_ANALYZE_LOGS = 'recursion-limit-analyze-logs',
UNSUPPORTED_LOG_SAMPLES_FORMAT = 'unsupported-log-samples-format',
UNPARSEABLE_CSV_DATA = 'unparseable-csv-data',
+ CEF_ERROR = 'cef-not-supported',
}
// Size limits
diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx
new file mode 100644
index 0000000000000..105bacf45e92c
--- /dev/null
+++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.test.tsx
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { render } from '@testing-library/react';
+import { ErrorMessage, isErrorMessageWithLink, MessageLink } from './error_with_link';
+
+describe('isErrorMessageWithLink', () => {
+ it('should return true when error is an ErrorMessageWithLink', () => {
+ const error = {
+ link: 'http://example.com',
+ errorMessage: 'An error occurred',
+ linkText: 'decode_cef',
+ };
+ expect(isErrorMessageWithLink(error)).toBe(true);
+ });
+
+ it('should return false when error is a string', () => {
+ const error = 'An error occurred';
+ expect(isErrorMessageWithLink(error)).toBe(false);
+ });
+
+ describe('MessageLink', () => {
+ it('should render link with correct href and text', () => {
+ const { getByText } = render();
+ const linkElement = getByText('decode_cef');
+ expect(linkElement).toBeInTheDocument();
+ expect(linkElement).toHaveAttribute('href', 'http://example.com');
+ });
+ });
+
+ describe('ErrorMessage', () => {
+ it('should render error message when error is a string', () => {
+ const error = 'An error occurred';
+ const { getByText } = render();
+ expect(getByText('An error occurred')).toBeInTheDocument();
+ });
+ });
+});
diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx
new file mode 100644
index 0000000000000..8791e9f03a0f8
--- /dev/null
+++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/error_with_link.tsx
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { EuiLink } from '@elastic/eui';
+import type { ErrorMessageWithLink } from '../../../../../../common/api/generation_error';
+
+interface ErrorMessageProps {
+ error: string | null | ErrorMessageWithLink;
+}
+
+interface MessageLinkProps {
+ link: string;
+ linkText: string;
+}
+
+export const isErrorMessageWithLink = (
+ error: string | ErrorMessageWithLink | null
+): error is ErrorMessageWithLink => {
+ return (
+ (error as ErrorMessageWithLink).link !== undefined &&
+ (error as ErrorMessageWithLink).linkText !== undefined &&
+ (error as ErrorMessageWithLink).errorMessage !== undefined
+ );
+};
+
+export const MessageLink = React.memo(({ link, linkText }) => {
+ return (
+
+ {linkText}
+
+ );
+});
+MessageLink.displayName = 'MessageLink';
+
+export const ErrorMessage = React.memo(({ error }) => {
+ return (
+ <>
+ {isErrorMessageWithLink(error) ? (
+ ,
+ }}
+ />
+ ) : typeof error === 'string' ? (
+ <>{error}
+ ) : null}
+
+ );
+});
+ErrorMessage.displayName = 'ErrorMessage';
diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx
index ba57d83618c13..9d8d52a5b8783 100644
--- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx
+++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/generation_modal.tsx
@@ -28,6 +28,7 @@ import * as i18n from './translations';
import type { OnComplete, ProgressItem } from './use_generation';
import { ProgressOrder, useGeneration } from './use_generation';
+import { ErrorMessage } from './error_with_link';
const progressText: Record = {
analyzeLogs: i18n.PROGRESS_ANALYZE_LOGS,
@@ -87,7 +88,7 @@ export const GenerationModal = React.memo(
iconType="alert"
data-test-subj="generationErrorCallout"
>
- {error}
+
) : (
diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
index ec90568da0ef9..8f2aab49622f0 100644
--- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
+++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
@@ -188,6 +188,13 @@ export const RETRY = i18n.translate('xpack.integrationAssistant.step.dataStream.
defaultMessage: 'Retry',
});
+export const DECODE_CEF_LINK = i18n.translate(
+ 'xpack.integrationAssistant.errors.cefFormat.decodeLink',
+ {
+ defaultMessage: 'CEF format not supported yet. Instead please use CEF Integration:',
+ }
+);
+
export const GENERATION_ERROR_TRANSLATION: Record<
GenerationErrorCode,
string | ((attributes: GenerationErrorAttributes) => string)
@@ -211,6 +218,11 @@ export const GENERATION_ERROR_TRANSLATION: Record<
defaultMessage: 'Unsupported log format in the samples.',
}
),
+ [GenerationErrorCode.CEF_ERROR]: i18n.translate('xpack.integrationAssistant.errors.cefError', {
+ // This is a default error message if the linking does not work.
+ defaultMessage:
+ 'CEF format detected. Please decode the CEF logs into JSON format using filebeat decode_cef processor.',
+ }),
[GenerationErrorCode.UNPARSEABLE_CSV_DATA]: (attributes) => {
if (
attributes.underlyingMessages !== undefined &&
diff --git a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx
index 566451d624c5e..de6b2eed038ec 100644
--- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx
+++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/use_generation.tsx
@@ -28,6 +28,8 @@ import type { State } from '../../state';
import * as i18n from './translations';
import { useTelemetry } from '../../../telemetry';
import type { AIConnector, IntegrationSettings } from '../../types';
+import type { ErrorMessageWithLink } from '../../../../../../common/api/generation_error';
+import { GenerationErrorCode } from '../../../../../../common/constants';
export type OnComplete = (result: State['result']) => void;
export const ProgressOrder = ['analyzeLogs', 'ecs', 'categorization', 'related'] as const;
@@ -48,12 +50,23 @@ interface RunGenerationProps {
// If the result is classified as a generation error, produce an error message
// as defined in the i18n file. Otherwise, return undefined.
-function generationErrorMessage(body: unknown | undefined): string | undefined {
+function generationErrorMessage(
+ body: unknown | undefined
+): string | ErrorMessageWithLink | undefined {
if (!isGenerationErrorBody(body)) {
return;
}
const errorCode = body.attributes.errorCode;
+ if (errorCode === GenerationErrorCode.CEF_ERROR) {
+ if (body.attributes.errorMessageWithLink !== undefined) {
+ return {
+ link: body.attributes.errorMessageWithLink.link,
+ errorMessage: i18n.DECODE_CEF_LINK,
+ linkText: body.attributes.errorMessageWithLink.linkText,
+ };
+ }
+ }
const translation = i18n.GENERATION_ERROR_TRANSLATION[errorCode];
return typeof translation === 'function' ? translation(body.attributes) : translation;
}
@@ -72,7 +85,7 @@ export const useGeneration = ({
const { reportGenerationComplete } = useTelemetry();
const { http, notifications } = useKibana().services;
const [progress, setProgress] = useState();
- const [error, setError] = useState(null);
+ const [error, setError] = useState(null);
const [isRequesting, setIsRequesting] = useState(true);
useEffect(() => {
diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts
index b6e777a87888a..09a7249a3786c 100644
--- a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts
+++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/prompts.ts
@@ -23,6 +23,7 @@ Follow these steps to do this:
b. If there is no csv header then set "header: false" and try to find good names for the columns in the "columns" array by looking into the values of data in those columns. For each column, if you are unable to find good name candidate for it then output an empty string, like in the example.
* 'structured': If the log samples have structured message body with key-value pairs then classify it as "name: structured". Look for a flat list of key-value pairs, often separated by some delimiters. Consider variations in formatting, such as quotes around values ("key=value", key="value"), special characters in keys or values, or escape sequences.
* 'unstructured': If the log samples have unstructured body like a free-form text then classify it as "name: unstructured".
+ * 'cef': If the log samples have Common Event Format (CEF) then classify it as "name: cef".
* 'unsupported': If you cannot put the format into any of the above categories then classify it with "name: unsupported".
2. Header: for structured and unstructured format:
- if the samples have any or all of priority, timestamp, loglevel, hostname, ipAddress, messageId in the beginning information then set "header: true".
diff --git a/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts b/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts
new file mode 100644
index 0000000000000..16889b30c1a11
--- /dev/null
+++ b/x-pack/plugins/integration_assistant/server/lib/errors/cef_error.ts
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaResponseFactory } from '@kbn/core/server';
+import {
+ GenerationErrorAttributes,
+ GenerationErrorBody,
+} from '../../../common/api/generation_error';
+import { ErrorThatHandlesItsOwnResponse } from './types';
+import { GenerationErrorCode } from '../../../common/constants';
+
+export class CefError extends Error implements ErrorThatHandlesItsOwnResponse {
+ private readonly errorCode: GenerationErrorCode = GenerationErrorCode.CEF_ERROR;
+ attributes: GenerationErrorAttributes;
+
+ constructor(message: string) {
+ super(message);
+ this.attributes = {
+ errorCode: this.errorCode,
+ errorMessageWithLink: {
+ linkText: 'cef-integration',
+ link: 'https://www.elastic.co/docs/current/integrations/cef',
+ errorMessage: '', // Will be set using translation in the UI.
+ },
+ };
+ }
+
+ public sendResponse(res: KibanaResponseFactory) {
+ const body: GenerationErrorBody = {
+ message: this.errorCode,
+ attributes: this.attributes,
+ };
+ return res.customError({
+ statusCode: 501,
+ body,
+ });
+ }
+}
diff --git a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts
index 34f05fcc82025..37926dac19156 100644
--- a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts
+++ b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts
@@ -19,6 +19,7 @@ import { withAvailability } from './with_availability';
import { isErrorThatHandlesItsOwnResponse, UnsupportedLogFormatError } from '../lib/errors';
import { handleCustomErrors } from './routes_util';
import { GenerationErrorCode } from '../../common/constants';
+import { CefError } from '../lib/errors/cef_error';
export function registerAnalyzeLogsRoutes(
router: IRouter
@@ -102,9 +103,16 @@ export function registerAnalyzeLogsRoutes(
.withConfig({ runName: 'Log Format' })
.invoke(logFormatParameters, options);
const graphLogFormat = graphResults.results.samplesFormat.name;
- if (graphLogFormat === 'unsupported') {
- throw new UnsupportedLogFormatError(GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT);
+
+ switch (graphLogFormat) {
+ case 'unsupported':
+ throw new UnsupportedLogFormatError(
+ GenerationErrorCode.UNSUPPORTED_LOG_SAMPLES_FORMAT
+ );
+ case 'cef':
+ throw new CefError(GenerationErrorCode.CEF_ERROR);
}
+
return res.ok({ body: AnalyzeLogsResponse.parse(graphResults) });
} catch (err) {
try {