From 702b207d2d93019e9efa3f00fdd48c9328c2f893 Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Fri, 5 Jan 2024 11:30:10 +0100 Subject: [PATCH] [Custom threshold] Add log rate analysis to the alert details page for one document count aggregation (#174031) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #171163 ## Summary This PR adds log rate analysis to the alert details page for one document count aggregation if the user has a license above platinum. This is a similar implementation in the log threshold alert details page. ![image](https://github.com/elastic/kibana/assets/12370520/29cd29bf-ead5-4574-8121-739d3ed11fe7) ## 🧪 How to test? - Create a Custom threshold rule with only one document count aggregation - Optional filter, document count aggregation filter, and group by information will be applied to the query of this component. - Go to the alert details page, you should see the log rate analysis on this page --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/custom_threshold_rule/types.ts | 5 + .../utils/get_interval_in_seconds.test.ts | 28 +++ .../common/utils/get_interval_in_seconds.ts | 32 +++ x-pack/plugins/observability/kibana.jsonc | 1 + .../alert_search_bar/alert_search_bar.tsx | 22 +- .../alert_details_app_section.test.tsx.snap | 0 .../alert_details_app_section.test.tsx | 8 +- .../alert_details_app_section.tsx | 28 ++- .../log_rate_analysis_query.test.ts.snap | 222 ++++++++++++++++ .../helpers/get_initial_analysis_start.ts | 58 +++++ .../helpers/log_rate_analysis_query.test.ts | 98 ++++++++ .../helpers/log_rate_analysis_query.ts | 62 +++++ .../log_rate_analysis.tsx | 238 ++++++++++++++++++ .../mocks/custom_threshold_rule.ts | 5 +- .../observability/public/hooks/use_license.ts | 4 +- .../public/pages/overview/overview.tsx | 18 +- x-pack/plugins/observability/public/plugin.ts | 3 + .../register_observability_rule_types.ts | 5 +- .../build_es_query/build_es_query.test.ts | 2 +- .../utils/build_es_query/build_es_query.ts | 14 +- .../custom_threshold/lib/evaluate_rule.ts | 3 +- .../lib/rules/custom_threshold/utils.ts | 26 -- x-pack/plugins/observability/tsconfig.json | 4 +- 23 files changed, 816 insertions(+), 70 deletions(-) create mode 100644 x-pack/plugins/observability/common/utils/get_interval_in_seconds.test.ts create mode 100644 x-pack/plugins/observability/common/utils/get_interval_in_seconds.ts rename x-pack/plugins/observability/public/components/custom_threshold/components/{ => alert_details_app_section}/__snapshots__/alert_details_app_section.test.tsx.snap (100%) rename x-pack/plugins/observability/public/components/custom_threshold/components/{ => alert_details_app_section}/alert_details_app_section.test.tsx (94%) rename x-pack/plugins/observability/public/components/custom_threshold/components/{ => alert_details_app_section}/alert_details_app_section.tsx (86%) create mode 100644 x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/__snapshots__/log_rate_analysis_query.test.ts.snap create mode 100644 x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/get_initial_analysis_start.ts create mode 100644 x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.test.ts create mode 100644 x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.ts create mode 100644 x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx diff --git a/x-pack/plugins/observability/common/custom_threshold_rule/types.ts b/x-pack/plugins/observability/common/custom_threshold_rule/types.ts index 7668e19dc3900..67849df1b59d7 100644 --- a/x-pack/plugins/observability/common/custom_threshold_rule/types.ts +++ b/x-pack/plugins/observability/common/custom_threshold_rule/types.ts @@ -107,6 +107,11 @@ export enum InfraFormatterType { percent = 'percent', } +// Custom threshold alert types + +// Alert fields['kibana.alert.group] type +export type GroupBy = Array<{ field: string; value: string }>; + /* * Utils * diff --git a/x-pack/plugins/observability/common/utils/get_interval_in_seconds.test.ts b/x-pack/plugins/observability/common/utils/get_interval_in_seconds.test.ts new file mode 100644 index 0000000000000..f2654a224ae86 --- /dev/null +++ b/x-pack/plugins/observability/common/utils/get_interval_in_seconds.test.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getIntervalInSeconds } from './get_interval_in_seconds'; + +describe('getIntervalInSeconds', () => { + const testData = [ + { interval: '5ms', result: 0.005 }, + { interval: '70s', result: 70 }, + { interval: '25m', result: 1500 }, + { interval: '10h', result: 36000 }, + { interval: '3d', result: 259200 }, + { interval: '1w', result: 604800 }, + { interval: '1y', result: 30758400 }, + ]; + + it.each(testData)('getIntervalInSeconds($interval) = $result', ({ interval, result }) => { + expect(getIntervalInSeconds(interval)).toBe(result); + }); + + it('Throws error if interval is not valid', () => { + expect(() => getIntervalInSeconds('invalid')).toThrow('Invalid interval string format.'); + }); +}); diff --git a/x-pack/plugins/observability/common/utils/get_interval_in_seconds.ts b/x-pack/plugins/observability/common/utils/get_interval_in_seconds.ts new file mode 100644 index 0000000000000..6ebdbb83bdc62 --- /dev/null +++ b/x-pack/plugins/observability/common/utils/get_interval_in_seconds.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const intervalUnits = ['y', 'M', 'w', 'd', 'h', 'm', 's', 'ms']; +const INTERVAL_STRING_RE = new RegExp('^([0-9\\.]*)\\s*(' + intervalUnits.join('|') + ')$'); + +interface UnitsToSeconds { + [unit: string]: number; +} + +const units: UnitsToSeconds = { + ms: 0.001, + s: 1, + m: 60, + h: 3600, + d: 86400, + w: 86400 * 7, + M: 86400 * 30, + y: 86400 * 356, +}; + +export const getIntervalInSeconds = (interval: string): number => { + const matches = interval.match(INTERVAL_STRING_RE); + if (matches) { + return parseFloat(matches[1]) * units[matches[2]]; + } + throw new Error('Invalid interval string format.'); +}; diff --git a/x-pack/plugins/observability/kibana.jsonc b/x-pack/plugins/observability/kibana.jsonc index 526c283c0f0be..c7fdb22b5f792 100644 --- a/x-pack/plugins/observability/kibana.jsonc +++ b/x-pack/plugins/observability/kibana.jsonc @@ -20,6 +20,7 @@ "dataViews", "dataViewEditor", "embeddable", + "fieldFormats", "uiActions", "presentationUtil", "exploratoryView", diff --git a/x-pack/plugins/observability/public/components/alert_search_bar/alert_search_bar.tsx b/x-pack/plugins/observability/public/components/alert_search_bar/alert_search_bar.tsx index d27e32970f2fa..ff69c8e2ff2c3 100644 --- a/x-pack/plugins/observability/public/components/alert_search_bar/alert_search_bar.tsx +++ b/x-pack/plugins/observability/public/components/alert_search_bar/alert_search_bar.tsx @@ -47,15 +47,15 @@ export function ObservabilityAlertSearchBar({ (alertStatus: AlertStatus) => { try { onEsQueryChange( - buildEsQuery( - { + buildEsQuery({ + timeRange: { to: rangeTo, from: rangeFrom, }, kuery, - [...getAlertStatusQuery(alertStatus), ...defaultSearchQueries], - getEsQueryConfig(uiSettings) - ) + queries: [...getAlertStatusQuery(alertStatus), ...defaultSearchQueries], + config: getEsQueryConfig(uiSettings), + }) ); } catch (error) { toasts.addError(error, { @@ -89,15 +89,15 @@ export function ObservabilityAlertSearchBar({ ({ dateRange, query }) => { try { // First try to create es query to make sure query is valid, then save it in state - const esQuery = buildEsQuery( - { + const esQuery = buildEsQuery({ + timeRange: { to: dateRange.to, from: dateRange.from, }, - query, - [...getAlertStatusQuery(status), ...defaultSearchQueries], - getEsQueryConfig(uiSettings) - ); + kuery: query, + queries: [...getAlertStatusQuery(status), ...defaultSearchQueries], + config: getEsQueryConfig(uiSettings), + }); if (query) onKueryChange(query); timeFilterService.setTime(dateRange); onRangeFromChange(dateRange.from); diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/__snapshots__/alert_details_app_section.test.tsx.snap b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/__snapshots__/alert_details_app_section.test.tsx.snap similarity index 100% rename from x-pack/plugins/observability/public/components/custom_threshold/components/__snapshots__/alert_details_app_section.test.tsx.snap rename to x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/__snapshots__/alert_details_app_section.test.tsx.snap diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section.test.tsx b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.test.tsx similarity index 94% rename from x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section.test.tsx rename to x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.test.tsx index 8747558251da3..c623d9aa15043 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section.test.tsx +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.test.tsx @@ -15,9 +15,9 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { buildCustomThresholdAlert, buildCustomThresholdRule, -} from '../mocks/custom_threshold_rule'; +} from '../../mocks/custom_threshold_rule'; import AlertDetailsAppSection from './alert_details_app_section'; -import { ExpressionChart } from './expression_chart'; +import { ExpressionChart } from '../expression_chart'; const mockedChartStartContract = chartPluginMock.createStartContract(); @@ -33,11 +33,11 @@ jest.mock('@kbn/observability-get-padded-alert-time-range-util', () => ({ }), })); -jest.mock('./expression_chart', () => ({ +jest.mock('../expression_chart', () => ({ ExpressionChart: jest.fn(() =>
), })); -jest.mock('../../../utils/kibana_react', () => ({ +jest.mock('../../../../utils/kibana_react', () => ({ useKibana: () => ({ services: { ...mockCoreMock.createStart(), diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section.tsx b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx similarity index 86% rename from x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section.tsx rename to x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx index ce9a651a7a1e3..3c95f0d6677b6 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section.tsx +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/alert_details_app_section.tsx @@ -25,15 +25,16 @@ import { Rule, RuleTypeParams } from '@kbn/alerting-plugin/common'; import { AlertAnnotation, AlertActiveTimeRangeAnnotation } from '@kbn/observability-alert-details'; import { getPaddedAlertTimeRange } from '@kbn/observability-get-padded-alert-time-range-util'; import { DataView } from '@kbn/data-views-plugin/common'; -import { MetricsExplorerChartType } from '../../../../common/custom_threshold_rule/types'; -import { useKibana } from '../../../utils/kibana_react'; -import { metricValueFormatter } from '../../../../common/custom_threshold_rule/metric_value_formatter'; -import { AlertSummaryField, TopAlert } from '../../..'; - -import { ExpressionChart } from './expression_chart'; -import { TIME_LABELS } from './criterion_preview_chart/criterion_preview_chart'; -import { Threshold } from './custom_threshold'; -import { AlertParams, CustomThresholdRuleTypeParams } from '../types'; +import { MetricsExplorerChartType } from '../../../../../common/custom_threshold_rule/types'; +import { useLicense } from '../../../../hooks/use_license'; +import { useKibana } from '../../../../utils/kibana_react'; +import { metricValueFormatter } from '../../../../../common/custom_threshold_rule/metric_value_formatter'; +import { AlertSummaryField, TopAlert } from '../../../..'; +import { AlertParams, CustomThresholdRuleTypeParams } from '../../types'; +import { ExpressionChart } from '../expression_chart'; +import { TIME_LABELS } from '../criterion_preview_chart/criterion_preview_chart'; +import { Threshold } from '../custom_threshold'; +import { LogRateAnalysis } from './log_rate_analysis'; // TODO Use a generic props for app sections https://github.com/elastic/kibana/issues/152690 export type CustomThresholdRule = Rule; @@ -57,8 +58,11 @@ export default function AlertDetailsAppSection({ ruleLink, setAlertSummaryFields, }: AppSectionProps) { - const { uiSettings, charts, data } = useKibana().services; + const services = useKibana().services; + const { uiSettings, charts, data } = services; const { euiTheme } = useEuiTheme(); + const { hasAtLeast } = useLicense(); + const hasLogRateAnalysisLicense = hasAtLeast('platinum'); const [dataView, setDataView] = useState(); const [, setDataViewError] = useState(); const ruleParams = rule.params as RuleTypeParams & AlertParams; @@ -83,6 +87,7 @@ export default function AlertDetailsAppSection({ key={ALERT_TIME_RANGE_ANNOTATION_ID} />, ]; + useEffect(() => { setAlertSummaryFields([ { @@ -181,6 +186,9 @@ export default function AlertDetailsAppSection({ ))} + {hasLogRateAnalysisLicense && ( + + )} ) : null; diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/__snapshots__/log_rate_analysis_query.test.ts.snap b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/__snapshots__/log_rate_analysis_query.test.ts.snap new file mode 100644 index 0000000000000..9a9eb41224222 --- /dev/null +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/__snapshots__/log_rate_analysis_query.test.ts.snap @@ -0,0 +1,222 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`buildEsQuery should generate correct es query for rule with multiple metrics 1`] = `undefined`; + +exports[`buildEsQuery should generate correct es query for rule with optional filer, count filter and WITHOUT group by 1`] = ` +Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "optionalFilter": "container-1", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "host.name": "host-1", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "host.name": "host-2", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + "must": Array [], + "must_not": Array [], + "should": Array [], + }, +} +`; + +exports[`buildEsQuery should generate correct es query for rule with optional filer, count filter and group by 1`] = ` +Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "optionalFilter": "container-1", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "host.name": "host-1", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "host.name": "host-2", + }, + }, + ], + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "groupByField": "groupByValue", + }, + }, + ], + }, + }, + ], + }, + }, + ], + "must": Array [], + "must_not": Array [], + "should": Array [], + }, +} +`; + +exports[`buildEsQuery should generate correct es query for rule with optional filer, count filter and multiple group by 1`] = ` +Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "filter": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "optionalFilter": "container-1", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "host.name": "host-1", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "host.name": "host-2", + }, + }, + ], + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "groupByField": "groupByValue", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "minimum_should_match": 1, + "should": Array [ + Object { + "match": Object { + "secondGroupByField": "secondGroupByValue", + }, + }, + ], + }, + }, + ], + }, + }, + ], + "must": Array [], + "must_not": Array [], + "should": Array [], + }, +} +`; diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/get_initial_analysis_start.ts b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/get_initial_analysis_start.ts new file mode 100644 index 0000000000000..08bc835b7f43e --- /dev/null +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/get_initial_analysis_start.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment, { Moment } from 'moment'; + +export interface GetInitialAnalysisStartArgs { + alertStart: Moment; + intervalFactor: number; + alertEnd?: Moment; +} + +export const getDeviationMax = ({ + alertStart, + intervalFactor, + alertEnd, +}: GetInitialAnalysisStartArgs) => { + if (alertEnd) { + return alertEnd + .clone() + .subtract(1 * intervalFactor, 'minutes') + .valueOf(); + } else if ( + alertStart + .clone() + .add(10 * intervalFactor, 'minutes') + .isAfter(moment(new Date())) + ) { + return moment(new Date()).valueOf(); + } else { + return alertStart + .clone() + .add(10 * intervalFactor, 'minutes') + .valueOf(); + } +}; + +export const getInitialAnalysisStart = (args: GetInitialAnalysisStartArgs) => { + const { alertStart, intervalFactor } = args; + return { + baselineMin: alertStart + .clone() + .subtract(13 * intervalFactor, 'minutes') + .valueOf(), + baselineMax: alertStart + .clone() + .subtract(2 * intervalFactor, 'minutes') + .valueOf(), + deviationMin: alertStart + .clone() + .subtract(1 * intervalFactor, 'minutes') + .valueOf(), + deviationMax: getDeviationMax(args), + }; +}; diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.test.ts b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.test.ts new file mode 100644 index 0000000000000..8169816800e4d --- /dev/null +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.test.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Aggregators, Comparator } from '../../../../../../common/custom_threshold_rule/types'; +import { CustomThresholdRuleTypeParams } from '../../../types'; +import { getLogRateAnalysisEQQuery } from './log_rate_analysis_query'; + +describe('buildEsQuery', () => { + const index = 'changedMockedIndex'; + const mockedParams: CustomThresholdRuleTypeParams = { + groupBy: ['host.hostname'], + searchConfiguration: { + index, + query: { query: 'optionalFilter: container-1', language: 'kuery' }, + }, + criteria: [ + { + metrics: [ + { + name: 'A', + aggType: Aggregators.COUNT, + filter: 'host.name: host-1 or host.name: host-2', + }, + ], + timeSize: 1, + timeUnit: 'm', + threshold: [90], + comparator: Comparator.GT, + }, + ], + }; + const mockedAlertWithMultipleGroups = { + fields: { + 'kibana.alert.group': [ + { + field: 'groupByField', + value: 'groupByValue', + }, + { + field: 'secondGroupByField', + value: 'secondGroupByValue', + }, + ], + }, + }; + const testData: Array<{ + title: string; + params: CustomThresholdRuleTypeParams; + alert: any; + }> = [ + { + title: 'rule with optional filer, count filter and group by', + params: mockedParams, + alert: { + fields: { + 'kibana.alert.group': [mockedAlertWithMultipleGroups.fields['kibana.alert.group'][0]], + }, + }, + }, + { + title: 'rule with optional filer, count filter and multiple group by', + params: mockedParams, + alert: mockedAlertWithMultipleGroups, + }, + { + title: 'rule with optional filer, count filter and WITHOUT group by', + params: mockedParams, + alert: {}, + }, + { + title: 'rule with multiple metrics', + params: { + ...mockedParams, + criteria: [ + { + metrics: [ + { name: 'A', aggType: Aggregators.COUNT, filter: 'host.name: host-1' }, + { name: 'B', aggType: Aggregators.AVERAGE, field: 'system.load.1' }, + ], + timeSize: 1, + timeUnit: 'm', + threshold: [90], + comparator: Comparator.GT, + }, + ], + }, + alert: {}, + }, + ]; + + test.each(testData)('should generate correct es query for $title', ({ alert, params }) => { + expect(getLogRateAnalysisEQQuery(alert, params)).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.ts b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.ts new file mode 100644 index 0000000000000..b9eef4643802a --- /dev/null +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/helpers/log_rate_analysis_query.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { Aggregators, GroupBy } from '../../../../../../common/custom_threshold_rule/types'; +import { buildEsQuery } from '../../../../../utils/build_es_query'; +import type { TopAlert } from '../../../../../typings/alerts'; +import type { CustomThresholdRuleTypeParams } from '../../../types'; +import type { CustomThresholdExpressionMetric } from '../../../../../../common/custom_threshold_rule/types'; + +const getKuery = ( + metrics: CustomThresholdExpressionMetric[], + filter?: string, + groupBy?: GroupBy +) => { + let query = ''; + const isOneCountConditionWithFilter = + metrics.length === 1 && metrics[0].aggType === 'count' && metrics[0].filter; + + if (filter && isOneCountConditionWithFilter) { + query = `(${filter}) and (${metrics[0].filter})`; + } else if (isOneCountConditionWithFilter) { + query = `(${metrics[0].filter!})`; + } else if (filter) { + query = `(${filter})`; + } + + if (groupBy) { + groupBy.forEach(({ field, value }) => { + query += ` and ${field}: ${value}`; + }); + } + + return query; +}; + +export const getLogRateAnalysisEQQuery = ( + alert: TopAlert>, + params: CustomThresholdRuleTypeParams +): QueryDslQueryContainer | undefined => { + // We only show log rate analysis for one condition with one count aggregation + if ( + params.criteria.length !== 1 || + params.criteria[0].metrics.length !== 1 || + params.criteria[0].metrics[0].aggType !== Aggregators.COUNT + ) { + return; + } + + const groupBy: GroupBy | undefined = get(alert, 'fields["kibana.alert.group"]'); + const optionalFilter: string | undefined = get(params.searchConfiguration, 'query.query'); + const boolQuery = buildEsQuery({ + kuery: getKuery(params.criteria[0].metrics, optionalFilter, groupBy), + }); + + return boolQuery; +}; diff --git a/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx new file mode 100644 index 0000000000000..7578f0979907f --- /dev/null +++ b/x-pack/plugins/observability/public/components/custom_threshold/components/alert_details_app_section/log_rate_analysis.tsx @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pick, orderBy } from 'lodash'; +import moment from 'moment'; +import React, { useEffect, useMemo, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; +import { + LOG_RATE_ANALYSIS_TYPE, + type LogRateAnalysisType, +} from '@kbn/aiops-utils/log_rate_analysis_type'; +import { LogRateAnalysisContent, type LogRateAnalysisResultsData } from '@kbn/aiops-plugin/public'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { type Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/public'; +import { ALERT_END } from '@kbn/rule-data-utils'; +import { CustomThresholdRuleTypeParams } from '../../types'; +import { TopAlert } from '../../../..'; +import { Color, colorTransformer } from '../../../../../common/custom_threshold_rule/color_palette'; +import { getLogRateAnalysisEQQuery } from './helpers/log_rate_analysis_query'; +import { getInitialAnalysisStart } from './helpers/get_initial_analysis_start'; + +export interface AlertDetailsLogRateAnalysisProps { + alert: TopAlert>; + dataView: any; + rule: Rule; + services: any; +} + +interface SignificantFieldValue { + field: string; + value: string | number; + docCount: number; + pValue: number | null; +} + +export function LogRateAnalysis({ + alert, + dataView, + rule, + services, +}: AlertDetailsLogRateAnalysisProps) { + const { + observabilityAIAssistant: { ObservabilityAIAssistantContextualInsight }, + } = services; + const [esSearchQuery, setEsSearchQuery] = useState(); + const [logRateAnalysisParams, setLogRateAnalysisParams] = useState< + | { logRateAnalysisType: LogRateAnalysisType; significantFieldValues: SignificantFieldValue[] } + | undefined + >(); + + useEffect(() => { + const esSearchRequest = getLogRateAnalysisEQQuery(alert, rule.params); + + if (esSearchRequest) { + setEsSearchQuery(esSearchRequest); + } + }, [alert, rule.params]); + + // Identify `intervalFactor` to adjust time ranges based on alert settings. + // The default time ranges for `initialAnalysisStart` are suitable for a `1m` lookback. + // If an alert would have a `5m` lookback, this would result in a factor of `5`. + const lookbackDuration = + alert.fields['kibana.alert.rule.parameters'] && + alert.fields['kibana.alert.rule.parameters'].timeSize && + alert.fields['kibana.alert.rule.parameters'].timeUnit + ? moment.duration( + alert.fields['kibana.alert.rule.parameters'].timeSize as number, + alert.fields['kibana.alert.rule.parameters'].timeUnit as any + ) + : moment.duration(1, 'm'); + const intervalFactor = Math.max(1, lookbackDuration.asSeconds() / 60); + + const alertStart = moment(alert.start); + const alertEnd = alert.fields[ALERT_END] ? moment(alert.fields[ALERT_END]) : undefined; + + const timeRange = { + min: alertStart.clone().subtract(15 * intervalFactor, 'minutes'), + max: alertEnd ? alertEnd.clone().add(1 * intervalFactor, 'minutes') : moment(new Date()), + }; + + const logRateAnalysisTitle = i18n.translate( + 'xpack.observability.customThreshold.alertDetails.logRateAnalysisTitle', + { + defaultMessage: 'Possible causes and remediations', + } + ); + + const onAnalysisCompleted = (analysisResults: LogRateAnalysisResultsData | undefined) => { + const significantFieldValues = orderBy( + analysisResults?.significantItems?.map((item) => ({ + field: item.fieldName, + value: item.fieldValue, + docCount: item.doc_count, + pValue: item.pValue, + })), + ['pValue', 'docCount'], + ['asc', 'asc'] + ).slice(0, 50); + + const logRateAnalysisType = analysisResults?.analysisType; + setLogRateAnalysisParams( + significantFieldValues && logRateAnalysisType + ? { logRateAnalysisType, significantFieldValues } + : undefined + ); + }; + + const messages = useMemo(() => { + const hasLogRateAnalysisParams = + logRateAnalysisParams && logRateAnalysisParams.significantFieldValues?.length > 0; + + if (!hasLogRateAnalysisParams) { + return undefined; + } + + const { logRateAnalysisType } = logRateAnalysisParams; + + const header = 'Field name,Field value,Doc count,p-value'; + const rows = logRateAnalysisParams.significantFieldValues + .map((item) => Object.values(item).join(',')) + .join('\n'); + + const content = `You are an observability expert using Elastic Observability Suite on call being consulted about a log threshold alert that got triggered by a ${logRateAnalysisType} in log messages. Your job is to take immediate action and proceed with both urgency and precision. + "Log Rate Analysis" is an AIOps feature that uses advanced statistical methods to identify reasons for increases and decreases in log rates. It makes it easy to find and investigate causes of unusual spikes or dips by using the analysis workflow view. + You are using "Log Rate Analysis" and ran the statistical analysis on the log messages which occured during the alert. + You received the following analysis results from "Log Rate Analysis" which list statistically significant co-occuring field/value combinations sorted from most significant (lower p-values) to least significant (higher p-values) that ${ + logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE + ? 'contribute to the log rate spike' + : 'are less or not present in the log rate dip' + }: + + ${ + logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE + ? 'The median log rate in the selected deviation time range is higher than the baseline. Therefore, the results shows statistically significant items within the deviation time range that are contributors to the spike. The "doc count" column refers to the amount of documents in the deviation time range.' + : 'The median log rate in the selected deviation time range is lower than the baseline. Therefore, the analysis results table shows statistically significant items within the baseline time range that are less in number or missing within the deviation time range. The "doc count" column refers to the amount of documents in the baseline time range.' + } + + ${header} + ${rows} + + Based on the above analysis results and your observability expert knowledge, output the following: + Analyse the type of these logs and explain their usual purpose (1 paragraph). + ${ + logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE + ? 'Based on the type of these logs do a root cause analysis on why the field and value combinations from the analysis results are causing this log rate spike (2 parapraphs)' + : 'Based on the type of these logs explain why the statistically significant field and value combinations are less in number or missing from the log rate dip with concrete examples based on the analysis results data which contains items that are present in the baseline time range and are missing or less in number in the deviation time range (2 paragraphs)' + }. + ${ + logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE + ? 'Recommend concrete remediations to resolve the root cause (3 bullet points).' + : '' + } + + Do not mention individual p-values from the analysis results. + Do not repeat the full list of field names and field values back to the user. + Do not guess, just say what you are sure of. Do not repeat the given instructions in your output.`; + + const now = new Date().toISOString(); + + return [ + { + '@timestamp': now, + message: { + content, + role: MessageRole.User, + }, + }, + ]; + }, [logRateAnalysisParams]); + + if (!dataView || !esSearchQuery) return null; + + return ( + + + + +

+ +

+ + + + + + + + {ObservabilityAIAssistantContextualInsight && messages ? ( + + + + ) : null} + + + ); +} diff --git a/x-pack/plugins/observability/public/components/custom_threshold/mocks/custom_threshold_rule.ts b/x-pack/plugins/observability/public/components/custom_threshold/mocks/custom_threshold_rule.ts index cfe41b6bd9d04..817e5dbadb978 100644 --- a/x-pack/plugins/observability/public/components/custom_threshold/mocks/custom_threshold_rule.ts +++ b/x-pack/plugins/observability/public/components/custom_threshold/mocks/custom_threshold_rule.ts @@ -8,7 +8,10 @@ import { v4 as uuidv4 } from 'uuid'; import { Aggregators, Comparator } from '../../../../common/custom_threshold_rule/types'; -import { CustomThresholdAlert, CustomThresholdRule } from '../components/alert_details_app_section'; +import { + CustomThresholdAlert, + CustomThresholdRule, +} from '../components/alert_details_app_section/alert_details_app_section'; export const buildCustomThresholdRule = ( rule: Partial = {} diff --git a/x-pack/plugins/observability/public/hooks/use_license.ts b/x-pack/plugins/observability/public/hooks/use_license.ts index f666dcd025bd3..f6ed1ebf8bab1 100644 --- a/x-pack/plugins/observability/public/hooks/use_license.ts +++ b/x-pack/plugins/observability/public/hooks/use_license.ts @@ -14,7 +14,7 @@ import { useKibana } from '../utils/kibana_react'; interface UseLicenseReturnValue { getLicense: () => ILicense | null; - hasAtLeast: (level: LicenseType) => boolean | undefined; + hasAtLeast: (level: LicenseType) => boolean; } export const useLicense = (): UseLicenseReturnValue => { @@ -25,7 +25,7 @@ export const useLicense = (): UseLicenseReturnValue => { getLicense: () => license, hasAtLeast: useCallback( (level: LicenseType) => { - if (!license) return; + if (!license) return false; return !!license && license.isAvailable && license.isActive && license.hasAtLeast(level); }, diff --git a/x-pack/plugins/observability/public/pages/overview/overview.tsx b/x-pack/plugins/observability/public/pages/overview/overview.tsx index 17121be51c668..5cc5e2ac967ea 100644 --- a/x-pack/plugins/observability/public/pages/overview/overview.tsx +++ b/x-pack/plugins/observability/public/pages/overview/overview.tsx @@ -73,8 +73,10 @@ export function OverviewPage() { const [esQuery, setEsQuery] = useState<{ bool: BoolQuery }>( buildEsQuery({ - from: relativeStart, - to: relativeEnd, + timeRange: { + from: relativeStart, + to: relativeEnd, + }, }) ); @@ -108,8 +110,10 @@ export function OverviewPage() { useEffect(() => { setEsQuery( buildEsQuery({ - from: relativeStart, - to: relativeEnd, + timeRange: { + from: relativeStart, + to: relativeEnd, + }, }) ); }, [relativeEnd, relativeStart]); @@ -117,8 +121,10 @@ export function OverviewPage() { const handleTimeRangeRefresh = useCallback(() => { setEsQuery( buildEsQuery({ - from: relativeStart, - to: relativeEnd, + timeRange: { + from: relativeStart, + to: relativeEnd, + }, }) ); }, [relativeEnd, relativeStart]); diff --git a/x-pack/plugins/observability/public/plugin.ts b/x-pack/plugins/observability/public/plugin.ts index 808573579cb99..03322f7817282 100644 --- a/x-pack/plugins/observability/public/plugin.ts +++ b/x-pack/plugins/observability/public/plugin.ts @@ -28,6 +28,7 @@ import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import { LOG_EXPLORER_LOCATOR_ID, LogExplorerLocatorParams } from '@kbn/deeplinks-observability'; import type { DiscoverStart } from '@kbn/discover-plugin/public'; import type { EmbeddableStart } from '@kbn/embeddable-plugin/public'; +import type { FieldFormatsSetup, FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { HomePublicPluginSetup, HomePublicPluginStart } from '@kbn/home-plugin/public'; import { i18n } from '@kbn/i18n'; import type { LensPublicStart } from '@kbn/lens-plugin/public'; @@ -114,6 +115,7 @@ export interface ConfigSchema { export type ObservabilityPublicSetup = ReturnType; export interface ObservabilityPublicPluginsSetup { data: DataPublicPluginSetup; + fieldFormats: FieldFormatsSetup; observabilityShared: ObservabilitySharedPluginSetup; observabilityAIAssistant: ObservabilityAIAssistantPluginSetup; share: SharePluginSetup; @@ -137,6 +139,7 @@ export interface ObservabilityPublicPluginsStart { discover: DiscoverStart; embeddable: EmbeddableStart; exploratoryView: ExploratoryViewPublicStart; + fieldFormats: FieldFormatsStart; guidedOnboarding?: GuidedOnboardingPluginStart; lens: LensPublicStart; licensing: LicensingPluginStart; diff --git a/x-pack/plugins/observability/public/rules/register_observability_rule_types.ts b/x-pack/plugins/observability/public/rules/register_observability_rule_types.ts index c9079991bdf4d..04519eba23d5f 100644 --- a/x-pack/plugins/observability/public/rules/register_observability_rule_types.ts +++ b/x-pack/plugins/observability/public/rules/register_observability_rule_types.ts @@ -156,7 +156,10 @@ export const registerObservabilityRuleTypes = async ( }; }, alertDetailsAppSection: lazy( - () => import('../components/custom_threshold/components/alert_details_app_section') + () => + import( + '../components/custom_threshold/components/alert_details_app_section/alert_details_app_section' + ) ), priority: 5, }); diff --git a/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.test.ts b/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.test.ts index 4bbacaa7bb1ad..3cb37893e6169 100644 --- a/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.test.ts +++ b/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.test.ts @@ -37,6 +37,6 @@ describe('buildEsQuery', () => { ]; test.each(testData)('should generate correct es query for %j', ({ kuery, timeRange }) => { - expect(buildEsQuery(timeRange, kuery)).toMatchSnapshot(); + expect(buildEsQuery({ timeRange, kuery })).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.ts b/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.ts index 1fd26690f86d5..acf6b4cfc4a7e 100644 --- a/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.ts +++ b/x-pack/plugins/observability/public/utils/build_es_query/build_es_query.ts @@ -9,12 +9,14 @@ import { buildEsQuery as kbnBuildEsQuery, TimeRange, Query, EsQueryConfig } from import { ALERT_TIME_RANGE } from '@kbn/rule-data-utils'; import { getTime } from '@kbn/data-plugin/common'; -export function buildEsQuery( - timeRange: TimeRange, - kuery?: string, - queries: Query[] = [], - config: EsQueryConfig = {} -) { +interface BuildEsQueryArgs { + timeRange?: TimeRange; + kuery?: string; + queries?: Query[]; + config?: EsQueryConfig; +} + +export function buildEsQuery({ timeRange, kuery, queries = [], config = {} }: BuildEsQueryArgs) { const timeFilter = timeRange && getTime(undefined, timeRange, { diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts index b1f790da9d146..f5b3f0adf1bdd 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/lib/evaluate_rule.ts @@ -9,7 +9,8 @@ import moment from 'moment'; import { ElasticsearchClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import { CustomMetricExpressionParams } from '../../../../../common/custom_threshold_rule/types'; -import { AdditionalContext, getIntervalInSeconds } from '../utils'; +import { getIntervalInSeconds } from '../../../../../common/utils/get_interval_in_seconds'; +import { AdditionalContext } from '../utils'; import { SearchConfigurationType } from '../types'; import { createTimerange } from './create_timerange'; import { getData } from './get_data'; diff --git a/x-pack/plugins/observability/server/lib/rules/custom_threshold/utils.ts b/x-pack/plugins/observability/server/lib/rules/custom_threshold/utils.ts index 66049ab431c5d..59d7cf93e0ea4 100644 --- a/x-pack/plugins/observability/server/lib/rules/custom_threshold/utils.ts +++ b/x-pack/plugins/observability/server/lib/rules/custom_threshold/utils.ts @@ -257,32 +257,6 @@ export const isTooManyBucketsPreviewException = ( ): value is TooManyBucketsPreviewExceptionMetadata => Boolean(value && value.TOO_MANY_BUCKETS_PREVIEW_EXCEPTION); -const intervalUnits = ['y', 'M', 'w', 'd', 'h', 'm', 's', 'ms']; -const INTERVAL_STRING_RE = new RegExp('^([0-9\\.]*)\\s*(' + intervalUnits.join('|') + ')$'); - -interface UnitsToSeconds { - [unit: string]: number; -} - -const units: UnitsToSeconds = { - ms: 0.001, - s: 1, - m: 60, - h: 3600, - d: 86400, - w: 86400 * 7, - M: 86400 * 30, - y: 86400 * 356, -}; - -export const getIntervalInSeconds = (interval: string): number => { - const matches = interval.match(INTERVAL_STRING_RE); - if (matches) { - return parseFloat(matches[1]) * units[matches[2]]; - } - throw new Error('Invalid interval string format.'); -}; - export const calculateRateTimeranges = (timerange: { to: number; from: number }) => { // This is the total number of milliseconds for the entire timerange const totalTime = timerange.to - timerange.from; diff --git a/x-pack/plugins/observability/tsconfig.json b/x-pack/plugins/observability/tsconfig.json index ba9ac59725adc..04f853572085f 100644 --- a/x-pack/plugins/observability/tsconfig.json +++ b/x-pack/plugins/observability/tsconfig.json @@ -102,7 +102,9 @@ "@kbn/core-elasticsearch-client-server-mocks", "@kbn/ingest-pipelines-plugin", "@kbn/core-saved-objects-api-server-mocks", - "@kbn/core-ui-settings-browser-mocks" + "@kbn/core-ui-settings-browser-mocks", + "@kbn/field-formats-plugin", + "@kbn/aiops-utils" ], "exclude": [ "target/**/*"