From d42cecd7a994f081bc5733a35d291e82982af3bc Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 Nov 2023 13:43:44 +0100 Subject: [PATCH 01/17] fallback if baseline or deviation has 0 docs --- .../analysis_handlers/index_info_handler.ts | 6 +- .../analysis_handlers/top_items_handler.ts | 212 ++++++++++++++++++ .../queries/fetch_index_info.ts | 52 ++--- .../queries/fetch_top_categories.ts | 71 ++++++ .../queries/fetch_top_terms.ts | 162 +++++++++++++ .../queries/get_random_docs_request.ts | 35 +++ .../queries/get_total_doc_count_request.ts | 26 +++ .../response_stream_factory.ts | 2 + .../route_handler_factory.ts | 4 +- 9 files changed, 538 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts index 6ae2a07055aec..14d2da84312c0 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts @@ -36,6 +36,7 @@ export const indexInfoHandlerFactory = const textFieldCandidates: string[] = []; let totalDocCount = 0; + let zeroDocsFallback = false; if (!requestBody.overrides?.remainingFieldCandidates) { logDebugMessage('Fetch index information.'); @@ -63,7 +64,8 @@ export const indexInfoHandlerFactory = fieldCandidates.push(...indexInfo.fieldCandidates); fieldCandidatesCount = fieldCandidates.length; textFieldCandidates.push(...indexInfo.textFieldCandidates); - totalDocCount = indexInfo.totalDocCount; + totalDocCount = indexInfo.deviationTotalDocCount; + zeroDocsFallback = indexInfo.zeroDocsFallback; } catch (e) { if (!isRequestAbortedError(e)) { logger.error(`Failed to fetch index information, got: \n${e.toString()}`); @@ -105,5 +107,5 @@ export const indexInfoHandlerFactory = } } - return { fieldCandidates, textFieldCandidates }; + return { fieldCandidates, textFieldCandidates, zeroDocsFallback }; }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts new file mode 100644 index 0000000000000..10f0b19f07925 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { queue } from 'async'; + +import { SIGNIFICANT_ITEM_TYPE, type SignificantItem } from '@kbn/ml-agg-utils'; +import { i18n } from '@kbn/i18n'; + +import { + addSignificantItemsAction, + updateLoadingStateAction, +} from '../../../../common/api/log_rate_analysis/actions'; + +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; + +import { fetchTopCategories } from '../queries/fetch_top_categories'; +import { fetchTopTerms } from '../queries/fetch_top_terms'; + +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '../../../../common/api/log_rate_analysis/schema'; + +import { + LOADED_FIELD_CANDIDATES, + MAX_CONCURRENT_QUERIES, + PROGRESS_STEP_P_VALUES, +} from '../response_stream_utils/constants'; +import type { ResponseStreamFetchOptions } from '../response_stream_factory'; + +export const topItemsHandlerFactory = + ({ + abortSignal, + client, + logDebugMessage, + logger, + requestBody, + responseStream, + stateHandler, + version, + }: ResponseStreamFetchOptions) => + async ({ + fieldCandidates, + textFieldCandidates, + }: { + fieldCandidates: string[]; + textFieldCandidates: string[]; + }) => { + let fieldCandidatesCount = fieldCandidates.length; + + // This will store the combined count of detected log patterns and keywords + let fieldValuePairsCount = 0; + + const topCategories: SignificantItem[] = []; + + if (version === '1') { + topCategories.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'1'>).overrides?.significantTerms?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.LOG_PATTERN + ) ?? []) + ); + } + + if (version === '2') { + topCategories.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'2'>).overrides?.significantItems?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.LOG_PATTERN + ) ?? []) + ); + } + + // Get categories of text fields + if (textFieldCandidates.length > 0) { + topCategories.push( + ...(await fetchTopCategories( + client, + requestBody, + textFieldCandidates, + logger, + stateHandler.sampleProbability(), + responseStream.pushError, + abortSignal + )) + ); + + if (topCategories.length > 0) { + responseStream.push(addSignificantItemsAction(topCategories, version)); + } + } + + const topTerms: SignificantItem[] = []; + + if (version === '1') { + topTerms.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'1'>).overrides?.significantTerms?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.KEYWORD + ) ?? []) + ); + } + + if (version === '2') { + topTerms.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'2'>).overrides?.significantItems?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.KEYWORD + ) ?? []) + ); + } + + const fieldsToSample = new Set(); + + let remainingFieldCandidates: string[]; + let loadingStepSizeTopTerms = PROGRESS_STEP_P_VALUES; + + if (requestBody.overrides?.remainingFieldCandidates) { + fieldCandidates.push(...requestBody.overrides?.remainingFieldCandidates); + remainingFieldCandidates = requestBody.overrides?.remainingFieldCandidates; + fieldCandidatesCount = fieldCandidates.length; + loadingStepSizeTopTerms = + LOADED_FIELD_CANDIDATES + + PROGRESS_STEP_P_VALUES - + (requestBody.overrides?.loaded ?? PROGRESS_STEP_P_VALUES); + } else { + remainingFieldCandidates = fieldCandidates; + } + + logDebugMessage('Fetch p-values.'); + + const topTermsQueue = queue(async function (fieldCandidate: string) { + stateHandler.loaded((1 / fieldCandidatesCount) * loadingStepSizeTopTerms, false); + + let fetchedTopTerms: Awaited>; + + try { + fetchedTopTerms = await fetchTopTerms( + client, + requestBody, + [fieldCandidate], + logger, + stateHandler.sampleProbability(), + responseStream.pushError, + abortSignal + ); + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error(`Failed to fetch p-values for '${fieldCandidate}', got: \n${e.toString()}`); + responseStream.pushError(`Failed to fetch p-values for '${fieldCandidate}'.`); + } + return; + } + + remainingFieldCandidates = remainingFieldCandidates.filter((d) => d !== fieldCandidate); + + if (fetchedTopTerms.length > 0) { + fetchedTopTerms.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); + topTerms.push(...fetchedTopTerms); + + responseStream.push(addSignificantItemsAction(fetchedTopTerms, version)); + } + + responseStream.push( + updateLoadingStateAction({ + ccsWarning: false, + loaded: stateHandler.loaded(), + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.identifiedFieldValuePairs', + { + defaultMessage: + 'Identified {fieldValuePairsCount, plural, one {# significant field/value pair} other {# significant field/value pairs}}.', + values: { + fieldValuePairsCount, + }, + } + ), + remainingFieldCandidates, + }) + ); + }, MAX_CONCURRENT_QUERIES); + + topTermsQueue.push(fieldCandidates, (err) => { + if (err) { + logger.error(`Failed to fetch p-values.', got: \n${err.toString()}`); + responseStream.pushError(`Failed to fetch p-values.`); + topTermsQueue.kill(); + responseStream.end(); + } else if (stateHandler.shouldStop()) { + logDebugMessage('shouldStop fetching p-values.'); + topTermsQueue.kill(); + responseStream.end(); + } + }); + await topTermsQueue.drain(); + + fieldValuePairsCount = topCategories.length + topTerms.length; + + if (fieldValuePairsCount === 0) { + logDebugMessage('Stopping analysis, did not find any categories or terms.'); + responseStream.endWithUpdatedLoadingState(); + return; + } + + return { + fieldValuePairsCount, + significantCategories: topCategories, + significantTerms: topTerms, + }; + }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts index 595e212159437..fd7975aac1b6e 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts @@ -12,14 +12,12 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; -import { getQueryWithParams } from './get_query_with_params'; -import { getRequestBase } from './get_request_base'; +import { getRandomDocsRequest } from './get_random_docs_request'; +import { getTotalDocCountRequest } from './get_total_doc_count_request'; // TODO Consolidate with duplicate `fetchPValues` in // `x-pack/plugins/apm/server/routes/correlations/queries/fetch_duration_field_candidates.ts` -const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; - const SUPPORTED_ES_FIELD_TYPES = [ ES_FIELD_TYPES.KEYWORD, ES_FIELD_TYPES.IP, @@ -28,30 +26,12 @@ const SUPPORTED_ES_FIELD_TYPES = [ const SUPPORTED_ES_FIELD_TYPES_TEXT = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.MATCH_ONLY_TEXT]; -export const getRandomDocsRequest = ( - params: AiopsLogRateAnalysisSchema -): estypes.SearchRequest => ({ - ...getRequestBase(params), - body: { - fields: ['*'], - _source: false, - query: { - function_score: { - query: getQueryWithParams({ params }), - // @ts-ignore - random_score: {}, - }, - }, - size: POPULATED_DOC_COUNT_SAMPLE_SIZE, - // Used to determine sample probability for follow up queries - track_total_hits: true, - }, -}); - interface IndexInfo { fieldCandidates: string[]; textFieldCandidates: string[]; - totalDocCount: number; + baselineTotalDocCount: number; + deviationTotalDocCount: number; + zeroDocsFallback: boolean; } export const fetchIndexInfo = async ( @@ -95,15 +75,24 @@ export const fetchIndexInfo = async ( allFieldNames.push(key); }); + // Get the total doc count for the baseline time range + const respBaselineTotalDocCount = await esClient.search( + getTotalDocCountRequest({ ...params, start: params.baselineMin, end: params.baselineMax }), + { + signal: abortSignal, + maxRetries: 0, + } + ); + // Only the deviation window will be used to identify field candidates and sample probability based on total doc count. - const resp = await esClient.search( + const respDeviationRandomDocs = await esClient.search( getRandomDocsRequest({ ...params, start: params.deviationMin, end: params.deviationMax }), { signal: abortSignal, maxRetries: 0, } ); - const sampledDocs = resp.hits.hits.map((d) => d.fields ?? {}); + const sampledDocs = respDeviationRandomDocs.hits.hits.map((d) => d.fields ?? {}); const textFieldCandidatesOverridesWithKeywordPostfix = textFieldCandidatesOverrides.map( (d) => `${d}.keyword` @@ -127,11 +116,16 @@ export const fetchIndexInfo = async ( } }); - const totalDocCount = (resp.hits.total as estypes.SearchTotalHits).value; + const baselineTotalDocCount = (respBaselineTotalDocCount.hits.total as estypes.SearchTotalHits) + .value; + const deviationTotalDocCount = (respDeviationRandomDocs.hits.total as estypes.SearchTotalHits) + .value; return { fieldCandidates: [...finalFieldCandidates], textFieldCandidates: [...finalTextFieldCandidates], - totalDocCount, + baselineTotalDocCount, + deviationTotalDocCount, + zeroDocsFallback: baselineTotalDocCount === 0 || deviationTotalDocCount === 0, }; }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts new file mode 100644 index 0000000000000..2933e5d3e414a --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniq } from 'lodash'; + +import { ElasticsearchClient } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import { type SignificantItem, SIGNIFICANT_ITEM_TYPE } from '@kbn/ml-agg-utils'; + +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { fetchCategories } from './fetch_categories'; + +export const fetchTopCategories = async ( + esClient: ElasticsearchClient, + params: AiopsLogRateAnalysisSchema, + fieldNames: string[], + logger: Logger, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1, + emitError: (m: string) => void, + abortSignal?: AbortSignal +) => { + const categoriesOverall = await fetchCategories( + esClient, + params, + fieldNames, + logger, + sampleProbability, + emitError, + abortSignal + ); + + if (categoriesOverall.length !== fieldNames.length) return []; + + const topCategories: SignificantItem[] = []; + + // Using for...of to allow `await` within the loop. + for (const [i, fieldName] of fieldNames.entries()) { + if (categoriesOverall[i].categories.length === 0) { + continue; + } + + // Get all unique keys + const allKeys: string[] = uniq(categoriesOverall[i].categories.map((cd) => cd.key)); + + allKeys.forEach((key) => { + const categoryData = categoriesOverall[i].categories.find((c) => c.key === key); + + topCategories.push({ + key, + fieldName, + fieldValue: categoryData?.examples[0] ?? '', + doc_count: categoryData?.count ?? 0, + bg_count: 0, + total_doc_count: 0, + total_bg_count: 0, + score: 0, + pValue: 1, + normalizedScore: 0, + type: SIGNIFICANT_ITEM_TYPE.LOG_PATTERN, + }); + }); + } + + return topCategories; +}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts new file mode 100644 index 0000000000000..d1b8007249136 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { uniqBy } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ElasticsearchClient } from '@kbn/core/server'; + +import type { Logger } from '@kbn/logging'; +import { type SignificantItem, SIGNIFICANT_ITEM_TYPE } from '@kbn/ml-agg-utils'; +import { + createRandomSamplerWrapper, + type RandomSamplerWrapper, +} from '@kbn/ml-random-sampler-utils'; + +import { RANDOM_SAMPLER_SEED } from '../../../../common/constants'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; + +import { getQueryWithParams } from './get_query_with_params'; +import { getRequestBase } from './get_request_base'; + +// TODO Consolidate with duplicate `fetchDurationFieldCandidates` in +// `x-pack/plugins/apm/server/routes/correlations/queries/fetch_failed_events_correlation_p_values.ts` + +export const getTopTermRequest = ( + params: AiopsLogRateAnalysisSchema, + fieldName: string, + { wrap }: RandomSamplerWrapper +): estypes.SearchRequest => { + const query = getQueryWithParams({ + params, + }); + + const timeFieldName = params.timeFieldName ?? '@timestamp'; + + let filter: estypes.QueryDslQueryContainer[] = []; + + if (query.bool && Array.isArray(query.bool.filter)) { + filter = query.bool.filter.filter((d) => Object.keys(d)[0] !== 'range'); + + query.bool.filter = [ + ...filter, + { + range: { + [timeFieldName]: { + gte: params.deviationMin, + lt: params.deviationMax, + format: 'epoch_millis', + }, + }, + }, + ]; + } + + const termAgg: Record<'log_rate_top_terms', estypes.AggregationsAggregationContainer> = { + log_rate_top_terms: { + terms: { + field: fieldName, + size: 10, + }, + }, + }; + + const body = { + query, + size: 0, + aggs: wrap(termAgg), + }; + + return { + ...getRequestBase(params), + body, + }; +}; + +interface Aggs extends estypes.AggregationsLongTermsAggregate { + buckets: estypes.AggregationsLongTermsBucket[]; +} + +export const fetchTopTerms = async ( + esClient: ElasticsearchClient, + params: AiopsLogRateAnalysisSchema, + fieldNames: string[], + logger: Logger, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1, + emitError: (m: string) => void, + abortSignal?: AbortSignal +): Promise => { + const randomSamplerWrapper = createRandomSamplerWrapper({ + probability: sampleProbability, + seed: RANDOM_SAMPLER_SEED, + }); + + const result: SignificantItem[] = []; + + const settledPromises = await Promise.allSettled( + fieldNames.map((fieldName) => + esClient.search(getTopTermRequest(params, fieldName, randomSamplerWrapper), { + signal: abortSignal, + maxRetries: 0, + }) + ) + ); + + function reportError(fieldName: string, error: unknown) { + if (!isRequestAbortedError(error)) { + logger.error( + `Failed to fetch term aggregation for fieldName "${fieldName}", got: \n${JSON.stringify( + error, + null, + 2 + )}` + ); + emitError(`Failed to fetch term aggregation for fieldName "${fieldName}".`); + } + } + + for (const [index, settledPromise] of settledPromises.entries()) { + const fieldName = fieldNames[index]; + + if (settledPromise.status === 'rejected') { + reportError(fieldName, settledPromise.reason); + // Still continue the analysis even if individual p-value queries fail. + continue; + } + + const resp = settledPromise.value; + + if (resp.aggregations === undefined) { + reportError(fieldName, resp); + // Still continue the analysis even if individual p-value queries fail. + continue; + } + + const overallResult = ( + randomSamplerWrapper.unwrap(resp.aggregations) as Record<'log_rate_top_terms', Aggs> + ).log_rate_top_terms; + + for (const bucket of overallResult.buckets) { + result.push({ + key: `${fieldName}:${String(bucket.key)}`, + type: SIGNIFICANT_ITEM_TYPE.KEYWORD, + fieldName, + fieldValue: String(bucket.key), + doc_count: bucket.doc_count, + bg_count: 0, + total_doc_count: 0, + total_bg_count: 0, + score: 0, + pValue: 1, + normalizedScore: 0, + }); + } + } + + return uniqBy(result, (d) => `${d.fieldName},${d.fieldValue}`); +}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts new file mode 100644 index 0000000000000..7c1abdfd52667 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { getQueryWithParams } from './get_query_with_params'; +import { getRequestBase } from './get_request_base'; + +const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; + +export const getRandomDocsRequest = ( + params: AiopsLogRateAnalysisSchema +): estypes.SearchRequest => ({ + ...getRequestBase(params), + body: { + fields: ['*'], + _source: false, + query: { + function_score: { + query: getQueryWithParams({ params }), + // @ts-ignore + random_score: {}, + }, + }, + size: POPULATED_DOC_COUNT_SAMPLE_SIZE, + // Used to determine sample probability for follow up queries + track_total_hits: true, + }, +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts new file mode 100644 index 0000000000000..e9b0fc853535f --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { getQueryWithParams } from './get_query_with_params'; +import { getRequestBase } from './get_request_base'; + +export const getTotalDocCountRequest = ( + params: AiopsLogRateAnalysisSchema +): estypes.SearchRequest => ({ + ...getRequestBase(params), + body: { + fields: ['*'], + _source: false, + query: getQueryWithParams({ params }), + size: 0, + track_total_hits: true, + }, +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts index f54c2e93dd29e..0dce9c7cf34d1 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts @@ -22,6 +22,7 @@ import { groupingHandlerFactory } from './analysis_handlers/grouping_handler'; import { histogramHandlerFactory } from './analysis_handlers/histogram_handler'; import { overridesHandlerFactory } from './analysis_handlers/overrides_handler'; import { significantItemsHandlerFactory } from './analysis_handlers/significant_items_handler'; +import { topItemsHandlerFactory } from './analysis_handlers/top_items_handler'; import { overallHistogramHandlerFactory } from './analysis_handlers/overall_histogram_handler'; import { logDebugMessageFactory, @@ -131,6 +132,7 @@ export const responseStreamFactory = (options: ResponseStr overallHistogramHandler: overallHistogramHandlerFactory(streamFetchOptions), overridesHandler: overridesHandlerFactory(streamFetchOptions), significantItemsHandler: significantItemsHandlerFactory(streamFetchOptions), + topItemsHandler: topItemsHandlerFactory(streamFetchOptions), }, responseWithHeaders, }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts index 3c1bcde38ebd6..e0a9b9c3e9803 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts @@ -92,7 +92,9 @@ export function routeHandlerFactory( } // Step 2: Significant categories and terms - const significantItemsObj = await analysis.significantItemsHandler(indexInfo); + const significantItemsObj = indexInfo.zeroDocsFallback + ? await analysis.topItemsHandler(indexInfo) + : await analysis.significantItemsHandler(indexInfo); if (!significantItemsObj) { return; From ef36e78023979b8d2ac99e897b7079c98cc4739a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 Nov 2023 14:34:25 +0100 Subject: [PATCH 02/17] fallback options for results tables --- .../common/api/log_rate_analysis/actions.ts | 18 ++++++- .../aiops/common/api/stream_reducer.ts | 4 ++ .../log_rate_analysis_results.tsx | 5 +- .../log_rate_analysis_results_table.tsx | 53 ++++++++++++------- ...log_rate_analysis_results_table_groups.tsx | 53 ++++++++++++------- .../analysis_handlers/index_info_handler.ts | 7 ++- 6 files changed, 97 insertions(+), 43 deletions(-) diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts index bd3afd3152ae5..2348b32c32f8d 100644 --- a/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts +++ b/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts @@ -36,6 +36,7 @@ export const API_ACTION_NAME = { RESET_ALL: 'reset_all', RESET_ERRORS: 'reset_errors', RESET_GROUPS: 'reset_groups', + SET_ZERO_DOCS_FALLBACK: 'set_zero_docs_fallback', UPDATE_LOADING_STATE: 'update_loading_state', } as const; export type ApiActionName = typeof API_ACTION_NAME[keyof typeof API_ACTION_NAME]; @@ -210,6 +211,20 @@ export function updateLoadingStateAction( }; } +interface ApiActionSetZeroDocsFallback { + type: typeof API_ACTION_NAME.SET_ZERO_DOCS_FALLBACK; + payload: boolean; +} + +export function setZeroDocsFallback( + payload: ApiActionSetZeroDocsFallback['payload'] +): ApiActionSetZeroDocsFallback { + return { + type: API_ACTION_NAME.SET_ZERO_DOCS_FALLBACK, + payload, + }; +} + export type AiopsLogRateAnalysisApiAction = | ApiActionAddSignificantItems | ApiActionAddSignificantItemsGroup @@ -220,4 +235,5 @@ export type AiopsLogRateAnalysisApiAction = | ApiActionResetAll | ApiActionResetErrors | ApiActionResetGroups - | ApiActionUpdateLoadingState; + | ApiActionUpdateLoadingState + | ApiActionSetZeroDocsFallback; diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.ts b/x-pack/plugins/aiops/common/api/stream_reducer.ts index 05d3fce52c22e..06da4d3d250ad 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.ts @@ -18,6 +18,7 @@ interface StreamState { loadingState: string; remainingFieldCandidates?: string[]; groupsMissing?: boolean; + zeroDocsFallback: boolean; } export const initialState: StreamState = { @@ -27,6 +28,7 @@ export const initialState: StreamState = { errors: [], loaded: 0, loadingState: '', + zeroDocsFallback: false, }; export function streamReducer( @@ -72,6 +74,8 @@ export function streamReducer( return initialState; case API_ACTION_NAME.UPDATE_LOADING_STATE: return { ...state, ...action.payload }; + case API_ACTION_NAME.SET_ZERO_DOCS_FALLBACK: + return { ...state, zeroDocsFallback: action.payload }; default: return state; } diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx index 457419c8b0501..21543d30120a4 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx @@ -209,7 +209,8 @@ export const LogRateAnalysisResults: FC = ({ { [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin } ); - const { significantItems } = data; + const { significantItems, zeroDocsFallback } = data; + useEffect( () => setUniqueFieldNames(uniq(significantItems.map((d) => d.fieldName)).sort()), [significantItems] @@ -490,6 +491,7 @@ export const LogRateAnalysisResults: FC = ({ searchQuery={searchQuery} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + zeroDocsFallback={zeroDocsFallback} /> ) : null} {showLogRateAnalysisResultsTable && !groupResults ? ( @@ -501,6 +503,7 @@ export const LogRateAnalysisResults: FC = ({ searchQuery={searchQuery} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + zeroDocsFallback={zeroDocsFallback} /> ) : null} diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx index d8845145f1ace..b0b4d699919be 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx @@ -51,7 +51,9 @@ const NOT_AVAILABLE = '--'; const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; const DEFAULT_SORT_FIELD = 'pValue'; +const DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK = 'doc_count'; const DEFAULT_SORT_DIRECTION = 'asc'; +const DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK = 'desc'; const TRUNCATE_TEXT_LINES = 3; @@ -66,6 +68,7 @@ interface LogRateAnalysisResultsTableProps { barColorOverride?: string; /** Optional color override for the highlighted bar color for charts */ barHighlightColorOverride?: string; + zeroDocsFallback?: boolean; } export const LogRateAnalysisResultsTable: FC = ({ @@ -77,6 +80,7 @@ export const LogRateAnalysisResultsTable: FC = timeRangeMs, barColorOverride, barHighlightColorOverride, + zeroDocsFallback = false, }) => { const euiTheme = useEuiTheme(); const primaryBackgroundColor = useEuiBackgroundColor('primary'); @@ -93,8 +97,12 @@ export const LogRateAnalysisResultsTable: FC = const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(DEFAULT_SORT_FIELD); - const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>(DEFAULT_SORT_DIRECTION); + const [sortField, setSortField] = useState( + zeroDocsFallback ? DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK : DEFAULT_SORT_FIELD + ); + const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>( + zeroDocsFallback ? DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK : DEFAULT_SORT_DIRECTION + ); const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext(); @@ -236,7 +244,10 @@ export const LogRateAnalysisResultsTable: FC = sortable: true, valign: 'middle', }, - { + ]; + + if (!zeroDocsFallback) { + columns.push({ 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnPValue', width: NARROW_COLUMN_WIDTH, field: 'pValue', @@ -260,8 +271,9 @@ export const LogRateAnalysisResultsTable: FC = render: (pValue: number | null) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, sortable: true, valign: 'middle', - }, - { + }); + + columns.push({ 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnImpact', width: NARROW_COLUMN_WIDTH, field: 'pValue', @@ -291,21 +303,22 @@ export const LogRateAnalysisResultsTable: FC = }, sortable: true, valign: 'middle', - }, - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnAction', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.actionsColumnName', { - defaultMessage: 'Actions', - }), - actions: [ - ...(viewInDiscoverAction ? [viewInDiscoverAction] : []), - ...(viewInLogPatternAnalysisAction ? [viewInLogPatternAnalysisAction] : []), - copyToClipBoardAction, - ], - width: ACTIONS_COLUMN_WIDTH, - valign: 'middle', - }, - ]; + }); + } + + columns.push({ + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnAction', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.actionsColumnName', { + defaultMessage: 'Actions', + }), + actions: [ + ...(viewInDiscoverAction ? [viewInDiscoverAction] : []), + ...(viewInLogPatternAnalysisAction ? [viewInLogPatternAnalysisAction] : []), + copyToClipBoardAction, + ], + width: ACTIONS_COLUMN_WIDTH, + valign: 'middle', + }); if (isExpandedRow === true) { columns.unshift({ diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx index f6961e49c2c78..957385780ceaa 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx @@ -50,7 +50,9 @@ const MAX_GROUP_BADGES = 5; const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; const DEFAULT_SORT_FIELD = 'pValue'; +const DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK = 'docCount'; const DEFAULT_SORT_DIRECTION = 'asc'; +const DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK = 'desc'; interface LogRateAnalysisResultsTableProps { significantItems: SignificantItem[]; @@ -63,6 +65,7 @@ interface LogRateAnalysisResultsTableProps { barColorOverride?: string; /** Optional color override for the highlighted bar color for charts */ barHighlightColorOverride?: string; + zeroDocsFallback?: boolean; } export const LogRateAnalysisResultsGroupsTable: FC = ({ @@ -74,11 +77,16 @@ export const LogRateAnalysisResultsGroupsTable: FC { const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(DEFAULT_SORT_FIELD); - const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>(DEFAULT_SORT_DIRECTION); + const [sortField, setSortField] = useState<'docCount' | 'pValue'>( + zeroDocsFallback ? DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK : DEFAULT_SORT_FIELD + ); + const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>( + zeroDocsFallback ? DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK : DEFAULT_SORT_DIRECTION + ); const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( {} ); @@ -297,7 +305,10 @@ export const LogRateAnalysisResultsGroupsTable: FC pValue?.toPrecision(3) ?? NOT_AVAILABLE, sortable: true, valign: 'top', - }, - { + }); + + columns.push({ 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnImpact', width: NARROW_COLUMN_WIDTH, field: 'pValue', @@ -355,21 +367,22 @@ export const LogRateAnalysisResultsGroupsTable: FC { if (tableSettings.page) { diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts index 14d2da84312c0..3c6ea4d022bb0 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts @@ -7,7 +7,10 @@ import { i18n } from '@kbn/i18n'; -import { updateLoadingStateAction } from '../../../../common/api/log_rate_analysis/actions'; +import { + updateLoadingStateAction, + setZeroDocsFallback, +} from '../../../../common/api/log_rate_analysis/actions'; import type { AiopsLogRateAnalysisApiVersion as ApiVersion } from '../../../../common/api/log_rate_analysis/schema'; import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; @@ -98,6 +101,8 @@ export const indexInfoHandlerFactory = }) ); + responseStream.push(setZeroDocsFallback(zeroDocsFallback)); + if (fieldCandidatesCount === 0) { responseStream.endWithUpdatedLoadingState(); } else if (stateHandler.shouldStop()) { From 7b69c31b7d556f2556ac9b40e28c013561bc3e78 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 Nov 2023 14:57:45 +0100 Subject: [PATCH 03/17] adapt analysis type callout for fallback cases --- .../log_rate_analysis_results.tsx | 35 ++------- .../log_rate_analysis_type_callout.tsx | 73 +++++++++++++++++++ 2 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx index 21543d30120a4..121c19be2ddb1 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx @@ -46,6 +46,7 @@ import { import { useLogRateAnalysisResultsTableRowContext } from '../log_rate_analysis_results_table/log_rate_analysis_results_table_row_provider'; import { FieldFilterPopover } from './field_filter_popover'; +import { LogRateAnalysisTypeCallOut } from './log_rate_analysis_type_callout'; const groupResultsMessage = i18n.translate( 'xpack.aiops.logRateAnalysis.resultsTable.groupedSwitchLabel.groupResults', @@ -363,37 +364,13 @@ export const LogRateAnalysisResults: FC = ({ /> - {showLogRateAnalysisResultsTable && ( + {showLogRateAnalysisResultsTable && currentAnalysisType !== undefined && ( <> - - {currentAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE - ? i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutTitle', { - defaultMessage: 'Analysis type: Log rate spike', - }) - : i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutTitle', { - defaultMessage: 'Analysis type: Log rate dip', - })} - - } - color="primary" - iconType="pin" - size="s" - > - - {currentAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE - ? i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContent', { - defaultMessage: - 'The median log rate in the selected deviation time range is higher than the baseline. Therefore, the analysis results table shows statistically significant items within the deviation time range that are contributors to the spike. The "doc count" column refers to the amount of documents in the deviation time range.', - }) - : i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContent', { - defaultMessage: - 'The median log rate in the selected deviation time range is lower than the baseline. Therefore, the analysis results table shows statistically significant items within the baseline time range that are less in number or missing within the deviation time range. The "doc count" column refers to the amount of documents in the baseline time range.', - })} - - + )} diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx new file mode 100644 index 0000000000000..c77c96a50a647 --- /dev/null +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FC } from 'react'; + +import { EuiCallOut, EuiText } from '@elastic/eui'; + +import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '@kbn/aiops-utils'; +import { i18n } from '@kbn/i18n'; + +interface LogRateAnalysisTypeCallOutProps { + analysisType: LogRateAnalysisType; + zeroDocsFallback: boolean; +} + +export const LogRateAnalysisTypeCallOut: FC = ({ + analysisType, + zeroDocsFallback, +}) => { + let callOutTitle: string; + let callOutText: string; + + if (!zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutTitle', { + defaultMessage: 'Analysis type: Log rate spike', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContent', { + defaultMessage: + 'The median log rate in the selected deviation time range is higher than the baseline. Therefore, the analysis results table shows statistically significant items within the deviation time range that are contributors to the spike. The "doc count" column refers to the amount of documents in the deviation time range.', + }); + } else if (!zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.DIP) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutTitle', { + defaultMessage: 'Analysis type: Log rate dip', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContent', { + defaultMessage: + 'The median log rate in the selected deviation time range is lower than the baseline. Therefore, the analysis results table shows statistically significant items within the baseline time range that are less in number or missing within the deviation time range. The "doc count" column refers to the amount of documents in the baseline time range.', + }); + } else if (zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeFallbackCallOutTitle', { + defaultMessage: 'Analysis type: Top items for deviation time range', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContentFallback', { + defaultMessage: + 'The baseline time range does not contain any documents. Therefor the results show top log message categories and field values for the deviation time range.', + }); + } else if (zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.DIP) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeDipFallbackCallOutTitle', { + defaultMessage: 'Analysis type: Top items for baseline time range', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContentFallback', { + defaultMessage: + 'The deviation time range does not contain any documents. Therefor the results show top log message categories and field values for the baseline time range.', + }); + } else { + return null; + } + + return ( + {callOutTitle}} + color="primary" + iconType="pin" + size="s" + > + {callOutText} + + ); +}; From a835a4b9c4f06c1ca9f881e64d6585e971bf0f6c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 Nov 2023 16:22:00 +0100 Subject: [PATCH 04/17] fix tests --- .../queries/fetch_index_info.test.ts | 57 ++----------------- .../queries/get_random_docs_request.test.ts | 56 ++++++++++++++++++ 2 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts index 0344005c869f8..9c196067a3eaa 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts @@ -11,55 +11,9 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { paramsSearchQueryMock } from './__mocks__/params_search_query'; -import { fetchIndexInfo, getRandomDocsRequest } from './fetch_index_info'; +import { fetchIndexInfo } from './fetch_index_info'; describe('fetch_index_info', () => { - describe('getRandomDocsRequest', () => { - it('returns the most basic request body for a sample of random documents', () => { - const req = getRandomDocsRequest(paramsSearchQueryMock); - - expect(req).toEqual({ - body: { - _source: false, - fields: ['*'], - query: { - function_score: { - query: { - bool: { - filter: [ - { - bool: { - filter: [], - minimum_should_match: 1, - must_not: [], - should: [{ term: { 'the-term': { value: 'the-value' } } }], - }, - }, - { - range: { - 'the-time-field-name': { - format: 'epoch_millis', - gte: 0, - lte: 50, - }, - }, - }, - ], - }, - }, - random_score: {}, - }, - }, - size: 1000, - track_total_hits: true, - }, - index: paramsSearchQueryMock.index, - ignore_throttled: undefined, - ignore_unavailable: true, - }); - }); - }); - describe('fetchFieldCandidates', () => { it('returns field candidates and total hits', async () => { const esClientFieldCapsMock = jest.fn(() => ({ @@ -99,13 +53,12 @@ describe('fetch_index_info', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const { totalDocCount, fieldCandidates } = await fetchIndexInfo( - esClientMock, - paramsSearchQueryMock - ); + const { baselineTotalDocCount, deviationTotalDocCount, fieldCandidates } = + await fetchIndexInfo(esClientMock, paramsSearchQueryMock); expect(fieldCandidates).toEqual(['myIpFieldName', 'myKeywordFieldName']); - expect(totalDocCount).toEqual(5000000); + expect(baselineTotalDocCount).toEqual(5000000); + expect(deviationTotalDocCount).toEqual(5000000); expect(esClientFieldCapsMock).toHaveBeenCalledTimes(1); expect(esClientSearchMock).toHaveBeenCalledTimes(1); }); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts new file mode 100644 index 0000000000000..6e68c789142c5 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { paramsSearchQueryMock } from './__mocks__/params_search_query'; + +import { getRandomDocsRequest } from './get_random_docs_request'; + +describe('getRandomDocsRequest', () => { + it('returns the most basic request body for a sample of random documents', () => { + const req = getRandomDocsRequest(paramsSearchQueryMock); + + expect(req).toEqual({ + body: { + _source: false, + fields: ['*'], + query: { + function_score: { + query: { + bool: { + filter: [ + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, + { + range: { + 'the-time-field-name': { + format: 'epoch_millis', + gte: 0, + lte: 50, + }, + }, + }, + ], + }, + }, + random_score: {}, + }, + }, + size: 1000, + track_total_hits: true, + }, + index: paramsSearchQueryMock.index, + ignore_throttled: undefined, + ignore_unavailable: true, + }); + }); +}); From 45be6e599ec235fa1dad81216b1378584148fbb7 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 Nov 2023 16:37:44 +0100 Subject: [PATCH 05/17] fix API integration test assertinos --- .../api_integration/apis/aiops/test_data.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 291779ed6c7b2..d5740624e4ea3 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -39,9 +39,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 35, + chunksLength: 36, chunksLengthGroupOnly: 5, - actionsLength: 34, + actionsLength: 35, actionsLengthGroupOnly: 4, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -93,9 +93,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 27, + chunksLength: 28, chunksLengthGroupOnly: 11, - actionsLength: 26, + actionsLength: 27, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -120,9 +120,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 30, + chunksLength: 31, chunksLengthGroupOnly: 11, - actionsLength: 29, + actionsLength: 30, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -147,9 +147,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 27, + chunksLength: 28, chunksLengthGroupOnly: 11, - actionsLength: 26, + actionsLength: 27, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -174,9 +174,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 30, + chunksLength: 31, chunksLengthGroupOnly: 11, - actionsLength: 29, + actionsLength: 30, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, From cf6436e2340106605079691d839f7fa5232475ee Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 Nov 2023 18:00:55 +0100 Subject: [PATCH 06/17] fix jest tests --- x-pack/plugins/aiops/common/api/stream_reducer.test.ts | 1 + .../routes/log_rate_analysis/queries/fetch_index_info.test.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.test.ts b/x-pack/plugins/aiops/common/api/stream_reducer.test.ts index f3dd6cce856c7..5344d2448463c 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.test.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.test.ts @@ -31,6 +31,7 @@ describe('streamReducer', () => { significantItems: [], significantItemsGroups: [], errors: [], + zeroDocsFallback: false, }); }); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts index 9c196067a3eaa..38377a9e0d32c 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts @@ -60,7 +60,7 @@ describe('fetch_index_info', () => { expect(baselineTotalDocCount).toEqual(5000000); expect(deviationTotalDocCount).toEqual(5000000); expect(esClientFieldCapsMock).toHaveBeenCalledTimes(1); - expect(esClientSearchMock).toHaveBeenCalledTimes(1); + expect(esClientSearchMock).toHaveBeenCalledTimes(2); }); }); }); From 657ba18490c8206563d3de79d66bac32e6e97c12 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 27 Nov 2023 12:42:56 +0100 Subject: [PATCH 07/17] fix typo --- .../log_rate_analysis/log_rate_analysis_type_callout.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx index c77c96a50a647..2760a27c3d224 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx @@ -46,7 +46,7 @@ export const LogRateAnalysisTypeCallOut: FC = ( }); callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContentFallback', { defaultMessage: - 'The baseline time range does not contain any documents. Therefor the results show top log message categories and field values for the deviation time range.', + 'The baseline time range does not contain any documents. Therefore the results show top log message categories and field values for the deviation time range.', }); } else if (zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.DIP) { callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeDipFallbackCallOutTitle', { @@ -54,7 +54,7 @@ export const LogRateAnalysisTypeCallOut: FC = ( }); callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContentFallback', { defaultMessage: - 'The deviation time range does not contain any documents. Therefor the results show top log message categories and field values for the baseline time range.', + 'The deviation time range does not contain any documents. Therefore the results show top log message categories and field values for the baseline time range.', }); } else { return null; From 729fe65fe53115a06de874d047adab07be9b1d66 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 28 Nov 2023 14:51:39 +0100 Subject: [PATCH 08/17] update functional tests --- .../apps/aiops/log_rate_analysis.ts | 49 ++- .../artificial_log_data_view_test_data.ts | 360 ++++++++++++++++++ .../farequote_data_view_test_data.ts | 28 ++ ...arequote_data_view_test_data_with_query.ts | 47 +++ .../kibana_logs_data_view_test_data.ts | 116 ++++++ .../apps/aiops/log_rate_analysis_test_data.ts | 290 +------------- x-pack/test/functional/apps/aiops/types.ts | 2 +- .../aiops/log_rate_analysis_data_generator.ts | 100 ++++- .../services/aiops/log_rate_analysis_page.ts | 22 +- 9 files changed, 706 insertions(+), 308 deletions(-) create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis.ts index 68c9256b382d0..f613349078b48 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis.ts @@ -149,12 +149,18 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { } // Wait for the analysis to finish - await aiops.logRateAnalysisPage.assertAnalysisComplete(testData.analysisType); + await aiops.logRateAnalysisPage.assertAnalysisComplete( + testData.analysisType, + testData.dataGenerator + ); // At this stage the baseline and deviation brush position should be stored in // the url state and a full browser refresh should restore the analysis. await browser.refresh(); - await aiops.logRateAnalysisPage.assertAnalysisComplete(testData.analysisType); + await aiops.logRateAnalysisPage.assertAnalysisComplete( + testData.analysisType, + testData.dataGenerator + ); // The group switch should be disabled by default await aiops.logRateAnalysisPage.assertLogRateAnalysisResultsGroupSwitchExists(false); @@ -167,8 +173,15 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const analysisGroupsTable = await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); - expect(orderBy(analysisGroupsTable, 'group')).to.be.eql( - orderBy(testData.expected.analysisGroupsTable, 'group') + + const actualAnalysisGroupsTable = orderBy(analysisGroupsTable, 'group'); + const expectedAnalysisGroupsTable = orderBy(testData.expected.analysisGroupsTable, 'group'); + + expect(actualAnalysisGroupsTable).to.be.eql( + expectedAnalysisGroupsTable, + `Expected analysis groups table to be ${JSON.stringify( + expectedAnalysisGroupsTable + )}, got ${JSON.stringify(actualAnalysisGroupsTable)}` ); await ml.testExecution.logTestStep('expand table row'); @@ -177,8 +190,18 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { if (!isTestDataExpectedWithSampleProbability(testData.expected)) { const analysisTable = await aiops.logRateAnalysisResultsTable.parseAnalysisTable(); - expect(orderBy(analysisTable, ['fieldName', 'fieldValue'])).to.be.eql( - orderBy(testData.expected.analysisTable, ['fieldName', 'fieldValue']) + + const actualAnalysisTable = orderBy(analysisTable, ['fieldName', 'fieldValue']); + const expectedAnalysisTable = orderBy(testData.expected.analysisTable, [ + 'fieldName', + 'fieldValue', + ]); + + expect(actualAnalysisTable).to.be.eql( + expectedAnalysisTable, + `Expected analysis table results to be ${JSON.stringify( + expectedAnalysisTable + )}, got ${JSON.stringify(actualAnalysisTable)}` ); } @@ -206,8 +229,18 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { if (!isTestDataExpectedWithSampleProbability(testData.expected)) { const filteredAnalysisGroupsTable = await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); - expect(orderBy(filteredAnalysisGroupsTable, 'group')).to.be.eql( - orderBy(testData.expected.filteredAnalysisGroupsTable, 'group') + + const actualFilteredAnalysisGroupsTable = orderBy(filteredAnalysisGroupsTable, 'group'); + const expectedFilteredAnalysisGroupsTable = orderBy( + testData.expected.filteredAnalysisGroupsTable, + 'group' + ); + + expect(actualFilteredAnalysisGroupsTable).to.be.eql( + expectedFilteredAnalysisGroupsTable, + `Expected filtered analysis groups table to be ${JSON.stringify( + expectedFilteredAnalysisGroupsTable + )}, got ${JSON.stringify(actualFilteredAnalysisGroupsTable)}` ); } } diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts new file mode 100644 index 0000000000000..94e23adf3351f --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts @@ -0,0 +1,360 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { LogRateAnalysisType } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +import type { LogRateAnalysisDataGenerator } from '../../../../services/aiops/log_rate_analysis_data_generator'; + +const REFERENCE_TS = 1669018354793; +const DAY_MS = 86400000; + +const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; +const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; + +export const getArtificialLogDataViewTestData = ( + analysisType: LogRateAnalysisType, + textField: boolean, + gaps: boolean +): TestData => { + function getAnalysisGroupsTable() { + if (gaps) { + return textField + ? [ + { + group: + 'message: an unexpected error occuredurl: home.phpuser: Maryresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'message: an unexpected error occuredurl: home.phpuser: Paulresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'message: an unexpected error occuredurl: login.phpuser: Paulresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'url: home.phpuser: Paulresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterurl: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterurl: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + ] + : [ + { group: 'response_code: 500url: home.phpuser: Maryversion: v1.0.0', docCount: '47' }, + { group: 'response_code: 500url: home.phpuser: Paulversion: v1.0.0', docCount: '59' }, + { group: 'response_code: 500url: login.phpuser: Maryversion: v1.0.0', docCount: '35' }, + { group: 'response_code: 500url: login.phpuser: Paulversion: v1.0.0', docCount: '39' }, + { group: 'user: Peterurl: home.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: home.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: login.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: login.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: user.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: user.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + ]; + } + + return [ + textField + ? { + group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', + docCount: '634', + } + : { + group: 'response_code: 500url: home.php', + docCount: '792', + }, + textField + ? { + group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500', + docCount: '632', + } + : { + group: 'url: login.phpresponse_code: 500', + docCount: '790', + }, + { + docCount: '636', + group: 'user: Peterurl: home.php', + }, + { + docCount: '632', + group: 'user: Peterurl: login.php', + }, + ]; + } + + function getFilteredAnalysisGroupsTable() { + if (gaps) { + return textField + ? [ + { + group: + 'message: an unexpected error occuredurl: home.phpresponse_code: 500version: v1.0.0', + docCount: '58', + }, + { + group: + 'message: an unexpected error occuredurl: login.phpresponse_code: 500version: v1.0.0', + docCount: '58', + }, + { + group: + 'response_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '46', + }, + { + group: + 'response_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '35', + }, + { + group: + 'response_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '63', + }, + { + group: + 'response_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '40', + }, + { + group: + 'url: home.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '48', + }, + { + group: + 'url: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '40', + }, + { + group: + 'url: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '51', + }, + { + group: + 'url: user.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '41', + }, + ] + : [ + { group: 'url: home.phpresponse_code: 200version: v1.0.0', docCount: '46' }, + { group: 'url: home.phpresponse_code: 404version: v1.0.0', docCount: '63' }, + { group: 'url: home.phpresponse_code: 500version: v1.0.0', docCount: '106' }, + { group: 'url: login.phpresponse_code: 200version: v1.0.0', docCount: '35' }, + { group: 'url: login.phpresponse_code: 404version: v1.0.0', docCount: '40' }, + { group: 'url: login.phpresponse_code: 500version: v1.0.0', docCount: '74' }, + { group: 'url: user.phpresponse_code: 200version: v1.0.0', docCount: '40' }, + { group: 'url: user.phpresponse_code: 404version: v1.0.0', docCount: '51' }, + { group: 'url: user.phpresponse_code: 500version: v1.0.0', docCount: '41' }, + ]; + } + + return textField + ? [ + { + group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', + docCount: '634', + }, + { + group: '* url: login.phpmessage: an unexpected error occuredresponse_code: 500', + docCount: '632', + }, + ] + : [ + { group: '* url: home.phpresponse_code: 500', docCount: '792' }, + { group: '* url: login.phpresponse_code: 500', docCount: '790' }, + ]; + } + + function getAnalysisTable() { + if (gaps) { + return textField + ? [ + { + fieldName: 'message', + fieldValue: 'Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'user', + fieldValue: 'Paul', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'version', + fieldValue: 'v1.0.0', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + ] + : [ + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'user', + fieldValue: 'Paul', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'version', + fieldValue: 'v1.0.0', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + ]; + } + + return [ + ...(textField + ? [ + { + fieldName: 'message', + fieldValue: 'an unexpected error occured', + logRate: 'Chart type:bar chart', + pValue: '0.00000100', + impact: 'Medium', + }, + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '3.61e-12', + impact: 'High', + }, + ] + : []), + { + fieldName: 'url', + fieldValue: 'home.php', + impact: 'Low', + logRate: 'Chart type:bar chart', + pValue: '0.00974', + }, + ...(textField + ? [] + : [ + { + fieldName: 'user', + fieldValue: 'Peter', + impact: 'High', + logRate: 'Chart type:bar chart', + pValue: '2.63e-21', + }, + ]), + ]; + } + + function getFieldSelectorPopover() { + if (gaps) { + return [...(textField ? ['message'] : []), 'response_code', 'url', 'user', 'version']; + } + return [...(textField ? ['message'] : []), 'response_code', 'url', 'user']; + } + + function getSuiteTitle() { + return `artificial logs with ${analysisType} and ${ + textField ? 'text field' : 'no text field' + } and ${gaps ? 'gaps' : 'no gaps'}`; + } + + function getDataGenerator(): LogRateAnalysisDataGenerator { + return `artificial_logs_with_${analysisType}_${textField ? 'textfield' : 'notextfield'}_${ + gaps ? 'gaps' : 'nogaps' + }`; + } + + return { + suiteTitle: getSuiteTitle(), + analysisType, + dataGenerator: getDataGenerator(), + isSavedSearch: false, + sourceIndexOrSavedSearch: getDataGenerator(), + brushBaselineTargetTimestamp: gaps ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2, + brushDeviationTargetTimestamp: gaps ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2, + brushIntervalFactor: gaps ? 1 : 10, + chartClickCoordinates: [-200, 30], + fieldSelectorSearch: 'user', + fieldSelectorApplyAvailable: true, + expected: { + totalDocCountFormatted: gaps ? '9,482' : '8,400', + analysisGroupsTable: getAnalysisGroupsTable(), + filteredAnalysisGroupsTable: getFilteredAnalysisGroupsTable(), + analysisTable: getAnalysisTable(), + fieldSelectorPopover: getFieldSelectorPopover(), + }, + }; +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts new file mode 100644 index 0000000000000..be019193f2ef1 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +export const farequoteDataViewTestData: TestData = { + suiteTitle: 'farequote with spike', + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + dataGenerator: 'farequote_with_spike', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_farequote', + brushDeviationTargetTimestamp: 1455033600000, + brushIntervalFactor: 1, + chartClickCoordinates: [0, 0], + fieldSelectorSearch: 'airline', + fieldSelectorApplyAvailable: false, + expected: { + totalDocCountFormatted: '86,374', + sampleProbabilityFormatted: '0.5', + fieldSelectorPopover: ['airline', 'custom_field.keyword'], + }, +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts new file mode 100644 index 0000000000000..653889c1eb0ca --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +export const farequoteDataViewTestDataWithQuery: TestData = { + suiteTitle: 'farequote with spike', + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + dataGenerator: 'farequote_with_spike', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_farequote', + brushDeviationTargetTimestamp: 1455033600000, + brushIntervalFactor: 1, + chartClickCoordinates: [0, 0], + fieldSelectorSearch: 'airline', + fieldSelectorApplyAvailable: false, + query: 'NOT airline:("SWR" OR "ACA" OR "AWE" OR "BAW" OR "JAL" OR "JBU" OR "JZA" OR "KLM")', + expected: { + totalDocCountFormatted: '48,799', + analysisGroupsTable: [ + { + docCount: '297', + group: '* airline: AAL', + }, + { + docCount: '100', + group: '* custom_field.keyword: deviation* airline: UAL', + }, + ], + analysisTable: [ + { + fieldName: 'airline', + fieldValue: 'AAL', + logRate: 'Chart type:bar chart', + pValue: '1.18e-8', + impact: 'High', + }, + ], + fieldSelectorPopover: ['airline', 'custom_field.keyword'], + }, +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts new file mode 100644 index 0000000000000..2d85fe1e64210 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +export const kibanaLogsDataViewTestData: TestData = { + suiteTitle: 'kibana sample data logs', + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + dataGenerator: 'kibana_sample_data_logs', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'kibana_sample_data_logs', + brushIntervalFactor: 1, + chartClickCoordinates: [235, 0], + fieldSelectorSearch: 'referer', + fieldSelectorApplyAvailable: true, + action: { + type: 'LogPatternAnalysis', + tableRowId: '1064853178', + expected: { + queryBar: + 'clientip:30.156.16.164 AND host.keyword:elastic-elastic-elastic.org AND ip:30.156.16.163 AND response.keyword:404 AND machine.os.keyword:win xp AND geo.dest:IN AND geo.srcdest:US\\:IN', + totalDocCount: 100, + }, + }, + expected: { + totalDocCountFormatted: '14,074', + analysisGroupsTable: [ + { + group: + '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* referer: http://www.elastic-elastic-elastic.com/success/timothy-l-kopra* response.keyword: 404Showing 5 out of 8 items. 8 items unique to this group.', + docCount: '100', + }, + ], + filteredAnalysisGroupsTable: [ + { + group: + '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* response.keyword: 404* machine.os.keyword: win xpShowing 5 out of 7 items. 7 items unique to this group.', + docCount: '100', + }, + ], + analysisTable: [ + { + fieldName: 'clientip', + fieldValue: '30.156.16.164', + logRate: 'Chart type:bar chart', + pValue: '3.10e-13', + impact: 'High', + }, + { + fieldName: 'geo.dest', + fieldValue: 'IN', + logRate: 'Chart type:bar chart', + pValue: '0.000716', + impact: 'Medium', + }, + { + fieldName: 'geo.srcdest', + fieldValue: 'US:IN', + logRate: 'Chart type:bar chart', + pValue: '0.000716', + impact: 'Medium', + }, + { + fieldName: 'host.keyword', + fieldValue: 'elastic-elastic-elastic.org', + logRate: 'Chart type:bar chart', + pValue: '7.14e-9', + impact: 'High', + }, + { + fieldName: 'ip', + fieldValue: '30.156.16.163', + logRate: 'Chart type:bar chart', + pValue: '3.28e-13', + impact: 'High', + }, + { + fieldName: 'machine.os.keyword', + fieldValue: 'win xp', + logRate: 'Chart type:bar chart', + pValue: '0.0000997', + impact: 'Medium', + }, + { + fieldName: 'referer', + fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', + logRate: 'Chart type:bar chart', + pValue: '4.74e-13', + impact: 'High', + }, + { + fieldName: 'response.keyword', + fieldValue: '404', + logRate: 'Chart type:bar chart', + pValue: '0.00000604', + impact: 'Medium', + }, + ], + fieldSelectorPopover: [ + 'clientip', + 'geo.dest', + 'geo.srcdest', + 'host.keyword', + 'ip', + 'machine.os.keyword', + 'referer', + 'response.keyword', + ], + }, +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts index 3fb1e00b95201..3c9644a93c64f 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts @@ -5,289 +5,23 @@ * 2.0. */ -import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '@kbn/aiops-utils'; +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; -import type { TestData } from './types'; - -export const kibanaLogsDataViewTestData: TestData = { - suiteTitle: 'kibana sample data logs', - analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, - dataGenerator: 'kibana_sample_data_logs', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'kibana_sample_data_logs', - brushIntervalFactor: 1, - chartClickCoordinates: [235, 0], - fieldSelectorSearch: 'referer', - fieldSelectorApplyAvailable: true, - action: { - type: 'LogPatternAnalysis', - tableRowId: '1064853178', - expected: { - queryBar: - 'clientip:30.156.16.164 AND host.keyword:elastic-elastic-elastic.org AND ip:30.156.16.163 AND response.keyword:404 AND machine.os.keyword:win xp AND geo.dest:IN AND geo.srcdest:US\\:IN', - totalDocCount: 100, - }, - }, - expected: { - totalDocCountFormatted: '14,074', - analysisGroupsTable: [ - { - group: - '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* referer: http://www.elastic-elastic-elastic.com/success/timothy-l-kopra* response.keyword: 404Showing 5 out of 8 items. 8 items unique to this group.', - docCount: '100', - }, - ], - filteredAnalysisGroupsTable: [ - { - group: - '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* response.keyword: 404* machine.os.keyword: win xpShowing 5 out of 7 items. 7 items unique to this group.', - docCount: '100', - }, - ], - analysisTable: [ - { - fieldName: 'clientip', - fieldValue: '30.156.16.164', - logRate: 'Chart type:bar chart', - pValue: '3.10e-13', - impact: 'High', - }, - { - fieldName: 'geo.dest', - fieldValue: 'IN', - logRate: 'Chart type:bar chart', - pValue: '0.000716', - impact: 'Medium', - }, - { - fieldName: 'geo.srcdest', - fieldValue: 'US:IN', - logRate: 'Chart type:bar chart', - pValue: '0.000716', - impact: 'Medium', - }, - { - fieldName: 'host.keyword', - fieldValue: 'elastic-elastic-elastic.org', - logRate: 'Chart type:bar chart', - pValue: '7.14e-9', - impact: 'High', - }, - { - fieldName: 'ip', - fieldValue: '30.156.16.163', - logRate: 'Chart type:bar chart', - pValue: '3.28e-13', - impact: 'High', - }, - { - fieldName: 'machine.os.keyword', - fieldValue: 'win xp', - logRate: 'Chart type:bar chart', - pValue: '0.0000997', - impact: 'Medium', - }, - { - fieldName: 'referer', - fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', - logRate: 'Chart type:bar chart', - pValue: '4.74e-13', - impact: 'High', - }, - { - fieldName: 'response.keyword', - fieldValue: '404', - logRate: 'Chart type:bar chart', - pValue: '0.00000604', - impact: 'Medium', - }, - ], - fieldSelectorPopover: [ - 'clientip', - 'geo.dest', - 'geo.srcdest', - 'host.keyword', - 'ip', - 'machine.os.keyword', - 'referer', - 'response.keyword', - ], - }, -}; - -export const farequoteDataViewTestData: TestData = { - suiteTitle: 'farequote with spike', - analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, - dataGenerator: 'farequote_with_spike', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'ft_farequote', - brushDeviationTargetTimestamp: 1455033600000, - brushIntervalFactor: 1, - chartClickCoordinates: [0, 0], - fieldSelectorSearch: 'airline', - fieldSelectorApplyAvailable: false, - expected: { - totalDocCountFormatted: '86,374', - sampleProbabilityFormatted: '0.5', - fieldSelectorPopover: ['airline', 'custom_field.keyword'], - }, -}; - -export const farequoteDataViewTestDataWithQuery: TestData = { - suiteTitle: 'farequote with spike', - analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, - dataGenerator: 'farequote_with_spike', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'ft_farequote', - brushDeviationTargetTimestamp: 1455033600000, - brushIntervalFactor: 1, - chartClickCoordinates: [0, 0], - fieldSelectorSearch: 'airline', - fieldSelectorApplyAvailable: false, - query: 'NOT airline:("SWR" OR "ACA" OR "AWE" OR "BAW" OR "JAL" OR "JBU" OR "JZA" OR "KLM")', - expected: { - totalDocCountFormatted: '48,799', - analysisGroupsTable: [ - { - docCount: '297', - group: '* airline: AAL', - }, - { - docCount: '100', - group: '* custom_field.keyword: deviation* airline: UAL', - }, - ], - analysisTable: [ - { - fieldName: 'airline', - fieldValue: 'AAL', - logRate: 'Chart type:bar chart', - pValue: '1.18e-8', - impact: 'High', - }, - ], - fieldSelectorPopover: ['airline', 'custom_field.keyword'], - }, -}; +import { kibanaLogsDataViewTestData } from './log_rate_analysis/test_data/kibana_logs_data_view_test_data'; +import { farequoteDataViewTestData } from './log_rate_analysis/test_data/farequote_data_view_test_data'; +import { farequoteDataViewTestDataWithQuery } from './log_rate_analysis/test_data/farequote_data_view_test_data_with_query'; +import { getArtificialLogDataViewTestData } from './log_rate_analysis/test_data/artificial_log_data_view_test_data'; -const REFERENCE_TS = 1669018354793; -const DAY_MS = 86400000; - -const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; -const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; - -const getArtificialLogDataViewTestData = ( - analysisType: LogRateAnalysisType, - textField: boolean -): TestData => ({ - suiteTitle: `artificial logs with ${analysisType} and ${ - textField ? 'text field' : 'no text field' - }`, - analysisType, - dataGenerator: `artificial_logs_with_${analysisType}_${textField ? 'textfield' : 'notextfield'}`, - isSavedSearch: false, - sourceIndexOrSavedSearch: `artificial_logs_with_${analysisType}_${ - textField ? 'textfield' : 'notextfield' - }`, - brushBaselineTargetTimestamp: BASELINE_TS + DAY_MS / 2, - brushDeviationTargetTimestamp: DEVIATION_TS + DAY_MS / 2, - brushIntervalFactor: 10, - chartClickCoordinates: [-200, 30], - fieldSelectorSearch: 'user', - fieldSelectorApplyAvailable: true, - expected: { - totalDocCountFormatted: '8,400', - analysisGroupsTable: [ - textField - ? { - group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', - docCount: '634', - } - : { - group: 'response_code: 500url: home.php', - docCount: '792', - }, - textField - ? { - group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500', - docCount: '632', - } - : { - group: 'url: login.phpresponse_code: 500', - docCount: '790', - }, - { - docCount: '636', - group: 'user: Peterurl: home.php', - }, - { - docCount: '632', - group: 'user: Peterurl: login.php', - }, - ], - filteredAnalysisGroupsTable: textField - ? [ - { - group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', - docCount: '634', - }, - { - group: '* url: login.phpmessage: an unexpected error occuredresponse_code: 500', - docCount: '632', - }, - ] - : [ - { group: '* url: home.phpresponse_code: 500', docCount: '792' }, - { group: '* url: login.phpresponse_code: 500', docCount: '790' }, - ], - analysisTable: [ - ...(textField - ? [ - { - fieldName: 'message', - fieldValue: 'an unexpected error occured', - logRate: 'Chart type:bar chart', - pValue: '0.00000100', - impact: 'Medium', - }, - { - fieldName: 'response_code', - fieldValue: '500', - logRate: 'Chart type:bar chart', - pValue: '3.61e-12', - impact: 'High', - }, - ] - : []), - { - fieldName: 'url', - fieldValue: 'home.php', - impact: 'Low', - logRate: 'Chart type:bar chart', - pValue: '0.00974', - }, - ...(textField - ? [] - : [ - { - fieldName: 'user', - fieldValue: 'Peter', - impact: 'High', - logRate: 'Chart type:bar chart', - pValue: '2.63e-21', - }, - ]), - ], - fieldSelectorPopover: [...(textField ? ['message'] : []), 'response_code', 'url', 'user'], - }, -}); +import type { TestData } from './types'; export const logRateAnalysisTestData: TestData[] = [ kibanaLogsDataViewTestData, farequoteDataViewTestData, farequoteDataViewTestDataWithQuery, - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, true), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, true), + getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, false, false), + getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, true, false), + getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, true, true), + getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, false, true), + getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, false, false), + getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, true, false), ]; diff --git a/x-pack/test/functional/apps/aiops/types.ts b/x-pack/test/functional/apps/aiops/types.ts index 45f376cb670e1..72a72b1d2bb37 100644 --- a/x-pack/test/functional/apps/aiops/types.ts +++ b/x-pack/test/functional/apps/aiops/types.ts @@ -8,7 +8,7 @@ import type { LogRateAnalysisType } from '@kbn/aiops-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import { LogRateAnalysisDataGenerator } from '../../services/aiops/log_rate_analysis_data_generator'; +import type { LogRateAnalysisDataGenerator } from '../../services/aiops/log_rate_analysis_data_generator'; interface TestDataTableActionLogPatternAnalysis { type: 'LogPatternAnalysis'; diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts index 48028b2ddbd1a..bb2d9e5d70d42 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts @@ -14,10 +14,14 @@ import { FtrProviderContext } from '../../ftr_provider_context'; const LOG_RATE_ANALYSYS_DATA_GENERATOR = { KIBANA_SAMPLE_DATA_LOGS: 'kibana_sample_data_logs', FAREQUOTE_WITH_SPIKE: 'farequote_with_spike', - ARTIFICIAL_LOGS_WITH_SPIKE_NOTEXTFIELD: 'artificial_logs_with_spike_notextfield', - ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD: 'artificial_logs_with_spike_textfield', - ARTIFICIAL_LOGS_WITH_DIP_NOTEXTFIELD: 'artificial_logs_with_dip_notextfield', - ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD: 'artificial_logs_with_dip_textfield', + ARTIFICIAL_LOGS_WITH_SPIKE_NOTEXTFIELD_GAPS: 'artificial_logs_with_spike_notextfield_gaps', + ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD_GAPS: 'artificial_logs_with_spike_textfield_gaps', + ARTIFICIAL_LOGS_WITH_DIP_NOTEXTFIELD_GAPS: 'artificial_logs_with_dip_notextfield_gaps', + ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD_GAPS: 'artificial_logs_with_dip_textfield_gaps', + ARTIFICIAL_LOGS_WITH_SPIKE_NOTEXTFIELD_NOGAPS: 'artificial_logs_with_spike_notextfield_nogaps', + ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD_NOGAPS: 'artificial_logs_with_spike_textfield_nogaps', + ARTIFICIAL_LOGS_WITH_DIP_NOTEXTFIELD_NOGAPS: 'artificial_logs_with_dip_notextfield_nogaps', + ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD_NOGAPS: 'artificial_logs_with_dip_textfield_nogaps', } as const; export type LogRateAnalysisDataGenerator = typeof LOG_RATE_ANALYSYS_DATA_GENERATOR[keyof typeof LOG_RATE_ANALYSYS_DATA_GENERATOR]; @@ -38,7 +42,7 @@ const DAY_MS = 86400000; const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; -function getMessage(timestamp: number, user: string, url: string, responseCode: string) { +function getTextFieldMessage(timestamp: number, user: string, url: string, responseCode: string) { const date = new Date(timestamp); return `${user} [${date.toLocaleString('en-US')}] "GET /${url} HTTP/1.1" ${responseCode}`; } @@ -46,12 +50,37 @@ function getMessage(timestamp: number, user: string, url: string, responseCode: function getArtificialLogsWithDeviation( index: string, deviationType: string, - includeTextField = false + includeTextField = false, + includeGaps = false ) { const bulkBody: estypes.BulkRequest['body'] = []; const action = { index: { _index: index } }; let tsOffset = 0; + if (includeGaps) { + const earliestDoc: GeneratedDoc = { + user: 'Peter', + response_code: '200', + url: 'login.php', + version: 'v1.0.0', + '@timestamp': BASELINE_TS - DAY_MS, + should_ignore_this_field: 'should_ignore_this_field', + }; + bulkBody.push(action); + bulkBody.push(earliestDoc); + + const latestDoc: GeneratedDoc = { + user: 'Peter', + response_code: '200', + url: 'login.php', + version: 'v1.0.0', + '@timestamp': DEVIATION_TS + 2 * DAY_MS, + should_ignore_this_field: 'should_ignore_this_field', + }; + bulkBody.push(action); + bulkBody.push(latestDoc); + } + // Creates docs evenly spread across baseline and deviation time frame [BASELINE_TS, DEVIATION_TS].forEach((ts) => { ['Peter', 'Paul', 'Mary'].forEach((user) => { @@ -66,8 +95,32 @@ function getArtificialLogsWithDeviation( ) ) { tsOffset = 0; - [...Array(100)].forEach(() => { - tsOffset += Math.round(DAY_MS / 100); + + let docCount = 100; + let responseCodeFactor = 1; + + if (includeGaps) { + if (responseCode === '404') { + responseCodeFactor = 2; + } else if (responseCode === '500') { + responseCodeFactor = 3; + } + + if (url === 'user.php') { + responseCodeFactor *= 2; + } else if (url === 'home.php') { + responseCodeFactor *= 3; + } + + if (user === 'Paul') { + docCount = 40 * responseCodeFactor; + } else if (user === 'Mary') { + docCount = 25 * responseCodeFactor; + } + } + + [...Array(docCount)].forEach(() => { + tsOffset += Math.round(DAY_MS / docCount); const timestamp = ts + tsOffset; const doc: GeneratedDoc = { user, @@ -79,7 +132,7 @@ function getArtificialLogsWithDeviation( }; if (includeTextField) { - doc.message = getMessage(timestamp, user, url, responseCode); + doc.message = getTextFieldMessage(timestamp, user, url, responseCode); } bulkBody.push(action); @@ -116,7 +169,7 @@ function getArtificialLogsWithDeviation( }; if (includeTextField) { - doc.message = getMessage(timestamp, 'Peter', url, responseCode); + doc.message = getTextFieldMessage(timestamp, 'Peter', url, responseCode); } bulkBody.push(action); @@ -204,10 +257,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); break; - case 'artificial_logs_with_spike_notextfield': - case 'artificial_logs_with_spike_textfield': - case 'artificial_logs_with_dip_notextfield': - case 'artificial_logs_with_dip_textfield': + case 'artificial_logs_with_spike_notextfield_nogaps': + case 'artificial_logs_with_spike_textfield_nogaps': + case 'artificial_logs_with_dip_notextfield_nogaps': + case 'artificial_logs_with_dip_textfield_nogaps': + case 'artificial_logs_with_spike_notextfield_gaps': + case 'artificial_logs_with_spike_textfield_gaps': + case 'artificial_logs_with_dip_notextfield_gaps': + case 'artificial_logs_with_dip_textfield_gaps': try { const indexExists = await es.indices.exists({ index: dataGenerator, @@ -240,10 +297,11 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider const dataGeneratorOptions = dataGenerator.split('_'); const deviationType = dataGeneratorOptions[3] ?? LOG_RATE_ANALYSIS_TYPE.SPIKE; const textField = dataGeneratorOptions[4] === 'textfield' ?? false; + const gaps = dataGeneratorOptions[5] === 'gaps' ?? false; await es.bulk({ refresh: 'wait_for', - body: getArtificialLogsWithDeviation(dataGenerator, deviationType, textField), + body: getArtificialLogsWithDeviation(dataGenerator, deviationType, textField, gaps), }); break; @@ -262,10 +320,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); break; - case 'artificial_logs_with_spike_notextfield': - case 'artificial_logs_with_spike_textfield': - case 'artificial_logs_with_dip_notextfield': - case 'artificial_logs_with_dip_textfield': + case 'artificial_logs_with_spike_notextfield_nogaps': + case 'artificial_logs_with_spike_textfield_nogaps': + case 'artificial_logs_with_dip_notextfield_nogaps': + case 'artificial_logs_with_dip_textfield_nogaps': + case 'artificial_logs_with_spike_notextfield_gaps': + case 'artificial_logs_with_spike_textfield_gaps': + case 'artificial_logs_with_dip_notextfield_gaps': + case 'artificial_logs_with_dip_textfield_gaps': try { await es.indices.delete({ index: dataGenerator, diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts index ea2251a475889..e814dcfd3ca51 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts @@ -11,6 +11,8 @@ import type { LogRateAnalysisType } from '@kbn/aiops-utils'; import type { FtrProviderContext } from '../../ftr_provider_context'; +import type { LogRateAnalysisDataGenerator } from './log_rate_analysis_data_generator'; + export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrProviderContext) { const browser = getService('browser'); const elasticChart = getService('elasticChart'); @@ -241,7 +243,12 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr }); }, - async assertAnalysisComplete(analisysType: LogRateAnalysisType) { + async assertAnalysisComplete( + analysisType: LogRateAnalysisType, + dataGenerator: LogRateAnalysisDataGenerator + ) { + const dataGeneratorParts = dataGenerator.split('_'); + const includeGaps = dataGeneratorParts[5] === 'gaps'; await retry.tryForTime(30 * 1000, async () => { await testSubjects.existOrFail('aiopsAnalysisComplete'); const currentProgressTitle = await testSubjects.getVisibleText('aiopsAnalysisComplete'); @@ -251,7 +258,18 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr const currentAnalysisTypeCalloutTitle = await testSubjects.getVisibleText( 'aiopsAnalysisTypeCalloutTitle' ); - expect(currentAnalysisTypeCalloutTitle).to.be(`Analysis type: Log rate ${analisysType}`); + + if (includeGaps && analysisType === 'spike') { + expect(currentAnalysisTypeCalloutTitle).to.be( + 'Analysis type: Top items for deviation time range' + ); + } else if (includeGaps && analysisType === 'dip') { + expect(currentAnalysisTypeCalloutTitle).to.be( + 'Analysis type: Top items for baseline time range' + ); + } else { + expect(currentAnalysisTypeCalloutTitle).to.be(`Analysis type: Log rate ${analysisType}`); + } }); }, From 5325647c384c3d98cc82d58616d5afa3b8952b1b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 28 Nov 2023 15:37:10 +0100 Subject: [PATCH 09/17] refactor mocks --- .../analysis_groups_table_notextfield_gaps.ts | 19 ++ ...nalysis_groups_table_notextfield_nogaps.ts | 25 ++ .../analysis_groups_table_textfield_gaps.ts | 59 ++++ .../analysis_groups_table_textfield_nogaps.ts | 25 ++ .../analysis_table_notextfield_gaps.ts | 37 +++ .../analysis_table_notextfield_nogaps.ts | 23 ++ .../analysis_table_textfield_gaps.ts | 44 +++ .../analysis_table_textfield_nogaps.ts | 30 ++ ..._analysis_groups_table_notextfield_gaps.ts | 18 ++ ...nalysis_groups_table_notextfield_nogaps.ts | 11 + ...ed_analysis_groups_table_textfield_gaps.ts | 57 ++++ ..._analysis_groups_table_textfield_nogaps.ts | 17 + .../artificial_log_data_view_test_data.ts | 298 ++---------------- 13 files changed, 386 insertions(+), 277 deletions(-) create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts create mode 100644 x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts new file mode 100644 index 0000000000000..1455dc1b073f1 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableNotextfieldGaps = [ + { group: 'response_code: 500url: home.phpuser: Maryversion: v1.0.0', docCount: '47' }, + { group: 'response_code: 500url: home.phpuser: Paulversion: v1.0.0', docCount: '59' }, + { group: 'response_code: 500url: login.phpuser: Maryversion: v1.0.0', docCount: '35' }, + { group: 'response_code: 500url: login.phpuser: Paulversion: v1.0.0', docCount: '39' }, + { group: 'user: Peterurl: home.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: home.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: login.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: login.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: user.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: user.phpresponse_code: 404version: v1.0.0', docCount: '30' }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts new file mode 100644 index 0000000000000..94586bb849d34 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableNotextfieldNogaps = [ + { + group: 'response_code: 500url: home.php', + docCount: '792', + }, + { + group: 'url: login.phpresponse_code: 500', + docCount: '790', + }, + { + docCount: '636', + group: 'user: Peterurl: home.php', + }, + { + docCount: '632', + group: 'user: Peterurl: login.php', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts new file mode 100644 index 0000000000000..3a9dfef39bb90 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableTextfieldGaps = [ + { + group: + 'message: an unexpected error occuredurl: home.phpuser: Maryresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'message: an unexpected error occuredurl: home.phpuser: Paulresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'message: an unexpected error occuredurl: login.phpuser: Paulresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'url: home.phpuser: Paulresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterurl: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterurl: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts new file mode 100644 index 0000000000000..f2e728a655675 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableTextfieldNogaps = [ + { + group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', + docCount: '634', + }, + { + group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500', + docCount: '632', + }, + { + docCount: '636', + group: 'user: Peterurl: home.php', + }, + { + docCount: '632', + group: 'user: Peterurl: login.php', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts new file mode 100644 index 0000000000000..b70a0e2286ba4 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableNotextfieldGaps = [ + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'user', + fieldValue: 'Paul', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'version', + fieldValue: 'v1.0.0', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts new file mode 100644 index 0000000000000..2065ea5d89236 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableNotextfieldNogaps = [ + { + fieldName: 'url', + fieldValue: 'home.php', + impact: 'Low', + logRate: 'Chart type:bar chart', + pValue: '0.00974', + }, + { + fieldName: 'user', + fieldValue: 'Peter', + impact: 'High', + logRate: 'Chart type:bar chart', + pValue: '2.63e-21', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts new file mode 100644 index 0000000000000..e07e4ef8740d6 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableTextfieldGaps = [ + { + fieldName: 'message', + fieldValue: 'Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'user', + fieldValue: 'Paul', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'version', + fieldValue: 'v1.0.0', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts new file mode 100644 index 0000000000000..06dafacbc3b4e --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableTextfieldNogaps = [ + { + fieldName: 'message', + fieldValue: 'an unexpected error occured', + logRate: 'Chart type:bar chart', + pValue: '0.00000100', + impact: 'Medium', + }, + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '3.61e-12', + impact: 'High', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + impact: 'Low', + logRate: 'Chart type:bar chart', + pValue: '0.00974', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts new file mode 100644 index 0000000000000..09d50e08497a6 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableNotextfieldGaps = [ + { group: 'url: home.phpresponse_code: 200version: v1.0.0', docCount: '46' }, + { group: 'url: home.phpresponse_code: 404version: v1.0.0', docCount: '63' }, + { group: 'url: home.phpresponse_code: 500version: v1.0.0', docCount: '106' }, + { group: 'url: login.phpresponse_code: 200version: v1.0.0', docCount: '35' }, + { group: 'url: login.phpresponse_code: 404version: v1.0.0', docCount: '40' }, + { group: 'url: login.phpresponse_code: 500version: v1.0.0', docCount: '74' }, + { group: 'url: user.phpresponse_code: 200version: v1.0.0', docCount: '40' }, + { group: 'url: user.phpresponse_code: 404version: v1.0.0', docCount: '51' }, + { group: 'url: user.phpresponse_code: 500version: v1.0.0', docCount: '41' }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts new file mode 100644 index 0000000000000..e276b64b8fd02 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableNotextfieldNogaps = [ + { group: '* url: home.phpresponse_code: 500', docCount: '792' }, + { group: '* url: login.phpresponse_code: 500', docCount: '790' }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts new file mode 100644 index 0000000000000..199e6b37e2cfa --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableTextfieldGaps = [ + { + group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500version: v1.0.0', + docCount: '58', + }, + { + group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500version: v1.0.0', + docCount: '58', + }, + { + group: + 'response_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '46', + }, + { + group: + 'response_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '35', + }, + { + group: + 'response_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '63', + }, + { + group: + 'response_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '40', + }, + { + group: + 'url: home.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '48', + }, + { + group: + 'url: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '40', + }, + { + group: + 'url: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '51', + }, + { + group: + 'url: user.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '41', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts new file mode 100644 index 0000000000000..044c88ecbc3eb --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableTextfieldNogaps = [ + { + group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', + docCount: '634', + }, + { + group: '* url: login.phpmessage: an unexpected error occuredresponse_code: 500', + docCount: '632', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts index 94e23adf3351f..b53a4a2183d87 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts @@ -11,6 +11,19 @@ import type { TestData } from '../../types'; import type { LogRateAnalysisDataGenerator } from '../../../../services/aiops/log_rate_analysis_data_generator'; +import { analysisGroupsTableTextfieldGaps } from './__mocks__/analysis_groups_table_textfield_gaps'; +import { analysisGroupsTableNotextfieldGaps } from './__mocks__/analysis_groups_table_notextfield_gaps'; +import { analysisGroupsTableTextfieldNogaps } from './__mocks__/analysis_groups_table_textfield_nogaps'; +import { analysisGroupsTableNotextfieldNogaps } from './__mocks__/analysis_groups_table_notextfield_nogaps'; +import { filteredAnalysisGroupsTableTextfieldGaps } from './__mocks__/filtered_analysis_groups_table_textfield_gaps'; +import { filteredAnalysisGroupsTableNotextfieldGaps } from './__mocks__/filtered_analysis_groups_table_notextfield_gaps'; +import { filteredAnalysisGroupsTableTextfieldNogaps } from './__mocks__/filtered_analysis_groups_table_textfield_nogaps'; +import { filteredAnalysisGroupsTableNotextfieldNogaps } from './__mocks__/filtered_analysis_groups_table_notextfield_nogaps'; +import { analysisTableTextfieldGaps } from './__mocks__/analysis_table_textfield_gaps'; +import { analysisTableNotextfieldGaps } from './__mocks__/analysis_table_notextfield_gaps'; +import { analysisTableTextfieldNogaps } from './__mocks__/analysis_table_textfield_nogaps'; +import { analysisTableNotextfieldNogaps } from './__mocks__/analysis_table_notextfield_nogaps'; + const REFERENCE_TS = 1669018354793; const DAY_MS = 86400000; @@ -24,298 +37,29 @@ export const getArtificialLogDataViewTestData = ( ): TestData => { function getAnalysisGroupsTable() { if (gaps) { - return textField - ? [ - { - group: - 'message: an unexpected error occuredurl: home.phpuser: Maryresponse_code: 500version: v1.0.0', - docCount: '29', - }, - { - group: - 'message: an unexpected error occuredurl: home.phpuser: Paulresponse_code: 500version: v1.0.0', - docCount: '29', - }, - { - group: - 'message: an unexpected error occuredurl: login.phpuser: Paulresponse_code: 500version: v1.0.0', - docCount: '29', - }, - { - group: - 'url: home.phpuser: Paulresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - { - group: - 'user: Peterresponse_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - { - group: - 'user: Peterresponse_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - { - group: - 'user: Peterresponse_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - { - group: - 'user: Peterresponse_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - { - group: - 'user: Peterurl: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - { - group: - 'user: Peterurl: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '30', - }, - ] - : [ - { group: 'response_code: 500url: home.phpuser: Maryversion: v1.0.0', docCount: '47' }, - { group: 'response_code: 500url: home.phpuser: Paulversion: v1.0.0', docCount: '59' }, - { group: 'response_code: 500url: login.phpuser: Maryversion: v1.0.0', docCount: '35' }, - { group: 'response_code: 500url: login.phpuser: Paulversion: v1.0.0', docCount: '39' }, - { group: 'user: Peterurl: home.phpresponse_code: 200version: v1.0.0', docCount: '30' }, - { group: 'user: Peterurl: home.phpresponse_code: 404version: v1.0.0', docCount: '30' }, - { group: 'user: Peterurl: login.phpresponse_code: 200version: v1.0.0', docCount: '30' }, - { group: 'user: Peterurl: login.phpresponse_code: 404version: v1.0.0', docCount: '30' }, - { group: 'user: Peterurl: user.phpresponse_code: 200version: v1.0.0', docCount: '30' }, - { group: 'user: Peterurl: user.phpresponse_code: 404version: v1.0.0', docCount: '30' }, - ]; + return textField ? analysisGroupsTableTextfieldGaps : analysisGroupsTableNotextfieldGaps; } - - return [ - textField - ? { - group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', - docCount: '634', - } - : { - group: 'response_code: 500url: home.php', - docCount: '792', - }, - textField - ? { - group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500', - docCount: '632', - } - : { - group: 'url: login.phpresponse_code: 500', - docCount: '790', - }, - { - docCount: '636', - group: 'user: Peterurl: home.php', - }, - { - docCount: '632', - group: 'user: Peterurl: login.php', - }, - ]; + return textField ? analysisGroupsTableTextfieldNogaps : analysisGroupsTableNotextfieldNogaps; } function getFilteredAnalysisGroupsTable() { if (gaps) { return textField - ? [ - { - group: - 'message: an unexpected error occuredurl: home.phpresponse_code: 500version: v1.0.0', - docCount: '58', - }, - { - group: - 'message: an unexpected error occuredurl: login.phpresponse_code: 500version: v1.0.0', - docCount: '58', - }, - { - group: - 'response_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '46', - }, - { - group: - 'response_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '35', - }, - { - group: - 'response_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '63', - }, - { - group: - 'response_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '40', - }, - { - group: - 'url: home.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '48', - }, - { - group: - 'url: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '40', - }, - { - group: - 'url: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '51', - }, - { - group: - 'url: user.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', - docCount: '41', - }, - ] - : [ - { group: 'url: home.phpresponse_code: 200version: v1.0.0', docCount: '46' }, - { group: 'url: home.phpresponse_code: 404version: v1.0.0', docCount: '63' }, - { group: 'url: home.phpresponse_code: 500version: v1.0.0', docCount: '106' }, - { group: 'url: login.phpresponse_code: 200version: v1.0.0', docCount: '35' }, - { group: 'url: login.phpresponse_code: 404version: v1.0.0', docCount: '40' }, - { group: 'url: login.phpresponse_code: 500version: v1.0.0', docCount: '74' }, - { group: 'url: user.phpresponse_code: 200version: v1.0.0', docCount: '40' }, - { group: 'url: user.phpresponse_code: 404version: v1.0.0', docCount: '51' }, - { group: 'url: user.phpresponse_code: 500version: v1.0.0', docCount: '41' }, - ]; + ? filteredAnalysisGroupsTableTextfieldGaps + : filteredAnalysisGroupsTableNotextfieldGaps; } return textField - ? [ - { - group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', - docCount: '634', - }, - { - group: '* url: login.phpmessage: an unexpected error occuredresponse_code: 500', - docCount: '632', - }, - ] - : [ - { group: '* url: home.phpresponse_code: 500', docCount: '792' }, - { group: '* url: login.phpresponse_code: 500', docCount: '790' }, - ]; + ? filteredAnalysisGroupsTableTextfieldNogaps + : filteredAnalysisGroupsTableNotextfieldNogaps; } function getAnalysisTable() { if (gaps) { - return textField - ? [ - { - fieldName: 'message', - fieldValue: 'Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'response_code', - fieldValue: '500', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'url', - fieldValue: 'home.php', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'user', - fieldValue: 'Paul', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'version', - fieldValue: 'v1.0.0', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - ] - : [ - { - fieldName: 'response_code', - fieldValue: '500', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'url', - fieldValue: 'home.php', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'user', - fieldValue: 'Paul', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - { - fieldName: 'version', - fieldValue: 'v1.0.0', - logRate: 'Chart type:bar chart', - pValue: '1.00', - impact: '', - }, - ]; + return textField ? analysisTableTextfieldGaps : analysisTableNotextfieldGaps; } - return [ - ...(textField - ? [ - { - fieldName: 'message', - fieldValue: 'an unexpected error occured', - logRate: 'Chart type:bar chart', - pValue: '0.00000100', - impact: 'Medium', - }, - { - fieldName: 'response_code', - fieldValue: '500', - logRate: 'Chart type:bar chart', - pValue: '3.61e-12', - impact: 'High', - }, - ] - : []), - { - fieldName: 'url', - fieldValue: 'home.php', - impact: 'Low', - logRate: 'Chart type:bar chart', - pValue: '0.00974', - }, - ...(textField - ? [] - : [ - { - fieldName: 'user', - fieldValue: 'Peter', - impact: 'High', - logRate: 'Chart type:bar chart', - pValue: '2.63e-21', - }, - ]), - ]; + return textField ? analysisTableTextfieldNogaps : analysisTableNotextfieldNogaps; } function getFieldSelectorPopover() { From e82554fc9022cd35cf8535c208ca5875b45d0e4d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 28 Nov 2023 15:45:54 +0100 Subject: [PATCH 10/17] fix API integration tests --- .../api_integration/apis/aiops/test_data.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index d5740624e4ea3..17ee18c136dce 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -78,14 +78,14 @@ export const getLogRateAnalysisTestData = (): Array(): Array(): Array(): Array Date: Tue, 28 Nov 2023 16:57:16 +0100 Subject: [PATCH 11/17] refactor --- .../artificial_log_data_view_test_data.ts | 28 +++++++++++---- .../apps/aiops/log_rate_analysis_test_data.ts | 36 +++++++++++++++---- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts index b53a4a2183d87..832c7a60993b6 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts @@ -30,11 +30,17 @@ const DAY_MS = 86400000; const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; -export const getArtificialLogDataViewTestData = ( - analysisType: LogRateAnalysisType, - textField: boolean, - gaps: boolean -): TestData => { +interface GetArtificialLogDataViewTestDataOptions { + analysisType: LogRateAnalysisType; + textField: boolean; + gaps: boolean; +} + +export const getArtificialLogDataViewTestData = ({ + analysisType, + textField, + gaps, +}: GetArtificialLogDataViewTestDataOptions): TestData => { function getAnalysisGroupsTable() { if (gaps) { return textField ? analysisGroupsTableTextfieldGaps : analysisGroupsTableNotextfieldGaps; @@ -81,14 +87,22 @@ export const getArtificialLogDataViewTestData = ( }`; } + function getBrushBaselineTargetTimestamp() { + return gaps ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2; + } + + function getBrushDeviationTargetTimestamp() { + return gaps ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2; + } + return { suiteTitle: getSuiteTitle(), analysisType, dataGenerator: getDataGenerator(), isSavedSearch: false, sourceIndexOrSavedSearch: getDataGenerator(), - brushBaselineTargetTimestamp: gaps ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2, - brushDeviationTargetTimestamp: gaps ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2, + brushBaselineTargetTimestamp: getBrushBaselineTargetTimestamp(), + brushDeviationTargetTimestamp: getBrushDeviationTargetTimestamp(), brushIntervalFactor: gaps ? 1 : 10, chartClickCoordinates: [-200, 30], fieldSelectorSearch: 'user', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts index 3c9644a93c64f..2523aa6866e8f 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts @@ -18,10 +18,34 @@ export const logRateAnalysisTestData: TestData[] = [ kibanaLogsDataViewTestData, farequoteDataViewTestData, farequoteDataViewTestDataWithQuery, - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, false, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, true, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, true, true), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, false, true), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, false, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, true, false), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: false, + gaps: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: true, + gaps: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: true, + gaps: true, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: false, + gaps: true, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: false, + gaps: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: true, + gaps: false, + }), ]; From 8c5c3f641a2a6789c92897ae00ec7c6188ba766d Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 29 Nov 2023 07:40:58 +0100 Subject: [PATCH 12/17] functional tests for fallback with dip --- .../artificial_log_data_view_test_data.ts | 8 ++++++++ .../apps/aiops/log_rate_analysis_test_data.ts | 18 ++++++++++++++---- .../aiops/log_rate_analysis_data_generator.ts | 6 +++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts index 832c7a60993b6..006661f5030dd 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts @@ -88,10 +88,18 @@ export const getArtificialLogDataViewTestData = ({ } function getBrushBaselineTargetTimestamp() { + if (analysisType === 'dip' && gaps) { + return DEVIATION_TS; + } + return gaps ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2; } function getBrushDeviationTargetTimestamp() { + if (analysisType === 'dip' && gaps) { + return DEVIATION_TS + DAY_MS * 1.5; + } + return gaps ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2; } diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts index 2523aa6866e8f..16d3339c1fe99 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts @@ -28,6 +28,16 @@ export const logRateAnalysisTestData: TestData[] = [ textField: true, gaps: false, }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: false, + gaps: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: true, + gaps: false, + }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, textField: true, @@ -40,12 +50,12 @@ export const logRateAnalysisTestData: TestData[] = [ }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, - textField: false, - gaps: false, + textField: true, + gaps: true, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, - textField: true, - gaps: false, + textField: false, + gaps: true, }), ]; diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts index bb2d9e5d70d42..49e6a06f46082 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts @@ -295,10 +295,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); const dataGeneratorOptions = dataGenerator.split('_'); - const deviationType = dataGeneratorOptions[3] ?? LOG_RATE_ANALYSIS_TYPE.SPIKE; + let deviationType = dataGeneratorOptions[3] ?? LOG_RATE_ANALYSIS_TYPE.SPIKE; const textField = dataGeneratorOptions[4] === 'textfield' ?? false; const gaps = dataGeneratorOptions[5] === 'gaps' ?? false; + if (gaps) { + deviationType = LOG_RATE_ANALYSIS_TYPE.SPIKE; + } + await es.bulk({ refresh: 'wait_for', body: getArtificialLogsWithDeviation(dataGenerator, deviationType, textField, gaps), From fcfffc6172965fe0c216aed7c2c570c95e46fe4e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 29 Nov 2023 08:35:40 +0100 Subject: [PATCH 13/17] rename gaps to zerodocsfallback --- ...eld_nogaps.ts => analysis_groups_table.ts} | 2 +- ....ts => analysis_groups_table_textfield.ts} | 2 +- ...roups_table_textfield_zerodocsfallback.ts} | 2 +- ...analysis_groups_table_zerodocsfallback.ts} | 2 +- ...otextfield_nogaps.ts => analysis_table.ts} | 2 +- ..._nogaps.ts => analysis_table_textfield.ts} | 2 +- ...lysis_table_textfield_zerodocsfallback.ts} | 2 +- ....ts => analysis_table_zerodocsfallback.ts} | 2 +- ...s.ts => filtered_analysis_groups_table.ts} | 2 +- ...ltered_analysis_groups_table_textfield.ts} | 2 +- ...roups_table_textfield_zerodocsfallback.ts} | 2 +- ...analysis_groups_table_zerodocsfallback.ts} | 2 +- .../artificial_log_data_view_test_data.ts | 72 +++++++++---------- .../apps/aiops/log_rate_analysis_test_data.ts | 16 ++--- .../aiops/log_rate_analysis_data_generator.ts | 69 ++++++++++-------- .../services/aiops/log_rate_analysis_page.ts | 6 +- 16 files changed, 99 insertions(+), 88 deletions(-) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_groups_table_notextfield_nogaps.ts => analysis_groups_table.ts} (91%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_groups_table_textfield_nogaps.ts => analysis_groups_table_textfield.ts} (92%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_groups_table_textfield_gaps.ts => analysis_groups_table_textfield_zerodocsfallback.ts} (96%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_groups_table_notextfield_gaps.ts => analysis_groups_table_zerodocsfallback.ts} (95%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_table_notextfield_nogaps.ts => analysis_table.ts} (91%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_table_textfield_nogaps.ts => analysis_table_textfield.ts} (93%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_table_textfield_gaps.ts => analysis_table_textfield_zerodocsfallback.ts} (94%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{analysis_table_notextfield_gaps.ts => analysis_table_zerodocsfallback.ts} (94%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{filtered_analysis_groups_table_notextfield_nogaps.ts => filtered_analysis_groups_table.ts} (86%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{filtered_analysis_groups_table_textfield_nogaps.ts => filtered_analysis_groups_table_textfield.ts} (89%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{filtered_analysis_groups_table_textfield_gaps.ts => filtered_analysis_groups_table_textfield_zerodocsfallback.ts} (96%) rename x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/{filtered_analysis_groups_table_notextfield_gaps.ts => filtered_analysis_groups_table_zerodocsfallback.ts} (94%) diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table.ts similarity index 91% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table.ts index 94586bb849d34..6780cab5cb209 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_nogaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisGroupsTableNotextfieldNogaps = [ +export const analysisGroupsTable = [ { group: 'response_code: 500url: home.php', docCount: '792', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield.ts similarity index 92% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield.ts index f2e728a655675..2f20ab7900386 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_nogaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisGroupsTableTextfieldNogaps = [ +export const analysisGroupsTableTextfield = [ { group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', docCount: '634', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts similarity index 96% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts index 3a9dfef39bb90..072c47eb41426 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_gaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisGroupsTableTextfieldGaps = [ +export const analysisGroupsTableTextfieldZerodocsfallback = [ { group: 'message: an unexpected error occuredurl: home.phpuser: Maryresponse_code: 500version: v1.0.0', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_zerodocsfallback.ts similarity index 95% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_zerodocsfallback.ts index 1455dc1b073f1..cd749df4f2921 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_notextfield_gaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_zerodocsfallback.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisGroupsTableNotextfieldGaps = [ +export const analysisGroupsTableZerodocsfallback = [ { group: 'response_code: 500url: home.phpuser: Maryversion: v1.0.0', docCount: '47' }, { group: 'response_code: 500url: home.phpuser: Paulversion: v1.0.0', docCount: '59' }, { group: 'response_code: 500url: login.phpuser: Maryversion: v1.0.0', docCount: '35' }, diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table.ts similarity index 91% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table.ts index 2065ea5d89236..797a9b0911489 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_nogaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisTableNotextfieldNogaps = [ +export const analysisTable = [ { fieldName: 'url', fieldValue: 'home.php', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield.ts similarity index 93% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield.ts index 06dafacbc3b4e..be77ef870d536 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_nogaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisTableTextfieldNogaps = [ +export const analysisTableTextfield = [ { fieldName: 'message', fieldValue: 'an unexpected error occured', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts similarity index 94% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts index e07e4ef8740d6..d7504cd5aacfc 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_gaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisTableTextfieldGaps = [ +export const analysisTableTextfieldZerodocsfallback = [ { fieldName: 'message', fieldValue: 'Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_zerodocsfallback.ts similarity index 94% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_zerodocsfallback.ts index b70a0e2286ba4..3d4f806db8f49 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_notextfield_gaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_zerodocsfallback.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const analysisTableNotextfieldGaps = [ +export const analysisTableZerodocsfallback = [ { fieldName: 'response_code', fieldValue: '500', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table.ts similarity index 86% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table.ts index e276b64b8fd02..97b1e2ce7169b 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_nogaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const filteredAnalysisGroupsTableNotextfieldNogaps = [ +export const filteredAnalysisGroupsTable = [ { group: '* url: home.phpresponse_code: 500', docCount: '792' }, { group: '* url: login.phpresponse_code: 500', docCount: '790' }, ]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield.ts similarity index 89% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield.ts index 044c88ecbc3eb..41c5efe2cb211 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_nogaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const filteredAnalysisGroupsTableTextfieldNogaps = [ +export const filteredAnalysisGroupsTableTextfield = [ { group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', docCount: '634', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts similarity index 96% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts index 199e6b37e2cfa..000b931a1b266 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_gaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const filteredAnalysisGroupsTableTextfieldGaps = [ +export const filteredAnalysisGroupsTableTextfieldZerodocsfallback = [ { group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500version: v1.0.0', docCount: '58', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_zerodocsfallback.ts similarity index 94% rename from x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts rename to x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_zerodocsfallback.ts index 09d50e08497a6..eaeb04b9c0204 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_notextfield_gaps.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_zerodocsfallback.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const filteredAnalysisGroupsTableNotextfieldGaps = [ +export const filteredAnalysisGroupsTableZerodocsfallback = [ { group: 'url: home.phpresponse_code: 200version: v1.0.0', docCount: '46' }, { group: 'url: home.phpresponse_code: 404version: v1.0.0', docCount: '63' }, { group: 'url: home.phpresponse_code: 500version: v1.0.0', docCount: '106' }, diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts index 006661f5030dd..6afbaab856424 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts @@ -11,18 +11,18 @@ import type { TestData } from '../../types'; import type { LogRateAnalysisDataGenerator } from '../../../../services/aiops/log_rate_analysis_data_generator'; -import { analysisGroupsTableTextfieldGaps } from './__mocks__/analysis_groups_table_textfield_gaps'; -import { analysisGroupsTableNotextfieldGaps } from './__mocks__/analysis_groups_table_notextfield_gaps'; -import { analysisGroupsTableTextfieldNogaps } from './__mocks__/analysis_groups_table_textfield_nogaps'; -import { analysisGroupsTableNotextfieldNogaps } from './__mocks__/analysis_groups_table_notextfield_nogaps'; -import { filteredAnalysisGroupsTableTextfieldGaps } from './__mocks__/filtered_analysis_groups_table_textfield_gaps'; -import { filteredAnalysisGroupsTableNotextfieldGaps } from './__mocks__/filtered_analysis_groups_table_notextfield_gaps'; -import { filteredAnalysisGroupsTableTextfieldNogaps } from './__mocks__/filtered_analysis_groups_table_textfield_nogaps'; -import { filteredAnalysisGroupsTableNotextfieldNogaps } from './__mocks__/filtered_analysis_groups_table_notextfield_nogaps'; -import { analysisTableTextfieldGaps } from './__mocks__/analysis_table_textfield_gaps'; -import { analysisTableNotextfieldGaps } from './__mocks__/analysis_table_notextfield_gaps'; -import { analysisTableTextfieldNogaps } from './__mocks__/analysis_table_textfield_nogaps'; -import { analysisTableNotextfieldNogaps } from './__mocks__/analysis_table_notextfield_nogaps'; +import { analysisGroupsTableTextfieldZerodocsfallback } from './__mocks__/analysis_groups_table_textfield_zerodocsfallback'; +import { analysisGroupsTableZerodocsfallback } from './__mocks__/analysis_groups_table_zerodocsfallback'; +import { analysisGroupsTableTextfield } from './__mocks__/analysis_groups_table_textfield'; +import { analysisGroupsTable } from './__mocks__/analysis_groups_table'; +import { filteredAnalysisGroupsTableTextfieldZerodocsfallback } from './__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback'; +import { filteredAnalysisGroupsTableZerodocsfallback } from './__mocks__/filtered_analysis_groups_table_zerodocsfallback'; +import { filteredAnalysisGroupsTableTextfield } from './__mocks__/filtered_analysis_groups_table_textfield'; +import { filteredAnalysisGroupsTable } from './__mocks__/filtered_analysis_groups_table'; +import { analysisTableTextfieldZerodocsfallback } from './__mocks__/analysis_table_textfield_zerodocsfallback'; +import { analysisTableZerodocsfallback } from './__mocks__/analysis_table_zerodocsfallback'; +import { analysisTableTextfield } from './__mocks__/analysis_table_textfield'; +import { analysisTable } from './__mocks__/analysis_table'; const REFERENCE_TS = 1669018354793; const DAY_MS = 86400000; @@ -33,43 +33,43 @@ const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; interface GetArtificialLogDataViewTestDataOptions { analysisType: LogRateAnalysisType; textField: boolean; - gaps: boolean; + zeroDocsFallback: boolean; } export const getArtificialLogDataViewTestData = ({ analysisType, textField, - gaps, + zeroDocsFallback, }: GetArtificialLogDataViewTestDataOptions): TestData => { function getAnalysisGroupsTable() { - if (gaps) { - return textField ? analysisGroupsTableTextfieldGaps : analysisGroupsTableNotextfieldGaps; + if (zeroDocsFallback) { + return textField + ? analysisGroupsTableTextfieldZerodocsfallback + : analysisGroupsTableZerodocsfallback; } - return textField ? analysisGroupsTableTextfieldNogaps : analysisGroupsTableNotextfieldNogaps; + return textField ? analysisGroupsTableTextfield : analysisGroupsTable; } function getFilteredAnalysisGroupsTable() { - if (gaps) { + if (zeroDocsFallback) { return textField - ? filteredAnalysisGroupsTableTextfieldGaps - : filteredAnalysisGroupsTableNotextfieldGaps; + ? filteredAnalysisGroupsTableTextfieldZerodocsfallback + : filteredAnalysisGroupsTableZerodocsfallback; } - return textField - ? filteredAnalysisGroupsTableTextfieldNogaps - : filteredAnalysisGroupsTableNotextfieldNogaps; + return textField ? filteredAnalysisGroupsTableTextfield : filteredAnalysisGroupsTable; } function getAnalysisTable() { - if (gaps) { - return textField ? analysisTableTextfieldGaps : analysisTableNotextfieldGaps; + if (zeroDocsFallback) { + return textField ? analysisTableTextfieldZerodocsfallback : analysisTableZerodocsfallback; } - return textField ? analysisTableTextfieldNogaps : analysisTableNotextfieldNogaps; + return textField ? analysisTableTextfield : analysisTable; } function getFieldSelectorPopover() { - if (gaps) { + if (zeroDocsFallback) { return [...(textField ? ['message'] : []), 'response_code', 'url', 'user', 'version']; } return [...(textField ? ['message'] : []), 'response_code', 'url', 'user']; @@ -78,29 +78,29 @@ export const getArtificialLogDataViewTestData = ({ function getSuiteTitle() { return `artificial logs with ${analysisType} and ${ textField ? 'text field' : 'no text field' - } and ${gaps ? 'gaps' : 'no gaps'}`; + } and ${zeroDocsFallback ? 'zero docs fallback' : 'no zero docs fallback'}`; } function getDataGenerator(): LogRateAnalysisDataGenerator { - return `artificial_logs_with_${analysisType}_${textField ? 'textfield' : 'notextfield'}_${ - gaps ? 'gaps' : 'nogaps' + return `artificial_logs_with_${analysisType}${textField ? '_textfield' : ''}${ + zeroDocsFallback ? '_zerodocsfallback' : '' }`; } function getBrushBaselineTargetTimestamp() { - if (analysisType === 'dip' && gaps) { + if (analysisType === 'dip' && zeroDocsFallback) { return DEVIATION_TS; } - return gaps ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2; + return zeroDocsFallback ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2; } function getBrushDeviationTargetTimestamp() { - if (analysisType === 'dip' && gaps) { + if (analysisType === 'dip' && zeroDocsFallback) { return DEVIATION_TS + DAY_MS * 1.5; } - return gaps ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2; + return zeroDocsFallback ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2; } return { @@ -111,12 +111,12 @@ export const getArtificialLogDataViewTestData = ({ sourceIndexOrSavedSearch: getDataGenerator(), brushBaselineTargetTimestamp: getBrushBaselineTargetTimestamp(), brushDeviationTargetTimestamp: getBrushDeviationTargetTimestamp(), - brushIntervalFactor: gaps ? 1 : 10, + brushIntervalFactor: zeroDocsFallback ? 1 : 10, chartClickCoordinates: [-200, 30], fieldSelectorSearch: 'user', fieldSelectorApplyAvailable: true, expected: { - totalDocCountFormatted: gaps ? '9,482' : '8,400', + totalDocCountFormatted: zeroDocsFallback ? '9,482' : '8,400', analysisGroupsTable: getAnalysisGroupsTable(), filteredAnalysisGroupsTable: getFilteredAnalysisGroupsTable(), analysisTable: getAnalysisTable(), diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts index 16d3339c1fe99..c3f33ff2aa5ba 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts @@ -21,41 +21,41 @@ export const logRateAnalysisTestData: TestData[] = [ getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, textField: false, - gaps: false, + zeroDocsFallback: false, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, textField: true, - gaps: false, + zeroDocsFallback: false, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, textField: false, - gaps: false, + zeroDocsFallback: false, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, textField: true, - gaps: false, + zeroDocsFallback: false, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, textField: true, - gaps: true, + zeroDocsFallback: true, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, textField: false, - gaps: true, + zeroDocsFallback: true, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, textField: true, - gaps: true, + zeroDocsFallback: true, }), getArtificialLogDataViewTestData({ analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, textField: false, - gaps: true, + zeroDocsFallback: true, }), ]; diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts index 49e6a06f46082..7e1ead80b4ff8 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts @@ -14,14 +14,16 @@ import { FtrProviderContext } from '../../ftr_provider_context'; const LOG_RATE_ANALYSYS_DATA_GENERATOR = { KIBANA_SAMPLE_DATA_LOGS: 'kibana_sample_data_logs', FAREQUOTE_WITH_SPIKE: 'farequote_with_spike', - ARTIFICIAL_LOGS_WITH_SPIKE_NOTEXTFIELD_GAPS: 'artificial_logs_with_spike_notextfield_gaps', - ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD_GAPS: 'artificial_logs_with_spike_textfield_gaps', - ARTIFICIAL_LOGS_WITH_DIP_NOTEXTFIELD_GAPS: 'artificial_logs_with_dip_notextfield_gaps', - ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD_GAPS: 'artificial_logs_with_dip_textfield_gaps', - ARTIFICIAL_LOGS_WITH_SPIKE_NOTEXTFIELD_NOGAPS: 'artificial_logs_with_spike_notextfield_nogaps', - ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD_NOGAPS: 'artificial_logs_with_spike_textfield_nogaps', - ARTIFICIAL_LOGS_WITH_DIP_NOTEXTFIELD_NOGAPS: 'artificial_logs_with_dip_notextfield_nogaps', - ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD_NOGAPS: 'artificial_logs_with_dip_textfield_nogaps', + ARTIFICIAL_LOGS_WITH_SPIKE_ZERODOCSFALLBACK: 'artificial_logs_with_spike_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD_ZERODOCSFALLBACK: + 'artificial_logs_with_spike_textfield_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_DIP_ZERODOCSFALLBACK: 'artificial_logs_with_dip_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD_ZERODOCSFALLBACK: + 'artificial_logs_with_dip_textfield_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_SPIKE: 'artificial_logs_with_spike', + ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD: 'artificial_logs_with_spike_textfield', + ARTIFICIAL_LOGS_WITH_DIP: 'artificial_logs_with_dip', + ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD: 'artificial_logs_with_dip_textfield', } as const; export type LogRateAnalysisDataGenerator = typeof LOG_RATE_ANALYSYS_DATA_GENERATOR[keyof typeof LOG_RATE_ANALYSYS_DATA_GENERATOR]; @@ -257,14 +259,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); break; - case 'artificial_logs_with_spike_notextfield_nogaps': - case 'artificial_logs_with_spike_textfield_nogaps': - case 'artificial_logs_with_dip_notextfield_nogaps': - case 'artificial_logs_with_dip_textfield_nogaps': - case 'artificial_logs_with_spike_notextfield_gaps': - case 'artificial_logs_with_spike_textfield_gaps': - case 'artificial_logs_with_dip_notextfield_gaps': - case 'artificial_logs_with_dip_textfield_gaps': + case 'artificial_logs_with_spike': + case 'artificial_logs_with_spike_textfield': + case 'artificial_logs_with_dip': + case 'artificial_logs_with_dip_textfield': + case 'artificial_logs_with_spike_zerodocsfallback': + case 'artificial_logs_with_spike_textfield_zerodocsfallback': + case 'artificial_logs_with_dip_zerodocsfallback': + case 'artificial_logs_with_dip_textfield_zerodocsfallback': try { const indexExists = await es.indices.exists({ index: dataGenerator, @@ -295,17 +297,26 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); const dataGeneratorOptions = dataGenerator.split('_'); - let deviationType = dataGeneratorOptions[3] ?? LOG_RATE_ANALYSIS_TYPE.SPIKE; - const textField = dataGeneratorOptions[4] === 'textfield' ?? false; - const gaps = dataGeneratorOptions[5] === 'gaps' ?? false; - if (gaps) { + let deviationType = dataGeneratorOptions.includes(LOG_RATE_ANALYSIS_TYPE.SPIKE) + ? LOG_RATE_ANALYSIS_TYPE.SPIKE + : LOG_RATE_ANALYSIS_TYPE.DIP; + + const textField = dataGeneratorOptions.includes('textfield'); + const zeroDocsFallback = dataGeneratorOptions.includes('zerodocsfallback'); + + if (zeroDocsFallback) { deviationType = LOG_RATE_ANALYSIS_TYPE.SPIKE; } await es.bulk({ refresh: 'wait_for', - body: getArtificialLogsWithDeviation(dataGenerator, deviationType, textField, gaps), + body: getArtificialLogsWithDeviation( + dataGenerator, + deviationType, + textField, + zeroDocsFallback + ), }); break; @@ -324,14 +335,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); break; - case 'artificial_logs_with_spike_notextfield_nogaps': - case 'artificial_logs_with_spike_textfield_nogaps': - case 'artificial_logs_with_dip_notextfield_nogaps': - case 'artificial_logs_with_dip_textfield_nogaps': - case 'artificial_logs_with_spike_notextfield_gaps': - case 'artificial_logs_with_spike_textfield_gaps': - case 'artificial_logs_with_dip_notextfield_gaps': - case 'artificial_logs_with_dip_textfield_gaps': + case 'artificial_logs_with_spike': + case 'artificial_logs_with_spike_textfield': + case 'artificial_logs_with_dip': + case 'artificial_logs_with_dip_textfield': + case 'artificial_logs_with_spike_zerodocsfallback': + case 'artificial_logs_with_spike_textfield_zerodocsfallback': + case 'artificial_logs_with_dip_zerodocsfallback': + case 'artificial_logs_with_dip_textfield_zerodocsfallback': try { await es.indices.delete({ index: dataGenerator, diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts index e814dcfd3ca51..a8733fb2114a7 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts @@ -248,7 +248,7 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr dataGenerator: LogRateAnalysisDataGenerator ) { const dataGeneratorParts = dataGenerator.split('_'); - const includeGaps = dataGeneratorParts[5] === 'gaps'; + const zeroDocsFallback = dataGeneratorParts.includes('zerodocsfallback'); await retry.tryForTime(30 * 1000, async () => { await testSubjects.existOrFail('aiopsAnalysisComplete'); const currentProgressTitle = await testSubjects.getVisibleText('aiopsAnalysisComplete'); @@ -259,11 +259,11 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr 'aiopsAnalysisTypeCalloutTitle' ); - if (includeGaps && analysisType === 'spike') { + if (zeroDocsFallback && analysisType === 'spike') { expect(currentAnalysisTypeCalloutTitle).to.be( 'Analysis type: Top items for deviation time range' ); - } else if (includeGaps && analysisType === 'dip') { + } else if (zeroDocsFallback && analysisType === 'dip') { expect(currentAnalysisTypeCalloutTitle).to.be( 'Analysis type: Top items for baseline time range' ); From 213b2e3f8d4f3775677e34ff229f4da8a593fab6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 29 Nov 2023 09:52:03 +0100 Subject: [PATCH 14/17] fix API integration tests --- .../api_integration/apis/aiops/test_data.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 17ee18c136dce..b0e441f84a71c 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -78,14 +78,14 @@ export const getLogRateAnalysisTestData = (): Array(): Array(): Array(): Array Date: Wed, 29 Nov 2023 10:28:40 +0100 Subject: [PATCH 15/17] adds API integration tests for zero docs fallback --- .../__mocks__/artificial_logs/top_terms.ts | 142 +++ .../artificial_logs/top_terms_groups.ts | 1086 +++++++++++++++++ .../aiops/log_rate_analysis_full_analysis.ts | 11 +- .../api_integration/apis/aiops/test_data.ts | 29 + 4 files changed, 1265 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts create mode 100644 x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts new file mode 100644 index 0000000000000..c597a18e2f050 --- /dev/null +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SignificantItem } from '@kbn/ml-agg-utils'; + +// Named topTerms since all these items are of type `keyword`. +export const topTerms: SignificantItem[] = [ + { + bg_count: 0, + doc_count: 5102, + fieldName: 'version', + fieldValue: 'v1.0.0', + key: 'version:v1.0.0', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 2272, + fieldName: 'response_code', + fieldValue: '500', + key: 'response_code:500', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 2197, + fieldName: 'url', + fieldValue: 'home.php', + key: 'url:home.php', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1981, + fieldName: 'user', + fieldValue: 'Peter', + key: 'user:Peter', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1773, + fieldName: 'user', + fieldValue: 'Paul', + key: 'user:Paul', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1574, + fieldName: 'url', + fieldValue: 'login.php', + key: 'url:login.php', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1569, + fieldName: 'response_code', + fieldValue: '404', + key: 'response_code:404', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1348, + fieldName: 'user', + fieldValue: 'Mary', + key: 'user:Mary', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1331, + fieldName: 'url', + fieldValue: 'user.php', + key: 'url:user.php', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1261, + fieldName: 'response_code', + fieldValue: '200', + key: 'response_code:200', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, +]; diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts new file mode 100644 index 0000000000000..582fded59d7bb --- /dev/null +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts @@ -0,0 +1,1086 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; + +export const topTermsGroups: SignificantItemGroup[] = [ + { + id: '1921491265', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 1097, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 602, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 602, + pValue: 1, + }, + { + id: '1354434627', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 1097, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 495, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 495, + pValue: 1, + }, + { + id: '1539462188', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 787, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 411, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 411, + pValue: 1, + }, + { + id: '1553868016', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 787, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 376, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 376, + pValue: 1, + }, + { + id: '3400778144', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 473, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 318, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 318, + pValue: 1, + }, + { + id: '2753923470', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 627, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 318, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 318, + pValue: 1, + }, + { + id: '611881711', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 420, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 317, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 317, + pValue: 1, + }, + { + id: '1605132418', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 523, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 317, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 317, + pValue: 1, + }, + { + id: '1389551523', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 368, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 316, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 316, + pValue: 1, + }, + { + id: '1156337696', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 419, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 316, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 316, + pValue: 1, + }, + { + id: '3582406482', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 627, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 190, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 190, + pValue: 1, + }, + { + id: '1758796965', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 388, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 190, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 190, + pValue: 1, + }, + { + id: '943892302', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 523, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 127, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 127, + pValue: 1, + }, + { + id: '3952579328', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 627, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 119, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 119, + pValue: 1, + }, + { + id: '1573710542', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 388, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 119, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 119, + pValue: 1, + }, + { + id: '1384949010', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 473, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 95, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 95, + pValue: 1, + }, + { + id: '3945828984', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 523, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 79, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 79, + pValue: 1, + }, + { + id: '3487185726', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 388, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 79, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 79, + pValue: 1, + }, + { + id: '113295987', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 420, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 63, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 63, + pValue: 1, + }, + { + id: '2896531546', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 419, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 63, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 63, + pValue: 1, + }, + { + id: '2376883532', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 473, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 60, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 60, + pValue: 1, + }, + { + id: '3927320460', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 420, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 40, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 40, + pValue: 1, + }, + { + id: '769745306', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 419, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 40, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 40, + pValue: 1, + }, + { + id: '2532116164', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 368, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 32, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 32, + pValue: 1, + }, + { + id: '1953946181', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 368, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 20, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 20, + pValue: 1, + }, +]; diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts index 3168dfdfc5ecf..352282b4ba41f 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts @@ -86,9 +86,14 @@ export default ({ getService }: FtrProviderContext) => { const groupActions = getGroupActions(data, apiVersion); const groups = groupActions.flatMap((d) => d.payload); - expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( - orderBy(testData.expected.groups, ['docCount'], ['desc']), - 'Grouping result does not match expected values.' + const actualGroups = orderBy(groups, ['docCount'], ['desc']); + const expectedGroups = orderBy(testData.expected.groups, ['docCount'], ['desc']); + + expect(actualGroups).to.eql( + expectedGroups, + `Grouping result does not match expected values. Expected ${JSON.stringify( + expectedGroups + )}, got ${JSON.stringify(actualGroups)}` ); const groupHistogramActions = getGroupHistogramActions(data, apiVersion); diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index b0e441f84a71c..2931532956449 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -12,6 +12,8 @@ import { significantTerms as artificialLogSignificantTerms } from '@kbn/aiops-pl import { significantLogPatterns as artificialLogSignificantLogPatterns } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/significant_log_patterns'; import { finalSignificantItemGroups as artificialLogsSignificantItemGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups'; import { finalSignificantItemGroupsTextfield as artificialLogsSignificantItemGroupsTextfield } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups_textfield'; +import { topTerms } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms'; +import { topTermsGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms_groups'; import type { AiopsLogRateAnalysisSchema, @@ -104,6 +106,33 @@ export const getLogRateAnalysisTestData = (): Array, + expected: { + chunksLength: 62, + chunksLengthGroupOnly: 11, + actionsLength: 61, + actionsLengthGroupOnly: 10, + noIndexChunksLength: 4, + noIndexActionsLength: 3, + significantItems: topTerms, + groups: topTermsGroups, + histogramLength: 20, + }, + }, { testName: 'artificial_logs_with_spike_textfield', dataGenerator: 'artificial_logs_with_spike_textfield', From ff9aed669db7d629b7cbd142df50c7e09043ab9f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 29 Nov 2023 12:48:01 +0100 Subject: [PATCH 16/17] adds API integration tests for zero docs fallback with dip --- .../api_integration/apis/aiops/test_data.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 2931532956449..4b29892169335 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -133,6 +133,33 @@ export const getLogRateAnalysisTestData = (): Array, + expected: { + chunksLength: 62, + chunksLengthGroupOnly: 11, + actionsLength: 61, + actionsLengthGroupOnly: 10, + noIndexChunksLength: 4, + noIndexActionsLength: 3, + significantItems: topTerms, + groups: topTermsGroups, + histogramLength: 20, + }, + }, { testName: 'artificial_logs_with_spike_textfield', dataGenerator: 'artificial_logs_with_spike_textfield', From 274468bf508b5508e46e2712cf062f131e5485f3 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 29 Nov 2023 16:21:18 +0000 Subject: [PATCH 17/17] fix assertions --- .../test/api_integration/apis/aiops/test_data.ts | 8 ++++---- ...is_groups_table_textfield_zerodocsfallback.ts | 14 +++++++------- .../analysis_table_textfield_zerodocsfallback.ts | 2 +- ...is_groups_table_textfield_zerodocsfallback.ts | 16 ++++++++-------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 4b29892169335..3c1793ab9efe3 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -123,9 +123,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { chunksLength: 62, - chunksLengthGroupOnly: 11, + chunksLengthGroupOnly: 32, actionsLength: 61, - actionsLengthGroupOnly: 10, + actionsLengthGroupOnly: 31, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: topTerms, @@ -150,9 +150,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { chunksLength: 62, - chunksLengthGroupOnly: 11, + chunksLengthGroupOnly: 32, actionsLength: 61, - actionsLengthGroupOnly: 10, + actionsLengthGroupOnly: 31, noIndexChunksLength: 4, noIndexActionsLength: 3, significantItems: topTerms, diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts index 072c47eb41426..4c9308f6c0ae2 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts @@ -23,37 +23,37 @@ export const analysisGroupsTableTextfieldZerodocsfallback = [ }, { group: - 'url: home.phpuser: Paulresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'url: home.phpuser: Paulresponse_code: 500message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, { group: - 'user: Peterresponse_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'user: Peterresponse_code: 200url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, { group: - 'user: Peterresponse_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'user: Peterresponse_code: 200url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, { group: - 'user: Peterresponse_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'user: Peterresponse_code: 404url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, { group: - 'user: Peterresponse_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'user: Peterresponse_code: 404url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, { group: - 'user: Peterurl: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'user: Peterurl: user.phpresponse_code: 200message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, { group: - 'user: Peterurl: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'user: Peterurl: user.phpresponse_code: 404message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '30', }, ]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts index d7504cd5aacfc..75c624a67321c 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts @@ -8,7 +8,7 @@ export const analysisTableTextfieldZerodocsfallback = [ { fieldName: 'message', - fieldValue: 'Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200', + fieldValue: 'Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200', logRate: 'Chart type:bar chart', pValue: '1.00', impact: '', diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts index 000b931a1b266..9ec9f0e19f89b 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts @@ -16,42 +16,42 @@ export const filteredAnalysisGroupsTableTextfieldZerodocsfallback = [ }, { group: - 'response_code: 200url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'response_code: 200url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '46', }, { group: - 'response_code: 200url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'response_code: 200url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '35', }, { group: - 'response_code: 404url: home.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'response_code: 404url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '63', }, { group: - 'response_code: 404url: login.phpmessage: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'response_code: 404url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '40', }, { group: - 'url: home.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'url: home.phpresponse_code: 500message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '48', }, { group: - 'url: user.phpresponse_code: 200message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'url: user.phpresponse_code: 200message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '40', }, { group: - 'url: user.phpresponse_code: 404message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'url: user.phpresponse_code: 404message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '51', }, { group: - 'url: user.phpresponse_code: 500message: Paul [11/19/2022, 9:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + 'url: user.phpresponse_code: 500message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', docCount: '41', }, ];