diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts new file mode 100644 index 0000000000000..c597a18e2f050 --- /dev/null +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SignificantItem } from '@kbn/ml-agg-utils'; + +// Named topTerms since all these items are of type `keyword`. +export const topTerms: SignificantItem[] = [ + { + bg_count: 0, + doc_count: 5102, + fieldName: 'version', + fieldValue: 'v1.0.0', + key: 'version:v1.0.0', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 2272, + fieldName: 'response_code', + fieldValue: '500', + key: 'response_code:500', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 2197, + fieldName: 'url', + fieldValue: 'home.php', + key: 'url:home.php', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1981, + fieldName: 'user', + fieldValue: 'Peter', + key: 'user:Peter', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1773, + fieldName: 'user', + fieldValue: 'Paul', + key: 'user:Paul', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1574, + fieldName: 'url', + fieldValue: 'login.php', + key: 'url:login.php', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1569, + fieldName: 'response_code', + fieldValue: '404', + key: 'response_code:404', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1348, + fieldName: 'user', + fieldValue: 'Mary', + key: 'user:Mary', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1331, + fieldName: 'url', + fieldValue: 'user.php', + key: 'url:user.php', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, + { + bg_count: 0, + doc_count: 1261, + fieldName: 'response_code', + fieldValue: '200', + key: 'response_code:200', + normalizedScore: 0, + pValue: 1, + score: 0, + total_bg_count: 0, + total_doc_count: 0, + type: 'keyword', + }, +]; diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts new file mode 100644 index 0000000000000..582fded59d7bb --- /dev/null +++ b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/top_terms_groups.ts @@ -0,0 +1,1086 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SignificantItemGroup } from '@kbn/ml-agg-utils'; + +export const topTermsGroups: SignificantItemGroup[] = [ + { + id: '1921491265', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 1097, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 602, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 602, + pValue: 1, + }, + { + id: '1354434627', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 1097, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 495, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 495, + pValue: 1, + }, + { + id: '1539462188', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 787, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 411, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 411, + pValue: 1, + }, + { + id: '1553868016', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 787, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 376, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 376, + pValue: 1, + }, + { + id: '3400778144', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 473, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 318, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 318, + pValue: 1, + }, + { + id: '2753923470', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 627, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 318, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 318, + pValue: 1, + }, + { + id: '611881711', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 420, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 317, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 317, + pValue: 1, + }, + { + id: '1605132418', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 523, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 317, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 317, + pValue: 1, + }, + { + id: '1389551523', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 368, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 316, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 316, + pValue: 1, + }, + { + id: '1156337696', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 419, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 316, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 316, + pValue: 1, + }, + { + id: '3582406482', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 627, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 190, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 190, + pValue: 1, + }, + { + id: '1758796965', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 388, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 190, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 190, + pValue: 1, + }, + { + id: '943892302', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 523, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 127, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 127, + pValue: 1, + }, + { + id: '3952579328', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 627, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 119, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 119, + pValue: 1, + }, + { + id: '1573710542', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 388, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 119, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 119, + pValue: 1, + }, + { + id: '1384949010', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 473, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 95, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 95, + pValue: 1, + }, + { + id: '3945828984', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 523, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 79, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 79, + pValue: 1, + }, + { + id: '3487185726', + group: [ + { + key: 'response_code:500', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '500', + docCount: 2272, + pValue: 1, + duplicate: 7, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 2272, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 388, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Peter', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Peter', + docCount: 79, + pValue: 1, + duplicate: 7, + }, + ], + docCount: 79, + pValue: 1, + }, + { + id: '113295987', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 420, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 63, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 63, + pValue: 1, + }, + { + id: '2896531546', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 419, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 63, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 63, + pValue: 1, + }, + { + id: '2376883532', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:home.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'home.php', + docCount: 473, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 60, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 60, + pValue: 1, + }, + { + id: '3927320460', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:user.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'user.php', + docCount: 420, + pValue: 1, + duplicate: 9, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 40, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 40, + pValue: 1, + }, + { + id: '769745306', + group: [ + { + key: 'response_code:404', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '404', + docCount: 1569, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1569, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 419, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 40, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 40, + pValue: 1, + }, + { + id: '2532116164', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 368, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Paul', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Paul', + docCount: 32, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 32, + pValue: 1, + }, + { + id: '1953946181', + group: [ + { + key: 'response_code:200', + type: 'keyword', + fieldName: 'response_code', + fieldValue: '200', + docCount: 1261, + pValue: 1, + duplicate: 9, + }, + { + key: 'version:v1.0.0', + type: 'keyword', + fieldName: 'version', + fieldValue: 'v1.0.0', + docCount: 1261, + pValue: 1, + duplicate: 25, + }, + { + key: 'url:login.php', + type: 'keyword', + fieldName: 'url', + fieldValue: 'login.php', + docCount: 368, + pValue: 1, + duplicate: 8, + }, + { + key: 'user:Mary', + type: 'keyword', + fieldName: 'user', + fieldValue: 'Mary', + docCount: 20, + pValue: 1, + duplicate: 9, + }, + ], + docCount: 20, + pValue: 1, + }, +]; diff --git a/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts b/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts index bd3afd3152ae5..2348b32c32f8d 100644 --- a/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts +++ b/x-pack/plugins/aiops/common/api/log_rate_analysis/actions.ts @@ -36,6 +36,7 @@ export const API_ACTION_NAME = { RESET_ALL: 'reset_all', RESET_ERRORS: 'reset_errors', RESET_GROUPS: 'reset_groups', + SET_ZERO_DOCS_FALLBACK: 'set_zero_docs_fallback', UPDATE_LOADING_STATE: 'update_loading_state', } as const; export type ApiActionName = typeof API_ACTION_NAME[keyof typeof API_ACTION_NAME]; @@ -210,6 +211,20 @@ export function updateLoadingStateAction( }; } +interface ApiActionSetZeroDocsFallback { + type: typeof API_ACTION_NAME.SET_ZERO_DOCS_FALLBACK; + payload: boolean; +} + +export function setZeroDocsFallback( + payload: ApiActionSetZeroDocsFallback['payload'] +): ApiActionSetZeroDocsFallback { + return { + type: API_ACTION_NAME.SET_ZERO_DOCS_FALLBACK, + payload, + }; +} + export type AiopsLogRateAnalysisApiAction = | ApiActionAddSignificantItems | ApiActionAddSignificantItemsGroup @@ -220,4 +235,5 @@ export type AiopsLogRateAnalysisApiAction = | ApiActionResetAll | ApiActionResetErrors | ApiActionResetGroups - | ApiActionUpdateLoadingState; + | ApiActionUpdateLoadingState + | ApiActionSetZeroDocsFallback; diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.test.ts b/x-pack/plugins/aiops/common/api/stream_reducer.test.ts index f3dd6cce856c7..5344d2448463c 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.test.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.test.ts @@ -31,6 +31,7 @@ describe('streamReducer', () => { significantItems: [], significantItemsGroups: [], errors: [], + zeroDocsFallback: false, }); }); diff --git a/x-pack/plugins/aiops/common/api/stream_reducer.ts b/x-pack/plugins/aiops/common/api/stream_reducer.ts index 05d3fce52c22e..06da4d3d250ad 100644 --- a/x-pack/plugins/aiops/common/api/stream_reducer.ts +++ b/x-pack/plugins/aiops/common/api/stream_reducer.ts @@ -18,6 +18,7 @@ interface StreamState { loadingState: string; remainingFieldCandidates?: string[]; groupsMissing?: boolean; + zeroDocsFallback: boolean; } export const initialState: StreamState = { @@ -27,6 +28,7 @@ export const initialState: StreamState = { errors: [], loaded: 0, loadingState: '', + zeroDocsFallback: false, }; export function streamReducer( @@ -72,6 +74,8 @@ export function streamReducer( return initialState; case API_ACTION_NAME.UPDATE_LOADING_STATE: return { ...state, ...action.payload }; + case API_ACTION_NAME.SET_ZERO_DOCS_FALLBACK: + return { ...state, zeroDocsFallback: action.payload }; default: return state; } diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx index 457419c8b0501..121c19be2ddb1 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_results.tsx @@ -46,6 +46,7 @@ import { import { useLogRateAnalysisResultsTableRowContext } from '../log_rate_analysis_results_table/log_rate_analysis_results_table_row_provider'; import { FieldFilterPopover } from './field_filter_popover'; +import { LogRateAnalysisTypeCallOut } from './log_rate_analysis_type_callout'; const groupResultsMessage = i18n.translate( 'xpack.aiops.logRateAnalysis.resultsTable.groupedSwitchLabel.groupResults', @@ -209,7 +210,8 @@ export const LogRateAnalysisResults: FC = ({ { [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin } ); - const { significantItems } = data; + const { significantItems, zeroDocsFallback } = data; + useEffect( () => setUniqueFieldNames(uniq(significantItems.map((d) => d.fieldName)).sort()), [significantItems] @@ -362,37 +364,13 @@ export const LogRateAnalysisResults: FC = ({ /> - {showLogRateAnalysisResultsTable && ( + {showLogRateAnalysisResultsTable && currentAnalysisType !== undefined && ( <> - - {currentAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE - ? i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutTitle', { - defaultMessage: 'Analysis type: Log rate spike', - }) - : i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutTitle', { - defaultMessage: 'Analysis type: Log rate dip', - })} - - } - color="primary" - iconType="pin" - size="s" - > - - {currentAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE - ? i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContent', { - defaultMessage: - 'The median log rate in the selected deviation time range is higher than the baseline. Therefore, the analysis results table shows statistically significant items within the deviation time range that are contributors to the spike. The "doc count" column refers to the amount of documents in the deviation time range.', - }) - : i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContent', { - defaultMessage: - 'The median log rate in the selected deviation time range is lower than the baseline. Therefore, the analysis results table shows statistically significant items within the baseline time range that are less in number or missing within the deviation time range. The "doc count" column refers to the amount of documents in the baseline time range.', - })} - - + )} @@ -490,6 +468,7 @@ export const LogRateAnalysisResults: FC = ({ searchQuery={searchQuery} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + zeroDocsFallback={zeroDocsFallback} /> ) : null} {showLogRateAnalysisResultsTable && !groupResults ? ( @@ -501,6 +480,7 @@ export const LogRateAnalysisResults: FC = ({ searchQuery={searchQuery} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + zeroDocsFallback={zeroDocsFallback} /> ) : null} diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx new file mode 100644 index 0000000000000..2760a27c3d224 --- /dev/null +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_type_callout.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FC } from 'react'; + +import { EuiCallOut, EuiText } from '@elastic/eui'; + +import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '@kbn/aiops-utils'; +import { i18n } from '@kbn/i18n'; + +interface LogRateAnalysisTypeCallOutProps { + analysisType: LogRateAnalysisType; + zeroDocsFallback: boolean; +} + +export const LogRateAnalysisTypeCallOut: FC = ({ + analysisType, + zeroDocsFallback, +}) => { + let callOutTitle: string; + let callOutText: string; + + if (!zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutTitle', { + defaultMessage: 'Analysis type: Log rate spike', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContent', { + defaultMessage: + 'The median log rate in the selected deviation time range is higher than the baseline. Therefore, the analysis results table shows statistically significant items within the deviation time range that are contributors to the spike. The "doc count" column refers to the amount of documents in the deviation time range.', + }); + } else if (!zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.DIP) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutTitle', { + defaultMessage: 'Analysis type: Log rate dip', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContent', { + defaultMessage: + 'The median log rate in the selected deviation time range is lower than the baseline. Therefore, the analysis results table shows statistically significant items within the baseline time range that are less in number or missing within the deviation time range. The "doc count" column refers to the amount of documents in the baseline time range.', + }); + } else if (zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeFallbackCallOutTitle', { + defaultMessage: 'Analysis type: Top items for deviation time range', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeSpikeCallOutContentFallback', { + defaultMessage: + 'The baseline time range does not contain any documents. Therefore the results show top log message categories and field values for the deviation time range.', + }); + } else if (zeroDocsFallback && analysisType === LOG_RATE_ANALYSIS_TYPE.DIP) { + callOutTitle = i18n.translate('xpack.aiops.analysis.analysisTypeDipFallbackCallOutTitle', { + defaultMessage: 'Analysis type: Top items for baseline time range', + }); + callOutText = i18n.translate('xpack.aiops.analysis.analysisTypeDipCallOutContentFallback', { + defaultMessage: + 'The deviation time range does not contain any documents. Therefore the results show top log message categories and field values for the baseline time range.', + }); + } else { + return null; + } + + return ( + {callOutTitle}} + color="primary" + iconType="pin" + size="s" + > + {callOutText} + + ); +}; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx index d8845145f1ace..b0b4d699919be 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx @@ -51,7 +51,9 @@ const NOT_AVAILABLE = '--'; const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; const DEFAULT_SORT_FIELD = 'pValue'; +const DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK = 'doc_count'; const DEFAULT_SORT_DIRECTION = 'asc'; +const DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK = 'desc'; const TRUNCATE_TEXT_LINES = 3; @@ -66,6 +68,7 @@ interface LogRateAnalysisResultsTableProps { barColorOverride?: string; /** Optional color override for the highlighted bar color for charts */ barHighlightColorOverride?: string; + zeroDocsFallback?: boolean; } export const LogRateAnalysisResultsTable: FC = ({ @@ -77,6 +80,7 @@ export const LogRateAnalysisResultsTable: FC = timeRangeMs, barColorOverride, barHighlightColorOverride, + zeroDocsFallback = false, }) => { const euiTheme = useEuiTheme(); const primaryBackgroundColor = useEuiBackgroundColor('primary'); @@ -93,8 +97,12 @@ export const LogRateAnalysisResultsTable: FC = const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(DEFAULT_SORT_FIELD); - const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>(DEFAULT_SORT_DIRECTION); + const [sortField, setSortField] = useState( + zeroDocsFallback ? DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK : DEFAULT_SORT_FIELD + ); + const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>( + zeroDocsFallback ? DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK : DEFAULT_SORT_DIRECTION + ); const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext(); @@ -236,7 +244,10 @@ export const LogRateAnalysisResultsTable: FC = sortable: true, valign: 'middle', }, - { + ]; + + if (!zeroDocsFallback) { + columns.push({ 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnPValue', width: NARROW_COLUMN_WIDTH, field: 'pValue', @@ -260,8 +271,9 @@ export const LogRateAnalysisResultsTable: FC = render: (pValue: number | null) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, sortable: true, valign: 'middle', - }, - { + }); + + columns.push({ 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnImpact', width: NARROW_COLUMN_WIDTH, field: 'pValue', @@ -291,21 +303,22 @@ export const LogRateAnalysisResultsTable: FC = }, sortable: true, valign: 'middle', - }, - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnAction', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.actionsColumnName', { - defaultMessage: 'Actions', - }), - actions: [ - ...(viewInDiscoverAction ? [viewInDiscoverAction] : []), - ...(viewInLogPatternAnalysisAction ? [viewInLogPatternAnalysisAction] : []), - copyToClipBoardAction, - ], - width: ACTIONS_COLUMN_WIDTH, - valign: 'middle', - }, - ]; + }); + } + + columns.push({ + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnAction', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.actionsColumnName', { + defaultMessage: 'Actions', + }), + actions: [ + ...(viewInDiscoverAction ? [viewInDiscoverAction] : []), + ...(viewInLogPatternAnalysisAction ? [viewInLogPatternAnalysisAction] : []), + copyToClipBoardAction, + ], + width: ACTIONS_COLUMN_WIDTH, + valign: 'middle', + }); if (isExpandedRow === true) { columns.unshift({ diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx index f6961e49c2c78..957385780ceaa 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table_groups.tsx @@ -50,7 +50,9 @@ const MAX_GROUP_BADGES = 5; const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; const DEFAULT_SORT_FIELD = 'pValue'; +const DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK = 'docCount'; const DEFAULT_SORT_DIRECTION = 'asc'; +const DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK = 'desc'; interface LogRateAnalysisResultsTableProps { significantItems: SignificantItem[]; @@ -63,6 +65,7 @@ interface LogRateAnalysisResultsTableProps { barColorOverride?: string; /** Optional color override for the highlighted bar color for charts */ barHighlightColorOverride?: string; + zeroDocsFallback?: boolean; } export const LogRateAnalysisResultsGroupsTable: FC = ({ @@ -74,11 +77,16 @@ export const LogRateAnalysisResultsGroupsTable: FC { const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(DEFAULT_SORT_FIELD); - const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>(DEFAULT_SORT_DIRECTION); + const [sortField, setSortField] = useState<'docCount' | 'pValue'>( + zeroDocsFallback ? DEFAULT_SORT_FIELD_ZERO_DOCS_FALLBACK : DEFAULT_SORT_FIELD + ); + const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>( + zeroDocsFallback ? DEFAULT_SORT_DIRECTION_ZERO_DOCS_FALLBACK : DEFAULT_SORT_DIRECTION + ); const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( {} ); @@ -297,7 +305,10 @@ export const LogRateAnalysisResultsGroupsTable: FC pValue?.toPrecision(3) ?? NOT_AVAILABLE, sortable: true, valign: 'top', - }, - { + }); + + columns.push({ 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnImpact', width: NARROW_COLUMN_WIDTH, field: 'pValue', @@ -355,21 +367,22 @@ export const LogRateAnalysisResultsGroupsTable: FC { if (tableSettings.page) { diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts index 6ae2a07055aec..3c6ea4d022bb0 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/index_info_handler.ts @@ -7,7 +7,10 @@ import { i18n } from '@kbn/i18n'; -import { updateLoadingStateAction } from '../../../../common/api/log_rate_analysis/actions'; +import { + updateLoadingStateAction, + setZeroDocsFallback, +} from '../../../../common/api/log_rate_analysis/actions'; import type { AiopsLogRateAnalysisApiVersion as ApiVersion } from '../../../../common/api/log_rate_analysis/schema'; import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; @@ -36,6 +39,7 @@ export const indexInfoHandlerFactory = const textFieldCandidates: string[] = []; let totalDocCount = 0; + let zeroDocsFallback = false; if (!requestBody.overrides?.remainingFieldCandidates) { logDebugMessage('Fetch index information.'); @@ -63,7 +67,8 @@ export const indexInfoHandlerFactory = fieldCandidates.push(...indexInfo.fieldCandidates); fieldCandidatesCount = fieldCandidates.length; textFieldCandidates.push(...indexInfo.textFieldCandidates); - totalDocCount = indexInfo.totalDocCount; + totalDocCount = indexInfo.deviationTotalDocCount; + zeroDocsFallback = indexInfo.zeroDocsFallback; } catch (e) { if (!isRequestAbortedError(e)) { logger.error(`Failed to fetch index information, got: \n${e.toString()}`); @@ -96,6 +101,8 @@ export const indexInfoHandlerFactory = }) ); + responseStream.push(setZeroDocsFallback(zeroDocsFallback)); + if (fieldCandidatesCount === 0) { responseStream.endWithUpdatedLoadingState(); } else if (stateHandler.shouldStop()) { @@ -105,5 +112,5 @@ export const indexInfoHandlerFactory = } } - return { fieldCandidates, textFieldCandidates }; + return { fieldCandidates, textFieldCandidates, zeroDocsFallback }; }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts new file mode 100644 index 0000000000000..10f0b19f07925 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { queue } from 'async'; + +import { SIGNIFICANT_ITEM_TYPE, type SignificantItem } from '@kbn/ml-agg-utils'; +import { i18n } from '@kbn/i18n'; + +import { + addSignificantItemsAction, + updateLoadingStateAction, +} from '../../../../common/api/log_rate_analysis/actions'; + +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; + +import { fetchTopCategories } from '../queries/fetch_top_categories'; +import { fetchTopTerms } from '../queries/fetch_top_terms'; + +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '../../../../common/api/log_rate_analysis/schema'; + +import { + LOADED_FIELD_CANDIDATES, + MAX_CONCURRENT_QUERIES, + PROGRESS_STEP_P_VALUES, +} from '../response_stream_utils/constants'; +import type { ResponseStreamFetchOptions } from '../response_stream_factory'; + +export const topItemsHandlerFactory = + ({ + abortSignal, + client, + logDebugMessage, + logger, + requestBody, + responseStream, + stateHandler, + version, + }: ResponseStreamFetchOptions) => + async ({ + fieldCandidates, + textFieldCandidates, + }: { + fieldCandidates: string[]; + textFieldCandidates: string[]; + }) => { + let fieldCandidatesCount = fieldCandidates.length; + + // This will store the combined count of detected log patterns and keywords + let fieldValuePairsCount = 0; + + const topCategories: SignificantItem[] = []; + + if (version === '1') { + topCategories.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'1'>).overrides?.significantTerms?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.LOG_PATTERN + ) ?? []) + ); + } + + if (version === '2') { + topCategories.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'2'>).overrides?.significantItems?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.LOG_PATTERN + ) ?? []) + ); + } + + // Get categories of text fields + if (textFieldCandidates.length > 0) { + topCategories.push( + ...(await fetchTopCategories( + client, + requestBody, + textFieldCandidates, + logger, + stateHandler.sampleProbability(), + responseStream.pushError, + abortSignal + )) + ); + + if (topCategories.length > 0) { + responseStream.push(addSignificantItemsAction(topCategories, version)); + } + } + + const topTerms: SignificantItem[] = []; + + if (version === '1') { + topTerms.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'1'>).overrides?.significantTerms?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.KEYWORD + ) ?? []) + ); + } + + if (version === '2') { + topTerms.push( + ...((requestBody as AiopsLogRateAnalysisSchema<'2'>).overrides?.significantItems?.filter( + (d) => d.type === SIGNIFICANT_ITEM_TYPE.KEYWORD + ) ?? []) + ); + } + + const fieldsToSample = new Set(); + + let remainingFieldCandidates: string[]; + let loadingStepSizeTopTerms = PROGRESS_STEP_P_VALUES; + + if (requestBody.overrides?.remainingFieldCandidates) { + fieldCandidates.push(...requestBody.overrides?.remainingFieldCandidates); + remainingFieldCandidates = requestBody.overrides?.remainingFieldCandidates; + fieldCandidatesCount = fieldCandidates.length; + loadingStepSizeTopTerms = + LOADED_FIELD_CANDIDATES + + PROGRESS_STEP_P_VALUES - + (requestBody.overrides?.loaded ?? PROGRESS_STEP_P_VALUES); + } else { + remainingFieldCandidates = fieldCandidates; + } + + logDebugMessage('Fetch p-values.'); + + const topTermsQueue = queue(async function (fieldCandidate: string) { + stateHandler.loaded((1 / fieldCandidatesCount) * loadingStepSizeTopTerms, false); + + let fetchedTopTerms: Awaited>; + + try { + fetchedTopTerms = await fetchTopTerms( + client, + requestBody, + [fieldCandidate], + logger, + stateHandler.sampleProbability(), + responseStream.pushError, + abortSignal + ); + } catch (e) { + if (!isRequestAbortedError(e)) { + logger.error(`Failed to fetch p-values for '${fieldCandidate}', got: \n${e.toString()}`); + responseStream.pushError(`Failed to fetch p-values for '${fieldCandidate}'.`); + } + return; + } + + remainingFieldCandidates = remainingFieldCandidates.filter((d) => d !== fieldCandidate); + + if (fetchedTopTerms.length > 0) { + fetchedTopTerms.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); + topTerms.push(...fetchedTopTerms); + + responseStream.push(addSignificantItemsAction(fetchedTopTerms, version)); + } + + responseStream.push( + updateLoadingStateAction({ + ccsWarning: false, + loaded: stateHandler.loaded(), + loadingState: i18n.translate( + 'xpack.aiops.logRateAnalysis.loadingState.identifiedFieldValuePairs', + { + defaultMessage: + 'Identified {fieldValuePairsCount, plural, one {# significant field/value pair} other {# significant field/value pairs}}.', + values: { + fieldValuePairsCount, + }, + } + ), + remainingFieldCandidates, + }) + ); + }, MAX_CONCURRENT_QUERIES); + + topTermsQueue.push(fieldCandidates, (err) => { + if (err) { + logger.error(`Failed to fetch p-values.', got: \n${err.toString()}`); + responseStream.pushError(`Failed to fetch p-values.`); + topTermsQueue.kill(); + responseStream.end(); + } else if (stateHandler.shouldStop()) { + logDebugMessage('shouldStop fetching p-values.'); + topTermsQueue.kill(); + responseStream.end(); + } + }); + await topTermsQueue.drain(); + + fieldValuePairsCount = topCategories.length + topTerms.length; + + if (fieldValuePairsCount === 0) { + logDebugMessage('Stopping analysis, did not find any categories or terms.'); + responseStream.endWithUpdatedLoadingState(); + return; + } + + return { + fieldValuePairsCount, + significantCategories: topCategories, + significantTerms: topTerms, + }; + }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts index 0344005c869f8..38377a9e0d32c 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.test.ts @@ -11,55 +11,9 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { paramsSearchQueryMock } from './__mocks__/params_search_query'; -import { fetchIndexInfo, getRandomDocsRequest } from './fetch_index_info'; +import { fetchIndexInfo } from './fetch_index_info'; describe('fetch_index_info', () => { - describe('getRandomDocsRequest', () => { - it('returns the most basic request body for a sample of random documents', () => { - const req = getRandomDocsRequest(paramsSearchQueryMock); - - expect(req).toEqual({ - body: { - _source: false, - fields: ['*'], - query: { - function_score: { - query: { - bool: { - filter: [ - { - bool: { - filter: [], - minimum_should_match: 1, - must_not: [], - should: [{ term: { 'the-term': { value: 'the-value' } } }], - }, - }, - { - range: { - 'the-time-field-name': { - format: 'epoch_millis', - gte: 0, - lte: 50, - }, - }, - }, - ], - }, - }, - random_score: {}, - }, - }, - size: 1000, - track_total_hits: true, - }, - index: paramsSearchQueryMock.index, - ignore_throttled: undefined, - ignore_unavailable: true, - }); - }); - }); - describe('fetchFieldCandidates', () => { it('returns field candidates and total hits', async () => { const esClientFieldCapsMock = jest.fn(() => ({ @@ -99,15 +53,14 @@ describe('fetch_index_info', () => { search: esClientSearchMock, } as unknown as ElasticsearchClient; - const { totalDocCount, fieldCandidates } = await fetchIndexInfo( - esClientMock, - paramsSearchQueryMock - ); + const { baselineTotalDocCount, deviationTotalDocCount, fieldCandidates } = + await fetchIndexInfo(esClientMock, paramsSearchQueryMock); expect(fieldCandidates).toEqual(['myIpFieldName', 'myKeywordFieldName']); - expect(totalDocCount).toEqual(5000000); + expect(baselineTotalDocCount).toEqual(5000000); + expect(deviationTotalDocCount).toEqual(5000000); expect(esClientFieldCapsMock).toHaveBeenCalledTimes(1); - expect(esClientSearchMock).toHaveBeenCalledTimes(1); + expect(esClientSearchMock).toHaveBeenCalledTimes(2); }); }); }); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts index 595e212159437..fd7975aac1b6e 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_index_info.ts @@ -12,14 +12,12 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; -import { getQueryWithParams } from './get_query_with_params'; -import { getRequestBase } from './get_request_base'; +import { getRandomDocsRequest } from './get_random_docs_request'; +import { getTotalDocCountRequest } from './get_total_doc_count_request'; // TODO Consolidate with duplicate `fetchPValues` in // `x-pack/plugins/apm/server/routes/correlations/queries/fetch_duration_field_candidates.ts` -const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; - const SUPPORTED_ES_FIELD_TYPES = [ ES_FIELD_TYPES.KEYWORD, ES_FIELD_TYPES.IP, @@ -28,30 +26,12 @@ const SUPPORTED_ES_FIELD_TYPES = [ const SUPPORTED_ES_FIELD_TYPES_TEXT = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.MATCH_ONLY_TEXT]; -export const getRandomDocsRequest = ( - params: AiopsLogRateAnalysisSchema -): estypes.SearchRequest => ({ - ...getRequestBase(params), - body: { - fields: ['*'], - _source: false, - query: { - function_score: { - query: getQueryWithParams({ params }), - // @ts-ignore - random_score: {}, - }, - }, - size: POPULATED_DOC_COUNT_SAMPLE_SIZE, - // Used to determine sample probability for follow up queries - track_total_hits: true, - }, -}); - interface IndexInfo { fieldCandidates: string[]; textFieldCandidates: string[]; - totalDocCount: number; + baselineTotalDocCount: number; + deviationTotalDocCount: number; + zeroDocsFallback: boolean; } export const fetchIndexInfo = async ( @@ -95,15 +75,24 @@ export const fetchIndexInfo = async ( allFieldNames.push(key); }); + // Get the total doc count for the baseline time range + const respBaselineTotalDocCount = await esClient.search( + getTotalDocCountRequest({ ...params, start: params.baselineMin, end: params.baselineMax }), + { + signal: abortSignal, + maxRetries: 0, + } + ); + // Only the deviation window will be used to identify field candidates and sample probability based on total doc count. - const resp = await esClient.search( + const respDeviationRandomDocs = await esClient.search( getRandomDocsRequest({ ...params, start: params.deviationMin, end: params.deviationMax }), { signal: abortSignal, maxRetries: 0, } ); - const sampledDocs = resp.hits.hits.map((d) => d.fields ?? {}); + const sampledDocs = respDeviationRandomDocs.hits.hits.map((d) => d.fields ?? {}); const textFieldCandidatesOverridesWithKeywordPostfix = textFieldCandidatesOverrides.map( (d) => `${d}.keyword` @@ -127,11 +116,16 @@ export const fetchIndexInfo = async ( } }); - const totalDocCount = (resp.hits.total as estypes.SearchTotalHits).value; + const baselineTotalDocCount = (respBaselineTotalDocCount.hits.total as estypes.SearchTotalHits) + .value; + const deviationTotalDocCount = (respDeviationRandomDocs.hits.total as estypes.SearchTotalHits) + .value; return { fieldCandidates: [...finalFieldCandidates], textFieldCandidates: [...finalTextFieldCandidates], - totalDocCount, + baselineTotalDocCount, + deviationTotalDocCount, + zeroDocsFallback: baselineTotalDocCount === 0 || deviationTotalDocCount === 0, }; }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts new file mode 100644 index 0000000000000..2933e5d3e414a --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_categories.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniq } from 'lodash'; + +import { ElasticsearchClient } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import { type SignificantItem, SIGNIFICANT_ITEM_TYPE } from '@kbn/ml-agg-utils'; + +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { fetchCategories } from './fetch_categories'; + +export const fetchTopCategories = async ( + esClient: ElasticsearchClient, + params: AiopsLogRateAnalysisSchema, + fieldNames: string[], + logger: Logger, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1, + emitError: (m: string) => void, + abortSignal?: AbortSignal +) => { + const categoriesOverall = await fetchCategories( + esClient, + params, + fieldNames, + logger, + sampleProbability, + emitError, + abortSignal + ); + + if (categoriesOverall.length !== fieldNames.length) return []; + + const topCategories: SignificantItem[] = []; + + // Using for...of to allow `await` within the loop. + for (const [i, fieldName] of fieldNames.entries()) { + if (categoriesOverall[i].categories.length === 0) { + continue; + } + + // Get all unique keys + const allKeys: string[] = uniq(categoriesOverall[i].categories.map((cd) => cd.key)); + + allKeys.forEach((key) => { + const categoryData = categoriesOverall[i].categories.find((c) => c.key === key); + + topCategories.push({ + key, + fieldName, + fieldValue: categoryData?.examples[0] ?? '', + doc_count: categoryData?.count ?? 0, + bg_count: 0, + total_doc_count: 0, + total_bg_count: 0, + score: 0, + pValue: 1, + normalizedScore: 0, + type: SIGNIFICANT_ITEM_TYPE.LOG_PATTERN, + }); + }); + } + + return topCategories; +}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts new file mode 100644 index 0000000000000..d1b8007249136 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/fetch_top_terms.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { uniqBy } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ElasticsearchClient } from '@kbn/core/server'; + +import type { Logger } from '@kbn/logging'; +import { type SignificantItem, SIGNIFICANT_ITEM_TYPE } from '@kbn/ml-agg-utils'; +import { + createRandomSamplerWrapper, + type RandomSamplerWrapper, +} from '@kbn/ml-random-sampler-utils'; + +import { RANDOM_SAMPLER_SEED } from '../../../../common/constants'; +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { isRequestAbortedError } from '../../../lib/is_request_aborted_error'; + +import { getQueryWithParams } from './get_query_with_params'; +import { getRequestBase } from './get_request_base'; + +// TODO Consolidate with duplicate `fetchDurationFieldCandidates` in +// `x-pack/plugins/apm/server/routes/correlations/queries/fetch_failed_events_correlation_p_values.ts` + +export const getTopTermRequest = ( + params: AiopsLogRateAnalysisSchema, + fieldName: string, + { wrap }: RandomSamplerWrapper +): estypes.SearchRequest => { + const query = getQueryWithParams({ + params, + }); + + const timeFieldName = params.timeFieldName ?? '@timestamp'; + + let filter: estypes.QueryDslQueryContainer[] = []; + + if (query.bool && Array.isArray(query.bool.filter)) { + filter = query.bool.filter.filter((d) => Object.keys(d)[0] !== 'range'); + + query.bool.filter = [ + ...filter, + { + range: { + [timeFieldName]: { + gte: params.deviationMin, + lt: params.deviationMax, + format: 'epoch_millis', + }, + }, + }, + ]; + } + + const termAgg: Record<'log_rate_top_terms', estypes.AggregationsAggregationContainer> = { + log_rate_top_terms: { + terms: { + field: fieldName, + size: 10, + }, + }, + }; + + const body = { + query, + size: 0, + aggs: wrap(termAgg), + }; + + return { + ...getRequestBase(params), + body, + }; +}; + +interface Aggs extends estypes.AggregationsLongTermsAggregate { + buckets: estypes.AggregationsLongTermsBucket[]; +} + +export const fetchTopTerms = async ( + esClient: ElasticsearchClient, + params: AiopsLogRateAnalysisSchema, + fieldNames: string[], + logger: Logger, + // The default value of 1 means no sampling will be used + sampleProbability: number = 1, + emitError: (m: string) => void, + abortSignal?: AbortSignal +): Promise => { + const randomSamplerWrapper = createRandomSamplerWrapper({ + probability: sampleProbability, + seed: RANDOM_SAMPLER_SEED, + }); + + const result: SignificantItem[] = []; + + const settledPromises = await Promise.allSettled( + fieldNames.map((fieldName) => + esClient.search(getTopTermRequest(params, fieldName, randomSamplerWrapper), { + signal: abortSignal, + maxRetries: 0, + }) + ) + ); + + function reportError(fieldName: string, error: unknown) { + if (!isRequestAbortedError(error)) { + logger.error( + `Failed to fetch term aggregation for fieldName "${fieldName}", got: \n${JSON.stringify( + error, + null, + 2 + )}` + ); + emitError(`Failed to fetch term aggregation for fieldName "${fieldName}".`); + } + } + + for (const [index, settledPromise] of settledPromises.entries()) { + const fieldName = fieldNames[index]; + + if (settledPromise.status === 'rejected') { + reportError(fieldName, settledPromise.reason); + // Still continue the analysis even if individual p-value queries fail. + continue; + } + + const resp = settledPromise.value; + + if (resp.aggregations === undefined) { + reportError(fieldName, resp); + // Still continue the analysis even if individual p-value queries fail. + continue; + } + + const overallResult = ( + randomSamplerWrapper.unwrap(resp.aggregations) as Record<'log_rate_top_terms', Aggs> + ).log_rate_top_terms; + + for (const bucket of overallResult.buckets) { + result.push({ + key: `${fieldName}:${String(bucket.key)}`, + type: SIGNIFICANT_ITEM_TYPE.KEYWORD, + fieldName, + fieldValue: String(bucket.key), + doc_count: bucket.doc_count, + bg_count: 0, + total_doc_count: 0, + total_bg_count: 0, + score: 0, + pValue: 1, + normalizedScore: 0, + }); + } + } + + return uniqBy(result, (d) => `${d.fieldName},${d.fieldValue}`); +}; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts new file mode 100644 index 0000000000000..6e68c789142c5 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { paramsSearchQueryMock } from './__mocks__/params_search_query'; + +import { getRandomDocsRequest } from './get_random_docs_request'; + +describe('getRandomDocsRequest', () => { + it('returns the most basic request body for a sample of random documents', () => { + const req = getRandomDocsRequest(paramsSearchQueryMock); + + expect(req).toEqual({ + body: { + _source: false, + fields: ['*'], + query: { + function_score: { + query: { + bool: { + filter: [ + { + bool: { + filter: [], + minimum_should_match: 1, + must_not: [], + should: [{ term: { 'the-term': { value: 'the-value' } } }], + }, + }, + { + range: { + 'the-time-field-name': { + format: 'epoch_millis', + gte: 0, + lte: 50, + }, + }, + }, + ], + }, + }, + random_score: {}, + }, + }, + size: 1000, + track_total_hits: true, + }, + index: paramsSearchQueryMock.index, + ignore_throttled: undefined, + ignore_unavailable: true, + }); + }); +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts new file mode 100644 index 0000000000000..7c1abdfd52667 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_random_docs_request.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { getQueryWithParams } from './get_query_with_params'; +import { getRequestBase } from './get_request_base'; + +const POPULATED_DOC_COUNT_SAMPLE_SIZE = 1000; + +export const getRandomDocsRequest = ( + params: AiopsLogRateAnalysisSchema +): estypes.SearchRequest => ({ + ...getRequestBase(params), + body: { + fields: ['*'], + _source: false, + query: { + function_score: { + query: getQueryWithParams({ params }), + // @ts-ignore + random_score: {}, + }, + }, + size: POPULATED_DOC_COUNT_SAMPLE_SIZE, + // Used to determine sample probability for follow up queries + track_total_hits: true, + }, +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts new file mode 100644 index 0000000000000..e9b0fc853535f --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/queries/get_total_doc_count_request.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { AiopsLogRateAnalysisSchema } from '../../../../common/api/log_rate_analysis/schema'; + +import { getQueryWithParams } from './get_query_with_params'; +import { getRequestBase } from './get_request_base'; + +export const getTotalDocCountRequest = ( + params: AiopsLogRateAnalysisSchema +): estypes.SearchRequest => ({ + ...getRequestBase(params), + body: { + fields: ['*'], + _source: false, + query: getQueryWithParams({ params }), + size: 0, + track_total_hits: true, + }, +}); diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts index f54c2e93dd29e..0dce9c7cf34d1 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/response_stream_factory.ts @@ -22,6 +22,7 @@ import { groupingHandlerFactory } from './analysis_handlers/grouping_handler'; import { histogramHandlerFactory } from './analysis_handlers/histogram_handler'; import { overridesHandlerFactory } from './analysis_handlers/overrides_handler'; import { significantItemsHandlerFactory } from './analysis_handlers/significant_items_handler'; +import { topItemsHandlerFactory } from './analysis_handlers/top_items_handler'; import { overallHistogramHandlerFactory } from './analysis_handlers/overall_histogram_handler'; import { logDebugMessageFactory, @@ -131,6 +132,7 @@ export const responseStreamFactory = (options: ResponseStr overallHistogramHandler: overallHistogramHandlerFactory(streamFetchOptions), overridesHandler: overridesHandlerFactory(streamFetchOptions), significantItemsHandler: significantItemsHandlerFactory(streamFetchOptions), + topItemsHandler: topItemsHandlerFactory(streamFetchOptions), }, responseWithHeaders, }; diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts index 3c1bcde38ebd6..e0a9b9c3e9803 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/route_handler_factory.ts @@ -92,7 +92,9 @@ export function routeHandlerFactory( } // Step 2: Significant categories and terms - const significantItemsObj = await analysis.significantItemsHandler(indexInfo); + const significantItemsObj = indexInfo.zeroDocsFallback + ? await analysis.topItemsHandler(indexInfo) + : await analysis.significantItemsHandler(indexInfo); if (!significantItemsObj) { return; diff --git a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts index 3168dfdfc5ecf..352282b4ba41f 100644 --- a/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts +++ b/x-pack/test/api_integration/apis/aiops/log_rate_analysis_full_analysis.ts @@ -86,9 +86,14 @@ export default ({ getService }: FtrProviderContext) => { const groupActions = getGroupActions(data, apiVersion); const groups = groupActions.flatMap((d) => d.payload); - expect(orderBy(groups, ['docCount'], ['desc'])).to.eql( - orderBy(testData.expected.groups, ['docCount'], ['desc']), - 'Grouping result does not match expected values.' + const actualGroups = orderBy(groups, ['docCount'], ['desc']); + const expectedGroups = orderBy(testData.expected.groups, ['docCount'], ['desc']); + + expect(actualGroups).to.eql( + expectedGroups, + `Grouping result does not match expected values. Expected ${JSON.stringify( + expectedGroups + )}, got ${JSON.stringify(actualGroups)}` ); const groupHistogramActions = getGroupHistogramActions(data, apiVersion); diff --git a/x-pack/test/api_integration/apis/aiops/test_data.ts b/x-pack/test/api_integration/apis/aiops/test_data.ts index 291779ed6c7b2..3c1793ab9efe3 100644 --- a/x-pack/test/api_integration/apis/aiops/test_data.ts +++ b/x-pack/test/api_integration/apis/aiops/test_data.ts @@ -12,6 +12,8 @@ import { significantTerms as artificialLogSignificantTerms } from '@kbn/aiops-pl import { significantLogPatterns as artificialLogSignificantLogPatterns } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/significant_log_patterns'; import { finalSignificantItemGroups as artificialLogsSignificantItemGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups'; import { finalSignificantItemGroupsTextfield as artificialLogsSignificantItemGroupsTextfield } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/final_significant_item_groups_textfield'; +import { topTerms } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms'; +import { topTermsGroups } from '@kbn/aiops-plugin/common/__mocks__/artificial_logs/top_terms_groups'; import type { AiopsLogRateAnalysisSchema, @@ -39,9 +41,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 35, + chunksLength: 36, chunksLengthGroupOnly: 5, - actionsLength: 34, + actionsLength: 35, actionsLengthGroupOnly: 4, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -78,14 +80,14 @@ export const getLogRateAnalysisTestData = (): Array(): Array, expected: { - chunksLength: 27, + chunksLength: 28, chunksLengthGroupOnly: 11, - actionsLength: 26, + actionsLength: 27, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -104,6 +106,60 @@ export const getLogRateAnalysisTestData = (): Array, + expected: { + chunksLength: 62, + chunksLengthGroupOnly: 32, + actionsLength: 61, + actionsLengthGroupOnly: 31, + noIndexChunksLength: 4, + noIndexActionsLength: 3, + significantItems: topTerms, + groups: topTermsGroups, + histogramLength: 20, + }, + }, + { + testName: 'artificial_logs_with_dip_zerodocsfallback', + dataGenerator: 'artificial_logs_with_dip_zerodocsfallback', + requestBody: { + start: 0, + end: 1768855600010, + searchQuery: '{"match_all":{}}', + timeFieldName: '@timestamp', + index: 'artificial_logs_with_dip_zerodocsfallback', + baselineMin: 1768855600000, + baselineMax: 1768855600010, + deviationMin: 1668855600000, + deviationMax: 1668924000000, + grouping: true, + } as AiopsLogRateAnalysisSchema, + expected: { + chunksLength: 62, + chunksLengthGroupOnly: 32, + actionsLength: 61, + actionsLengthGroupOnly: 31, + noIndexChunksLength: 4, + noIndexActionsLength: 3, + significantItems: topTerms, + groups: topTermsGroups, + histogramLength: 20, + }, + }, { testName: 'artificial_logs_with_spike_textfield', dataGenerator: 'artificial_logs_with_spike_textfield', @@ -120,9 +176,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 30, + chunksLength: 31, chunksLengthGroupOnly: 11, - actionsLength: 29, + actionsLength: 30, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -132,14 +188,14 @@ export const getLogRateAnalysisTestData = (): Array(): Array, expected: { - chunksLength: 27, + chunksLength: 28, chunksLengthGroupOnly: 11, - actionsLength: 26, + actionsLength: 27, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, @@ -174,9 +230,9 @@ export const getLogRateAnalysisTestData = (): Array, expected: { - chunksLength: 30, + chunksLength: 31, chunksLengthGroupOnly: 11, - actionsLength: 29, + actionsLength: 30, actionsLengthGroupOnly: 10, noIndexChunksLength: 4, noIndexActionsLength: 3, diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis.ts index 68c9256b382d0..f613349078b48 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis.ts @@ -149,12 +149,18 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { } // Wait for the analysis to finish - await aiops.logRateAnalysisPage.assertAnalysisComplete(testData.analysisType); + await aiops.logRateAnalysisPage.assertAnalysisComplete( + testData.analysisType, + testData.dataGenerator + ); // At this stage the baseline and deviation brush position should be stored in // the url state and a full browser refresh should restore the analysis. await browser.refresh(); - await aiops.logRateAnalysisPage.assertAnalysisComplete(testData.analysisType); + await aiops.logRateAnalysisPage.assertAnalysisComplete( + testData.analysisType, + testData.dataGenerator + ); // The group switch should be disabled by default await aiops.logRateAnalysisPage.assertLogRateAnalysisResultsGroupSwitchExists(false); @@ -167,8 +173,15 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const analysisGroupsTable = await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); - expect(orderBy(analysisGroupsTable, 'group')).to.be.eql( - orderBy(testData.expected.analysisGroupsTable, 'group') + + const actualAnalysisGroupsTable = orderBy(analysisGroupsTable, 'group'); + const expectedAnalysisGroupsTable = orderBy(testData.expected.analysisGroupsTable, 'group'); + + expect(actualAnalysisGroupsTable).to.be.eql( + expectedAnalysisGroupsTable, + `Expected analysis groups table to be ${JSON.stringify( + expectedAnalysisGroupsTable + )}, got ${JSON.stringify(actualAnalysisGroupsTable)}` ); await ml.testExecution.logTestStep('expand table row'); @@ -177,8 +190,18 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { if (!isTestDataExpectedWithSampleProbability(testData.expected)) { const analysisTable = await aiops.logRateAnalysisResultsTable.parseAnalysisTable(); - expect(orderBy(analysisTable, ['fieldName', 'fieldValue'])).to.be.eql( - orderBy(testData.expected.analysisTable, ['fieldName', 'fieldValue']) + + const actualAnalysisTable = orderBy(analysisTable, ['fieldName', 'fieldValue']); + const expectedAnalysisTable = orderBy(testData.expected.analysisTable, [ + 'fieldName', + 'fieldValue', + ]); + + expect(actualAnalysisTable).to.be.eql( + expectedAnalysisTable, + `Expected analysis table results to be ${JSON.stringify( + expectedAnalysisTable + )}, got ${JSON.stringify(actualAnalysisTable)}` ); } @@ -206,8 +229,18 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { if (!isTestDataExpectedWithSampleProbability(testData.expected)) { const filteredAnalysisGroupsTable = await aiops.logRateAnalysisResultsGroupsTable.parseAnalysisTable(); - expect(orderBy(filteredAnalysisGroupsTable, 'group')).to.be.eql( - orderBy(testData.expected.filteredAnalysisGroupsTable, 'group') + + const actualFilteredAnalysisGroupsTable = orderBy(filteredAnalysisGroupsTable, 'group'); + const expectedFilteredAnalysisGroupsTable = orderBy( + testData.expected.filteredAnalysisGroupsTable, + 'group' + ); + + expect(actualFilteredAnalysisGroupsTable).to.be.eql( + expectedFilteredAnalysisGroupsTable, + `Expected filtered analysis groups table to be ${JSON.stringify( + expectedFilteredAnalysisGroupsTable + )}, got ${JSON.stringify(actualFilteredAnalysisGroupsTable)}` ); } } diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table.ts new file mode 100644 index 0000000000000..6780cab5cb209 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTable = [ + { + group: 'response_code: 500url: home.php', + docCount: '792', + }, + { + group: 'url: login.phpresponse_code: 500', + docCount: '790', + }, + { + docCount: '636', + group: 'user: Peterurl: home.php', + }, + { + docCount: '632', + group: 'user: Peterurl: login.php', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield.ts new file mode 100644 index 0000000000000..2f20ab7900386 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableTextfield = [ + { + group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', + docCount: '634', + }, + { + group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500', + docCount: '632', + }, + { + docCount: '636', + group: 'user: Peterurl: home.php', + }, + { + docCount: '632', + group: 'user: Peterurl: login.php', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts new file mode 100644 index 0000000000000..4c9308f6c0ae2 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_textfield_zerodocsfallback.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableTextfieldZerodocsfallback = [ + { + group: + 'message: an unexpected error occuredurl: home.phpuser: Maryresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'message: an unexpected error occuredurl: home.phpuser: Paulresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'message: an unexpected error occuredurl: login.phpuser: Paulresponse_code: 500version: v1.0.0', + docCount: '29', + }, + { + group: + 'url: home.phpuser: Paulresponse_code: 500message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 200url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 200url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 404url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterresponse_code: 404url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterurl: user.phpresponse_code: 200message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, + { + group: + 'user: Peterurl: user.phpresponse_code: 404message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '30', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_zerodocsfallback.ts new file mode 100644 index 0000000000000..cd749df4f2921 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_groups_table_zerodocsfallback.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisGroupsTableZerodocsfallback = [ + { group: 'response_code: 500url: home.phpuser: Maryversion: v1.0.0', docCount: '47' }, + { group: 'response_code: 500url: home.phpuser: Paulversion: v1.0.0', docCount: '59' }, + { group: 'response_code: 500url: login.phpuser: Maryversion: v1.0.0', docCount: '35' }, + { group: 'response_code: 500url: login.phpuser: Paulversion: v1.0.0', docCount: '39' }, + { group: 'user: Peterurl: home.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: home.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: login.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: login.phpresponse_code: 404version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: user.phpresponse_code: 200version: v1.0.0', docCount: '30' }, + { group: 'user: Peterurl: user.phpresponse_code: 404version: v1.0.0', docCount: '30' }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table.ts new file mode 100644 index 0000000000000..797a9b0911489 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTable = [ + { + fieldName: 'url', + fieldValue: 'home.php', + impact: 'Low', + logRate: 'Chart type:bar chart', + pValue: '0.00974', + }, + { + fieldName: 'user', + fieldValue: 'Peter', + impact: 'High', + logRate: 'Chart type:bar chart', + pValue: '2.63e-21', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield.ts new file mode 100644 index 0000000000000..be77ef870d536 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableTextfield = [ + { + fieldName: 'message', + fieldValue: 'an unexpected error occured', + logRate: 'Chart type:bar chart', + pValue: '0.00000100', + impact: 'Medium', + }, + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '3.61e-12', + impact: 'High', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + impact: 'Low', + logRate: 'Chart type:bar chart', + pValue: '0.00974', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts new file mode 100644 index 0000000000000..75c624a67321c --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_textfield_zerodocsfallback.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableTextfieldZerodocsfallback = [ + { + fieldName: 'message', + fieldValue: 'Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'user', + fieldValue: 'Paul', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'version', + fieldValue: 'v1.0.0', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_zerodocsfallback.ts new file mode 100644 index 0000000000000..3d4f806db8f49 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/analysis_table_zerodocsfallback.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const analysisTableZerodocsfallback = [ + { + fieldName: 'response_code', + fieldValue: '500', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'url', + fieldValue: 'home.php', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'user', + fieldValue: 'Paul', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, + { + fieldName: 'version', + fieldValue: 'v1.0.0', + logRate: 'Chart type:bar chart', + pValue: '1.00', + impact: '', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table.ts new file mode 100644 index 0000000000000..97b1e2ce7169b --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTable = [ + { group: '* url: home.phpresponse_code: 500', docCount: '792' }, + { group: '* url: login.phpresponse_code: 500', docCount: '790' }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield.ts new file mode 100644 index 0000000000000..41c5efe2cb211 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableTextfield = [ + { + group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', + docCount: '634', + }, + { + group: '* url: login.phpmessage: an unexpected error occuredresponse_code: 500', + docCount: '632', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts new file mode 100644 index 0000000000000..9ec9f0e19f89b --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableTextfieldZerodocsfallback = [ + { + group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500version: v1.0.0', + docCount: '58', + }, + { + group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500version: v1.0.0', + docCount: '58', + }, + { + group: + 'response_code: 200url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '46', + }, + { + group: + 'response_code: 200url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '35', + }, + { + group: + 'response_code: 404url: home.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '63', + }, + { + group: + 'response_code: 404url: login.phpmessage: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '40', + }, + { + group: + 'url: home.phpresponse_code: 500message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '48', + }, + { + group: + 'url: user.phpresponse_code: 200message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '40', + }, + { + group: + 'url: user.phpresponse_code: 404message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '51', + }, + { + group: + 'url: user.phpresponse_code: 500message: Paul [11/19/2022, 8:00:34 AM] "GET /home.php HTTP/1.1" 200version: v1.0.0', + docCount: '41', + }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_zerodocsfallback.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_zerodocsfallback.ts new file mode 100644 index 0000000000000..eaeb04b9c0204 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/__mocks__/filtered_analysis_groups_table_zerodocsfallback.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const filteredAnalysisGroupsTableZerodocsfallback = [ + { group: 'url: home.phpresponse_code: 200version: v1.0.0', docCount: '46' }, + { group: 'url: home.phpresponse_code: 404version: v1.0.0', docCount: '63' }, + { group: 'url: home.phpresponse_code: 500version: v1.0.0', docCount: '106' }, + { group: 'url: login.phpresponse_code: 200version: v1.0.0', docCount: '35' }, + { group: 'url: login.phpresponse_code: 404version: v1.0.0', docCount: '40' }, + { group: 'url: login.phpresponse_code: 500version: v1.0.0', docCount: '74' }, + { group: 'url: user.phpresponse_code: 200version: v1.0.0', docCount: '40' }, + { group: 'url: user.phpresponse_code: 404version: v1.0.0', docCount: '51' }, + { group: 'url: user.phpresponse_code: 500version: v1.0.0', docCount: '41' }, +]; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts new file mode 100644 index 0000000000000..6afbaab856424 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/artificial_log_data_view_test_data.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { LogRateAnalysisType } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +import type { LogRateAnalysisDataGenerator } from '../../../../services/aiops/log_rate_analysis_data_generator'; + +import { analysisGroupsTableTextfieldZerodocsfallback } from './__mocks__/analysis_groups_table_textfield_zerodocsfallback'; +import { analysisGroupsTableZerodocsfallback } from './__mocks__/analysis_groups_table_zerodocsfallback'; +import { analysisGroupsTableTextfield } from './__mocks__/analysis_groups_table_textfield'; +import { analysisGroupsTable } from './__mocks__/analysis_groups_table'; +import { filteredAnalysisGroupsTableTextfieldZerodocsfallback } from './__mocks__/filtered_analysis_groups_table_textfield_zerodocsfallback'; +import { filteredAnalysisGroupsTableZerodocsfallback } from './__mocks__/filtered_analysis_groups_table_zerodocsfallback'; +import { filteredAnalysisGroupsTableTextfield } from './__mocks__/filtered_analysis_groups_table_textfield'; +import { filteredAnalysisGroupsTable } from './__mocks__/filtered_analysis_groups_table'; +import { analysisTableTextfieldZerodocsfallback } from './__mocks__/analysis_table_textfield_zerodocsfallback'; +import { analysisTableZerodocsfallback } from './__mocks__/analysis_table_zerodocsfallback'; +import { analysisTableTextfield } from './__mocks__/analysis_table_textfield'; +import { analysisTable } from './__mocks__/analysis_table'; + +const REFERENCE_TS = 1669018354793; +const DAY_MS = 86400000; + +const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; +const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; + +interface GetArtificialLogDataViewTestDataOptions { + analysisType: LogRateAnalysisType; + textField: boolean; + zeroDocsFallback: boolean; +} + +export const getArtificialLogDataViewTestData = ({ + analysisType, + textField, + zeroDocsFallback, +}: GetArtificialLogDataViewTestDataOptions): TestData => { + function getAnalysisGroupsTable() { + if (zeroDocsFallback) { + return textField + ? analysisGroupsTableTextfieldZerodocsfallback + : analysisGroupsTableZerodocsfallback; + } + return textField ? analysisGroupsTableTextfield : analysisGroupsTable; + } + + function getFilteredAnalysisGroupsTable() { + if (zeroDocsFallback) { + return textField + ? filteredAnalysisGroupsTableTextfieldZerodocsfallback + : filteredAnalysisGroupsTableZerodocsfallback; + } + + return textField ? filteredAnalysisGroupsTableTextfield : filteredAnalysisGroupsTable; + } + + function getAnalysisTable() { + if (zeroDocsFallback) { + return textField ? analysisTableTextfieldZerodocsfallback : analysisTableZerodocsfallback; + } + + return textField ? analysisTableTextfield : analysisTable; + } + + function getFieldSelectorPopover() { + if (zeroDocsFallback) { + return [...(textField ? ['message'] : []), 'response_code', 'url', 'user', 'version']; + } + return [...(textField ? ['message'] : []), 'response_code', 'url', 'user']; + } + + function getSuiteTitle() { + return `artificial logs with ${analysisType} and ${ + textField ? 'text field' : 'no text field' + } and ${zeroDocsFallback ? 'zero docs fallback' : 'no zero docs fallback'}`; + } + + function getDataGenerator(): LogRateAnalysisDataGenerator { + return `artificial_logs_with_${analysisType}${textField ? '_textfield' : ''}${ + zeroDocsFallback ? '_zerodocsfallback' : '' + }`; + } + + function getBrushBaselineTargetTimestamp() { + if (analysisType === 'dip' && zeroDocsFallback) { + return DEVIATION_TS; + } + + return zeroDocsFallback ? BASELINE_TS - DAY_MS / 2 : BASELINE_TS + DAY_MS / 2; + } + + function getBrushDeviationTargetTimestamp() { + if (analysisType === 'dip' && zeroDocsFallback) { + return DEVIATION_TS + DAY_MS * 1.5; + } + + return zeroDocsFallback ? DEVIATION_TS : DEVIATION_TS + DAY_MS / 2; + } + + return { + suiteTitle: getSuiteTitle(), + analysisType, + dataGenerator: getDataGenerator(), + isSavedSearch: false, + sourceIndexOrSavedSearch: getDataGenerator(), + brushBaselineTargetTimestamp: getBrushBaselineTargetTimestamp(), + brushDeviationTargetTimestamp: getBrushDeviationTargetTimestamp(), + brushIntervalFactor: zeroDocsFallback ? 1 : 10, + chartClickCoordinates: [-200, 30], + fieldSelectorSearch: 'user', + fieldSelectorApplyAvailable: true, + expected: { + totalDocCountFormatted: zeroDocsFallback ? '9,482' : '8,400', + analysisGroupsTable: getAnalysisGroupsTable(), + filteredAnalysisGroupsTable: getFilteredAnalysisGroupsTable(), + analysisTable: getAnalysisTable(), + fieldSelectorPopover: getFieldSelectorPopover(), + }, + }; +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts new file mode 100644 index 0000000000000..be019193f2ef1 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +export const farequoteDataViewTestData: TestData = { + suiteTitle: 'farequote with spike', + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + dataGenerator: 'farequote_with_spike', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_farequote', + brushDeviationTargetTimestamp: 1455033600000, + brushIntervalFactor: 1, + chartClickCoordinates: [0, 0], + fieldSelectorSearch: 'airline', + fieldSelectorApplyAvailable: false, + expected: { + totalDocCountFormatted: '86,374', + sampleProbabilityFormatted: '0.5', + fieldSelectorPopover: ['airline', 'custom_field.keyword'], + }, +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts new file mode 100644 index 0000000000000..653889c1eb0ca --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/farequote_data_view_test_data_with_query.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +export const farequoteDataViewTestDataWithQuery: TestData = { + suiteTitle: 'farequote with spike', + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + dataGenerator: 'farequote_with_spike', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'ft_farequote', + brushDeviationTargetTimestamp: 1455033600000, + brushIntervalFactor: 1, + chartClickCoordinates: [0, 0], + fieldSelectorSearch: 'airline', + fieldSelectorApplyAvailable: false, + query: 'NOT airline:("SWR" OR "ACA" OR "AWE" OR "BAW" OR "JAL" OR "JBU" OR "JZA" OR "KLM")', + expected: { + totalDocCountFormatted: '48,799', + analysisGroupsTable: [ + { + docCount: '297', + group: '* airline: AAL', + }, + { + docCount: '100', + group: '* custom_field.keyword: deviation* airline: UAL', + }, + ], + analysisTable: [ + { + fieldName: 'airline', + fieldValue: 'AAL', + logRate: 'Chart type:bar chart', + pValue: '1.18e-8', + impact: 'High', + }, + ], + fieldSelectorPopover: ['airline', 'custom_field.keyword'], + }, +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts new file mode 100644 index 0000000000000..2d85fe1e64210 --- /dev/null +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis/test_data/kibana_logs_data_view_test_data.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; + +import type { TestData } from '../../types'; + +export const kibanaLogsDataViewTestData: TestData = { + suiteTitle: 'kibana sample data logs', + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + dataGenerator: 'kibana_sample_data_logs', + isSavedSearch: false, + sourceIndexOrSavedSearch: 'kibana_sample_data_logs', + brushIntervalFactor: 1, + chartClickCoordinates: [235, 0], + fieldSelectorSearch: 'referer', + fieldSelectorApplyAvailable: true, + action: { + type: 'LogPatternAnalysis', + tableRowId: '1064853178', + expected: { + queryBar: + 'clientip:30.156.16.164 AND host.keyword:elastic-elastic-elastic.org AND ip:30.156.16.163 AND response.keyword:404 AND machine.os.keyword:win xp AND geo.dest:IN AND geo.srcdest:US\\:IN', + totalDocCount: 100, + }, + }, + expected: { + totalDocCountFormatted: '14,074', + analysisGroupsTable: [ + { + group: + '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* referer: http://www.elastic-elastic-elastic.com/success/timothy-l-kopra* response.keyword: 404Showing 5 out of 8 items. 8 items unique to this group.', + docCount: '100', + }, + ], + filteredAnalysisGroupsTable: [ + { + group: + '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* response.keyword: 404* machine.os.keyword: win xpShowing 5 out of 7 items. 7 items unique to this group.', + docCount: '100', + }, + ], + analysisTable: [ + { + fieldName: 'clientip', + fieldValue: '30.156.16.164', + logRate: 'Chart type:bar chart', + pValue: '3.10e-13', + impact: 'High', + }, + { + fieldName: 'geo.dest', + fieldValue: 'IN', + logRate: 'Chart type:bar chart', + pValue: '0.000716', + impact: 'Medium', + }, + { + fieldName: 'geo.srcdest', + fieldValue: 'US:IN', + logRate: 'Chart type:bar chart', + pValue: '0.000716', + impact: 'Medium', + }, + { + fieldName: 'host.keyword', + fieldValue: 'elastic-elastic-elastic.org', + logRate: 'Chart type:bar chart', + pValue: '7.14e-9', + impact: 'High', + }, + { + fieldName: 'ip', + fieldValue: '30.156.16.163', + logRate: 'Chart type:bar chart', + pValue: '3.28e-13', + impact: 'High', + }, + { + fieldName: 'machine.os.keyword', + fieldValue: 'win xp', + logRate: 'Chart type:bar chart', + pValue: '0.0000997', + impact: 'Medium', + }, + { + fieldName: 'referer', + fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', + logRate: 'Chart type:bar chart', + pValue: '4.74e-13', + impact: 'High', + }, + { + fieldName: 'response.keyword', + fieldValue: '404', + logRate: 'Chart type:bar chart', + pValue: '0.00000604', + impact: 'Medium', + }, + ], + fieldSelectorPopover: [ + 'clientip', + 'geo.dest', + 'geo.srcdest', + 'host.keyword', + 'ip', + 'machine.os.keyword', + 'referer', + 'response.keyword', + ], + }, +}; diff --git a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts index 3fb1e00b95201..c3f33ff2aa5ba 100644 --- a/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts +++ b/x-pack/test/functional/apps/aiops/log_rate_analysis_test_data.ts @@ -5,289 +5,57 @@ * 2.0. */ -import { LOG_RATE_ANALYSIS_TYPE, type LogRateAnalysisType } from '@kbn/aiops-utils'; +import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-utils'; -import type { TestData } from './types'; - -export const kibanaLogsDataViewTestData: TestData = { - suiteTitle: 'kibana sample data logs', - analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, - dataGenerator: 'kibana_sample_data_logs', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'kibana_sample_data_logs', - brushIntervalFactor: 1, - chartClickCoordinates: [235, 0], - fieldSelectorSearch: 'referer', - fieldSelectorApplyAvailable: true, - action: { - type: 'LogPatternAnalysis', - tableRowId: '1064853178', - expected: { - queryBar: - 'clientip:30.156.16.164 AND host.keyword:elastic-elastic-elastic.org AND ip:30.156.16.163 AND response.keyword:404 AND machine.os.keyword:win xp AND geo.dest:IN AND geo.srcdest:US\\:IN', - totalDocCount: 100, - }, - }, - expected: { - totalDocCountFormatted: '14,074', - analysisGroupsTable: [ - { - group: - '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* referer: http://www.elastic-elastic-elastic.com/success/timothy-l-kopra* response.keyword: 404Showing 5 out of 8 items. 8 items unique to this group.', - docCount: '100', - }, - ], - filteredAnalysisGroupsTable: [ - { - group: - '* clientip: 30.156.16.164* host.keyword: elastic-elastic-elastic.org* ip: 30.156.16.163* response.keyword: 404* machine.os.keyword: win xpShowing 5 out of 7 items. 7 items unique to this group.', - docCount: '100', - }, - ], - analysisTable: [ - { - fieldName: 'clientip', - fieldValue: '30.156.16.164', - logRate: 'Chart type:bar chart', - pValue: '3.10e-13', - impact: 'High', - }, - { - fieldName: 'geo.dest', - fieldValue: 'IN', - logRate: 'Chart type:bar chart', - pValue: '0.000716', - impact: 'Medium', - }, - { - fieldName: 'geo.srcdest', - fieldValue: 'US:IN', - logRate: 'Chart type:bar chart', - pValue: '0.000716', - impact: 'Medium', - }, - { - fieldName: 'host.keyword', - fieldValue: 'elastic-elastic-elastic.org', - logRate: 'Chart type:bar chart', - pValue: '7.14e-9', - impact: 'High', - }, - { - fieldName: 'ip', - fieldValue: '30.156.16.163', - logRate: 'Chart type:bar chart', - pValue: '3.28e-13', - impact: 'High', - }, - { - fieldName: 'machine.os.keyword', - fieldValue: 'win xp', - logRate: 'Chart type:bar chart', - pValue: '0.0000997', - impact: 'Medium', - }, - { - fieldName: 'referer', - fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', - logRate: 'Chart type:bar chart', - pValue: '4.74e-13', - impact: 'High', - }, - { - fieldName: 'response.keyword', - fieldValue: '404', - logRate: 'Chart type:bar chart', - pValue: '0.00000604', - impact: 'Medium', - }, - ], - fieldSelectorPopover: [ - 'clientip', - 'geo.dest', - 'geo.srcdest', - 'host.keyword', - 'ip', - 'machine.os.keyword', - 'referer', - 'response.keyword', - ], - }, -}; - -export const farequoteDataViewTestData: TestData = { - suiteTitle: 'farequote with spike', - analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, - dataGenerator: 'farequote_with_spike', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'ft_farequote', - brushDeviationTargetTimestamp: 1455033600000, - brushIntervalFactor: 1, - chartClickCoordinates: [0, 0], - fieldSelectorSearch: 'airline', - fieldSelectorApplyAvailable: false, - expected: { - totalDocCountFormatted: '86,374', - sampleProbabilityFormatted: '0.5', - fieldSelectorPopover: ['airline', 'custom_field.keyword'], - }, -}; - -export const farequoteDataViewTestDataWithQuery: TestData = { - suiteTitle: 'farequote with spike', - analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, - dataGenerator: 'farequote_with_spike', - isSavedSearch: false, - sourceIndexOrSavedSearch: 'ft_farequote', - brushDeviationTargetTimestamp: 1455033600000, - brushIntervalFactor: 1, - chartClickCoordinates: [0, 0], - fieldSelectorSearch: 'airline', - fieldSelectorApplyAvailable: false, - query: 'NOT airline:("SWR" OR "ACA" OR "AWE" OR "BAW" OR "JAL" OR "JBU" OR "JZA" OR "KLM")', - expected: { - totalDocCountFormatted: '48,799', - analysisGroupsTable: [ - { - docCount: '297', - group: '* airline: AAL', - }, - { - docCount: '100', - group: '* custom_field.keyword: deviation* airline: UAL', - }, - ], - analysisTable: [ - { - fieldName: 'airline', - fieldValue: 'AAL', - logRate: 'Chart type:bar chart', - pValue: '1.18e-8', - impact: 'High', - }, - ], - fieldSelectorPopover: ['airline', 'custom_field.keyword'], - }, -}; +import { kibanaLogsDataViewTestData } from './log_rate_analysis/test_data/kibana_logs_data_view_test_data'; +import { farequoteDataViewTestData } from './log_rate_analysis/test_data/farequote_data_view_test_data'; +import { farequoteDataViewTestDataWithQuery } from './log_rate_analysis/test_data/farequote_data_view_test_data_with_query'; +import { getArtificialLogDataViewTestData } from './log_rate_analysis/test_data/artificial_log_data_view_test_data'; -const REFERENCE_TS = 1669018354793; -const DAY_MS = 86400000; - -const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; -const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; - -const getArtificialLogDataViewTestData = ( - analysisType: LogRateAnalysisType, - textField: boolean -): TestData => ({ - suiteTitle: `artificial logs with ${analysisType} and ${ - textField ? 'text field' : 'no text field' - }`, - analysisType, - dataGenerator: `artificial_logs_with_${analysisType}_${textField ? 'textfield' : 'notextfield'}`, - isSavedSearch: false, - sourceIndexOrSavedSearch: `artificial_logs_with_${analysisType}_${ - textField ? 'textfield' : 'notextfield' - }`, - brushBaselineTargetTimestamp: BASELINE_TS + DAY_MS / 2, - brushDeviationTargetTimestamp: DEVIATION_TS + DAY_MS / 2, - brushIntervalFactor: 10, - chartClickCoordinates: [-200, 30], - fieldSelectorSearch: 'user', - fieldSelectorApplyAvailable: true, - expected: { - totalDocCountFormatted: '8,400', - analysisGroupsTable: [ - textField - ? { - group: 'message: an unexpected error occuredurl: home.phpresponse_code: 500', - docCount: '634', - } - : { - group: 'response_code: 500url: home.php', - docCount: '792', - }, - textField - ? { - group: 'message: an unexpected error occuredurl: login.phpresponse_code: 500', - docCount: '632', - } - : { - group: 'url: login.phpresponse_code: 500', - docCount: '790', - }, - { - docCount: '636', - group: 'user: Peterurl: home.php', - }, - { - docCount: '632', - group: 'user: Peterurl: login.php', - }, - ], - filteredAnalysisGroupsTable: textField - ? [ - { - group: '* url: home.phpmessage: an unexpected error occuredresponse_code: 500', - docCount: '634', - }, - { - group: '* url: login.phpmessage: an unexpected error occuredresponse_code: 500', - docCount: '632', - }, - ] - : [ - { group: '* url: home.phpresponse_code: 500', docCount: '792' }, - { group: '* url: login.phpresponse_code: 500', docCount: '790' }, - ], - analysisTable: [ - ...(textField - ? [ - { - fieldName: 'message', - fieldValue: 'an unexpected error occured', - logRate: 'Chart type:bar chart', - pValue: '0.00000100', - impact: 'Medium', - }, - { - fieldName: 'response_code', - fieldValue: '500', - logRate: 'Chart type:bar chart', - pValue: '3.61e-12', - impact: 'High', - }, - ] - : []), - { - fieldName: 'url', - fieldValue: 'home.php', - impact: 'Low', - logRate: 'Chart type:bar chart', - pValue: '0.00974', - }, - ...(textField - ? [] - : [ - { - fieldName: 'user', - fieldValue: 'Peter', - impact: 'High', - logRate: 'Chart type:bar chart', - pValue: '2.63e-21', - }, - ]), - ], - fieldSelectorPopover: [...(textField ? ['message'] : []), 'response_code', 'url', 'user'], - }, -}); +import type { TestData } from './types'; export const logRateAnalysisTestData: TestData[] = [ kibanaLogsDataViewTestData, farequoteDataViewTestData, farequoteDataViewTestDataWithQuery, - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.SPIKE, true), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, false), - getArtificialLogDataViewTestData(LOG_RATE_ANALYSIS_TYPE.DIP, true), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: false, + zeroDocsFallback: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: true, + zeroDocsFallback: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: false, + zeroDocsFallback: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: true, + zeroDocsFallback: false, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: true, + zeroDocsFallback: true, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.SPIKE, + textField: false, + zeroDocsFallback: true, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: true, + zeroDocsFallback: true, + }), + getArtificialLogDataViewTestData({ + analysisType: LOG_RATE_ANALYSIS_TYPE.DIP, + textField: false, + zeroDocsFallback: true, + }), ]; diff --git a/x-pack/test/functional/apps/aiops/types.ts b/x-pack/test/functional/apps/aiops/types.ts index 45f376cb670e1..72a72b1d2bb37 100644 --- a/x-pack/test/functional/apps/aiops/types.ts +++ b/x-pack/test/functional/apps/aiops/types.ts @@ -8,7 +8,7 @@ import type { LogRateAnalysisType } from '@kbn/aiops-utils'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import { LogRateAnalysisDataGenerator } from '../../services/aiops/log_rate_analysis_data_generator'; +import type { LogRateAnalysisDataGenerator } from '../../services/aiops/log_rate_analysis_data_generator'; interface TestDataTableActionLogPatternAnalysis { type: 'LogPatternAnalysis'; diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts index 48028b2ddbd1a..7e1ead80b4ff8 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_data_generator.ts @@ -14,9 +14,15 @@ import { FtrProviderContext } from '../../ftr_provider_context'; const LOG_RATE_ANALYSYS_DATA_GENERATOR = { KIBANA_SAMPLE_DATA_LOGS: 'kibana_sample_data_logs', FAREQUOTE_WITH_SPIKE: 'farequote_with_spike', - ARTIFICIAL_LOGS_WITH_SPIKE_NOTEXTFIELD: 'artificial_logs_with_spike_notextfield', + ARTIFICIAL_LOGS_WITH_SPIKE_ZERODOCSFALLBACK: 'artificial_logs_with_spike_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD_ZERODOCSFALLBACK: + 'artificial_logs_with_spike_textfield_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_DIP_ZERODOCSFALLBACK: 'artificial_logs_with_dip_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD_ZERODOCSFALLBACK: + 'artificial_logs_with_dip_textfield_zerodocsfallback', + ARTIFICIAL_LOGS_WITH_SPIKE: 'artificial_logs_with_spike', ARTIFICIAL_LOGS_WITH_SPIKE_TEXTFIELD: 'artificial_logs_with_spike_textfield', - ARTIFICIAL_LOGS_WITH_DIP_NOTEXTFIELD: 'artificial_logs_with_dip_notextfield', + ARTIFICIAL_LOGS_WITH_DIP: 'artificial_logs_with_dip', ARTIFICIAL_LOGS_WITH_DIP_TEXTFIELD: 'artificial_logs_with_dip_textfield', } as const; export type LogRateAnalysisDataGenerator = @@ -38,7 +44,7 @@ const DAY_MS = 86400000; const DEVIATION_TS = REFERENCE_TS - DAY_MS * 2; const BASELINE_TS = DEVIATION_TS - DAY_MS * 1; -function getMessage(timestamp: number, user: string, url: string, responseCode: string) { +function getTextFieldMessage(timestamp: number, user: string, url: string, responseCode: string) { const date = new Date(timestamp); return `${user} [${date.toLocaleString('en-US')}] "GET /${url} HTTP/1.1" ${responseCode}`; } @@ -46,12 +52,37 @@ function getMessage(timestamp: number, user: string, url: string, responseCode: function getArtificialLogsWithDeviation( index: string, deviationType: string, - includeTextField = false + includeTextField = false, + includeGaps = false ) { const bulkBody: estypes.BulkRequest['body'] = []; const action = { index: { _index: index } }; let tsOffset = 0; + if (includeGaps) { + const earliestDoc: GeneratedDoc = { + user: 'Peter', + response_code: '200', + url: 'login.php', + version: 'v1.0.0', + '@timestamp': BASELINE_TS - DAY_MS, + should_ignore_this_field: 'should_ignore_this_field', + }; + bulkBody.push(action); + bulkBody.push(earliestDoc); + + const latestDoc: GeneratedDoc = { + user: 'Peter', + response_code: '200', + url: 'login.php', + version: 'v1.0.0', + '@timestamp': DEVIATION_TS + 2 * DAY_MS, + should_ignore_this_field: 'should_ignore_this_field', + }; + bulkBody.push(action); + bulkBody.push(latestDoc); + } + // Creates docs evenly spread across baseline and deviation time frame [BASELINE_TS, DEVIATION_TS].forEach((ts) => { ['Peter', 'Paul', 'Mary'].forEach((user) => { @@ -66,8 +97,32 @@ function getArtificialLogsWithDeviation( ) ) { tsOffset = 0; - [...Array(100)].forEach(() => { - tsOffset += Math.round(DAY_MS / 100); + + let docCount = 100; + let responseCodeFactor = 1; + + if (includeGaps) { + if (responseCode === '404') { + responseCodeFactor = 2; + } else if (responseCode === '500') { + responseCodeFactor = 3; + } + + if (url === 'user.php') { + responseCodeFactor *= 2; + } else if (url === 'home.php') { + responseCodeFactor *= 3; + } + + if (user === 'Paul') { + docCount = 40 * responseCodeFactor; + } else if (user === 'Mary') { + docCount = 25 * responseCodeFactor; + } + } + + [...Array(docCount)].forEach(() => { + tsOffset += Math.round(DAY_MS / docCount); const timestamp = ts + tsOffset; const doc: GeneratedDoc = { user, @@ -79,7 +134,7 @@ function getArtificialLogsWithDeviation( }; if (includeTextField) { - doc.message = getMessage(timestamp, user, url, responseCode); + doc.message = getTextFieldMessage(timestamp, user, url, responseCode); } bulkBody.push(action); @@ -116,7 +171,7 @@ function getArtificialLogsWithDeviation( }; if (includeTextField) { - doc.message = getMessage(timestamp, 'Peter', url, responseCode); + doc.message = getTextFieldMessage(timestamp, 'Peter', url, responseCode); } bulkBody.push(action); @@ -204,10 +259,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); break; - case 'artificial_logs_with_spike_notextfield': + case 'artificial_logs_with_spike': case 'artificial_logs_with_spike_textfield': - case 'artificial_logs_with_dip_notextfield': + case 'artificial_logs_with_dip': case 'artificial_logs_with_dip_textfield': + case 'artificial_logs_with_spike_zerodocsfallback': + case 'artificial_logs_with_spike_textfield_zerodocsfallback': + case 'artificial_logs_with_dip_zerodocsfallback': + case 'artificial_logs_with_dip_textfield_zerodocsfallback': try { const indexExists = await es.indices.exists({ index: dataGenerator, @@ -238,12 +297,26 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider }); const dataGeneratorOptions = dataGenerator.split('_'); - const deviationType = dataGeneratorOptions[3] ?? LOG_RATE_ANALYSIS_TYPE.SPIKE; - const textField = dataGeneratorOptions[4] === 'textfield' ?? false; + + let deviationType = dataGeneratorOptions.includes(LOG_RATE_ANALYSIS_TYPE.SPIKE) + ? LOG_RATE_ANALYSIS_TYPE.SPIKE + : LOG_RATE_ANALYSIS_TYPE.DIP; + + const textField = dataGeneratorOptions.includes('textfield'); + const zeroDocsFallback = dataGeneratorOptions.includes('zerodocsfallback'); + + if (zeroDocsFallback) { + deviationType = LOG_RATE_ANALYSIS_TYPE.SPIKE; + } await es.bulk({ refresh: 'wait_for', - body: getArtificialLogsWithDeviation(dataGenerator, deviationType, textField), + body: getArtificialLogsWithDeviation( + dataGenerator, + deviationType, + textField, + zeroDocsFallback + ), }); break; @@ -262,10 +335,14 @@ export function LogRateAnalysisDataGeneratorProvider({ getService }: FtrProvider await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote'); break; - case 'artificial_logs_with_spike_notextfield': + case 'artificial_logs_with_spike': case 'artificial_logs_with_spike_textfield': - case 'artificial_logs_with_dip_notextfield': + case 'artificial_logs_with_dip': case 'artificial_logs_with_dip_textfield': + case 'artificial_logs_with_spike_zerodocsfallback': + case 'artificial_logs_with_spike_textfield_zerodocsfallback': + case 'artificial_logs_with_dip_zerodocsfallback': + case 'artificial_logs_with_dip_textfield_zerodocsfallback': try { await es.indices.delete({ index: dataGenerator, diff --git a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts index ea2251a475889..a8733fb2114a7 100644 --- a/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts +++ b/x-pack/test/functional/services/aiops/log_rate_analysis_page.ts @@ -11,6 +11,8 @@ import type { LogRateAnalysisType } from '@kbn/aiops-utils'; import type { FtrProviderContext } from '../../ftr_provider_context'; +import type { LogRateAnalysisDataGenerator } from './log_rate_analysis_data_generator'; + export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrProviderContext) { const browser = getService('browser'); const elasticChart = getService('elasticChart'); @@ -241,7 +243,12 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr }); }, - async assertAnalysisComplete(analisysType: LogRateAnalysisType) { + async assertAnalysisComplete( + analysisType: LogRateAnalysisType, + dataGenerator: LogRateAnalysisDataGenerator + ) { + const dataGeneratorParts = dataGenerator.split('_'); + const zeroDocsFallback = dataGeneratorParts.includes('zerodocsfallback'); await retry.tryForTime(30 * 1000, async () => { await testSubjects.existOrFail('aiopsAnalysisComplete'); const currentProgressTitle = await testSubjects.getVisibleText('aiopsAnalysisComplete'); @@ -251,7 +258,18 @@ export function LogRateAnalysisPageProvider({ getService, getPageObject }: FtrPr const currentAnalysisTypeCalloutTitle = await testSubjects.getVisibleText( 'aiopsAnalysisTypeCalloutTitle' ); - expect(currentAnalysisTypeCalloutTitle).to.be(`Analysis type: Log rate ${analisysType}`); + + if (zeroDocsFallback && analysisType === 'spike') { + expect(currentAnalysisTypeCalloutTitle).to.be( + 'Analysis type: Top items for deviation time range' + ); + } else if (zeroDocsFallback && analysisType === 'dip') { + expect(currentAnalysisTypeCalloutTitle).to.be( + 'Analysis type: Top items for baseline time range' + ); + } else { + expect(currentAnalysisTypeCalloutTitle).to.be(`Analysis type: Log rate ${analysisType}`); + } }); },