[8.11] [ML] AIOps: Functional/API integration tests for text field su…

…pport for log rate analysis (#168177) (#168516) # Backport This will backport the following commits from `main` to `8.11`: - [[ML] AIOps: Functional/API integration tests for text field support for log rate analysis (#168177)](#168177)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Walter Rafelsberger <[email protected]>
elastic · Oct 10, 2023 · a88e7d2 · a88e7d2
1 parent abb04cd
commit a88e7d2
Show file tree

Hide file tree

Showing 25 changed files with 603 additions and 142 deletions.
diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_item_sets.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_item_sets.ts
@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import type { ItemsetResult } from '../../types';
+import type { ItemSet } from '../../types';
 
-export const filteredFrequentItemSets: ItemsetResult[] = [
+export const filteredFrequentItemSets: ItemSet[] = [
   {
     set: { response_code: '500', url: 'home.php' },
     size: 2,

diff --git a/...plugins/aiops/common/__mocks__/artificial_logs/final_significant_term_groups_textfield.ts b/...plugins/aiops/common/__mocks__/artificial_logs/final_significant_term_groups_textfield.ts
@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SignificantTermGroup } from '@kbn/ml-agg-utils';
+
+export const finalSignificantTermGroupsTextfield: SignificantTermGroup[] = [
+  {
+    docCount: 636,
+    group: [
+      {
+        docCount: 792,
+        duplicate: 2,
+        fieldName: 'url',
+        fieldValue: 'home.php',
+        key: 'url:home.php',
+        pValue: 0.00974308761016614,
+        type: 'keyword',
+      },
+      {
+        docCount: 636,
+        duplicate: 2,
+        fieldName: 'user',
+        fieldValue: 'Peter',
+        key: 'user:Peter',
+        pValue: 0.00974308761016614,
+        type: 'keyword',
+      },
+    ],
+    id: '2091742187',
+    pValue: 0.00974308761016614,
+  },
+  {
+    docCount: 634,
+    group: [
+      {
+        docCount: 1266,
+        duplicate: 2,
+        fieldName: 'response_code',
+        fieldValue: '500',
+        key: 'response_code:500',
+        pValue: 0.012783309213417932,
+        type: 'keyword',
+      },
+      {
+        docCount: 792,
+        duplicate: 2,
+        fieldName: 'url',
+        fieldValue: 'home.php',
+        key: 'url:home.php',
+        pValue: 0.00974308761016614,
+        type: 'keyword',
+      },
+      {
+        docCount: 634,
+        duplicate: 2,
+        fieldName: 'message',
+        fieldValue: 'an unexpected error occured',
+        key: 'an unexpected error occured',
+        pValue: 0.00974308761016614,
+        type: 'log_pattern',
+      },
+    ],
+    id: '1528268618',
+    pValue: 0.00974308761016614,
+  },
+  {
+    docCount: 632,
+    group: [
+      {
+        docCount: 1266,
+        duplicate: 2,
+        fieldName: 'response_code',
+        fieldValue: '500',
+        key: 'response_code:500',
+        pValue: 0.012783309213417932,
+        type: 'keyword',
+      },
+      {
+        docCount: 790,
+        duplicate: 2,
+        fieldName: 'url',
+        fieldValue: 'login.php',
+        key: 'url:login.php',
+        pValue: 0.012783309213417932,
+        type: 'keyword',
+      },
+      {
+        docCount: 632,
+        duplicate: 2,
+        fieldName: 'message',
+        fieldValue: 'an unexpected error occured',
+        key: 'an unexpected error occured',
+        pValue: 0.012783309213417932,
+        type: 'log_pattern',
+      },
+    ],
+    id: '2619569380',
+    pValue: 0.012783309213417932,
+  },
+  {
+    docCount: 632,
+    group: [
+      {
+        docCount: 790,
+        duplicate: 2,
+        fieldName: 'url',
+        fieldValue: 'login.php',
+        key: 'url:login.php',
+        pValue: 0.012783309213417932,
+        type: 'keyword',
+      },
+      {
+        docCount: 632,
+        duplicate: 2,
+        fieldName: 'user',
+        fieldValue: 'Peter',
+        key: 'user:Peter',
+        pValue: 0.012783309213417932,
+        type: 'keyword',
+      },
+    ],
+    id: '1937394803',
+    pValue: 0.012783309213417932,
+  },
+];
diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_item_sets.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_item_sets.ts
@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import type { ItemsetResult } from '../../types';
+import type { ItemSet } from '../../types';
 
-export const frequentItemSets: ItemsetResult[] = [
+export const frequentItemSets: ItemSet[] = [
   {
     set: { response_code: '500', url: 'home.php' },
     size: 2,

diff --git a/x-pack/plugins/aiops/common/__mocks__/artificial_logs/significant_log_patterns.ts b/x-pack/plugins/aiops/common/__mocks__/artificial_logs/significant_log_patterns.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SignificantTerm } from '@kbn/ml-agg-utils';
+
+export const significantLogPatterns: SignificantTerm[] = [
+  {
+    bg_count: 0,
+    doc_count: 1266,
+    fieldName: 'message',
+    fieldValue: 'an unexpected error occured',
+    key: 'an unexpected error occured',
+    normalizedScore: 0,
+    pValue: 0.000001,
+    score: -13.815510557964274,
+    total_bg_count: 1975,
+    total_doc_count: 4669,
+    type: 'log_pattern',
+  },
+];
diff --git a/x-pack/plugins/aiops/common/constants.ts b/x-pack/plugins/aiops/common/constants.ts
@@ -5,10 +5,16 @@
  * 2.0.
  */
 
-/**
- * The p-value threshold to be used for statistically significant items.
- */
-export const LOG_RATE_ANALYSIS_P_VALUE_THRESHOLD = 0.02;
+export const LOG_RATE_ANALYSIS_SETTINGS = {
+  /**
+   * The p-value threshold to be used for statistically significant items.
+   */
+  P_VALUE_THRESHOLD: 0.02,
+  /**
+   * The minimum support value to be used for the frequent item sets aggration.
+   */
+  FREQUENT_ITEMS_SETS_MINIMUM_SUPPORT: 0.001,
+} as const;
 
 /**
  * For the technical preview of Log Rate Analysis we use a hard coded seed.

diff --git a/x-pack/plugins/aiops/common/types.ts b/x-pack/plugins/aiops/common/types.ts
@@ -14,7 +14,7 @@ export interface SignificantTermDuplicateGroup {
 
 export type FieldValuePairCounts = Record<string, Record<string, number>>;
 
-export interface ItemsetResult {
+export interface ItemSet {
   set: Record<FieldValuePair['fieldName'], FieldValuePair['fieldValue']>;
   size: number;
   maxPValue: number;
@@ -23,6 +23,12 @@ export interface ItemsetResult {
   total_doc_count: number;
 }
 
+export interface FetchFrequentItemSetsResponse {
+  fields: string[];
+  itemSets: ItemSet[];
+  totalDocCount: number;
+}
+
 interface SimpleHierarchicalTreeNodeSet extends FieldValuePair {
   key: string;
   type: SignificantTermType;

diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis.ts
@@ -506,7 +506,7 @@ export const defineLogRateAnalysisRoute = (
                 );
 
                 try {
-                  const { fields, df } = await fetchFrequentItemSets(
+                  const { fields, itemSets } = await fetchFrequentItemSets(
                     client,
                     request.body.index,
                     JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer,
@@ -520,23 +520,26 @@ export const defineLogRateAnalysisRoute = (
                     abortSignal
                   );
 
-                  if (significantCategories.length > 0) {
-                    const { fields: significantCategoriesFields, df: significantCategoriesDf } =
-                      await fetchTerms2CategoriesCounts(
-                        client,
-                        request.body,
-                        JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer,
-                        significantTerms,
-                        significantCategories,
-                        request.body.deviationMin,
-                        request.body.deviationMax,
-                        logger,
-                        pushError,
-                        abortSignal
-                      );
+                  if (significantCategories.length > 0 && significantTerms.length > 0) {
+                    const {
+                      fields: significantCategoriesFields,
+                      itemSets: significantCategoriesItemSets,
+                    } = await fetchTerms2CategoriesCounts(
+                      client,
+                      request.body,
+                      JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer,
+                      significantTerms,
+                      itemSets,
+                      significantCategories,
+                      request.body.deviationMin,
+                      request.body.deviationMax,
+                      logger,
+                      pushError,
+                      abortSignal
+                    );
 
                     fields.push(...significantCategoriesFields);
-                    df.push(...significantCategoriesDf);
+                    itemSets.push(...significantCategoriesItemSets);
                   }
 
                   if (shouldStop) {
@@ -545,9 +548,9 @@ export const defineLogRateAnalysisRoute = (
                     return;
                   }
 
-                  if (fields.length > 0 && df.length > 0) {
+                  if (fields.length > 0 && itemSets.length > 0) {
                     const significantTermGroups = getSignificantTermGroups(
-                      df,
+                      itemSets,
                       [...significantTerms, ...significantCategories],
                       fields
                     );
@@ -757,7 +760,11 @@ export const defineLogRateAnalysisRoute = (
               }
 
               // histograms for text field patterns
-              if (overallTimeSeries !== undefined && significantCategories.length > 0) {
+              if (
+                overallTimeSeries !== undefined &&
+                significantCategories.length > 0 &&
+                !request.body.overrides?.regroupOnly
+              ) {
                 const significantCategoriesHistogramQueries = significantCategories.map((d) => {
                   const histogramQuery = getHistogramQuery(request.body);
                   const categoryQuery = getCategoryQuery(d.fieldName, [

diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_categories.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_categories.ts
@@ -33,11 +33,14 @@ export const getCategoryRequest = (
   fieldName: string,
   from: number | undefined,
   to: number | undefined,
+  filter: estypes.QueryDslQueryContainer,
   { wrap }: RandomSamplerWrapper
 ): estypes.SearchRequest => {
   const { index, timeFieldName } = params;
   const query = getQueryWithParams({
     params,
+    termFilters: undefined,
+    filter,
   });
   const { params: request } = createCategoryRequest(
     index,
@@ -63,6 +66,7 @@ export const fetchCategories = async (
   fieldNames: string[],
   from: number | undefined,
   to: number | undefined,
+  filter: estypes.QueryDslQueryContainer,
   logger: Logger,
   // The default value of 1 means no sampling will be used
   sampleProbability: number = 1,
@@ -78,7 +82,7 @@ export const fetchCategories = async (
 
   const settledPromises = await Promise.allSettled(
     fieldNames.map((fieldName) => {
-      const request = getCategoryRequest(params, fieldName, from, to, randomSamplerWrapper);
+      const request = getCategoryRequest(params, fieldName, from, to, filter, randomSamplerWrapper);
       return esClient.search(request, {
         signal: abortSignal,
         maxRetries: 0,

diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_item_sets.ts
@@ -15,8 +15,12 @@ import type { Logger } from '@kbn/logging';
 import { type SignificantTerm } from '@kbn/ml-agg-utils';
 import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
 
-import { RANDOM_SAMPLER_SEED } from '../../../common/constants';
-import type { SignificantTermDuplicateGroup, ItemsetResult } from '../../../common/types';
+import { RANDOM_SAMPLER_SEED, LOG_RATE_ANALYSIS_SETTINGS } from '../../../common/constants';
+import type {
+  SignificantTermDuplicateGroup,
+  ItemSet,
+  FetchFrequentItemSetsResponse,
+} from '../../../common/types';
 
 interface FrequentItemSetsAggregation extends estypes.AggregationsSamplerAggregation {
   fi: {
@@ -74,7 +78,7 @@ export async function fetchFrequentItemSets(
   sampleProbability: number = 1,
   emitError: (m: string) => void,
   abortSignal?: AbortSignal
-) {
+): Promise<FetchFrequentItemSetsResponse> {
   // Sort significant terms by ascending p-value, necessary to apply the field limit correctly.
   const sortedSignificantTerms = significantTerms.slice().sort((a, b) => {
     return (a.pValue ?? 0) - (b.pValue ?? 0);
@@ -103,7 +107,7 @@ export async function fetchFrequentItemSets(
       frequent_item_sets: {
         minimum_set_size: 2,
         size: 200,
-        minimum_support: 0.001,
+        minimum_support: LOG_RATE_ANALYSIS_SETTINGS.FREQUENT_ITEMS_SETS_MINIMUM_SUPPORT,
         fields: getFrequentItemSetsAggFields(sortedSignificantTerms),
       },
     },
@@ -138,7 +142,7 @@ export async function fetchFrequentItemSets(
     emitError(`Failed to fetch frequent_item_sets.`);
     return {
       fields: [],
-      df: [],
+      itemSets: [],
       totalDocCount: 0,
     };
   }
@@ -158,10 +162,10 @@ export async function fetchFrequentItemSets(
   const fiss = frequentItemSets.fi.buckets;
   fiss.length = maximum;
 
-  const results: ItemsetResult[] = [];
+  const results: ItemSet[] = [];
 
   fiss.forEach((fis) => {
-    const result: ItemsetResult = {
+    const result: ItemSet = {
       set: {},
       size: 0,
       maxPValue: 0,
@@ -203,7 +207,7 @@ export async function fetchFrequentItemSets(
 
   return {
     fields: uniqueFields,
-    df: results,
+    itemSets: results,
     totalDocCount: totalDocCountFi,
   };
 }