[SLO] Account for the built-in delay for burn rate alerting

elastic · Oct 16, 2023 · b487618 · b487618
1 parent eef222f
commit b487618
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 21 deletions.
diff --git a/x-pack/plugins/observability/public/components/slo/error_rate_chart/error_rate_chart.tsx b/x-pack/plugins/observability/public/components/slo/error_rate_chart/error_rate_chart.tsx
@@ -10,6 +10,7 @@ import { SLOResponse } from '@kbn/slo-schema';
 import moment from 'moment';
 import React from 'react';
 import { useKibana } from '../../../utils/kibana_react';
+import { getDelayInSecondsFromSLO } from '../../../utils/slo/get_delay_in_seconds_from_slo';
 import { useLensDefinition } from './use_lens_definition';
 
 interface Props {
@@ -22,14 +23,18 @@ export function ErrorRateChart({ slo, fromRange }: Props) {
     lens: { EmbeddableComponent },
   } = useKibana().services;
   const lensDef = useLensDefinition(slo);
+  const delayInSeconds = getDelayInSecondsFromSLO(slo);
+
+  const from = moment(fromRange).subtract(delayInSeconds, 'seconds').toISOString();
+  const to = moment().subtract(delayInSeconds, 'seconds').toISOString();
 
   return (
     <EmbeddableComponent
       id="sloErrorRateChart"
       style={{ height: 190 }}
       timeRange={{
-        from: fromRange.toISOString(),
-        to: moment().toISOString(),
+        from,
+        to,
       }}
       attributes={lensDef}
       viewMode={ViewMode.VIEW}

diff --git a/x-pack/plugins/observability/public/components/slo/error_rate_chart/use_lens_definition.ts b/x-pack/plugins/observability/public/components/slo/error_rate_chart/use_lens_definition.ts
@@ -7,11 +7,15 @@
 
 import { useEuiTheme } from '@elastic/eui';
 import { TypedLensByValueInput } from '@kbn/lens-plugin/public';
-import { ALL_VALUE, SLOResponse } from '@kbn/slo-schema';
+import { ALL_VALUE, SLOResponse, timeslicesBudgetingMethodSchema } from '@kbn/slo-schema';
 
 export function useLensDefinition(slo: SLOResponse): TypedLensByValueInput['attributes'] {
   const { euiTheme } = useEuiTheme();
 
+  const interval = timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)
+    ? slo.objective.timesliceWindow
+    : '60s';
+
   return {
     title: 'SLO Error Rate',
     description: '',
@@ -125,7 +129,7 @@ export function useLensDefinition(slo: SLOResponse): TypedLensByValueInput['attr
                   scale: 'interval',
                   params: {
                     // @ts-ignore
-                    interval: 'auto',
+                    interval,
                     includeEmptyRows: true,
                     dropPartials: false,
                   },

diff --git a/x-pack/plugins/observability/public/utils/slo/get_delay_in_seconds_from_slo.ts b/x-pack/plugins/observability/public/utils/slo/get_delay_in_seconds_from_slo.ts
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SLOResponse, timeslicesBudgetingMethodSchema, durationType } from '@kbn/slo-schema';
+import { isLeft } from 'fp-ts/lib/Either';
+
+export function getDelayInSecondsFromSLO(slo: SLOResponse) {
+  const fixedInterval = timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)
+    ? durationStringToSeconds(slo.objective.timesliceWindow)
+    : 60;
+  const syncDelay = durationStringToSeconds(slo.settings.syncDelay);
+  const frequency = durationStringToSeconds(slo.settings.frequency);
+  return fixedInterval + syncDelay + frequency;
+}
+
+function durationStringToSeconds(duration: string | undefined) {
+  if (!duration) {
+    return 0;
+  }
+  const result = durationType.decode(duration);
+  if (isLeft(result)) {
+    throw new Error(`Invalid duration string: ${duration}`);
+  }
+  return result.right.asSeconds();
+}
diff --git a/x-pack/plugins/observability/server/domain/services/get_delay_in_seconds_from_slo.ts b/x-pack/plugins/observability/server/domain/services/get_delay_in_seconds_from_slo.ts
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { timeslicesBudgetingMethodSchema } from '@kbn/slo-schema';
+import { SLO } from '../models';
+
+export function getDelayInSecondsFromSLO(slo: SLO) {
+  const fixedInterval = timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)
+    ? slo.objective.timesliceWindow!.asSeconds()
+    : 60;
+  const syncDelay = slo.settings.syncDelay.asSeconds();
+  const frequency = slo.settings.frequency.asSeconds();
+  return fixedInterval + syncDelay + frequency;
+}
diff --git a/x-pack/plugins/observability/server/lib/rules/slo_burn_rate/lib/build_query.ts b/x-pack/plugins/observability/server/lib/rules/slo_burn_rate/lib/build_query.ts
@@ -9,6 +9,7 @@ import moment from 'moment';
 import { timeslicesBudgetingMethodSchema } from '@kbn/slo-schema';
 import { Duration, SLO, toDurationUnit, toMomentUnitOfTime } from '../../../../domain/models';
 import { BurnRateRuleParams, WindowSchema } from '../types';
+import { getDelayInSecondsFromSLO } from '../../../../domain/services/get_delay_in_seconds_from_slo';
 
 type BurnRateWindowWithDuration = WindowSchema & {
   longDuration: Duration;
@@ -99,10 +100,15 @@ function buildWindowAgg(
   };
 }
 
-function buildWindowAggs(startedAt: Date, slo: SLO, burnRateWindows: BurnRateWindowWithDuration[]) {
+function buildWindowAggs(
+  startedAt: Date,
+  slo: SLO,
+  burnRateWindows: BurnRateWindowWithDuration[],
+  delayInSeconds = 0
+) {
   return burnRateWindows.reduce((acc, winDef, index) => {
-    const shortDateRange = getLookbackDateRange(startedAt, winDef.shortDuration);
-    const longDateRange = getLookbackDateRange(startedAt, winDef.longDuration);
+    const shortDateRange = getLookbackDateRange(startedAt, winDef.shortDuration, delayInSeconds);
+    const longDateRange = getLookbackDateRange(startedAt, winDef.longDuration, delayInSeconds);
     const windowId = generateWindowId(index);
     return {
       ...acc,
@@ -155,6 +161,7 @@ export function buildQuery(
   params: BurnRateRuleParams,
   afterKey?: EvaluationAfterKey
 ) {
+  const delayInSeconds = getDelayInSecondsFromSLO(slo);
   const burnRateWindows = params.windows.map((winDef) => {
     return {
       ...winDef,
@@ -169,7 +176,11 @@ export function buildQuery(
   const longestLookbackWindow = burnRateWindows.reduce((acc, winDef) => {
     return winDef.longDuration.isShorterThan(acc.longDuration) ? acc : winDef;
   }, burnRateWindows[0]);
-  const longestDateRange = getLookbackDateRange(startedAt, longestLookbackWindow.longDuration);
+  const longestDateRange = getLookbackDateRange(
+    startedAt,
+    longestLookbackWindow.longDuration,
+    delayInSeconds
+  );
 
   return {
     size: 0,
@@ -197,17 +208,21 @@ export function buildQuery(
           sources: [{ instanceId: { terms: { field: 'slo.instanceId' } } }],
         },
         aggs: {
-          ...buildWindowAggs(startedAt, slo, burnRateWindows),
+          ...buildWindowAggs(startedAt, slo, burnRateWindows, delayInSeconds),
           ...buildEvaluation(burnRateWindows),
         },
       },
     },
   };
 }
 
-function getLookbackDateRange(startedAt: Date, duration: Duration): { from: Date; to: Date } {
+function getLookbackDateRange(
+  startedAt: Date,
+  duration: Duration,
+  delayInSeconds = 0
+): { from: Date; to: Date } {
   const unit = toMomentUnitOfTime(duration.unit);
-  const now = moment(startedAt).startOf('minute');
+  const now = moment(startedAt).subtract(delayInSeconds, 'seconds').startOf('minute');
   const from = now.clone().subtract(duration.value, unit);
   const to = now.clone();
 

diff --git a/x-pack/plugins/observability/server/services/slo/sli_client.ts b/x-pack/plugins/observability/server/services/slo/sli_client.ts
@@ -25,6 +25,7 @@ import moment from 'moment';
 import { SLO_DESTINATION_INDEX_PATTERN } from '../../assets/constants';
 import { DateRange, Duration, IndicatorData, SLO } from '../../domain/models';
 import { InternalQueryError } from '../../errors';
+import { getDelayInSecondsFromSLO } from '../../domain/services/get_delay_in_seconds_from_slo';
 
 export interface SLIClient {
   fetchSLIDataFrom(
@@ -55,13 +56,14 @@ export class DefaultSLIClient implements SLIClient {
       a.duration.isShorterThan(b.duration) ? 1 : -1
     );
     const longestLookbackWindow = sortedLookbackWindows[0];
-    const longestDateRange = getLookbackDateRange(longestLookbackWindow.duration);
+    const delayInSeconds = getDelayInSecondsFromSLO(slo);
+    const longestDateRange = getLookbackDateRange(longestLookbackWindow.duration, delayInSeconds);
 
     if (occurrencesBudgetingMethodSchema.is(slo.budgetingMethod)) {
       const result = await this.esClient.search<unknown, EsAggregations>({
         ...commonQuery(slo, instanceId, longestDateRange),
         index: SLO_DESTINATION_INDEX_PATTERN,
-        aggs: toLookbackWindowsAggregationsQuery(sortedLookbackWindows),
+        aggs: toLookbackWindowsAggregationsQuery(sortedLookbackWindows, delayInSeconds),
       });
 
       return handleWindowedResult(result.aggregations, lookbackWindows);
@@ -71,7 +73,7 @@ export class DefaultSLIClient implements SLIClient {
       const result = await this.esClient.search<unknown, EsAggregations>({
         ...commonQuery(slo, instanceId, longestDateRange),
         index: SLO_DESTINATION_INDEX_PATTERN,
-        aggs: toLookbackWindowsSlicedAggregationsQuery(slo, sortedLookbackWindows),
+        aggs: toLookbackWindowsSlicedAggregationsQuery(sortedLookbackWindows, delayInSeconds),
       });
 
       return handleWindowedResult(result.aggregations, lookbackWindows);
@@ -110,14 +112,24 @@ function commonQuery(
   };
 }
 
-function toLookbackWindowsAggregationsQuery(sortedLookbackWindow: LookbackWindow[]) {
+function toLookbackWindowsAggregationsQuery(
+  sortedLookbackWindow: LookbackWindow[],
+  delayInSeconds = 0
+) {
   return sortedLookbackWindow.reduce<Record<string, AggregationsAggregationContainer>>(
     (acc, lookbackWindow) => ({
       ...acc,
       [lookbackWindow.name]: {
         date_range: {
           field: '@timestamp',
-          ranges: [{ from: `now-${lookbackWindow.duration.format()}/m`, to: 'now/m' }],
+          ranges: [
+            {
+              from: `now-${lookbackWindow.duration.format()}${
+                delayInSeconds > 0 ? '-' + delayInSeconds + 's' : ''
+              }/m`,
+              to: `now${delayInSeconds > 0 ? '-' + delayInSeconds + 's' : ''}/m`,
+            },
+          ],
         },
         aggs: {
           good: { sum: { field: 'slo.numerator' } },
@@ -129,7 +141,10 @@ function toLookbackWindowsAggregationsQuery(sortedLookbackWindow: LookbackWindow
   );
 }
 
-function toLookbackWindowsSlicedAggregationsQuery(slo: SLO, lookbackWindows: LookbackWindow[]) {
+function toLookbackWindowsSlicedAggregationsQuery(
+  lookbackWindows: LookbackWindow[],
+  delayInSeconds = 0
+) {
   return lookbackWindows.reduce<Record<string, AggregationsAggregationContainer>>(
     (acc, lookbackWindow) => ({
       ...acc,
@@ -138,8 +153,10 @@ function toLookbackWindowsSlicedAggregationsQuery(slo: SLO, lookbackWindows: Loo
           field: '@timestamp',
           ranges: [
             {
-              from: `now-${lookbackWindow.duration.format()}/m`,
-              to: 'now/m',
+              from: `now-${lookbackWindow.duration.format()}${
+                delayInSeconds > 0 ? '-' + delayInSeconds + 's' : ''
+              }/m`,
+              to: `now${delayInSeconds > 0 ? '-' + delayInSeconds + 's' : ''}/m`,
             },
           ],
         },
@@ -192,9 +209,9 @@ function handleWindowedResult(
   return indicatorDataPerLookbackWindow;
 }
 
-function getLookbackDateRange(duration: Duration): { from: Date; to: Date } {
+function getLookbackDateRange(duration: Duration, delayInSeconds = 0): { from: Date; to: Date } {
   const unit = toMomentUnitOfTime(duration.unit);
-  const now = moment.utc().startOf('minute');
+  const now = moment.utc().subtract(delayInSeconds, 'seconds').startOf('minute');
   const from = now.clone().subtract(duration.value, unit);
   const to = now.clone();