[Synthetics] Fix heatmap on monitor detail/history page for very large doc counts

x-pack/plugins/observability_solution/synthetics/common/constants/synthetics/rest_api.ts

@@ -25,7 +25,7 @@ export enum SYNTHETICS_API_URLS {
 
   SYNTHETICS_OVERVIEW = '/internal/synthetics/overview',
   PINGS = '/internal/synthetics/pings',
-  PING_STATUSES = '/internal/synthetics/ping_statuses',
+  MONITOR_STATUS_HEATMAP = '/internal/synthetics/ping_heatmap',
   OVERVIEW_TRENDS = '/internal/synthetics/overview_trends',
   OVERVIEW_STATUS = `/internal/synthetics/overview_status`,
   INDEX_SIZE = `/internal/synthetics/index_size`,

x-pack/plugins/observability_solution/synthetics/common/runtime_types/ping/ping.ts

@@ -255,40 +255,13 @@ export const PingStateType = t.type({
 export type Ping = t.TypeOf<typeof PingType>;
 export type PingState = t.TypeOf<typeof PingStateType>;
 
-export const PingStatusType = t.intersection([
-  t.type({
-    timestamp: t.string,
-    docId: t.string,
-    config_id: t.string,
-    locationId: t.string,
-    summary: t.partial({
-      down: t.number,
-      up: t.number,
-    }),
-  }),
-  t.partial({
-    error: PingErrorType,
-  }),
-]);
-
-export type PingStatus = t.TypeOf<typeof PingStatusType>;
-
 export const PingsResponseType = t.type({
   total: t.number,
   pings: t.array(PingType),
 });
 
 export type PingsResponse = t.TypeOf<typeof PingsResponseType>;
 
-export const PingStatusesResponseType = t.type({
-  total: t.number,
-  pings: t.array(PingStatusType),
-  from: t.string,
-  to: t.string,
-});
-
-export type PingStatusesResponse = t.TypeOf<typeof PingStatusesResponseType>;
-
 export const GetPingsParamsType = t.intersection([
   t.type({
     dateRange: DateRangeType,
@@ -306,3 +279,17 @@ export const GetPingsParamsType = t.intersection([
 ]);
 
 export type GetPingsParams = t.TypeOf<typeof GetPingsParamsType>;
+
+export const MonitorStatusHeatmapBucketType = t.type({
+  doc_count: t.number,
+  down: t.type({
+    value: t.number,
+  }),
+  up: t.type({
+    value: t.number,
+  }),
+  key: t.number,
+  key_as_string: t.string,
+});
+
+export type MonitorStatusHeatmapBucket = t.TypeOf<typeof MonitorStatusHeatmapBucketType>;

x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.ts

@@ -16,7 +16,7 @@ import {
   COLOR_MODES_STANDARD,
 } from '@elastic/eui';
 import type { BrushEvent } from '@elastic/charts';
-import { PingStatus } from '../../../../../../common/runtime_types';
+import { MonitorStatusHeatmapBucket } from '../../../../../../common/runtime_types';
 
 export const SUCCESS_VIZ_COLOR = VISUALIZATION_COLORS[0];
 export const DANGER_VIZ_COLOR = VISUALIZATION_COLORS[VISUALIZATION_COLORS.length - 1];
@@ -114,28 +114,26 @@ export function createTimeBuckets(intervalMinutes: number, from: number, to: num
 
 export function createStatusTimeBins(
   timeBuckets: MonitorStatusTimeBucket[],
-  pingStatuses: PingStatus[]
+  heatmapData: MonitorStatusHeatmapBucket[]
 ): MonitorStatusTimeBin[] {
-  let iPingStatus = 0;
-  return (timeBuckets ?? []).map((bucket) => {
-    const currentBin: MonitorStatusTimeBin = {
-      start: bucket.start,
-      end: bucket.end,
-      ups: 0,
-      downs: 0,
-      value: 0,
+  return timeBuckets.map(({ start, end }) => {
+    const { ups, downs } = heatmapData
+      .filter(({ key }) => key >= start && key <= end)
+      .reduce(
+        (acc, cur) => ({
+          ups: acc.ups + cur.up.value,
+          downs: acc.downs + cur.down.value,
+        }),
+        { ups: 0, downs: 0 }
+      );
+
+    return {
+      start,
+      end,
+      ups,
+      downs,
+      value: ups + downs === 0 ? 0 : getStatusEffectiveValue(ups, downs),
     };
-    while (
-      iPingStatus < pingStatuses.length &&
-      moment(pingStatuses[iPingStatus].timestamp).valueOf() < bucket.end
-    ) {
-      currentBin.ups += pingStatuses[iPingStatus]?.summary.up ?? 0;
-      currentBin.downs += pingStatuses[iPingStatus]?.summary.down ?? 0;
-      currentBin.value = getStatusEffectiveValue(currentBin.ups, currentBin.downs);
-      iPingStatus++;
-    }
-
-    return currentBin;
   });
 }
 

x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_panel.tsx

@@ -5,12 +5,11 @@
  * 2.0.
  */
 
-import React, { useMemo } from 'react';
+import React, { useMemo, useRef } from 'react';
 
-import { EuiPanel, useEuiTheme, EuiResizeObserver, EuiSpacer } from '@elastic/eui';
+import { EuiPanel, useEuiTheme, EuiResizeObserver, EuiSpacer, EuiProgress } from '@elastic/eui';
 import { Chart, Settings, Heatmap, ScaleType, Tooltip, LEGACY_LIGHT_THEME } from '@elastic/charts';
 import { i18n } from '@kbn/i18n';
-import { usePingStatusesIsLoading } from '../hooks/use_ping_statuses';
 import { MonitorStatusHeader } from './monitor_status_header';
 import { MonitorStatusCellTooltip } from './monitor_status_cell_tooltip';
 import { MonitorStatusLegend } from './monitor_status_legend';
@@ -32,9 +31,9 @@ export const MonitorStatusPanel = ({
   onBrushed,
 }: MonitorStatusPanelProps) => {
   const { euiTheme, colorMode } = useEuiTheme();
-  const { timeBins, handleResize, getTimeBinByXValue, xDomain, intervalByWidth } =
-    useMonitorStatusData({ from, to });
-  const isPingStatusesLoading = usePingStatusesIsLoading();
+  const initialSizeRef = useRef<HTMLDivElement | null>(null);
+  const { loading, timeBins, handleResize, getTimeBinByXValue, xDomain, minsPerBin } =
+    useMonitorStatusData({ from, to, initialSizeRef });
 
   const heatmap = useMemo(() => {
     return getMonitorStatusChartTheme(euiTheme, brushable);
@@ -53,61 +52,66 @@ export const MonitorStatusPanel = ({
 
       <EuiSpacer size="m" />
 
-      <EuiResizeObserver onResize={handleResize}>
-        {(resizeRef) => (
-          <div ref={resizeRef}>
-            <Chart
-              size={{
-                height: 60,
-              }}
-            >
-              <Tooltip
-                customTooltip={({ values }) => (
-                  <MonitorStatusCellTooltip
-                    timeBin={getTimeBinByXValue(values?.[0]?.datum?.x)}
-                    isLoading={isPingStatusesLoading}
+      <div ref={initialSizeRef}>
+        <EuiResizeObserver onResize={(e) => handleResize(e)}>
+          {(resizeRef) => (
+            <div ref={resizeRef}>
+              {minsPerBin && (
+                <Chart
+                  size={{
+                    height: 60,
+                  }}
+                >
+                  <Tooltip
+                    customTooltip={({ values }) => (
+                      <MonitorStatusCellTooltip
+                        timeBin={getTimeBinByXValue(values?.[0]?.datum?.x)}
+                        isLoading={loading}
+                      />
+                    )}
                   />
-                )}
-              />
-              <Settings
-                showLegend={false}
-                xDomain={xDomain}
-                theme={{ heatmap }}
-                // TODO connect to charts.theme service see src/plugins/charts/public/services/theme/README.md
-                baseTheme={LEGACY_LIGHT_THEME}
-                onBrushEnd={(brushArea) => {
-                  onBrushed?.(getBrushData(brushArea));
-                }}
-                locale={i18n.getLocale()}
-              />
-              <Heatmap
-                id="monitor-details-monitor-status-chart"
-                colorScale={{
-                  type: 'bands',
-                  bands: getColorBands(euiTheme, colorMode),
-                }}
-                data={timeBins}
-                xAccessor={(timeBin) => timeBin.end}
-                yAccessor={() => 'T'}
-                valueAccessor={(timeBin) => timeBin.value}
-                valueFormatter={(d) => d.toFixed(2)}
-                xAxisLabelFormatter={getXAxisLabelFormatter(intervalByWidth)}
-                timeZone="UTC"
-                xScale={{
-                  type: ScaleType.Time,
-                  interval: {
-                    type: 'calendar',
-                    unit: 'm',
-                    value: intervalByWidth,
-                  },
-                }}
-              />
-            </Chart>
-          </div>
-        )}
-      </EuiResizeObserver>
+                  <Settings
+                    showLegend={false}
+                    xDomain={xDomain}
+                    theme={{ heatmap }}
+                    // TODO connect to charts.theme service see src/plugins/charts/public/services/theme/README.md
+                    baseTheme={LEGACY_LIGHT_THEME}
+                    onBrushEnd={(brushArea) => {
+                      onBrushed?.(getBrushData(brushArea));
+                    }}
+                    locale={i18n.getLocale()}
+                  />
+                  <Heatmap
+                    id="monitor-details-monitor-status-chart"
+                    colorScale={{
+                      type: 'bands',
+                      bands: getColorBands(euiTheme, colorMode),
+                    }}
+                    data={timeBins}
+                    xAccessor={({ end }) => end}
+                    yAccessor={() => 'T'}
+                    valueAccessor={(timeBin) => timeBin.value}
+                    valueFormatter={(d) => d.toFixed(2)}
+                    xAxisLabelFormatter={getXAxisLabelFormatter(minsPerBin)}
+                    timeZone="UTC"
+                    xScale={{
+                      type: ScaleType.Time,
+                      interval: {
+                        type: 'calendar',
+                        unit: 'm',
+                        value: minsPerBin,
+                      },
+                    }}
+                  />
+                </Chart>
+              )}
+            </div>
+          )}
+        </EuiResizeObserver>
+      </div>
 
       <MonitorStatusLegend brushable={brushable} />
+      {loading && <EuiProgress size="xs" color="accent" />}
     </EuiPanel>
   );
 };