[core.metrics] Add support for multiple processes in ops metrics & stats API; deprecate process field

src/core/public/core_app/status/lib/load_status.test.ts

@@ -9,6 +9,7 @@
 import { StatusResponse } from '../../../../types/status';
 import { httpServiceMock } from '../../../http/http_service.mock';
 import { notificationServiceMock } from '../../../notifications/notifications_service.mock';
+import { mocked } from '../../../../server/metrics/event_loop_delays/event_loop_delays_monitor.mocks';
 import { loadStatus } from './load_status';
 
 const mockedResponse: StatusResponse = {
@@ -61,6 +62,8 @@ const mockedResponse: StatusResponse = {
       },
     },
     process: {
+      name: 'server_worker' as const,
+      pid: 1,
       memory: {
         heap: {
           size_limit: 1000000,
@@ -70,9 +73,26 @@ const mockedResponse: StatusResponse = {
         resident_set_size_in_bytes: 1,
       },
       event_loop_delay: 1,
-      pid: 1,
+      event_loop_delay_histogram: mocked.createHistogram(),
       uptime_in_millis: 1,
     },
+    processes: [
+      {
+        name: 'server_worker' as const,
+        pid: 1,
+        memory: {
+          heap: {
+            size_limit: 1000000,
+            used_in_bytes: 100,
+            total_in_bytes: 0,
+          },
+          resident_set_size_in_bytes: 1,
+        },
+        event_loop_delay: 1,
+        event_loop_delay_histogram: mocked.createHistogram(),
+        uptime_in_millis: 1,
+      },
+    ],
     response_times: {
       avg_in_millis: 4000,
       max_in_millis: 8000,

src/core/server/index.ts

@@ -378,7 +378,9 @@ export type {
   OpsProcessMetrics,
   MetricsServiceSetup,
   MetricsServiceStart,
+  IntervalHistogram,
 } from './metrics';
+export { EventLoopDelaysMonitor } from './metrics';
 
 export type { I18nServiceSetup } from './i18n';
 export type {

src/core/server/metrics/collectors/collector.mock.ts

@@ -8,8 +8,10 @@
 
 import { MetricsCollector } from './types';
 
-const createCollector = (collectReturnValue: any = {}): jest.Mocked<MetricsCollector<any>> => {
-  const collector: jest.Mocked<MetricsCollector<any>> = {
+const createCollector = <T = any>(
+  collectReturnValue: any = {}
+): jest.Mocked<MetricsCollector<T>> => {
+  const collector: jest.Mocked<MetricsCollector<T>> = {
     collect: jest.fn().mockResolvedValue(collectReturnValue),
     reset: jest.fn(),
   };

src/core/server/metrics/collectors/mocks.ts

@@ -6,7 +6,8 @@
  * Side Public License, v 1.
  */
 
-import { MetricsCollector } from './types';
+import type { MetricsCollector } from './types';
+import { createMockOpsProcessMetrics } from './process.mocks';
 
 const createMock = () => {
   const mocked: jest.Mocked<MetricsCollector<any>> = {
@@ -21,4 +22,5 @@ const createMock = () => {
 
 export const collectorMock = {
   create: createMock,
+  createOpsProcessMetrics: createMockOpsProcessMetrics,
 };

src/core/server/metrics/collectors/process.mocks.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { mocked } from '../event_loop_delays/event_loop_delays_monitor.mocks';
+import type { OpsProcessMetrics } from './types';
+
+export function createMockOpsProcessMetrics(): OpsProcessMetrics {
+  const histogram = mocked.createHistogram();
+
+  return {
+    name: 'server_worker' as const,
+    memory: {
+      heap: { total_in_bytes: 1, used_in_bytes: 1, size_limit: 1 },
+      resident_set_size_in_bytes: 1,
+    },
+    event_loop_delay: 1,
+    event_loop_delay_histogram: histogram,
+    pid: 1,
+    uptime_in_millis: 1,
+  };
+}

src/core/server/metrics/collectors/process.test.ts

@@ -9,6 +9,7 @@
 import v8, { HeapInfo } from 'v8';
 import { ProcessMetricsCollector } from './process';
 
+/* eslint-disable dot-notation */
 describe('ProcessMetricsCollector', () => {
   let collector: ProcessMetricsCollector;
 
@@ -20,28 +21,34 @@ describe('ProcessMetricsCollector', () => {
     jest.restoreAllMocks();
   });
 
-  it('collects pid from the process', async () => {
-    const metrics = await collector.collect();
+  it('collects pid from the process', () => {
+    const metrics = collector.collect();
 
-    expect(metrics.pid).toEqual(process.pid);
+    expect(metrics).toHaveLength(1);
+    expect(metrics[0].pid).toEqual(process.pid);
   });
 
-  it('collects event loop delay', async () => {
-    const metrics = await collector.collect();
-
-    expect(metrics.event_loop_delay).toBeGreaterThan(0);
+  it('collects event loop delay', () => {
+    const mockEventLoopDelayMonitor = { collect: jest.fn().mockReturnValue({ mean: 13 }) };
+    // @ts-expect-error-next-line readonly private method.
+    collector['eventLoopDelayMonitor'] = mockEventLoopDelayMonitor;
+    const metrics = collector.collect();
+    expect(metrics).toHaveLength(1);
+    expect(metrics[0].event_loop_delay).toBe(13);
+    expect(mockEventLoopDelayMonitor.collect).toBeCalledTimes(1);
   });
 
-  it('collects uptime info from the process', async () => {
+  it('collects uptime info from the process', () => {
     const uptime = 58986;
     jest.spyOn(process, 'uptime').mockImplementation(() => uptime);
 
-    const metrics = await collector.collect();
+    const metrics = collector.collect();
 
-    expect(metrics.uptime_in_millis).toEqual(uptime * 1000);
+    expect(metrics).toHaveLength(1);
+    expect(metrics[0].uptime_in_millis).toEqual(uptime * 1000);
   });
 
-  it('collects memory info from the process', async () => {
+  it('collects memory info from the process', () => {
     const heapTotal = 58986;
     const heapUsed = 4688;
     const heapSizeLimit = 5788;
@@ -61,11 +68,12 @@ describe('ProcessMetricsCollector', () => {
         } as HeapInfo)
     );
 
-    const metrics = await collector.collect();
+    const metrics = collector.collect();
 
-    expect(metrics.memory.heap.total_in_bytes).toEqual(heapTotal);
-    expect(metrics.memory.heap.used_in_bytes).toEqual(heapUsed);
-    expect(metrics.memory.heap.size_limit).toEqual(heapSizeLimit);
-    expect(metrics.memory.resident_set_size_in_bytes).toEqual(rss);
+    expect(metrics).toHaveLength(1);
+    expect(metrics[0].memory.heap.total_in_bytes).toEqual(heapTotal);
+    expect(metrics[0].memory.heap.used_in_bytes).toEqual(heapUsed);
+    expect(metrics[0].memory.heap.size_limit).toEqual(heapSizeLimit);
+    expect(metrics[0].memory.resident_set_size_in_bytes).toEqual(rss);
   });
 });

src/core/server/metrics/collectors/process.ts

@@ -7,15 +7,23 @@
  */
 
 import v8 from 'v8';
-import { Bench } from '@hapi/hoek';
 import { OpsProcessMetrics, MetricsCollector } from './types';
+import { EventLoopDelaysMonitor } from '../event_loop_delays';
 
-export class ProcessMetricsCollector implements MetricsCollector<OpsProcessMetrics> {
-  public async collect(): Promise<OpsProcessMetrics> {
+export class ProcessMetricsCollector implements MetricsCollector<OpsProcessMetrics[]> {
+  static getMainThreadMetrics(processes: OpsProcessMetrics[]): undefined | OpsProcessMetrics {
+    return processes.find(({ name }) => name === 'server_worker');
+  }
+
+  private readonly eventLoopDelayMonitor = new EventLoopDelaysMonitor();
+
+  private getCurrentPidMetrics(): OpsProcessMetrics {
+    const eventLoopDelayHistogram = this.eventLoopDelayMonitor.collect();
     const heapStats = v8.getHeapStatistics();
     const memoryUsage = process.memoryUsage();
-    const [eventLoopDelay] = await Promise.all([getEventLoopDelay()]);
+
     return {
+      name: 'server_worker' as const,
       memory: {
         heap: {
           total_in_bytes: memoryUsage.heapTotal,
@@ -25,19 +33,17 @@ export class ProcessMetricsCollector implements MetricsCollector<OpsProcessMetri
         resident_set_size_in_bytes: memoryUsage.rss,
       },
       pid: process.pid,
-      event_loop_delay: eventLoopDelay,
+      event_loop_delay: eventLoopDelayHistogram.mean,
+      event_loop_delay_histogram: eventLoopDelayHistogram,
       uptime_in_millis: process.uptime() * 1000,
     };
   }
 
-  public reset() {}
-}
+  public collect(): OpsProcessMetrics[] {
+    return [this.getCurrentPidMetrics()];
+  }
 
-const getEventLoopDelay = (): Promise<number> => {
-  const bench = new Bench();
-  return new Promise((resolve) => {
-    setImmediate(() => {
-      return resolve(bench.elapsed());
-    });
-  });
-};
+  public reset() {
+    this.eventLoopDelayMonitor.reset();
+  }
+}

src/core/server/metrics/collectors/types.ts

@@ -5,11 +5,12 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+import type { IntervalHistogram } from '../event_loop_delays';
 
 /** Base interface for all metrics gatherers */
 export interface MetricsCollector<T> {
   /** collect the data currently gathered by the collector */
-  collect(): Promise<T>;
+  collect(): Promise<T> | T;
   /** reset the internal state of the collector */
   reset(): void;
 }
@@ -19,6 +20,10 @@ export interface MetricsCollector<T> {
  * @public
  */
 export interface OpsProcessMetrics {
+  /** pid of the kibana process */
+  pid: number;
+  /** name of process (example: 'coordinator' | 'server_worker' | 'task_worker' | 'reporting_worker') */
+  name: 'coordinator' | 'server_worker';
   /** process memory usage */
   memory: {
     /** heap memory usage */
@@ -33,10 +38,10 @@ export interface OpsProcessMetrics {
     /** node rss */
     resident_set_size_in_bytes: number;
   };
-  /** node event loop delay */
+  /** mean event loop delay since last collection*/
   event_loop_delay: number;
-  /** pid of the kibana process */
-  pid: number;
+  /** node event loop delay histogram since last collection */
+  event_loop_delay_histogram: IntervalHistogram;
   /** uptime of the kibana process */
   uptime_in_millis: number;
 }

src/core/server/metrics/event_loop_delays/__mocks__/perf_hooks.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { mocked } from '../event_loop_delays_monitor.mocks';
+
+export const mockMonitor = {
+  enable: jest.fn(),
+  percentile: jest.fn(),
+  disable: jest.fn(),
+  reset: jest.fn(),
+};
+
+export const monitorEventLoopDelay = jest.fn().mockReturnValue({
+  ...mockMonitor,
+  ...mocked.createHistogram(),
+});

src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.mocks.ts → src/core/server/metrics/event_loop_delays/event_loop_delays_monitor.mocks.ts

@@ -6,23 +6,8 @@
  * Side Public License, v 1.
  */
 import moment from 'moment';
-import type { IntervalHistogram } from './event_loop_delays';
-
-export const mockMonitorEnable = jest.fn();
-export const mockMonitorPercentile = jest.fn();
-export const mockMonitorReset = jest.fn();
-export const mockMonitorDisable = jest.fn();
-export const monitorEventLoopDelay = jest.fn().mockReturnValue({
-  enable: mockMonitorEnable,
-  percentile: mockMonitorPercentile,
-  disable: mockMonitorDisable,
-  reset: mockMonitorReset,
-  ...createMockHistogram(),
-});
-
-jest.doMock('perf_hooks', () => ({
-  monitorEventLoopDelay,
-}));
+import type { IntervalHistogram } from 'kibana/server';
+import type { EventLoopDelaysMonitor } from './event_loop_delays_monitor';
 
 function createMockHistogram(overwrites: Partial<IntervalHistogram> = {}): IntervalHistogram {
   const now = moment();
@@ -45,6 +30,22 @@ function createMockHistogram(overwrites: Partial<IntervalHistogram> = {}): Inter
   };
 }
 
+function createMockEventLoopDelaysMonitor() {
+  const mockCollect = jest.fn();
+  const MockEventLoopDelaysMonitor: jest.MockedClass<
+    typeof EventLoopDelaysMonitor
+  > = jest.fn().mockReturnValue({
+    collect: mockCollect,
+    reset: jest.fn(),
+    stop: jest.fn(),
+  });
+
+  mockCollect.mockReturnValue(createMockHistogram());
+
+  return new MockEventLoopDelaysMonitor();
+}
+
 export const mocked = {
   createHistogram: createMockHistogram,
+  createEventLoopDelaysMonitor: createMockEventLoopDelaysMonitor,
 };