diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/constants.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/constants.ts new file mode 100644 index 0000000000000..1753c87c9d005 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/constants.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Roll daily indices every 24h + */ +export const ROLL_DAILY_INDICES_INTERVAL = 24 * 60 * 60 * 1000; + +/** + * Start rolling indices after 5 minutes up + */ +export const ROLL_INDICES_START = 5 * 60 * 1000; + +/** + * Reset the event loop delay historgram every 1 hour + */ +export const MONITOR_EVENT_LOOP_DELAYS_INTERVAL = 1 * 60 * 60 * 1000; + +/** + * Reset the event loop delay historgram every 24h + */ +export const MONITOR_EVENT_LOOP_DELAYS_RESET = 24 * 60 * 60 * 1000; + +/** + * Start monitoring the event loop delays after 1 minute + */ +export const MONITOR_EVENT_LOOP_DELAYS_START = 1 * 60 * 1000; + +/** + * Event loop monitoring sampling rate in milliseconds. + */ +export const MONITOR_EVENT_LOOP_DELAYS_RESOLUTION = 10; diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.mocks.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.mocks.ts new file mode 100644 index 0000000000000..6b03d3cc5cbd1 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.mocks.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import moment from 'moment'; +import type { IntervalHistogram } from './event_loop_delays'; + +export const mockMonitorEnable = jest.fn(); +export const mockMonitorPercentile = jest.fn(); +export const mockMonitorReset = jest.fn(); +export const mockMonitorDisable = jest.fn(); +export const monitorEventLoopDelay = jest.fn().mockReturnValue({ + enable: mockMonitorEnable, + percentile: mockMonitorPercentile, + disable: mockMonitorDisable, + reset: mockMonitorReset, +}); + +jest.doMock('perf_hooks', () => ({ + monitorEventLoopDelay, +})); + +function createMockHistogram(overwrites: Partial = {}): IntervalHistogram { + const now = moment(); + + return { + min: 9093120, + max: 53247999, + mean: 11993238.600747818, + exceeds: 0, + stddev: 1168191.9357543814, + fromTimestamp: now.startOf('day').toISOString(), + lastUpdatedAt: now.toISOString(), + percentiles: { + '50': 12607487, + '75': 12615679, + '95': 12648447, + '99': 12713983, + }, + ...overwrites, + }; +} + +export const mocked = { + createHistogram: createMockHistogram, +}; diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.test.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.test.ts new file mode 100644 index 0000000000000..d03236a9756b3 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Subject } from 'rxjs'; + +import { + mockMonitorEnable, + mockMonitorPercentile, + monitorEventLoopDelay, + mockMonitorReset, + mockMonitorDisable, +} from './event_loop_delays.mocks'; +import { savedObjectsRepositoryMock } from '../../../../../core/server/mocks'; +import { startTrackingEventLoopDelaysUsage, EventLoopDelaysCollector } from './event_loop_delays'; + +describe('EventLoopDelaysCollector', () => { + jest.useFakeTimers('modern'); + const mockNow = jest.getRealSystemTime(); + jest.setSystemTime(mockNow); + + beforeEach(() => jest.clearAllMocks()); + afterAll(() => jest.useRealTimers()); + + test('#constructor enables monitoring', () => { + new EventLoopDelaysCollector(); + expect(monitorEventLoopDelay).toBeCalledWith({ resolution: 10 }); + expect(mockMonitorEnable).toBeCalledTimes(1); + }); + + test('#collect returns event loop delays histogram', () => { + const eventLoopDelaysCollector = new EventLoopDelaysCollector(); + const histogramData = eventLoopDelaysCollector.collect(); + expect(mockMonitorPercentile).toHaveBeenNthCalledWith(1, 50); + expect(mockMonitorPercentile).toHaveBeenNthCalledWith(2, 75); + expect(mockMonitorPercentile).toHaveBeenNthCalledWith(3, 95); + expect(mockMonitorPercentile).toHaveBeenNthCalledWith(4, 99); + + expect(Object.keys(histogramData)).toMatchInlineSnapshot(` + Array [ + "min", + "max", + "mean", + "exceeds", + "stddev", + "fromTimestamp", + "lastUpdatedAt", + "percentiles", + ] + `); + }); + test('#reset resets histogram data', () => { + const eventLoopDelaysCollector = new EventLoopDelaysCollector(); + eventLoopDelaysCollector.reset(); + expect(mockMonitorReset).toBeCalledTimes(1); + }); + test('#stop disables monitoring event loop delays', () => { + const eventLoopDelaysCollector = new EventLoopDelaysCollector(); + eventLoopDelaysCollector.stop(); + expect(mockMonitorDisable).toBeCalledTimes(1); + }); +}); + +describe('startTrackingEventLoopDelaysUsage', () => { + const mockInternalRepository = savedObjectsRepositoryMock.create(); + const stopMonitoringEventLoop$ = new Subject(); + + beforeAll(() => jest.useFakeTimers('modern')); + beforeEach(() => jest.clearAllMocks()); + afterEach(() => stopMonitoringEventLoop$.next()); + + it('initializes EventLoopDelaysCollector and starts timer', () => { + const collectionStartDelay = 1000; + startTrackingEventLoopDelaysUsage( + mockInternalRepository, + stopMonitoringEventLoop$, + collectionStartDelay + ); + + expect(monitorEventLoopDelay).toBeCalledTimes(1); + expect(mockMonitorPercentile).toBeCalledTimes(0); + jest.advanceTimersByTime(collectionStartDelay); + expect(mockMonitorPercentile).toBeCalled(); + }); + + it('stores event loop delays every collectionInterval duration', () => { + const collectionStartDelay = 100; + const collectionInterval = 1000; + startTrackingEventLoopDelaysUsage( + mockInternalRepository, + stopMonitoringEventLoop$, + collectionStartDelay, + collectionInterval + ); + + expect(mockInternalRepository.create).toBeCalledTimes(0); + jest.advanceTimersByTime(collectionStartDelay); + expect(mockInternalRepository.create).toBeCalledTimes(1); + jest.advanceTimersByTime(collectionInterval); + expect(mockInternalRepository.create).toBeCalledTimes(2); + jest.advanceTimersByTime(collectionInterval); + expect(mockInternalRepository.create).toBeCalledTimes(3); + }); + + it('resets histogram every histogramReset duration', () => { + const collectionStartDelay = 0; + const collectionInterval = 1000; + const histogramReset = 5000; + startTrackingEventLoopDelaysUsage( + mockInternalRepository, + stopMonitoringEventLoop$, + collectionStartDelay, + collectionInterval, + histogramReset + ); + + expect(mockMonitorReset).toBeCalledTimes(0); + jest.advanceTimersByTime(collectionInterval * 5); + expect(mockMonitorReset).toBeCalledTimes(1); + jest.advanceTimersByTime(collectionInterval * 5); + expect(mockMonitorReset).toBeCalledTimes(2); + }); + + it('stops monitoring event loop delays once stopMonitoringEventLoop$.next is called', () => { + startTrackingEventLoopDelaysUsage(mockInternalRepository, stopMonitoringEventLoop$); + + expect(mockMonitorDisable).toBeCalledTimes(0); + stopMonitoringEventLoop$.next(); + expect(mockMonitorDisable).toBeCalledTimes(1); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.ts new file mode 100644 index 0000000000000..655cba580fc5d --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { EventLoopDelayMonitor } from 'perf_hooks'; +import { monitorEventLoopDelay } from 'perf_hooks'; +import { takeUntil, finalize, map } from 'rxjs/operators'; +import { Observable, timer } from 'rxjs'; +import type { ISavedObjectsRepository } from 'kibana/server'; +import { + MONITOR_EVENT_LOOP_DELAYS_START, + MONITOR_EVENT_LOOP_DELAYS_INTERVAL, + MONITOR_EVENT_LOOP_DELAYS_RESET, + MONITOR_EVENT_LOOP_DELAYS_RESOLUTION, +} from './constants'; +import { storeHistogram } from './saved_objects'; + +export interface IntervalHistogram { + fromTimestamp: string; + lastUpdatedAt: string; + min: number; + max: number; + mean: number; + exceeds: number; + stddev: number; + percentiles: { + 50: number; + 75: number; + 95: number; + 99: number; + }; +} + +export class EventLoopDelaysCollector { + private readonly loopMonitor: EventLoopDelayMonitor; + private fromTimestamp: Date; + + constructor() { + const monitor = monitorEventLoopDelay({ + resolution: MONITOR_EVENT_LOOP_DELAYS_RESOLUTION, + }); + monitor.enable(); + this.fromTimestamp = new Date(); + this.loopMonitor = monitor; + } + + public collect(): IntervalHistogram { + const { min, max, mean, exceeds, stddev } = this.loopMonitor; + + return { + min, + max, + mean, + exceeds, + stddev, + fromTimestamp: this.fromTimestamp.toISOString(), + lastUpdatedAt: new Date().toISOString(), + percentiles: { + 50: this.loopMonitor.percentile(50), + 75: this.loopMonitor.percentile(75), + 95: this.loopMonitor.percentile(95), + 99: this.loopMonitor.percentile(99), + }, + }; + } + + public reset() { + this.loopMonitor.reset(); + this.fromTimestamp = new Date(); + } + + public stop() { + this.loopMonitor.disable(); + } +} + +/** + * The monitoring of the event loop starts immediately. + * The first collection of the histogram happens after 1 minute. + * The daily histogram data is updated every 1 hour. + */ +export function startTrackingEventLoopDelaysUsage( + internalRepository: ISavedObjectsRepository, + stopMonitoringEventLoop$: Observable, + collectionStartDelay = MONITOR_EVENT_LOOP_DELAYS_START, + collectionInterval = MONITOR_EVENT_LOOP_DELAYS_INTERVAL, + histogramReset = MONITOR_EVENT_LOOP_DELAYS_RESET +) { + const eventLoopDelaysCollector = new EventLoopDelaysCollector(); + + const resetOnCount = Math.ceil(histogramReset / collectionInterval); + timer(collectionStartDelay, collectionInterval) + .pipe( + map((i) => (i + 1) % resetOnCount === 0), + takeUntil(stopMonitoringEventLoop$), + finalize(() => eventLoopDelaysCollector.stop()) + ) + .subscribe(async (shouldReset) => { + const histogram = eventLoopDelaysCollector.collect(); + if (shouldReset) { + eventLoopDelaysCollector.reset(); + } + await storeHistogram(histogram, internalRepository); + }); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays_usage_collector.test.ts new file mode 100644 index 0000000000000..06c51f6afa3a8 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays_usage_collector.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + Collector, + createUsageCollectionSetupMock, + createCollectorFetchContextMock, +} from '../../../../usage_collection/server/mocks'; +import { registerEventLoopDelaysCollector } from './event_loop_delays_usage_collector'; +import { loggingSystemMock, savedObjectsRepositoryMock } from '../../../../../core/server/mocks'; +import type { SavedObjectsFindResponse } from '../../../../../core/server'; + +const logger = loggingSystemMock.createLogger(); + +describe('registerEventLoopDelaysCollector', () => { + let collector: Collector; + const mockRegisterType = jest.fn(); + const mockInternalRepository = savedObjectsRepositoryMock.create(); + const mockGetSavedObjectsClient = () => mockInternalRepository; + + const usageCollectionMock = createUsageCollectionSetupMock(); + usageCollectionMock.makeUsageCollector.mockImplementation((config) => { + collector = new Collector(logger, config); + return createUsageCollectionSetupMock().makeUsageCollector(config); + }); + + const collectorFetchContext = createCollectorFetchContextMock(); + + beforeAll(() => { + registerEventLoopDelaysCollector( + logger, + usageCollectionMock, + mockRegisterType, + mockGetSavedObjectsClient + ); + }); + + it('registers event_loop_delays collector', () => { + expect(collector).not.toBeUndefined(); + expect(collector.type).toBe('event_loop_delays'); + }); + + it('registers savedObjectType "event_loop_delays_daily"', () => { + expect(mockRegisterType).toBeCalledTimes(1); + expect(mockRegisterType).toBeCalledWith( + expect.objectContaining({ + name: 'event_loop_delays_daily', + }) + ); + }); + + it('returns objects from event_loop_delays_daily from fetch function', async () => { + const mockFind = jest.fn().mockResolvedValue(({ + saved_objects: [{ attributes: { test: 1 } }], + } as unknown) as SavedObjectsFindResponse); + mockInternalRepository.find = mockFind; + const fetchResult = await collector.fetch(collectorFetchContext); + + expect(fetchResult).toMatchInlineSnapshot(` + Object { + "daily": Array [ + Object { + "test": 1, + }, + ], + } + `); + expect(mockFind).toBeCalledTimes(1); + expect(mockFind.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "sortField": "updated_at", + "sortOrder": "desc", + "type": "event_loop_delays_daily", + }, + ] + `); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays_usage_collector.ts new file mode 100644 index 0000000000000..774e021d7a549 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/event_loop_delays_usage_collector.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { timer } from 'rxjs'; +import { SavedObjectsServiceSetup, ISavedObjectsRepository, Logger } from 'kibana/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { rollDailyData } from './rollups'; +import { registerSavedObjectTypes, EventLoopDelaysDaily } from './saved_objects'; +import { eventLoopDelaysUsageSchema, EventLoopDelaysUsageReport } from './schema'; +import { SAVED_OBJECTS_DAILY_TYPE } from './saved_objects'; +import { ROLL_DAILY_INDICES_INTERVAL, ROLL_INDICES_START } from './constants'; + +export function registerEventLoopDelaysCollector( + logger: Logger, + usageCollection: UsageCollectionSetup, + registerType: SavedObjectsServiceSetup['registerType'], + getSavedObjectsClient: () => ISavedObjectsRepository | undefined +) { + registerSavedObjectTypes(registerType); + + timer(ROLL_INDICES_START, ROLL_DAILY_INDICES_INTERVAL).subscribe(() => + rollDailyData(logger, getSavedObjectsClient()) + ); + + const collector = usageCollection.makeUsageCollector({ + type: 'event_loop_delays', + isReady: () => typeof getSavedObjectsClient() !== 'undefined', + schema: eventLoopDelaysUsageSchema, + fetch: async () => { + const internalRepository = getSavedObjectsClient(); + if (!internalRepository) { + return { daily: [] }; + } + + const { saved_objects: savedObjects } = await internalRepository.find({ + type: SAVED_OBJECTS_DAILY_TYPE, + sortField: 'updated_at', + sortOrder: 'desc', + }); + + return { + daily: savedObjects.map((savedObject) => savedObject.attributes), + }; + }, + }); + + usageCollection.registerCollector(collector); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/index.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/index.ts new file mode 100644 index 0000000000000..693b173c2759e --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { startTrackingEventLoopDelaysUsage } from './event_loop_delays'; +export { registerEventLoopDelaysCollector } from './event_loop_delays_usage_collector'; +export { SAVED_OBJECTS_DAILY_TYPE } from './saved_objects'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/daily.test.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/daily.test.ts new file mode 100644 index 0000000000000..cb59d6a44b07e --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/daily.test.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { rollDailyData } from './daily'; +import { loggingSystemMock, savedObjectsRepositoryMock } from '../../../../../../core/server/mocks'; +import type { SavedObjectsFindResponse } from '../../../../../../core/server'; + +describe('rollDailyData', () => { + const logger = loggingSystemMock.createLogger(); + const mockSavedObjectsClient = savedObjectsRepositoryMock.create(); + + beforeEach(() => jest.clearAllMocks()); + + it('returns false if no savedObjectsClient', async () => { + await rollDailyData(logger, undefined); + expect(mockSavedObjectsClient.find).toBeCalledTimes(0); + }); + + it('calls delete on documents older than 3 days', async () => { + mockSavedObjectsClient.find.mockResolvedValueOnce({ + saved_objects: [{ id: 'test_id_1' }, { id: 'test_id_2' }], + } as SavedObjectsFindResponse); + + await rollDailyData(logger, mockSavedObjectsClient); + + expect(mockSavedObjectsClient.find).toHaveBeenCalledTimes(1); + expect(mockSavedObjectsClient.delete).toBeCalledTimes(2); + expect(mockSavedObjectsClient.delete).toHaveBeenNthCalledWith( + 1, + 'event_loop_delays_daily', + 'test_id_1' + ); + expect(mockSavedObjectsClient.delete).toHaveBeenNthCalledWith( + 2, + 'event_loop_delays_daily', + 'test_id_2' + ); + }); + + it('calls logger.debug on repository find error', async () => { + const mockError = new Error('find error'); + mockSavedObjectsClient.find.mockRejectedValueOnce(mockError); + + await rollDailyData(logger, mockSavedObjectsClient); + expect(logger.debug).toBeCalledTimes(2); + expect(logger.debug).toHaveBeenNthCalledWith( + 1, + 'Failed to rollup transactional to daily entries' + ); + expect(logger.debug).toHaveBeenNthCalledWith(2, mockError); + }); + + it('settles all deletes before logging failures', async () => { + const mockError1 = new Error('delete error 1'); + const mockError2 = new Error('delete error 2'); + mockSavedObjectsClient.find.mockResolvedValueOnce({ + saved_objects: [{ id: 'test_id_1' }, { id: 'test_id_2' }, { id: 'test_id_3' }], + } as SavedObjectsFindResponse); + + mockSavedObjectsClient.delete.mockRejectedValueOnce(mockError1); + mockSavedObjectsClient.delete.mockResolvedValueOnce(true); + mockSavedObjectsClient.delete.mockRejectedValueOnce(mockError2); + + await rollDailyData(logger, mockSavedObjectsClient); + expect(mockSavedObjectsClient.delete).toBeCalledTimes(3); + expect(logger.debug).toBeCalledTimes(2); + expect(logger.debug).toHaveBeenNthCalledWith( + 1, + 'Failed to rollup transactional to daily entries' + ); + expect(logger.debug).toHaveBeenNthCalledWith(2, [ + { reason: mockError1, status: 'rejected' }, + { reason: mockError2, status: 'rejected' }, + ]); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/daily.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/daily.ts new file mode 100644 index 0000000000000..29072335d272b --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/daily.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Logger } from '@kbn/logging'; +import { ISavedObjectsRepository } from '../../../../../../core/server'; +import { deleteHistogramSavedObjects } from '../saved_objects'; + +/** + * daily rollup function. Deletes histogram saved objects older than 3 days + * @param logger + * @param savedObjectsClient + */ +export async function rollDailyData( + logger: Logger, + savedObjectsClient?: ISavedObjectsRepository +): Promise { + if (!savedObjectsClient) { + return; + } + try { + const settledDeletes = await deleteHistogramSavedObjects(savedObjectsClient); + const failedDeletes = settledDeletes.filter(({ status }) => status !== 'fulfilled'); + if (failedDeletes.length) { + throw failedDeletes; + } + } catch (err) { + logger.debug(`Failed to rollup transactional to daily entries`); + logger.debug(err); + } +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/index.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/index.ts new file mode 100644 index 0000000000000..4523069a820e7 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { rollDailyData } from './daily'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/integration_tests/daily_rollups.test.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/integration_tests/daily_rollups.test.ts new file mode 100644 index 0000000000000..8c227f260da6e --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/rollups/integration_tests/daily_rollups.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Logger, ISavedObjectsRepository } from '../../../../../../../core/server'; +import { + createTestServers, + TestElasticsearchUtils, + TestKibanaUtils, + createRootWithCorePlugins, +} from '../../../../../../../core/test_helpers/kbn_server'; +import { rollDailyData } from '../daily'; +import { mocked } from '../../event_loop_delays.mocks'; + +import { + SAVED_OBJECTS_DAILY_TYPE, + serializeSavedObjectId, + EventLoopDelaysDaily, +} from '../../saved_objects'; +import moment from 'moment'; + +const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), +}); + +function createRawObject(date: moment.MomentInput) { + const pid = Math.round(Math.random() * 10000); + return { + type: SAVED_OBJECTS_DAILY_TYPE, + id: serializeSavedObjectId({ pid, date }), + attributes: { + ...mocked.createHistogram({ + fromTimestamp: moment(date).startOf('day').toISOString(), + lastUpdatedAt: moment(date).toISOString(), + }), + processId: pid, + }, + }; +} + +const rawEventLoopDelaysDaily = [ + createRawObject(moment.now()), + createRawObject(moment.now()), + createRawObject(moment().subtract(1, 'days')), + createRawObject(moment().subtract(3, 'days')), +]; + +const outdatedRawEventLoopDelaysDaily = [ + createRawObject(moment().subtract(5, 'days')), + createRawObject(moment().subtract(7, 'days')), +]; + +describe('daily rollups integration test', () => { + let esServer: TestElasticsearchUtils; + let root: TestKibanaUtils['root']; + let internalRepository: ISavedObjectsRepository; + let logger: Logger; + + beforeAll(async () => { + esServer = await startES(); + root = createRootWithCorePlugins(); + + await root.setup(); + const start = await root.start(); + logger = root.logger.get('test dailt rollups'); + internalRepository = start.savedObjects.createInternalRepository([SAVED_OBJECTS_DAILY_TYPE]); + + await internalRepository.bulkCreate( + [...rawEventLoopDelaysDaily, ...outdatedRawEventLoopDelaysDaily], + { refresh: true } + ); + }); + + afterAll(async () => { + await esServer.stop(); + await root.shutdown(); + }); + + it('deletes documents older that 3 days from the saved objects repository', async () => { + await rollDailyData(logger, internalRepository); + const { + total, + saved_objects: savedObjects, + } = await internalRepository.find({ type: SAVED_OBJECTS_DAILY_TYPE }); + expect(total).toBe(rawEventLoopDelaysDaily.length); + expect(savedObjects.map(({ id, type, attributes }) => ({ id, type, attributes }))).toEqual( + rawEventLoopDelaysDaily + ); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/saved_objects.test.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/saved_objects.test.ts new file mode 100644 index 0000000000000..022040615bd45 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/saved_objects.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + storeHistogram, + serializeSavedObjectId, + deleteHistogramSavedObjects, +} from './saved_objects'; +import { savedObjectsRepositoryMock } from '../../../../../core/server/mocks'; +import type { SavedObjectsFindResponse } from '../../../../../core/server/'; +import { mocked } from './event_loop_delays.mocks'; + +describe('serializeSavedObjectId', () => { + it('returns serialized id', () => { + const id = serializeSavedObjectId({ date: 1623233091278, pid: 123 }); + expect(id).toBe('123::09062021'); + }); +}); + +describe('storeHistogram', () => { + const mockHistogram = mocked.createHistogram(); + const mockInternalRepository = savedObjectsRepositoryMock.create(); + + jest.useFakeTimers('modern'); + const mockNow = jest.getRealSystemTime(); + jest.setSystemTime(mockNow); + + beforeEach(() => jest.clearAllMocks()); + afterAll(() => jest.useRealTimers()); + + it('stores histogram data in a savedObject', async () => { + await storeHistogram(mockHistogram, mockInternalRepository); + const pid = process.pid; + const id = serializeSavedObjectId({ date: mockNow, pid }); + + expect(mockInternalRepository.create).toBeCalledWith( + 'event_loop_delays_daily', + { ...mockHistogram, processId: pid }, + { id, overwrite: true } + ); + }); +}); + +describe('deleteHistogramSavedObjects', () => { + const mockInternalRepository = savedObjectsRepositoryMock.create(); + + beforeEach(() => { + jest.clearAllMocks(); + mockInternalRepository.find.mockResolvedValue({ + saved_objects: [{ id: 'test_obj_1' }, { id: 'test_obj_1' }], + } as SavedObjectsFindResponse); + }); + + it('builds filter query based on time range passed in days', async () => { + await deleteHistogramSavedObjects(mockInternalRepository); + await deleteHistogramSavedObjects(mockInternalRepository, 20); + expect(mockInternalRepository.find.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "filter": "event_loop_delays_daily.attributes.lastUpdatedAt < \\"now-3d/d\\"", + "type": "event_loop_delays_daily", + }, + ], + Array [ + Object { + "filter": "event_loop_delays_daily.attributes.lastUpdatedAt < \\"now-20d/d\\"", + "type": "event_loop_delays_daily", + }, + ], + ] + `); + }); + + it('loops over saved objects and deletes them', async () => { + mockInternalRepository.delete.mockImplementation(async (type, id) => { + return id; + }); + + const results = await deleteHistogramSavedObjects(mockInternalRepository); + expect(results).toMatchInlineSnapshot(` + Array [ + Object { + "status": "fulfilled", + "value": "test_obj_1", + }, + Object { + "status": "fulfilled", + "value": "test_obj_1", + }, + ] + `); + }); + + it('settles all promises even if some of the deletes fail.', async () => { + mockInternalRepository.delete.mockImplementationOnce(async (type, id) => { + throw new Error('Intentional failure'); + }); + mockInternalRepository.delete.mockImplementationOnce(async (type, id) => { + return id; + }); + + const results = await deleteHistogramSavedObjects(mockInternalRepository); + expect(results).toMatchInlineSnapshot(` + Array [ + Object { + "reason": [Error: Intentional failure], + "status": "rejected", + }, + Object { + "status": "fulfilled", + "value": "test_obj_1", + }, + ] + `); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/saved_objects.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/saved_objects.ts new file mode 100644 index 0000000000000..610a6697da364 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/saved_objects.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + SavedObjectAttributes, + SavedObjectsServiceSetup, + ISavedObjectsRepository, +} from 'kibana/server'; +import moment from 'moment'; +import type { IntervalHistogram } from './event_loop_delays'; + +export const SAVED_OBJECTS_DAILY_TYPE = 'event_loop_delays_daily'; + +export interface EventLoopDelaysDaily extends SavedObjectAttributes, IntervalHistogram { + processId: number; +} + +export function registerSavedObjectTypes(registerType: SavedObjectsServiceSetup['registerType']) { + registerType({ + name: SAVED_OBJECTS_DAILY_TYPE, + hidden: true, + namespaceType: 'agnostic', + mappings: { + dynamic: false, + properties: { + // This type requires `lastUpdatedAt` to be indexed so we can use it when rolling up totals (lastUpdatedAt < now-90d) + lastUpdatedAt: { type: 'date' }, + }, + }, + }); +} + +export function serializeSavedObjectId({ date, pid }: { date: moment.MomentInput; pid: number }) { + const formattedDate = moment(date).format('DDMMYYYY'); + + return `${pid}::${formattedDate}`; +} + +export async function deleteHistogramSavedObjects( + internalRepository: ISavedObjectsRepository, + daysTimeRange = 3 +) { + const { saved_objects: savedObjects } = await internalRepository.find({ + type: SAVED_OBJECTS_DAILY_TYPE, + filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.lastUpdatedAt < "now-${daysTimeRange}d/d"`, + }); + + return await Promise.allSettled( + savedObjects.map(async (savedObject) => { + return await internalRepository.delete(SAVED_OBJECTS_DAILY_TYPE, savedObject.id); + }) + ); +} + +export async function storeHistogram( + histogram: IntervalHistogram, + internalRepository: ISavedObjectsRepository +) { + const pid = process.pid; + const id = serializeSavedObjectId({ date: histogram.lastUpdatedAt, pid }); + + return await internalRepository.create( + SAVED_OBJECTS_DAILY_TYPE, + { ...histogram, processId: pid }, + { id, overwrite: true } + ); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/schema.ts new file mode 100644 index 0000000000000..319e8c77438b8 --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/event_loop_delays/schema.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { MakeSchemaFrom } from 'src/plugins/usage_collection/server'; + +export interface EventLoopDelaysUsageReport { + daily: Array<{ + processId: number; + lastUpdatedAt: string; + fromTimestamp: string; + min: number; + max: number; + mean: number; + exceeds: number; + stddev: number; + percentiles: { + '50': number; + '75': number; + '95': number; + '99': number; + }; + }>; +} + +export const eventLoopDelaysUsageSchema: MakeSchemaFrom = { + daily: { + type: 'array', + items: { + processId: { + type: 'long', + _meta: { + description: 'The process id of the monitored kibana instance.', + }, + }, + fromTimestamp: { + type: 'date', + _meta: { + description: 'Timestamp at which the histogram started monitoring.', + }, + }, + lastUpdatedAt: { + type: 'date', + _meta: { + description: 'Latest timestamp this histogram object was updated this day.', + }, + }, + min: { + type: 'long', + _meta: { + description: 'The minimum recorded event loop delay.', + }, + }, + max: { + type: 'long', + _meta: { + description: 'The maximum recorded event loop delay.', + }, + }, + mean: { + type: 'long', + _meta: { + description: 'The mean of the recorded event loop delays.', + }, + }, + exceeds: { + type: 'long', + _meta: { + description: + 'The number of times the event loop delay exceeded the maximum 1 hour eventloop delay threshold.', + }, + }, + stddev: { + type: 'long', + _meta: { + description: 'The standard deviation of the recorded event loop delays.', + }, + }, + percentiles: { + '50': { + type: 'long', + _meta: { + description: 'The 50th accumulated percentile distribution', + }, + }, + '75': { + type: 'long', + _meta: { + description: 'The 75th accumulated percentile distribution', + }, + }, + '95': { + type: 'long', + _meta: { + description: 'The 95th accumulated percentile distribution', + }, + }, + '99': { + type: 'long', + _meta: { + description: 'The 99th accumulated percentile distribution', + }, + }, + }, + }, + }, +}; diff --git a/src/plugins/kibana_usage_collection/server/collectors/index.ts b/src/plugins/kibana_usage_collection/server/collectors/index.ts index 761989938e56d..e4ed24611bfa8 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/index.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/index.ts @@ -28,3 +28,4 @@ export { registerUsageCountersRollups, registerUsageCountersUsageCollector, } from './usage_counters'; +export { registerEventLoopDelaysCollector } from './event_loop_delays'; diff --git a/src/plugins/kibana_usage_collection/server/plugin.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts index 2100b9bbb918b..1584366a42dc1 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -16,7 +16,6 @@ import { createUsageCollectionSetupMock, } from '../../usage_collection/server/mocks'; import { cloudDetailsMock } from './mocks'; - import { plugin } from './'; describe('kibana_usage_collection', () => { @@ -105,6 +104,10 @@ describe('kibana_usage_collection', () => { "isReady": true, "type": "localization", }, + Object { + "isReady": false, + "type": "event_loop_delays", + }, ] `); }); diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index da6445ce957d8..4ec717c48610e 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -22,6 +22,10 @@ import type { CoreUsageDataStart, } from 'src/core/server'; import { SavedObjectsClient } from '../../../core/server'; +import { + startTrackingEventLoopDelaysUsage, + SAVED_OBJECTS_DAILY_TYPE, +} from './collectors/event_loop_delays'; import { registerApplicationUsageCollector, registerKibanaUsageCollector, @@ -39,6 +43,7 @@ import { registerUsageCountersRollups, registerUsageCountersUsageCollector, registerSavedObjectsCountUsageCollector, + registerEventLoopDelaysCollector, } from './collectors'; interface KibanaUsageCollectionPluginsDepsSetup { @@ -54,46 +59,46 @@ export class KibanaUsageCollectionPlugin implements Plugin { private uiSettingsClient?: IUiSettingsClient; private metric$: Subject; private coreUsageData?: CoreUsageDataStart; - private stopUsingUiCounterIndicies$: Subject; + private pluginStop$: Subject; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); this.legacyConfig$ = initializerContext.config.legacy.globalConfig$; this.metric$ = new Subject(); - this.stopUsingUiCounterIndicies$ = new Subject(); + this.pluginStop$ = new Subject(); } public setup(coreSetup: CoreSetup, { usageCollection }: KibanaUsageCollectionPluginsDepsSetup) { usageCollection.createUsageCounter('uiCounters'); - this.registerUsageCollectors( usageCollection, coreSetup, this.metric$, - this.stopUsingUiCounterIndicies$, + this.pluginStop$, coreSetup.savedObjects.registerType.bind(coreSetup.savedObjects) ); } public start(core: CoreStart) { const { savedObjects, uiSettings } = core; - this.savedObjectsClient = savedObjects.createInternalRepository(); + this.savedObjectsClient = savedObjects.createInternalRepository([SAVED_OBJECTS_DAILY_TYPE]); const savedObjectsClient = new SavedObjectsClient(this.savedObjectsClient); this.uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient); core.metrics.getOpsMetrics$().subscribe(this.metric$); this.coreUsageData = core.coreUsageData; + startTrackingEventLoopDelaysUsage(this.savedObjectsClient, this.pluginStop$.asObservable()); } public stop() { this.metric$.complete(); - this.stopUsingUiCounterIndicies$.complete(); + this.pluginStop$.complete(); } private registerUsageCollectors( usageCollection: UsageCollectionSetup, coreSetup: CoreSetup, metric$: Subject, - stopUsingUiCounterIndicies$: Subject, + pluginStop$: Subject, registerType: SavedObjectsRegisterType ) { const getSavedObjectsClient = () => this.savedObjectsClient; @@ -101,12 +106,8 @@ export class KibanaUsageCollectionPlugin implements Plugin { const getCoreUsageDataService = () => this.coreUsageData!; registerUiCounterSavedObjectType(coreSetup.savedObjects); - registerUiCountersRollups( - this.logger.get('ui-counters'), - stopUsingUiCounterIndicies$, - getSavedObjectsClient - ); - registerUiCountersUsageCollector(usageCollection, stopUsingUiCounterIndicies$); + registerUiCountersRollups(this.logger.get('ui-counters'), pluginStop$, getSavedObjectsClient); + registerUiCountersUsageCollector(usageCollection, pluginStop$); registerUsageCountersRollups(this.logger.get('usage-counters-rollup'), getSavedObjectsClient); registerUsageCountersUsageCollector(usageCollection); @@ -127,5 +128,11 @@ export class KibanaUsageCollectionPlugin implements Plugin { registerCoreUsageCollector(usageCollection, getCoreUsageDataService); registerConfigUsageCollector(usageCollection, getCoreUsageDataService); registerLocalizationUsageCollector(usageCollection, coreSetup.i18n); + registerEventLoopDelaysCollector( + this.logger.get('event-loop-delays'), + usageCollection, + registerType, + getSavedObjectsClient + ); } } diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 6ab550389a12d..99c6dcb40e57d 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7900,6 +7900,93 @@ } } }, + "event_loop_delays": { + "properties": { + "daily": { + "type": "array", + "items": { + "properties": { + "processId": { + "type": "long", + "_meta": { + "description": "The process id of the monitored kibana instance." + } + }, + "fromTimestamp": { + "type": "date", + "_meta": { + "description": "Timestamp at which the histogram started monitoring." + } + }, + "lastUpdatedAt": { + "type": "date", + "_meta": { + "description": "Latest timestamp this histogram object was updated this day." + } + }, + "min": { + "type": "long", + "_meta": { + "description": "The minimum recorded event loop delay." + } + }, + "max": { + "type": "long", + "_meta": { + "description": "The maximum recorded event loop delay." + } + }, + "mean": { + "type": "long", + "_meta": { + "description": "The mean of the recorded event loop delays." + } + }, + "exceeds": { + "type": "long", + "_meta": { + "description": "The number of times the event loop delay exceeded the maximum 1 hour eventloop delay threshold." + } + }, + "stddev": { + "type": "long", + "_meta": { + "description": "The standard deviation of the recorded event loop delays." + } + }, + "percentiles": { + "properties": { + "50": { + "type": "long", + "_meta": { + "description": "The 50th accumulated percentile distribution" + } + }, + "75": { + "type": "long", + "_meta": { + "description": "The 75th accumulated percentile distribution" + } + }, + "95": { + "type": "long", + "_meta": { + "description": "The 95th accumulated percentile distribution" + } + }, + "99": { + "type": "long", + "_meta": { + "description": "The 99th accumulated percentile distribution" + } + } + } + } + } + } + } + } + }, "localization": { "properties": { "locale": {