From 05698771709585879eedc38665d5d0604cb6e78e Mon Sep 17 00:00:00 2001 From: Miriam <31922082+MiriamAparicio@users.noreply.github.com> Date: Thu, 14 Nov 2024 18:50:37 +0000 Subject: [PATCH] [obsUX][APM] Migrate APM metrics tests to agnostic deployment tests (#200175) ### Summary Closes https://github.com/elastic/kibana/issues/198976 Part of https://github.com/elastic/kibana/issues/193245 This PR contains the changes to migrate metrics test folder to Deployment-agnostic testing strategy. #### How to test Serverless ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM" ``` It's recommended to be run against [MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki) Stateful ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM" ``` Checks - [ ] (OPTIONAL, only if a test has been unskipped) Run flaky test suite - [x] local run for serverless - [x] local run for stateful - [x] MKI run for serverless --- .../apis/observability/apm/index.ts | 1 + .../apis/observability/apm/metrics/index.ts | 19 + .../apm}/metrics/memory/generate_data.ts | 0 .../metrics/memory/memory_metrics.spec.ts | 15 +- .../apm/metrics/metrics_charts.spec.ts | 461 +++++++++++++++++ .../apm}/metrics/serverless/generate_data.ts | 0 .../serverless_active_instances.spec.ts | 15 +- .../serverless_functions_overview.spec.ts | 16 +- .../serverless_metrics_charts.spec.ts | 331 +++++++++++++ .../serverless/serverless_summary.spec.ts | 110 +++++ .../es_archiver/metrics_8.0.0/mappings.json | 7 +- .../tests/metrics/metrics_charts.spec.ts | 462 ------------------ .../serverless_metrics_charts.spec.ts | 333 ------------- .../serverless/serverless_summary.spec.ts | 111 ----- 14 files changed, 948 insertions(+), 933 deletions(-) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/index.ts rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/metrics/memory/generate_data.ts (100%) rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/metrics/memory/memory_metrics.spec.ts (85%) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/metrics_charts.spec.ts rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/metrics/serverless/generate_data.ts (100%) rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/metrics/serverless/serverless_active_instances.spec.ts (89%) rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/metrics/serverless/serverless_functions_overview.spec.ts (84%) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_metrics_charts.spec.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_summary.spec.ts delete mode 100644 x-pack/test/apm_api_integration/tests/metrics/metrics_charts.spec.ts delete mode 100644 x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_metrics_charts.spec.ts delete mode 100644 x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_summary.spec.ts diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts index d115e10f2373a..b46b2c3f10637 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts @@ -23,6 +23,7 @@ export default function apmApiIntegrationTests({ loadTestFile(require.resolve('./correlations')); loadTestFile(require.resolve('./entities')); loadTestFile(require.resolve('./cold_start')); + loadTestFile(require.resolve('./metrics')); loadTestFile(require.resolve('./services')); loadTestFile(require.resolve('./historical_data')); loadTestFile(require.resolve('./observability_overview')); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/index.ts new file mode 100644 index 0000000000000..454e3b16e9aad --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) { + describe('metrics', () => { + loadTestFile(require.resolve('./metrics_charts.spec.ts')); + loadTestFile(require.resolve('./memory/memory_metrics.spec.ts')); + loadTestFile(require.resolve('./serverless/serverless_active_instances.spec.ts')); + loadTestFile(require.resolve('./serverless/serverless_functions_overview.spec.ts')); + loadTestFile(require.resolve('./serverless/serverless_metrics_charts.spec.ts')); + loadTestFile(require.resolve('./serverless/serverless_summary.spec.ts')); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/memory/generate_data.ts similarity index 100% rename from x-pack/test/apm_api_integration/tests/metrics/memory/generate_data.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/memory/generate_data.ts diff --git a/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/memory/memory_metrics.spec.ts similarity index 85% rename from x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/memory/memory_metrics.spec.ts index e0f8fc1cf28c2..0bcfff6395fef 100644 --- a/x-pack/test/apm_api_integration/tests/metrics/memory/memory_metrics.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/memory/memory_metrics.spec.ts @@ -6,13 +6,13 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { config, generateData } from './generate_data'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); const start = new Date('2023-01-01T00:00:00.000Z').getTime(); const end = new Date('2023-01-01T00:15:00.000Z').getTime() - 1; @@ -33,9 +33,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } - // FLAKY: https://github.com/elastic/kibana/issues/176990 - registry.when('Memory', { config: 'trial', archives: [] }, () => { + describe('Memory', () => { + let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); await generateData({ start, end, apmSynthtraceEsClient }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/metrics_charts.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/metrics_charts.spec.ts new file mode 100644 index 0000000000000..f801113fdf823 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/metrics_charts.spec.ts @@ -0,0 +1,461 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { first } from 'lodash'; +import { GenericMetricsChart } from '@kbn/apm-plugin/server/routes/metrics/fetch_and_transform_metrics'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; +import { SupertestReturnType } from '../../../../services/apm_api'; + +type ChartResponse = SupertestReturnType<'GET /internal/apm/services/{serviceName}/metrics/charts'>; + +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); + + describe('Metrics charts when data is loaded', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['metrics_8.0.0']); + }); + + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['metrics_8.0.0']); + }); + + describe('for opbeans-node', () => { + describe('returns metrics data', () => { + let chartsResponse: ChartResponse; + before(async () => { + chartsResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', + params: { + path: { serviceName: 'opbeans-node' }, + query: { + start: '2020-09-08T14:50:00.000Z', + end: '2020-09-08T14:55:00.000Z', + agentName: 'nodejs', + environment: 'ENVIRONMENT_ALL', + kuery: ``, + }, + }, + }); + }); + + it('contains CPU usage and System memory usage chart data', async () => { + expect(chartsResponse.status).to.be(200); + expectSnapshot(chartsResponse.body.charts.map((chart) => chart.title)).toMatchInline(` + Array [ + "CPU usage", + "System memory usage", + ] + `); + }); + + describe('CPU usage', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find(({ key }) => key === 'cpu_usage_chart'); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "System max", + "System average", + "Process max", + "Process average", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0.714, + 0.3877, + 0.75, + 0.2543, + ] + `); + }); + }); + + describe("System memory usage (using 'system.memory' fields to calculate the memory usage)", () => { + let systemMemoryUsageChart: GenericMetricsChart | undefined; + before(() => { + systemMemoryUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'memory_usage_chart' + ); + }); + + it('has correct series', () => { + expect(systemMemoryUsageChart).to.not.empty(); + expectSnapshot(systemMemoryUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "Max", + "Average", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(systemMemoryUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0.722093920925555, + 0.718173546796348, + ] + `); + }); + }); + }); + }); + + describe('for opbeans-java', () => { + describe('returns metrics data', () => { + let chartsResponse: ChartResponse; + before(async () => { + chartsResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', + params: { + path: { serviceName: 'opbeans-java' }, + query: { + start: '2020-09-08T14:55:30.000Z', + end: '2020-09-08T15:00:00.000Z', + agentName: 'java', + environment: 'ENVIRONMENT_ALL', + kuery: ``, + }, + }, + }); + }); + + it('has correct chart data', async () => { + expect(chartsResponse.status).to.be(200); + expectSnapshot(chartsResponse.body.charts.map((chart) => chart.title)).toMatchInline(` + Array [ + "CPU usage", + "System memory usage", + "Heap Memory", + "Non-Heap Memory", + "Thread Count", + "Garbage collection per minute", + "Garbage collection time spent per minute", + ] + `); + }); + + describe('CPU usage', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find(({ key }) => key === 'cpu_usage_chart'); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "System max", + "System average", + "Process max", + "Process average", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0.203, + 0.178777777777778, + 0.01, + 0.009, + ] + `); + }); + + it('has the correct rate', async () => { + const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); + expectSnapshot(yValues).toMatchInline(` + Array [ + 0.193, + 0.193, + 0.009, + 0.009, + ] + `); + }); + }); + + describe("System memory usage (using 'system.process.cgroup' fields to calculate the memory usage)", () => { + let systemMemoryUsageChart: GenericMetricsChart | undefined; + before(() => { + systemMemoryUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'memory_usage_chart' + ); + }); + + it('has correct series', () => { + expect(systemMemoryUsageChart).to.not.empty(); + expectSnapshot(systemMemoryUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "Max", + "Average", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(systemMemoryUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0.707924703557837, + 0.705395980841182, + ] + `); + }); + + it('has the correct rate', async () => { + const yValues = systemMemoryUsageChart?.series.map((serie) => first(serie.data)?.y); + expectSnapshot(yValues).toMatchInline(` + Array [ + 0.707924703557837, + 0.707924703557837, + ] + `); + }); + }); + + describe('Heap Memory', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'heap_memory_area_chart' + ); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "Avg. used", + "Avg. committed", + "Avg. limit", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 222501617.777778, + 374341632, + 1560281088, + ] + `); + }); + + it('has the correct rate', async () => { + const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); + expectSnapshot(yValues).toMatchInline(` + Array [ + 211472896, + 374341632, + 1560281088, + ] + `); + }); + }); + + describe('Non-Heap Memory', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'non_heap_memory_area_chart' + ); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "Avg. used", + "Avg. committed", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 138573397.333333, + 147677639.111111, + ] + `); + }); + + it('has the correct rate', async () => { + const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); + expectSnapshot(yValues).toMatchInline(` + Array [ + 138162752, + 147386368, + ] + `); + }); + }); + + describe('Thread Count', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'thread_count_line_chart' + ); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "Avg. count", + "Max count", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 44.4444444444444, + 45, + ] + `); + }); + + it('has the correct rate', async () => { + const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); + expectSnapshot(yValues).toMatchInline(` + Array [ + 44, + 44, + ] + `); + }); + }); + + describe('Garbage collection per minute', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'gc_rate_line_chart' + ); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "G1 Old Generation", + "G1 Young Generation", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0, + 3, + ] + `); + }); + }); + + describe('Garbage collection time spent per minute', () => { + let cpuUsageChart: GenericMetricsChart | undefined; + before(() => { + cpuUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'gc_time_line_chart' + ); + }); + + it('has correct series', () => { + expect(cpuUsageChart).to.not.empty(); + expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "G1 Old Generation", + "G1 Young Generation", + ] + `); + }); + + it('has correct series overall values', () => { + expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0, + 37500, + ] + `); + }); + }); + }); + + // 9223372036854771712 = memory limit for a c-group when no memory limit is specified + it('calculates system memory usage using system total field when cgroup limit is equal to 9223372036854771712', async () => { + const chartsResponse = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', + params: { + path: { serviceName: 'opbeans-java' }, + query: { + start: '2020-09-08T15:00:30.000Z', + end: '2020-09-08T15:05:00.000Z', + agentName: 'java', + environment: 'ENVIRONMENT_ALL', + kuery: ``, + }, + }, + }); + + const systemMemoryUsageChart = chartsResponse.body.charts.find( + ({ key }) => key === 'memory_usage_chart' + ); + + expect(systemMemoryUsageChart).to.not.empty(); + expectSnapshot(systemMemoryUsageChart?.series.map(({ title }) => title)).toMatchInline(` + Array [ + "Max", + "Average", + ] + `); + expectSnapshot(systemMemoryUsageChart?.series.map(({ overallValue }) => overallValue)) + .toMatchInline(` + Array [ + 0.114523896426499, + 0.114002376090415, + ] + `); + + const yValues = systemMemoryUsageChart?.series.map((serie) => first(serie.data)?.y); + expectSnapshot(yValues).toMatchInline(` + Array [ + 0.11383724014064, + 0.11383724014064, + ] + `); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/metrics/serverless/generate_data.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/generate_data.ts similarity index 100% rename from x-pack/test/apm_api_integration/tests/metrics/serverless/generate_data.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/generate_data.ts diff --git a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_active_instances.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_active_instances.spec.ts similarity index 89% rename from x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_active_instances.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_active_instances.spec.ts index 1b15e03c91987..b490482b4dd52 100644 --- a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_active_instances.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_active_instances.spec.ts @@ -6,15 +6,15 @@ */ import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import expect from '@kbn/expect'; import { sumBy } from 'lodash'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; import { config, expectedValues, generateData } from './generate_data'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; @@ -36,8 +36,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } - // FLAKY: https://github.com/elastic/kibana/issues/177639 - registry.when('Serverless active instances', { config: 'basic', archives: [] }, () => { + describe('Serverless active instances', () => { const { memoryTotal, billedDurationMs, @@ -47,8 +46,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { } = config; const { expectedMemoryUsed } = expectedValues; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); await generateData({ start, end, apmSynthtraceEsClient }); }); diff --git a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_functions_overview.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_functions_overview.spec.ts similarity index 84% rename from x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_functions_overview.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_functions_overview.spec.ts index 94792228a2859..3acd0921d2602 100644 --- a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_functions_overview.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_functions_overview.spec.ts @@ -7,13 +7,13 @@ import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { config, expectedValues, generateData } from './generate_data'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; @@ -34,8 +34,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } - // FLAKY: https://github.com/elastic/kibana/issues/177641 - registry.when('Serverless functions overview', { config: 'basic', archives: [] }, () => { + describe('Serverless functions overview', () => { const { memoryTotal, billedDurationMs, @@ -45,7 +44,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { } = config; const { expectedMemoryUsed } = expectedValues; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); await generateData({ start, end, apmSynthtraceEsClient }); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_metrics_charts.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_metrics_charts.spec.ts new file mode 100644 index 0000000000000..7f1e3c2a05004 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_metrics_charts.spec.ts @@ -0,0 +1,331 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { meanBy, sumBy } from 'lodash'; +import { Coordinate } from '@kbn/apm-plugin/typings/timeseries'; +import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; +import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { generateData, config } from './generate_data'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; + +function isNotNullOrZeroCoordinate(coordinate: Coordinate) { + return coordinate.y !== null && coordinate.y !== 0; +} + +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + const numberOfTransactionsCreated = 15; + + async function callApi(serviceName: string, serverlessId?: string) { + return await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services/{serviceName}/metrics/serverless/charts', + params: { + path: { serviceName }, + query: { + environment: 'test', + kuery: '', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.OneMinute, + bucketSizeInSeconds: 60, + ...(serverlessId ? { serverlessId } : {}), + }, + }, + }); + } + + describe('Serverless metrics charts', () => { + describe('when data is not loaded', () => { + let serverlessMetrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/charts'>; + before(async () => { + const response = await callApi('lambda-python'); + serverlessMetrics = response.body; + }); + + it('returns empty', () => { + serverlessMetrics.charts.forEach((chart) => { + expect(chart.series).to.be.empty(); + }); + }); + }); + + describe('when data is loaded', () => { + const { + memoryTotal, + memoryFree, + billedDurationMs, + coldStartDurationPython, + transactionDuration, + pythonServerlessFunctionNames, + serverlessId, + } = config; + + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await generateData({ start, end, apmSynthtraceEsClient }); + }); + + after(() => apmSynthtraceEsClient.clean()); + + describe('Python service', () => { + let serverlessMetrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/charts'>; + before(async () => { + const response = await callApi('lambda-python'); + serverlessMetrics = response.body; + }); + + it('returns all metrics chart', () => { + expect(serverlessMetrics.charts.length).to.be.greaterThan(0); + expect(serverlessMetrics.charts.map(({ title }) => title).sort()).to.eql([ + 'Cold start duration', + 'Cold starts', + 'Compute usage', + 'Lambda Duration', + 'System memory usage', + ]); + }); + + describe('Avg. Duration', () => { + const transactionDurationInMicroSeconds = transactionDuration * 1000; + [ + { title: 'Billed Duration', expectedValue: billedDurationMs * 1000 }, + { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const avgDurationMetric = serverlessMetrics.charts.find((chart) => { + return chart.key === 'avg_duration'; + }); + const series = avgDurationMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + let metricsChart: (typeof serverlessMetrics.charts)[0] | undefined; + + describe('Cold start duration', () => { + before(() => { + metricsChart = serverlessMetrics.charts.find((chart) => { + return chart.key === 'cold_start_duration'; + }); + }); + it('returns correct overall value', () => { + expect(metricsChart?.series[0].overallValue).to.equal(coldStartDurationPython * 1000); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), + 'y' + ); + expect(meanValue).to.equal(coldStartDurationPython * 1000); + }); + }); + + describe('Cold start count', () => { + before(() => { + metricsChart = serverlessMetrics.charts.find((chart) => { + return chart.key === 'cold_start_count'; + }); + }); + + it('returns correct overall value', () => { + expect(metricsChart?.series[0].overallValue).to.equal( + numberOfTransactionsCreated * pythonServerlessFunctionNames.length + ); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), + 'y' + ); + expect(sumValue).to.equal( + numberOfTransactionsCreated * pythonServerlessFunctionNames.length + ); + }); + }); + + describe('memory usage', () => { + const expectedFreeMemory = 1 - memoryFree / memoryTotal; + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const memoryUsageMetric = serverlessMetrics.charts.find((chart) => { + return chart.key === 'memory_usage_chart'; + }); + const series = memoryUsageMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Compute usage', () => { + const GBSeconds = 1024 * 1024 * 1024 * 1000; + let computeUsageMetric: (typeof serverlessMetrics.charts)[0] | undefined; + before(() => { + computeUsageMetric = serverlessMetrics.charts.find((chart) => { + return chart.key === 'compute_usage'; + }); + }); + it('returns correct overall value', () => { + const expectedValue = + ((memoryTotal * billedDurationMs) / GBSeconds) * numberOfTransactionsCreated * 2; + expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); + }); + + it('returns correct mean value', () => { + const expectedValue = ((memoryTotal * billedDurationMs) / GBSeconds) * 2; + const meanValue = meanBy( + computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(meanValue).to.equal(expectedValue); + }); + }); + }); + + describe('detailed metrics', () => { + let serverlessMetrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/charts'>; + before(async () => { + const response = await callApi( + 'lambda-python', + `${serverlessId}${pythonServerlessFunctionNames[0]}` + ); + serverlessMetrics = response.body; + }); + + it('returns all metrics chart', () => { + expect(serverlessMetrics.charts.length).to.be.greaterThan(0); + expect(serverlessMetrics.charts.map(({ title }) => title).sort()).to.eql([ + 'Cold start duration', + 'Cold starts', + 'Compute usage', + 'Lambda Duration', + 'System memory usage', + ]); + }); + + describe('Avg. Duration', () => { + const transactionDurationInMicroSeconds = transactionDuration * 1000; + [ + { title: 'Billed Duration', expectedValue: billedDurationMs * 1000 }, + { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const avgDurationMetric = serverlessMetrics.charts.find((chart) => { + return chart.key === 'avg_duration'; + }); + const series = avgDurationMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + let metricsChart: (typeof serverlessMetrics.charts)[0] | undefined; + + describe('Cold start duration', () => { + before(() => { + metricsChart = serverlessMetrics.charts.find((chart) => { + return chart.key === 'cold_start_duration'; + }); + }); + it('returns correct overall value', () => { + expect(metricsChart?.series[0].overallValue).to.equal(coldStartDurationPython * 1000); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), + 'y' + ); + expect(meanValue).to.equal(coldStartDurationPython * 1000); + }); + }); + + describe('Cold start count', () => { + before(() => { + metricsChart = serverlessMetrics.charts.find((chart) => { + return chart.key === 'cold_start_count'; + }); + }); + + it('returns correct overall value', () => { + expect(metricsChart?.series[0].overallValue).to.equal(numberOfTransactionsCreated); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated); + }); + }); + + describe('memory usage', () => { + const expectedFreeMemory = 1 - memoryFree / memoryTotal; + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const memoryUsageMetric = serverlessMetrics.charts.find((chart) => { + return chart.key === 'memory_usage_chart'; + }); + const series = memoryUsageMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Compute usage', () => { + const GBSeconds = 1024 * 1024 * 1024 * 1000; + let computeUsageMetric: (typeof serverlessMetrics.charts)[0] | undefined; + before(() => { + computeUsageMetric = serverlessMetrics.charts.find((chart) => { + return chart.key === 'compute_usage'; + }); + }); + it('returns correct overall value', () => { + const expectedValue = + ((memoryTotal * billedDurationMs) / GBSeconds) * numberOfTransactionsCreated; + expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); + }); + + it('returns correct mean value', () => { + const expectedValue = (memoryTotal * billedDurationMs) / GBSeconds; + const meanValue = meanBy( + computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(meanValue).to.equal(expectedValue); + }); + }); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_summary.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_summary.spec.ts new file mode 100644 index 0000000000000..c291ffab47648 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/metrics/serverless/serverless_summary.spec.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import expect from '@kbn/expect'; +import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { config, expectedValues, generateData } from './generate_data'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context'; + +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + async function callApi(serviceName: string, serverlessId?: string) { + return await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/{serviceName}/metrics/serverless/summary`, + params: { + path: { serviceName }, + query: { + environment: 'test', + kuery: '', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + ...(serverlessId ? { serverlessId } : {}), + }, + }, + }); + } + + describe('Serverless overview', () => { + describe('when data is not loaded', () => { + let serverlessSummary: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/summary'>; + before(async () => { + const response = await callApi('lambda-python'); + serverlessSummary = response.body; + }); + + it('returns empty', () => { + expect(serverlessSummary).to.be.empty(); + }); + }); + + describe('when data is loaded', () => { + const { billedDurationMs, pythonServerlessFunctionNames, faasDuration, serverlessId } = + config; + const { expectedMemoryUsedRate } = expectedValues; + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await generateData({ start, end, apmSynthtraceEsClient }); + }); + + after(() => apmSynthtraceEsClient.clean()); + + describe('Python service', () => { + let serverlessSummary: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/summary'>; + before(async () => { + const response = await callApi('lambda-python'); + serverlessSummary = response.body; + }); + + it('returns correct memory avg', () => { + expect(serverlessSummary.memoryUsageAvgRate).to.eql(expectedMemoryUsedRate); + }); + it('returns correct serverless function total', () => { + expect(serverlessSummary.serverlessFunctionsTotal).to.eql( + pythonServerlessFunctionNames.length + ); + }); + it('returns correct serverless duration avg', () => { + expect(serverlessSummary.serverlessDurationAvg).to.eql(faasDuration); + }); + it('returns correct billed duration avg', () => { + expect(serverlessSummary.billedDurationAvg).to.eql(billedDurationMs); + }); + }); + + describe('detailed metrics', () => { + let serverlessSummary: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/summary'>; + before(async () => { + const response = await callApi( + 'lambda-python', + `${serverlessId}${pythonServerlessFunctionNames[0]}` + ); + serverlessSummary = response.body; + }); + + it('returns correct memory avg', () => { + expect(serverlessSummary.memoryUsageAvgRate).to.eql(expectedMemoryUsedRate); + }); + it('returns correct serverless function total', () => { + expect(serverlessSummary.serverlessFunctionsTotal).to.eql(1); + }); + it('returns correct serverless duration avg', () => { + expect(serverlessSummary.serverlessDurationAvg).to.eql(faasDuration); + }); + it('returns correct billed duration avg', () => { + expect(serverlessSummary.billedDurationAvg).to.eql(billedDurationMs); + }); + }); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/metrics_8.0.0/mappings.json b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/metrics_8.0.0/mappings.json index 5cc5a0032cd31..b1044d198a9a3 100644 --- a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/metrics_8.0.0/mappings.json +++ b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/metrics_8.0.0/mappings.json @@ -4073,10 +4073,6 @@ "index": { "auto_expand_replicas": "0-1", "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-8.0.0-metric" - }, "mapping": { "total_fields": { "limit": "2000" @@ -4085,8 +4081,7 @@ "max_docvalue_fields_search": "200", "number_of_replicas": "0", "number_of_shards": "1", - "priority": "100", - "refresh_interval": "1ms" + "priority": "100" } } } diff --git a/x-pack/test/apm_api_integration/tests/metrics/metrics_charts.spec.ts b/x-pack/test/apm_api_integration/tests/metrics/metrics_charts.spec.ts deleted file mode 100644 index 92e7c8a80bcb1..0000000000000 --- a/x-pack/test/apm_api_integration/tests/metrics/metrics_charts.spec.ts +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { first } from 'lodash'; -import { GenericMetricsChart } from '@kbn/apm-plugin/server/routes/metrics/fetch_and_transform_metrics'; -import { SupertestReturnType } from '../../common/apm_api_supertest'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -type ChartResponse = SupertestReturnType<'GET /internal/apm/services/{serviceName}/metrics/charts'>; - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - - registry.when( - 'Metrics charts when data is loaded', - { config: 'basic', archives: ['metrics_8.0.0'] }, - () => { - describe('for opbeans-node', () => { - describe('returns metrics data', () => { - let chartsResponse: ChartResponse; - before(async () => { - chartsResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', - params: { - path: { serviceName: 'opbeans-node' }, - query: { - start: '2020-09-08T14:50:00.000Z', - end: '2020-09-08T14:55:00.000Z', - agentName: 'nodejs', - environment: 'ENVIRONMENT_ALL', - kuery: ``, - }, - }, - }); - }); - - it('contains CPU usage and System memory usage chart data', async () => { - expect(chartsResponse.status).to.be(200); - expectSnapshot(chartsResponse.body.charts.map((chart) => chart.title)).toMatchInline(` - Array [ - "CPU usage", - "System memory usage", - ] - `); - }); - - describe('CPU usage', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'cpu_usage_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "System max", - "System average", - "Process max", - "Process average", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0.714, - 0.3877, - 0.75, - 0.2543, - ] - `); - }); - }); - - describe("System memory usage (using 'system.memory' fields to calculate the memory usage)", () => { - let systemMemoryUsageChart: GenericMetricsChart | undefined; - before(() => { - systemMemoryUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'memory_usage_chart' - ); - }); - - it('has correct series', () => { - expect(systemMemoryUsageChart).to.not.empty(); - expectSnapshot(systemMemoryUsageChart?.series.map(({ title }) => title)) - .toMatchInline(` - Array [ - "Max", - "Average", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(systemMemoryUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0.722093920925555, - 0.718173546796348, - ] - `); - }); - }); - }); - }); - - describe('for opbeans-java', () => { - describe('returns metrics data', () => { - let chartsResponse: ChartResponse; - before(async () => { - chartsResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start: '2020-09-08T14:55:30.000Z', - end: '2020-09-08T15:00:00.000Z', - agentName: 'java', - environment: 'ENVIRONMENT_ALL', - kuery: ``, - }, - }, - }); - }); - - it('has correct chart data', async () => { - expect(chartsResponse.status).to.be(200); - expectSnapshot(chartsResponse.body.charts.map((chart) => chart.title)).toMatchInline(` - Array [ - "CPU usage", - "System memory usage", - "Heap Memory", - "Non-Heap Memory", - "Thread Count", - "Garbage collection per minute", - "Garbage collection time spent per minute", - ] - `); - }); - - describe('CPU usage', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'cpu_usage_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "System max", - "System average", - "Process max", - "Process average", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0.203, - 0.178777777777778, - 0.01, - 0.009, - ] - `); - }); - - it('has the correct rate', async () => { - const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); - expectSnapshot(yValues).toMatchInline(` - Array [ - 0.193, - 0.193, - 0.009, - 0.009, - ] - `); - }); - }); - - describe("System memory usage (using 'system.process.cgroup' fields to calculate the memory usage)", () => { - let systemMemoryUsageChart: GenericMetricsChart | undefined; - before(() => { - systemMemoryUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'memory_usage_chart' - ); - }); - - it('has correct series', () => { - expect(systemMemoryUsageChart).to.not.empty(); - expectSnapshot(systemMemoryUsageChart?.series.map(({ title }) => title)) - .toMatchInline(` - Array [ - "Max", - "Average", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(systemMemoryUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0.707924703557837, - 0.705395980841182, - ] - `); - }); - - it('has the correct rate', async () => { - const yValues = systemMemoryUsageChart?.series.map((serie) => first(serie.data)?.y); - expectSnapshot(yValues).toMatchInline(` - Array [ - 0.707924703557837, - 0.707924703557837, - ] - `); - }); - }); - - describe('Heap Memory', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'heap_memory_area_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "Avg. used", - "Avg. committed", - "Avg. limit", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 222501617.777778, - 374341632, - 1560281088, - ] - `); - }); - - it('has the correct rate', async () => { - const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); - expectSnapshot(yValues).toMatchInline(` - Array [ - 211472896, - 374341632, - 1560281088, - ] - `); - }); - }); - - describe('Non-Heap Memory', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'non_heap_memory_area_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "Avg. used", - "Avg. committed", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 138573397.333333, - 147677639.111111, - ] - `); - }); - - it('has the correct rate', async () => { - const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); - expectSnapshot(yValues).toMatchInline(` - Array [ - 138162752, - 147386368, - ] - `); - }); - }); - - describe('Thread Count', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'thread_count_line_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "Avg. count", - "Max count", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 44.4444444444444, - 45, - ] - `); - }); - - it('has the correct rate', async () => { - const yValues = cpuUsageChart?.series.map((serie) => first(serie.data)?.y); - expectSnapshot(yValues).toMatchInline(` - Array [ - 44, - 44, - ] - `); - }); - }); - - describe('Garbage collection per minute', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'gc_rate_line_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "G1 Old Generation", - "G1 Young Generation", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0, - 3, - ] - `); - }); - }); - - describe('Garbage collection time spent per minute', () => { - let cpuUsageChart: GenericMetricsChart | undefined; - before(() => { - cpuUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'gc_time_line_chart' - ); - }); - - it('has correct series', () => { - expect(cpuUsageChart).to.not.empty(); - expectSnapshot(cpuUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "G1 Old Generation", - "G1 Young Generation", - ] - `); - }); - - it('has correct series overall values', () => { - expectSnapshot(cpuUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0, - 37500, - ] - `); - }); - }); - }); - - // 9223372036854771712 = memory limit for a c-group when no memory limit is specified - it('calculates system memory usage using system total field when cgroup limit is equal to 9223372036854771712', async () => { - const chartsResponse = await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', - params: { - path: { serviceName: 'opbeans-java' }, - query: { - start: '2020-09-08T15:00:30.000Z', - end: '2020-09-08T15:05:00.000Z', - agentName: 'java', - environment: 'ENVIRONMENT_ALL', - kuery: ``, - }, - }, - }); - - const systemMemoryUsageChart = chartsResponse.body.charts.find( - ({ key }) => key === 'memory_usage_chart' - ); - - expect(systemMemoryUsageChart).to.not.empty(); - expectSnapshot(systemMemoryUsageChart?.series.map(({ title }) => title)).toMatchInline(` - Array [ - "Max", - "Average", - ] - `); - expectSnapshot(systemMemoryUsageChart?.series.map(({ overallValue }) => overallValue)) - .toMatchInline(` - Array [ - 0.114523896426499, - 0.114002376090415, - ] - `); - - const yValues = systemMemoryUsageChart?.series.map((serie) => first(serie.data)?.y); - expectSnapshot(yValues).toMatchInline(` - Array [ - 0.11383724014064, - 0.11383724014064, - ] - `); - }); - }); - } - ); -} diff --git a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_metrics_charts.spec.ts b/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_metrics_charts.spec.ts deleted file mode 100644 index be823d8cc9898..0000000000000 --- a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_metrics_charts.spec.ts +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { meanBy, sumBy } from 'lodash'; -import { Coordinate } from '@kbn/apm-plugin/typings/timeseries'; -import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; -import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { generateData, config } from './generate_data'; - -function isNotNullOrZeroCoordinate(coordinate: Coordinate) { - return coordinate.y !== null && coordinate.y !== 0; -} - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); - - const start = new Date('2021-01-01T00:00:00.000Z').getTime(); - const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; - const numberOfTransactionsCreated = 15; - - async function callApi(serviceName: string, serverlessId?: string) { - return await apmApiClient.readUser({ - endpoint: 'GET /internal/apm/services/{serviceName}/metrics/serverless/charts', - params: { - path: { serviceName }, - query: { - environment: 'test', - kuery: '', - start: new Date(start).toISOString(), - end: new Date(end).toISOString(), - documentType: ApmDocumentType.TransactionMetric, - rollupInterval: RollupInterval.OneMinute, - bucketSizeInSeconds: 60, - ...(serverlessId ? { serverlessId } : {}), - }, - }, - }); - } - - registry.when( - 'Serverless metrics charts when data is not loaded', - { config: 'basic', archives: [] }, - () => { - let serverlessMetrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/charts'>; - before(async () => { - const response = await callApi('lambda-python'); - serverlessMetrics = response.body; - }); - - it('returns empty', () => { - serverlessMetrics.charts.forEach((chart) => { - expect(chart.series).to.be.empty(); - }); - }); - } - ); - - // FLAKY: https://github.com/elastic/kibana/issues/177642 - registry.when('Serverless metrics charts', { config: 'basic', archives: [] }, () => { - const { - memoryTotal, - memoryFree, - billedDurationMs, - coldStartDurationPython, - transactionDuration, - pythonServerlessFunctionNames, - serverlessId, - } = config; - - // eslint-disable-next-line mocha/no-sibling-hooks - before(async () => { - await generateData({ start, end, apmSynthtraceEsClient }); - }); - - after(() => apmSynthtraceEsClient.clean()); - - describe('Python service', () => { - let serverlessMetrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/charts'>; - before(async () => { - const response = await callApi('lambda-python'); - serverlessMetrics = response.body; - }); - - it('returns all metrics chart', () => { - expect(serverlessMetrics.charts.length).to.be.greaterThan(0); - expect(serverlessMetrics.charts.map(({ title }) => title).sort()).to.eql([ - 'Cold start duration', - 'Cold starts', - 'Compute usage', - 'Lambda Duration', - 'System memory usage', - ]); - }); - - describe('Avg. Duration', () => { - const transactionDurationInMicroSeconds = transactionDuration * 1000; - [ - { title: 'Billed Duration', expectedValue: billedDurationMs * 1000 }, - { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, - ].map(({ title, expectedValue }) => - it(`returns correct ${title} value`, () => { - const avgDurationMetric = serverlessMetrics.charts.find((chart) => { - return chart.key === 'avg_duration'; - }); - const series = avgDurationMetric?.series.find((item) => item.title === title); - expect(series?.overallValue).to.eql(expectedValue); - const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); - expect(meanValue).to.eql(expectedValue); - }) - ); - }); - - let metricsChart: (typeof serverlessMetrics.charts)[0] | undefined; - - describe('Cold start duration', () => { - before(() => { - metricsChart = serverlessMetrics.charts.find((chart) => { - return chart.key === 'cold_start_duration'; - }); - }); - it('returns correct overall value', () => { - expect(metricsChart?.series[0].overallValue).to.equal(coldStartDurationPython * 1000); - }); - - it('returns correct mean value', () => { - const meanValue = meanBy( - metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), - 'y' - ); - expect(meanValue).to.equal(coldStartDurationPython * 1000); - }); - }); - - describe('Cold start count', () => { - before(() => { - metricsChart = serverlessMetrics.charts.find((chart) => { - return chart.key === 'cold_start_count'; - }); - }); - - it('returns correct overall value', () => { - expect(metricsChart?.series[0].overallValue).to.equal( - numberOfTransactionsCreated * pythonServerlessFunctionNames.length - ); - }); - - it('returns correct sum value', () => { - const sumValue = sumBy( - metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), - 'y' - ); - expect(sumValue).to.equal( - numberOfTransactionsCreated * pythonServerlessFunctionNames.length - ); - }); - }); - - describe('memory usage', () => { - const expectedFreeMemory = 1 - memoryFree / memoryTotal; - [ - { title: 'Max', expectedValue: expectedFreeMemory }, - { title: 'Average', expectedValue: expectedFreeMemory }, - ].map(({ title, expectedValue }) => - it(`returns correct ${title} value`, () => { - const memoryUsageMetric = serverlessMetrics.charts.find((chart) => { - return chart.key === 'memory_usage_chart'; - }); - const series = memoryUsageMetric?.series.find((item) => item.title === title); - expect(series?.overallValue).to.eql(expectedValue); - const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); - expect(meanValue).to.eql(expectedValue); - }) - ); - }); - - describe('Compute usage', () => { - const GBSeconds = 1024 * 1024 * 1024 * 1000; - let computeUsageMetric: (typeof serverlessMetrics.charts)[0] | undefined; - before(() => { - computeUsageMetric = serverlessMetrics.charts.find((chart) => { - return chart.key === 'compute_usage'; - }); - }); - it('returns correct overall value', () => { - const expectedValue = - ((memoryTotal * billedDurationMs) / GBSeconds) * numberOfTransactionsCreated * 2; - expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); - }); - - it('returns correct mean value', () => { - const expectedValue = ((memoryTotal * billedDurationMs) / GBSeconds) * 2; - const meanValue = meanBy( - computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), - 'y' - ); - expect(meanValue).to.equal(expectedValue); - }); - }); - }); - - describe('detailed metrics', () => { - let serverlessMetrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/charts'>; - before(async () => { - const response = await callApi( - 'lambda-python', - `${serverlessId}${pythonServerlessFunctionNames[0]}` - ); - serverlessMetrics = response.body; - }); - - it('returns all metrics chart', () => { - expect(serverlessMetrics.charts.length).to.be.greaterThan(0); - expect(serverlessMetrics.charts.map(({ title }) => title).sort()).to.eql([ - 'Cold start duration', - 'Cold starts', - 'Compute usage', - 'Lambda Duration', - 'System memory usage', - ]); - }); - - describe('Avg. Duration', () => { - const transactionDurationInMicroSeconds = transactionDuration * 1000; - [ - { title: 'Billed Duration', expectedValue: billedDurationMs * 1000 }, - { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, - ].map(({ title, expectedValue }) => - it(`returns correct ${title} value`, () => { - const avgDurationMetric = serverlessMetrics.charts.find((chart) => { - return chart.key === 'avg_duration'; - }); - const series = avgDurationMetric?.series.find((item) => item.title === title); - expect(series?.overallValue).to.eql(expectedValue); - const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); - expect(meanValue).to.eql(expectedValue); - }) - ); - }); - - let metricsChart: (typeof serverlessMetrics.charts)[0] | undefined; - - describe('Cold start duration', () => { - before(() => { - metricsChart = serverlessMetrics.charts.find((chart) => { - return chart.key === 'cold_start_duration'; - }); - }); - it('returns correct overall value', () => { - expect(metricsChart?.series[0].overallValue).to.equal(coldStartDurationPython * 1000); - }); - - it('returns correct mean value', () => { - const meanValue = meanBy( - metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), - 'y' - ); - expect(meanValue).to.equal(coldStartDurationPython * 1000); - }); - }); - - describe('Cold start count', () => { - before(() => { - metricsChart = serverlessMetrics.charts.find((chart) => { - return chart.key === 'cold_start_count'; - }); - }); - - it('returns correct overall value', () => { - expect(metricsChart?.series[0].overallValue).to.equal(numberOfTransactionsCreated); - }); - - it('returns correct sum value', () => { - const sumValue = sumBy( - metricsChart?.series[0]?.data.filter(isNotNullOrZeroCoordinate), - 'y' - ); - expect(sumValue).to.equal(numberOfTransactionsCreated); - }); - }); - - describe('memory usage', () => { - const expectedFreeMemory = 1 - memoryFree / memoryTotal; - [ - { title: 'Max', expectedValue: expectedFreeMemory }, - { title: 'Average', expectedValue: expectedFreeMemory }, - ].map(({ title, expectedValue }) => - it(`returns correct ${title} value`, () => { - const memoryUsageMetric = serverlessMetrics.charts.find((chart) => { - return chart.key === 'memory_usage_chart'; - }); - const series = memoryUsageMetric?.series.find((item) => item.title === title); - expect(series?.overallValue).to.eql(expectedValue); - const meanValue = meanBy(series?.data.filter(isNotNullOrZeroCoordinate), 'y'); - expect(meanValue).to.eql(expectedValue); - }) - ); - }); - - describe('Compute usage', () => { - const GBSeconds = 1024 * 1024 * 1024 * 1000; - let computeUsageMetric: (typeof serverlessMetrics.charts)[0] | undefined; - before(() => { - computeUsageMetric = serverlessMetrics.charts.find((chart) => { - return chart.key === 'compute_usage'; - }); - }); - it('returns correct overall value', () => { - const expectedValue = - ((memoryTotal * billedDurationMs) / GBSeconds) * numberOfTransactionsCreated; - expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); - }); - - it('returns correct mean value', () => { - const expectedValue = (memoryTotal * billedDurationMs) / GBSeconds; - const meanValue = meanBy( - computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), - 'y' - ); - expect(meanValue).to.equal(expectedValue); - }); - }); - }); - }); -} diff --git a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_summary.spec.ts b/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_summary.spec.ts deleted file mode 100644 index 065597eed1709..0000000000000 --- a/x-pack/test/apm_api_integration/tests/metrics/serverless/serverless_summary.spec.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { config, expectedValues, generateData } from './generate_data'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); - - const start = new Date('2021-01-01T00:00:00.000Z').getTime(); - const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; - async function callApi(serviceName: string, serverlessId?: string) { - return await apmApiClient.readUser({ - endpoint: `GET /internal/apm/services/{serviceName}/metrics/serverless/summary`, - params: { - path: { serviceName }, - query: { - environment: 'test', - kuery: '', - start: new Date(start).toISOString(), - end: new Date(end).toISOString(), - ...(serverlessId ? { serverlessId } : {}), - }, - }, - }); - } - - registry.when( - 'Serverless overview when data is not loaded', - { config: 'basic', archives: [] }, - () => { - let serverlessSummary: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/summary'>; - before(async () => { - const response = await callApi('lambda-python'); - serverlessSummary = response.body; - }); - - it('returns empty', () => { - expect(serverlessSummary).to.be.empty(); - }); - } - ); - - // FLAKY: https://github.com/elastic/kibana/issues/177650 - registry.when('Serverless overview', { config: 'basic', archives: [] }, () => { - const { billedDurationMs, pythonServerlessFunctionNames, faasDuration, serverlessId } = config; - const { expectedMemoryUsedRate } = expectedValues; - - // eslint-disable-next-line mocha/no-sibling-hooks - before(async () => { - await generateData({ start, end, apmSynthtraceEsClient }); - }); - - after(() => apmSynthtraceEsClient.clean()); - - describe('Python service', () => { - let serverlessSummary: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/summary'>; - before(async () => { - const response = await callApi('lambda-python'); - serverlessSummary = response.body; - }); - - it('returns correct memory avg', () => { - expect(serverlessSummary.memoryUsageAvgRate).to.eql(expectedMemoryUsedRate); - }); - it('returns correct serverless function total', () => { - expect(serverlessSummary.serverlessFunctionsTotal).to.eql( - pythonServerlessFunctionNames.length - ); - }); - it('returns correct serverless duration avg', () => { - expect(serverlessSummary.serverlessDurationAvg).to.eql(faasDuration); - }); - it('returns correct billed duration avg', () => { - expect(serverlessSummary.billedDurationAvg).to.eql(billedDurationMs); - }); - }); - - describe('detailed metrics', () => { - let serverlessSummary: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/serverless/summary'>; - before(async () => { - const response = await callApi( - 'lambda-python', - `${serverlessId}${pythonServerlessFunctionNames[0]}` - ); - serverlessSummary = response.body; - }); - - it('returns correct memory avg', () => { - expect(serverlessSummary.memoryUsageAvgRate).to.eql(expectedMemoryUsedRate); - }); - it('returns correct serverless function total', () => { - expect(serverlessSummary.serverlessFunctionsTotal).to.eql(1); - }); - it('returns correct serverless duration avg', () => { - expect(serverlessSummary.serverlessDurationAvg).to.eql(faasDuration); - }); - it('returns correct billed duration avg', () => { - expect(serverlessSummary.billedDurationAvg).to.eql(billedDurationMs); - }); - }); - }); -}