[8.x] [Data Usage] setup integration tests (#197112)

x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts

@@ -123,6 +123,13 @@ export const UsageMetricsAutoOpsResponseSchema = {
       ),
     }),
 };
-export type UsageMetricsAutoOpsResponseSchemaBody = TypeOf<
+export type UsageMetricsAutoOpsResponseMetricSeries = TypeOf<
   typeof UsageMetricsAutoOpsResponseSchema.body
->;
+>['metrics'][MetricTypes][number];
+
+export type UsageMetricsAutoOpsResponseSchemaBody = Omit<
+  TypeOf<typeof UsageMetricsAutoOpsResponseSchema.body>,
+  'metrics'
+> & {
+  metrics: Partial<Record<MetricTypes, UsageMetricsAutoOpsResponseMetricSeries[]>>;
+};

x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts

@@ -6,7 +6,6 @@
  */
 
 import { RequestHandler } from '@kbn/core/server';
-import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types';
 import {
   MetricTypes,
   UsageMetricsAutoOpsResponseSchemaBody,
@@ -44,12 +43,22 @@ export const getUsageMetricsHandler = (
           new CustomHttpRequestError('[request body.dataStreams]: no data streams selected', 400)
         );
       }
+      let dataStreamsResponse;
 
-      const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse =
-        await esClient.indices.getDataStream({
+      try {
+        // Attempt to fetch data streams
+        const { data_streams: dataStreams } = await esClient.indices.getDataStream({
           name: requestDsNames,
           expand_wildcards: 'all',
         });
+        dataStreamsResponse = dataStreams;
+      } catch (error) {
+        return errorHandler(
+          logger,
+          response,
+          new CustomHttpRequestError('Failed to retrieve data streams', 400)
+        );
+      }
       const metrics = await dataUsageService.getMetrics({
         from,
         to,
@@ -69,7 +78,7 @@ export const getUsageMetricsHandler = (
   };
 };
 
-function transformMetricsData(
+export function transformMetricsData(
   data: UsageMetricsAutoOpsResponseSchemaBody
 ): UsageMetricsResponseSchemaBody {
   return {

x-pack/plugins/data_usage/server/services/autoops_api.ts

@@ -13,6 +13,7 @@ import type { AxiosError, AxiosRequestConfig } from 'axios';
 import axios from 'axios';
 import { LogMeta } from '@kbn/core/server';
 import {
+  UsageMetricsAutoOpsResponseSchema,
   UsageMetricsAutoOpsResponseSchemaBody,
   UsageMetricsRequestBody,
 } from '../../common/rest_types';
@@ -134,8 +135,10 @@ export class AutoOpsAPIService {
       }
     );
 
+    const validatedResponse = UsageMetricsAutoOpsResponseSchema.body().validate(response.data);
+
     logger.debug(`[AutoOps API] Successfully created an autoops agent ${response}`);
-    return response;
+    return validatedResponse;
   }
 
   private createTlsConfig(autoopsConfig: AutoOpsConfig | undefined) {

x-pack/plugins/data_usage/server/services/index.ts

@@ -41,7 +41,7 @@ export class DataUsageService {
         metricTypes,
         dataStreams,
       });
-      return response.data;
+      return response;
     } catch (error) {
       if (error instanceof ValidationError) {
         throw new AutoOpsError(error.message);

x-pack/test_serverless/api_integration/test_suites/common/data_usage/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Serverless Data Usage APIs', function () {
+    this.tags(['esGate']);
+
+    loadTestFile(require.resolve('./tests/data_streams'));
+    loadTestFile(require.resolve('./tests/metrics'));
+  });
+}

x-pack/test_serverless/api_integration/test_suites/common/data_usage/mock_api.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createServer } from '@mswjs/http-middleware';
+import { UsageMetricsAutoOpsResponseSchemaBody } from '@kbn/data-usage-plugin/common/rest_types';
+import { http, HttpResponse, StrictResponse } from 'msw';
+import { mockAutoOpsResponse } from './mock_data';
+
+export const setupMockServer = () => {
+  const server = createServer(autoOpsHandler);
+  return server;
+};
+
+const autoOpsHandler = http.post(
+  '/',
+  async ({ request }): Promise<StrictResponse<UsageMetricsAutoOpsResponseSchemaBody>> => {
+    return HttpResponse.json(mockAutoOpsResponse);
+  }
+);

x-pack/test_serverless/api_integration/test_suites/common/data_usage/mock_data.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const mockAutoOpsResponse = {
+  metrics: {
+    ingest_rate: [
+      {
+        name: 'metrics-system.cpu-default',
+        data: [
+          [1726858530000, 13756849],
+          [1726862130000, 14657904],
+        ],
+      },
+      {
+        name: 'logs-nginx.access-default',
+        data: [
+          [1726858530000, 12894623],
+          [1726862130000, 14436905],
+        ],
+      },
+    ],
+    storage_retained: [
+      {
+        name: 'metrics-system.cpu-default',
+        data: [
+          [1726858530000, 12576413],
+          [1726862130000, 13956423],
+        ],
+      },
+      {
+        name: 'logs-nginx.access-default',
+        data: [
+          [1726858530000, 12894623],
+          [1726862130000, 14436905],
+        ],
+      },
+    ],
+  },
+};

x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/data_streams.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { SupertestWithRoleScope } from '@kbn/test-suites-xpack/api_integration/deployment_agnostic/services/role_scoped_supertest';
+import { DataStreamsResponseBodySchemaBody } from '@kbn/data-usage-plugin/common/rest_types';
+import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '@kbn/data-usage-plugin/common';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ getService }: FtrProviderContext) {
+  const svlDatastreamsHelpers = getService('svlDatastreamsHelpers');
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  let supertestAdminWithCookieCredentials: SupertestWithRoleScope;
+  const testDataStreamName = 'test-data-stream';
+  describe(`GET ${DATA_USAGE_DATA_STREAMS_API_ROUTE}`, function () {
+    // due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default
+    this.tags(['skipMKI']);
+    before(async () => {
+      await svlDatastreamsHelpers.createDataStream(testDataStreamName);
+      supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
+        'admin',
+        {
+          useCookieHeader: true,
+          withInternalHeaders: true,
+        }
+      );
+    });
+    after(async () => {
+      await svlDatastreamsHelpers.deleteDataStream(testDataStreamName);
+    });
+
+    it('returns created data streams', async () => {
+      const res = await supertestAdminWithCookieCredentials
+        .get(DATA_USAGE_DATA_STREAMS_API_ROUTE)
+        .set('elastic-api-version', '1');
+      const dataStreams: DataStreamsResponseBodySchemaBody = res.body;
+      const foundStream = dataStreams.find((stream) => stream.name === testDataStreamName);
+      expect(foundStream?.name).to.be(testDataStreamName);
+      expect(foundStream?.storageSizeBytes).to.be(0);
+      expect(res.statusCode).to.be(200);
+    });
+    it('returns system indices', async () => {
+      const res = await supertestAdminWithCookieCredentials
+        .get(DATA_USAGE_DATA_STREAMS_API_ROUTE)
+        .set('elastic-api-version', '1');
+      const dataStreams: DataStreamsResponseBodySchemaBody = res.body;
+      const systemDataStreams = dataStreams.filter((stream) => stream.name.startsWith('.'));
+      expect(systemDataStreams.length).to.be.greaterThan(0);
+      expect(res.statusCode).to.be(200);
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/metrics.ts

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import http from 'http';
+
+import { SupertestWithRoleScope } from '@kbn/test-suites-xpack/api_integration/deployment_agnostic/services/role_scoped_supertest';
+import { UsageMetricsRequestBody } from '@kbn/data-usage-plugin/common/rest_types';
+import { DATA_USAGE_METRICS_API_ROUTE } from '@kbn/data-usage-plugin/common';
+import { transformMetricsData } from '@kbn/data-usage-plugin/server/routes/internal/usage_metrics_handler';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+import { setupMockServer } from '../mock_api';
+import { mockAutoOpsResponse } from '../mock_data';
+
+export default function ({ getService }: FtrProviderContext) {
+  const svlDatastreamsHelpers = getService('svlDatastreamsHelpers');
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  let supertestAdminWithCookieCredentials: SupertestWithRoleScope;
+  const mockAutoopsApiService = setupMockServer();
+  describe('Metrics', function () {
+    let mockApiServer: http.Server;
+    // due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default
+    this.tags(['skipMKI']);
+
+    before(async () => {
+      mockApiServer = mockAutoopsApiService.listen(9000);
+      supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
+        'admin',
+        {
+          useCookieHeader: true,
+          withInternalHeaders: true,
+        }
+      );
+    });
+
+    after(() => {
+      mockApiServer.close();
+    });
+    describe(`POST ${DATA_USAGE_METRICS_API_ROUTE}`, () => {
+      const testDataStreamName = 'test-data-stream';
+      before(async () => await svlDatastreamsHelpers.createDataStream(testDataStreamName));
+      after(async () => await svlDatastreamsHelpers.deleteDataStream(testDataStreamName));
+      it('returns 400 with non-existent data streams', async () => {
+        const requestBody: UsageMetricsRequestBody = {
+          from: 'now-24h/h',
+          to: 'now',
+          metricTypes: ['ingest_rate', 'storage_retained'],
+          dataStreams: ['invalid-data-stream'],
+        };
+        const res = await supertestAdminWithCookieCredentials
+          .post(DATA_USAGE_METRICS_API_ROUTE)
+          .set('elastic-api-version', '1')
+          .send(requestBody);
+        expect(res.statusCode).to.be(400);
+        expect(res.body.message).to.be('Failed to retrieve data streams');
+      });
+
+      it('returns 400 when requesting no data streams', async () => {
+        const requestBody = {
+          from: 'now-24h/h',
+          to: 'now',
+          metricTypes: ['ingest_rate'],
+          dataStreams: [],
+        };
+        const res = await supertestAdminWithCookieCredentials
+          .post(DATA_USAGE_METRICS_API_ROUTE)
+          .set('elastic-api-version', '1')
+          .send(requestBody);
+        expect(res.statusCode).to.be(400);
+        expect(res.body.message).to.be('[request body.dataStreams]: no data streams selected');
+      });
+
+      it('returns 400 when requesting an invalid metric type', async () => {
+        const requestBody = {
+          from: 'now-24h/h',
+          to: 'now',
+          metricTypes: [testDataStreamName],
+          dataStreams: ['datastream'],
+        };
+        const res = await supertestAdminWithCookieCredentials
+          .post(DATA_USAGE_METRICS_API_ROUTE)
+          .set('elastic-api-version', '1')
+          .send(requestBody);
+        expect(res.statusCode).to.be(400);
+        expect(res.body.message).to.be(
+          '[request body.metricTypes]: must be one of ingest_rate, storage_retained, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate'
+        );
+      });
+
+      it('returns 200 with valid request', async () => {
+        const requestBody: UsageMetricsRequestBody = {
+          from: 'now-24h/h',
+          to: 'now',
+          metricTypes: ['ingest_rate', 'storage_retained'],
+          dataStreams: [testDataStreamName],
+        };
+        const res = await supertestAdminWithCookieCredentials
+          .post(DATA_USAGE_METRICS_API_ROUTE)
+          .set('elastic-api-version', '1')
+          .send(requestBody);
+        expect(res.statusCode).to.be(200);
+        expect(res.body).to.eql(transformMetricsData(mockAutoOpsResponse));
+      });
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts

@@ -32,6 +32,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       require.resolve('../../common/console'),
       require.resolve('../../common/saved_objects_management'),
       require.resolve('../../common/telemetry'),
+      require.resolve('../../common/data_usage'),
     ],
     junit: {
       reportName: 'Serverless Observability API Integration Tests - Common Group 1',

x-pack/test_serverless/api_integration/test_suites/observability/config.ts

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
 import { createTestConfig } from '../../config.base';
 import { services as apmServices } from './apm_api_integration/common/services';
 import { services as datasetQualityServices } from './dataset_quality_api_integration/common/services';
@@ -26,5 +26,12 @@ export default createTestConfig({
     '--xpack.uptime.service.manifestUrl=mockDevUrl',
     // useful for testing (also enabled in MKI QA)
     '--coreApp.allowDynamicConfigOverrides=true',
+    '--xpack.dataUsage.enabled=true',
+    // dataUsage.autoops* config is set in kibana controller
+    '--xpack.dataUsage.autoops.enabled=true',
+    '--xpack.dataUsage.autoops.api.url=http://localhost:9000',
+    `--xpack.dataUsage.autoops.api.tls.certificate=${KBN_CERT_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.key=${KBN_KEY_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.ca=${CA_CERT_PATH}`,
   ],
 });

x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts

@@ -30,6 +30,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       require.resolve('../../common/console'),
       require.resolve('../../common/saved_objects_management'),
       require.resolve('../../common/telemetry'),
+      require.resolve('../../common/data_usage'),
     ],
     junit: {
       reportName: 'Serverless Search API Integration Tests - Common Group 1',

x-pack/test_serverless/api_integration/test_suites/search/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
 import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
@@ -21,5 +22,12 @@ export default createTestConfig({
   kbnServerArgs: [
     // useful for testing (also enabled in MKI QA)
     '--coreApp.allowDynamicConfigOverrides=true',
+    '--xpack.dataUsage.enabled=true',
+    // dataUsage.autoops* config is set in kibana controller
+    '--xpack.dataUsage.autoops.enabled=true',
+    '--xpack.dataUsage.autoops.api.url=http://localhost:9000',
+    `--xpack.dataUsage.autoops.api.tls.certificate=${KBN_CERT_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.key=${KBN_KEY_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.ca=${CA_CERT_PATH}`,
   ],
 });