[Data Usage] setup integration tests

.github/CODEOWNERS

@@ -1568,6 +1568,7 @@ x-pack/test/api_integration/apis/management/index_management/inference_endpoints
 /x-pack/test/accessibility/apps/group3/rollup_jobs.ts @elastic/kibana-management
 /x-pack/test/accessibility/apps/group3/upgrade_assistant.ts @elastic/kibana-management
 /x-pack/test/accessibility/apps/group3/watcher.ts @elastic/kibana-management
+/x-pack/test_serverless/**/test_suites/common/data_usage/ @elastic/security-solution @elastic/obs-ai-assistant
 
 #CC# /x-pack/plugins/cross_cluster_replication/ @elastic/kibana-management
 

x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts

@@ -123,6 +123,13 @@ export const UsageMetricsAutoOpsResponseSchema = {
       ),
     }),
 };
-export type UsageMetricsAutoOpsResponseSchemaBody = TypeOf<
+export type UsageMetricsAutoOpsResponseMetricSeries = TypeOf<
   typeof UsageMetricsAutoOpsResponseSchema.body
->;
+>['metrics'][MetricTypes][number];
+
+export type UsageMetricsAutoOpsResponseSchemaBody = Omit<
+  TypeOf<typeof UsageMetricsAutoOpsResponseSchema.body>,
+  'metrics'
+> & {
+  metrics: Partial<Record<MetricTypes, UsageMetricsAutoOpsResponseMetricSeries[]>>;
+};

x-pack/plugins/data_usage/server/services/autoops_api.ts

@@ -13,6 +13,7 @@ import type { AxiosError, AxiosRequestConfig } from 'axios';
 import axios from 'axios';
 import { LogMeta } from '@kbn/core/server';
 import {
+  UsageMetricsAutoOpsResponseSchema,
   UsageMetricsAutoOpsResponseSchemaBody,
   UsageMetricsRequestBody,
 } from '../../common/rest_types';
@@ -134,8 +135,10 @@ export class AutoOpsAPIService {
       }
     );
 
+    const validatedResponse = UsageMetricsAutoOpsResponseSchema.body().validate(response.data);
+
     logger.debug(`[AutoOps API] Created an autoops agent ${response}`);
-    return response;
+    return validatedResponse;
   }
 
   private createTlsConfig(autoopsConfig: AutoOpsConfig | undefined) {

x-pack/plugins/data_usage/server/services/index.ts

@@ -38,6 +38,6 @@ export class DataUsageService {
       metricTypes,
       dataStreams,
     });
-    return response.data;
+    return response;
   }
 }

x-pack/test_serverless/api_integration/test_suites/common/data_usage/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Serverless Data Usage APIs', function () {
+    this.tags(['esGate']);
+
+    loadTestFile(require.resolve('./tests/data_streams'));
+    loadTestFile(require.resolve('./tests/metrics'));
+  });
+}

x-pack/test_serverless/api_integration/test_suites/common/data_usage/mock_api.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createServer } from '@mswjs/http-middleware';
+import { UsageMetricsAutoOpsResponseSchemaBody } from '@kbn/data-usage-plugin/common/rest_types';
+
+import { http, HttpResponse, StrictResponse } from 'msw';
+
+export const setupMockServer = () => {
+  const server = createServer(autoOpsHandler);
+  return server;
+};
+
+const autoOpsHandler = http.post(
+  '/',
+  async ({ request }): Promise<StrictResponse<UsageMetricsAutoOpsResponseSchemaBody>> => {
+    return HttpResponse.json({
+      metrics: {
+        ingest_rate: [
+          {
+            name: 'metrics-system.cpu-default',
+            data: [
+              [1726858530000, 13756849],
+              [1726862130000, 14657904],
+            ],
+          },
+          {
+            name: 'logs-nginx.access-default',
+            data: [
+              [1726858530000, 12894623],
+              [1726862130000, 14436905],
+            ],
+          },
+        ],
+        storage_retained: [
+          {
+            name: 'metrics-system.cpu-default',
+            data: [
+              [1726858530000, 12576413],
+              [1726862130000, 13956423],
+            ],
+          },
+          {
+            name: 'logs-nginx.access-default',
+            data: [
+              [1726858530000, 12894623],
+              [1726862130000, 14436905],
+            ],
+          },
+        ],
+      },
+    });
+  }
+);

x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/data_streams.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { SupertestWithRoleScope } from '@kbn/test-suites-xpack/api_integration/deployment_agnostic/services/role_scoped_supertest';
+import { DataStreamsResponseBodySchemaBody } from '@kbn/data-usage-plugin/common/rest_types';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+const API_PATH = '/internal/api/data_usage/data_streams';
+export default function ({ getService }: FtrProviderContext) {
+  const svlDatastreamsHelpers = getService('svlDatastreamsHelpers');
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  let supertestAdminWithCookieCredentials: SupertestWithRoleScope;
+  const testDataStreamName = 'test-data-stream';
+  describe(`GET ${API_PATH}`, function () {
+    // due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default
+    this.tags(['skipMKI']);
+    before(async () => {
+      await svlDatastreamsHelpers.createDataStream(testDataStreamName);
+      supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
+        'admin',
+        {
+          useCookieHeader: true,
+          withInternalHeaders: true,
+        }
+      );
+    });
+    after(async () => {
+      await svlDatastreamsHelpers.deleteDataStream(testDataStreamName);
+    });
+
+    it('returns created data streams', async () => {
+      const res = await supertestAdminWithCookieCredentials
+        .get(API_PATH)
+        .set('elastic-api-version', '1');
+      const dataStreams: DataStreamsResponseBodySchemaBody = res.body;
+      const foundStream = dataStreams.find((stream) => stream.name === testDataStreamName);
+      expect(foundStream?.name).to.be(testDataStreamName);
+      expect(foundStream?.storageSizeBytes).to.be(0);
+      expect(res.statusCode).to.be(200);
+    });
+    it('returns system indices', async () => {
+      const res = await supertestAdminWithCookieCredentials
+        .get(API_PATH)
+        .set('elastic-api-version', '1');
+      const dataStreams: DataStreamsResponseBodySchemaBody = res.body;
+      const systemDataStreams = dataStreams.filter((stream) => stream.name.startsWith('.'));
+      expect(systemDataStreams.length).to.be.greaterThan(0);
+      expect(res.statusCode).to.be(200);
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/metrics.ts

@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import http from 'http';
+
+import { SupertestWithRoleScope } from '@kbn/test-suites-xpack/api_integration/deployment_agnostic/services/role_scoped_supertest';
+import { UsageMetricsRequestBody } from '@kbn/data-usage-plugin/common/rest_types';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+import { setupMockServer } from '../mock_api';
+
+const API_PATH = '/internal/api/data_usage/metrics';
+export default function ({ getService }: FtrProviderContext) {
+  const svlDatastreamsHelpers = getService('svlDatastreamsHelpers');
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  let supertestAdminWithCookieCredentials: SupertestWithRoleScope;
+  const mockAutoopsApiService = setupMockServer();
+  describe('Metrics', function () {
+    let mockApiServer: http.Server;
+    // due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default
+    this.tags(['skipMKI']);
+
+    before(async () => {
+      mockApiServer = mockAutoopsApiService.listen(9000);
+      supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
+        'admin',
+        {
+          useCookieHeader: true,
+          withInternalHeaders: true,
+        }
+      );
+    });
+
+    after(() => {
+      mockApiServer.close();
+    });
+    describe(`POST ${API_PATH}`, () => {
+      const testDataStreamName = 'test-data-stream';
+      before(async () => await svlDatastreamsHelpers.createDataStream(testDataStreamName));
+      after(async () => await svlDatastreamsHelpers.deleteDataStream(testDataStreamName));
+      it('returns 500 with non-existent data streams', async () => {
+        const requestBody: UsageMetricsRequestBody = {
+          from: 'now-24h/h',
+          to: 'now',
+          metricTypes: ['ingest_rate', 'storage_retained'],
+          dataStreams: ['invalid-data-stream'],
+        };
+        const res = await supertestAdminWithCookieCredentials
+          .post(API_PATH)
+          .set('elastic-api-version', '1')
+          .send(requestBody);
+        expect(res.statusCode).to.be(500);
+      });
+
+      it('returns 200 with valid request', async () => {
+        const requestBody: UsageMetricsRequestBody = {
+          from: 'now-24h/h',
+          to: 'now',
+          metricTypes: ['ingest_rate', 'storage_retained'],
+          dataStreams: [testDataStreamName],
+        };
+        const res = await supertestAdminWithCookieCredentials
+          .post(API_PATH)
+          .set('elastic-api-version', '1')
+          .send(requestBody);
+        expect(res.statusCode).to.be(200);
+        // TODO: decide on how to generate data
+        expect(res.body).to.eql({
+          metrics: {
+            ingest_rate: [
+              {
+                data: [
+                  {
+                    x: 1726858530000,
+                    y: 13756849,
+                  },
+                  {
+                    x: 1726862130000,
+                    y: 14657904,
+                  },
+                ],
+                name: 'metrics-system.cpu-default',
+              },
+              {
+                data: [
+                  {
+                    x: 1726858530000,
+                    y: 12894623,
+                  },
+                  {
+                    x: 1726862130000,
+                    y: 14436905,
+                  },
+                ],
+                name: 'logs-nginx.access-default',
+              },
+            ],
+            storage_retained: [
+              {
+                data: [
+                  {
+                    x: 1726858530000,
+                    y: 12576413,
+                  },
+                  {
+                    x: 1726862130000,
+                    y: 13956423,
+                  },
+                ],
+                name: 'metrics-system.cpu-default',
+              },
+              {
+                data: [
+                  {
+                    x: 1726858530000,
+                    y: 12894623,
+                  },
+                  {
+                    x: 1726862130000,
+                    y: 14436905,
+                  },
+                ],
+                name: 'logs-nginx.access-default',
+              },
+            ],
+          },
+        });
+      });
+    });
+  });
+}

x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts

@@ -32,6 +32,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       require.resolve('../../common/console'),
       require.resolve('../../common/saved_objects_management'),
       require.resolve('../../common/telemetry'),
+      require.resolve('../../common/data_usage'),
     ],
     junit: {
       reportName: 'Serverless Observability API Integration Tests - Common Group 1',

x-pack/test_serverless/api_integration/test_suites/observability/config.ts

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
 import { createTestConfig } from '../../config.base';
 import { services as apmServices } from './apm_api_integration/common/services';
 import { services as datasetQualityServices } from './dataset_quality_api_integration/common/services';
@@ -26,5 +26,12 @@ export default createTestConfig({
     '--xpack.uptime.service.manifestUrl=mockDevUrl',
     // useful for testing (also enabled in MKI QA)
     '--coreApp.allowDynamicConfigOverrides=true',
+    '--xpack.dataUsage.enabled=true',
+    // dataUsage.autoops* config is set in kibana controller
+    '--xpack.dataUsage.autoops.enabled=true',
+    '--xpack.dataUsage.autoops.api.url=http://localhost:9000',
+    `--xpack.dataUsage.autoops.api.tls.certificate=${KBN_CERT_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.key=${KBN_KEY_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.ca=${CA_CERT_PATH}`,
   ],
 });

x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts

@@ -30,6 +30,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       require.resolve('../../common/console'),
       require.resolve('../../common/saved_objects_management'),
       require.resolve('../../common/telemetry'),
+      require.resolve('../../common/data_usage'),
     ],
     junit: {
       reportName: 'Serverless Search API Integration Tests - Common Group 1',

x-pack/test_serverless/api_integration/test_suites/search/config.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
 import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
@@ -21,5 +22,12 @@ export default createTestConfig({
   kbnServerArgs: [
     // useful for testing (also enabled in MKI QA)
     '--coreApp.allowDynamicConfigOverrides=true',
+    '--xpack.dataUsage.enabled=true',
+    // dataUsage.autoops* config is set in kibana controller
+    '--xpack.dataUsage.autoops.enabled=true',
+    '--xpack.dataUsage.autoops.api.url=http://localhost:9000',
+    `--xpack.dataUsage.autoops.api.tls.certificate=${KBN_CERT_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.key=${KBN_KEY_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.ca=${CA_CERT_PATH}`,
   ],
 });

x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts

@@ -32,6 +32,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       require.resolve('../../common/console'),
       require.resolve('../../common/saved_objects_management'),
       require.resolve('../../common/telemetry'),
+      require.resolve('../../common/data_usage'),
     ],
     junit: {
       reportName: 'Serverless Security API Integration Tests - Common Group 1',