[APM] Foundation for migrating APM tests to deployment agnostic appro…

…ach (#198775)

part of #193245
closes #198958

## Summary

This PR lays the foundation to start migrating APM API integration tests
to the deployment agnostic approach.


### How to test

- Serverless

```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM API tests"
```

It's recommended to be run against
[MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki)

- Stateful
```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM API tests"
```

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>

x-pack/test/apm_api_integration/tests/agent_explorer/agent_explorer.spec.ts → x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/agent_explorer/agent_explorer.spec.ts

@@ -8,13 +8,13 @@ import expect from '@kbn/expect';
 import { apm, timerange } from '@kbn/apm-synthtrace-client';
 import { APIClientRequestParamsOf } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
 import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
+import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
 import { keyBy } from 'lodash';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
 
-export default function ApiTest({ getService }: FtrProviderContext) {
-  const registry = getService('registry');
-  const apmApiClient = getService('apmApiClient');
-  const apmSynthtraceEsClient = getService('apmSynthtraceEsClient');
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
+  const synthtrace = getService('synthtrace');
 
   const start = new Date('2021-01-01T00:00:00.000Z').getTime();
   const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
@@ -42,18 +42,22 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
   }
 
-  registry.when('Agent explorer when data is not loaded', { config: 'basic', archives: [] }, () => {
-    it('handles empty state', async () => {
-      const { status, body } = await callApi();
+  describe('Agent explorer', () => {
+    describe('when data is not loaded', () => {
+      it('handles empty state', async () => {
+        const { status, body } = await callApi();
 
-      expect(status).to.be(200);
-      expect(body.items).to.be.empty();
+        expect(status).to.be(200);
+        expect(body.items).to.be.empty();
+      });
     });
-  });
 
-  registry.when('Agent explorer', { config: 'basic', archives: [] }, () => {
     describe('when data is loaded', () => {
+      let apmSynthtraceEsClient: ApmSynthtraceEsClient;
+
       before(async () => {
+        apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient();
+
         const serviceOtelJava = apm
           .service({
             name: otelJavaServiceName,

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/agent_explorer/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
+  describe('agent_explorer', () => {
+    loadTestFile(require.resolve('./agent_explorer.spec.ts'));
+    loadTestFile(require.resolve('./latest_agent_versions.spec.ts'));
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/agent_explorer/latest_agent_versions.spec.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { ElasticApmAgentLatestVersion } from '@kbn/apm-plugin/common/agent_explorer';
+import expect from '@kbn/expect';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
+  const nodeAgentName = 'nodejs';
+  const unlistedAgentName = 'unlistedAgent';
+
+  async function callApi() {
+    return await apmApiClient.readUser({
+      endpoint: 'GET /internal/apm/get_latest_agent_versions',
+    });
+  }
+
+  describe('Agent latest versions when configuration is defined', () => {
+    it('returns a version when agent is listed in the file', async () => {
+      const { status, body } = await callApi();
+      expect(status).to.be(200);
+
+      const agents = body.data;
+
+      const nodeAgent = agents[nodeAgentName] as ElasticApmAgentLatestVersion;
+      expect(nodeAgent?.latest_version).not.to.be(undefined);
+    });
+
+    it('returns undefined when agent is not listed in the file', async () => {
+      const { status, body } = await callApi();
+      expect(status).to.be(200);
+
+      const agents = body.data;
+
+      // @ts-ignore
+      const unlistedAgent = agents[unlistedAgentName] as ElasticApmAgentLatestVersion;
+      expect(unlistedAgent?.latest_version).to.be(undefined);
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
+
+export default function apmApiIntegrationTests({
+  loadTestFile,
+}: DeploymentAgnosticFtrProviderContext) {
+  describe('APM', function () {
+    loadTestFile(require.resolve('./agent_explorer'));
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/infra_asset_count.ts

@@ -48,6 +48,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
         before(async () => {
           supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
             withInternalHeaders: true,
+            useCookieHeader: true,
           });
           await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/logs_and_metrics');
         });

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/infra_custom_dashboards.ts

@@ -28,6 +28,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
     });
 

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/ip_to_hostname.ts

@@ -19,6 +19,7 @@ export default function ipToHostNameTest({ getService }: DeploymentAgnosticFtrPr
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
       await esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs');
     });

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/metrics_overview_top.ts

@@ -25,6 +25,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
       await esArchiver.load('x-pack/test/functional/es_archives/infra/7.0.0/hosts');
     });

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/metrics_process_list.ts

@@ -24,6 +24,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
       await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
     });

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/metrics_process_list_chart.ts

@@ -25,6 +25,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
       await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
     });

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/node_details.ts

@@ -36,6 +36,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
       await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/pods_only');
     });

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/services.ts

@@ -106,10 +106,10 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
 
-      const version = (await synthtrace.apmSynthtraceKibanaClient.installApmPackage()).version;
-      synthtraceApmClient = await synthtrace.createApmSynthtraceEsClient(version);
+      synthtraceApmClient = await synthtrace.createApmSynthtraceEsClient();
     });
     after(async () => {
       await synthtrace.apmSynthtraceKibanaClient.uninstallApmPackage();

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/snapshot.ts

@@ -37,6 +37,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
     });
     after(async () => {

x-pack/test/api_integration/deployment_agnostic/apis/observability/infra/sources.ts

@@ -39,6 +39,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
+        useCookieHeader: true,
       });
       await esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs');
       await kibanaServer.savedObjects.cleanStandardList();

x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.index.ts

@@ -20,5 +20,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
     loadTestFile(require.resolve('../../apis/painless_lab'));
     loadTestFile(require.resolve('../../apis/saved_objects_management'));
     loadTestFile(require.resolve('../../apis/observability/slo'));
+    loadTestFile(require.resolve('../../apis/observability/apm'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.index.ts

@@ -14,5 +14,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
     loadTestFile(require.resolve('../../apis/observability/dataset_quality'));
     loadTestFile(require.resolve('../../apis/observability/slo'));
     loadTestFile(require.resolve('../../apis/observability/infra'));
+    loadTestFile(require.resolve('../../apis/observability/apm'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/services/apm_api.ts

@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { format } from 'url';
+import request from 'superagent';
+import type {
+  APIReturnType,
+  APIClientRequestParamsOf,
+} from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import { APIEndpoint } from '@kbn/apm-plugin/server';
+import { formatRequest } from '@kbn/server-route-repository';
+import { RoleCredentials } from '@kbn/ftr-common-functional-services';
+import type { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context';
+
+const INTERNAL_API_REGEX = /^\S+\s(\/)?internal\/[^\s]*$/;
+
+type InternalApi = `${string} /internal/${string}`;
+interface ExternalEndpointParams {
+  roleAuthc: RoleCredentials;
+}
+
+type Options<TEndpoint extends APIEndpoint> = (TEndpoint extends InternalApi
+  ? {}
+  : ExternalEndpointParams) & {
+  type?: 'form-data';
+  endpoint: TEndpoint;
+  spaceId?: string;
+} & APIClientRequestParamsOf<TEndpoint> & {
+    params?: { query?: { _inspect?: boolean } };
+  };
+
+function isPublicApi<TEndpoint extends APIEndpoint>(
+  options: Options<TEndpoint>
+): options is Options<TEndpoint> & ExternalEndpointParams {
+  return !INTERNAL_API_REGEX.test(options.endpoint);
+}
+
+function createApmApiClient({ getService }: DeploymentAgnosticFtrProviderContext, role: string) {
+  const supertestWithoutAuth = getService('supertestWithoutAuth');
+  const samlAuth = getService('samlAuth');
+  const logger = getService('log');
+
+  return async <TEndpoint extends APIEndpoint>(
+    options: Options<TEndpoint>
+  ): Promise<SupertestReturnType<TEndpoint>> => {
+    const { endpoint, type } = options;
+
+    const params = 'params' in options ? (options.params as Record<string, any>) : {};
+
+    const credentials = isPublicApi(options)
+      ? options.roleAuthc.apiKeyHeader
+      : await samlAuth.getM2MApiCookieCredentialsWithRoleScope(role);
+
+    const headers: Record<string, string> = {
+      ...samlAuth.getInternalRequestHeader(),
+      ...credentials,
+    };
+
+    const { method, pathname, version } = formatRequest(endpoint, params.path);
+    const pathnameWithSpaceId = options.spaceId ? `/s/${options.spaceId}${pathname}` : pathname;
+    const url = format({ pathname: pathnameWithSpaceId, query: params?.query });
+
+    logger.debug(`Calling APM API: ${method.toUpperCase()} ${url}`);
+
+    if (version) {
+      headers['Elastic-Api-Version'] = version;
+    }
+
+    let res: request.Response;
+    if (type === 'form-data') {
+      const fields: Array<[string, any]> = Object.entries(params.body);
+      const formDataRequest = supertestWithoutAuth[method](url)
+        .set(headers)
+        .set('Content-type', 'multipart/form-data');
+
+      for (const field of fields) {
+        void formDataRequest.field(field[0], field[1]);
+      }
+
+      res = await formDataRequest;
+    } else if (params.body) {
+      res = await supertestWithoutAuth[method](url).send(params.body).set(headers);
+    } else {
+      res = await supertestWithoutAuth[method](url).set(headers);
+    }
+
+    // supertest doesn't throw on http errors
+    if (res?.status !== 200) {
+      throw new ApmApiError(res, endpoint);
+    }
+
+    return res;
+  };
+}
+
+type ApiErrorResponse = Omit<request.Response, 'body'> & {
+  body: {
+    statusCode: number;
+    error: string;
+    message: string;
+    attributes: object;
+  };
+};
+
+export type ApmApiSupertest = ReturnType<typeof createApmApiClient>;
+
+export class ApmApiError extends Error {
+  res: ApiErrorResponse;
+
+  constructor(res: request.Response, endpoint: string) {
+    super(
+      `Unhandled ApmApiError.
+Status: "${res.status}"
+Endpoint: "${endpoint}"
+Body: ${JSON.stringify(res.body)}`
+    );
+
+    this.res = res;
+  }
+}
+
+export interface SupertestReturnType<TEndpoint extends APIEndpoint> {
+  status: number;
+  body: APIReturnType<TEndpoint>;
+}
+
+export function ApmApiProvider(context: DeploymentAgnosticFtrProviderContext) {
+  return {
+    readUser: createApmApiClient(context, 'viewer'),
+    adminUser: createApmApiClient(context, 'admin'),
+    writeUser: createApmApiClient(context, 'editor'),
+  };
+}

x-pack/test/api_integration/deployment_agnostic/services/index.ts

@@ -13,6 +13,7 @@ import { PackageApiProvider } from './package_api';
 import { RoleScopedSupertestProvider, SupertestWithRoleScope } from './role_scoped_supertest';
 import { SloApiProvider } from './slo_api';
 import { SynthtraceProvider } from './synthtrace';
+import { ApmApiProvider } from './apm_api';
 
 export type {
   InternalRequestHeader,
@@ -31,6 +32,7 @@ export const services = {
   roleScopedSupertest: RoleScopedSupertestProvider,
   // create a new deployment-agnostic service and load here
   synthtrace: SynthtraceProvider,
+  apmApi: ApmApiProvider,
 };
 
 export type SupertestWithRoleScopeType = SupertestWithRoleScope;

x-pack/test/api_integration/deployment_agnostic/services/synthtrace.ts

@@ -39,7 +39,8 @@ export function SynthtraceProvider({ getService }: DeploymentAgnosticFtrProvider
   return {
     apmSynthtraceKibanaClient,
     createLogsSynthtraceEsClient: () => getLogsSynthtraceEsClient(client),
-    async createApmSynthtraceEsClient(packageVersion: string) {
+    async createApmSynthtraceEsClient() {
+      const packageVersion = (await apmSynthtraceKibanaClient.installApmPackage()).version;
       return getApmSynthtraceEsClient({ client, packageVersion });
     },
   };