Add foundation to run apm tests

Migrate agent_explorer test

Update latest_agent_versions test

x-pack/test/apm_api_integration/tests/agent_explorer/agent_explorer.spec.ts → x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/agent_explorer/agent_explorer.spec.ts

@@ -8,13 +8,14 @@ import expect from '@kbn/expect';
 import { apm, timerange } from '@kbn/apm-synthtrace-client';
 import { APIClientRequestParamsOf } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
 import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
+import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
 import { keyBy } from 'lodash';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
 
-export default function ApiTest({ getService }: FtrProviderContext) {
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apiApi = getService('apmApi');
   const registry = getService('registry');
-  const apmApiClient = getService('apmApiClient');
-  const apmSynthtraceEsClient = getService('apmSynthtraceEsClient');
+  const synthtrace = getService('synthtrace');
 
   const start = new Date('2021-01-01T00:00:00.000Z').getTime();
   const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1;
@@ -27,7 +28,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       APIClientRequestParamsOf<'GET /internal/apm/get_agents_per_service'>['params']
     >
   ) {
-    return await apmApiClient.readUser({
+    const client = await apiApi.createApmApiClient();
+    return await client.readUser({
       endpoint: 'GET /internal/apm/get_agents_per_service',
       params: {
         query: {
@@ -42,7 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
   }
 
-  registry.when('Agent explorer when data is not loaded', { config: 'basic', archives: [] }, () => {
+  registry.when('Agent explorer when data is not loaded', () => {
     it('handles empty state', async () => {
       const { status, body } = await callApi();
 
@@ -51,9 +53,14 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
   });
 
-  registry.when('Agent explorer', { config: 'basic', archives: [] }, () => {
+  registry.when('Agent explorer', () => {
     describe('when data is loaded', () => {
+      let apmSynthtraceEsClient: ApmSynthtraceEsClient;
+
       before(async () => {
+        const version = (await synthtrace.apmSynthtraceKibanaClient.installApmPackage()).version;
+        apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(version);
+
         const serviceOtelJava = apm
           .service({
             name: otelJavaServiceName,

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import globby from 'globby';
+import path from 'path';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
+
+const cwd = path.join(__dirname);
+const envGrepFiles = process.env.APM_TEST_GREP_FILES as string;
+
+function getGlobPattern() {
+  try {
+    const envGrepFilesParsed = JSON.parse(envGrepFiles as string) as string[];
+    return envGrepFilesParsed.map((pattern) => {
+      return pattern.includes('spec') ? `**/${pattern}**` : `**/${pattern}**.spec.ts`;
+    });
+  } catch (e) {
+    // ignore
+  }
+  return '**/*.spec.ts';
+}
+
+export default function apmApiIntegrationTests({
+  getService,
+  loadTestFile,
+}: DeploymentAgnosticFtrProviderContext) {
+  // DO NOT SKIP
+  // Skipping here will skip the entire apm api test suite
+  // Instead skip (flaky) tests individually
+  // Failing: See https://github.com/elastic/kibana/issues/176948
+  describe('APM API tests', function () {
+    const registry = getService('registry');
+    const filePattern = getGlobPattern();
+    const tests = globby.sync(filePattern, { cwd });
+
+    if (envGrepFiles) {
+      // eslint-disable-next-line no-console
+      console.log(
+        `\nCommand "--grep-files=${filePattern}" matched ${tests.length} file(s):\n${tests
+          .map((name) => ` - ${name}`)
+          .join('\n')}\n`
+      );
+    }
+
+    tests.forEach((testName) => {
+      describe(testName, () => {
+        loadTestFile(require.resolve(`./${testName}`));
+        registry.run();
+      });
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.index.ts

@@ -19,5 +19,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
     loadTestFile(require.resolve('../../apis/painless_lab'));
     loadTestFile(require.resolve('../../apis/saved_objects_management'));
     loadTestFile(require.resolve('../../apis/observability/slo'));
+    loadTestFile(require.resolve('../../apis/observability/apm'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts

@@ -10,6 +10,7 @@ import { createServerlessTestConfig } from '../../default_configs/serverless.con
 export default createServerlessTestConfig({
   serverlessProject: 'oblt',
   testFiles: [require.resolve('./oblt.index.ts')],
+  servicesRequiredForTestAnalysis: ['registry'],
   junit: {
     reportName: 'Serverless Observability - Deployment-agnostic API Integration Tests',
   },

x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.index.ts

@@ -13,5 +13,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
     loadTestFile(require.resolve('../../apis/observability/alerting'));
     loadTestFile(require.resolve('../../apis/observability/dataset_quality'));
     loadTestFile(require.resolve('../../apis/observability/slo'));
+    loadTestFile(require.resolve('../../apis/observability/apm'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts

@@ -9,6 +9,7 @@ import { createStatefulTestConfig } from '../../default_configs/stateful.config.
 
 export default createStatefulTestConfig({
   testFiles: [require.resolve('./oblt.index.ts')],
+  servicesRequiredForTestAnalysis: ['registry'],
   junit: {
     reportName: 'Stateful Observability - Deployment-agnostic API Integration Tests',
   },

x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts

@@ -16,6 +16,7 @@ interface CreateTestConfigOptions<T extends DeploymentAgnosticCommonServices> {
   esServerArgs?: string[];
   kbnServerArgs?: string[];
   services?: T;
+  servicesRequiredForTestAnalysis?: string[];
   testFiles: string[];
   junit: { reportName: string };
   suiteTags?: { include?: string[]; exclude?: string[] };
@@ -85,6 +86,7 @@ export function createServerlessTestConfig<T extends DeploymentAgnosticCommonSer
         // services can be customized, but must extend DeploymentAgnosticCommonServices
         ...(options.services || services),
       },
+      servicesRequiredForTestAnalysis: options.servicesRequiredForTestAnalysis,
       dockerServers: defineDockerServersConfig({
         registry: {
           enabled: !!dockerRegistryPort,

x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts

@@ -30,6 +30,7 @@ interface CreateTestConfigOptions<T extends DeploymentAgnosticCommonServices> {
   kbnServerArgs?: string[];
   services?: T;
   testFiles: string[];
+  servicesRequiredForTestAnalysis?: string[];
   junit: { reportName: string };
   suiteTags?: { include?: string[]; exclude?: string[] };
 }
@@ -100,6 +101,7 @@ export function createStatefulTestConfig<T extends DeploymentAgnosticCommonServi
       security: { disableTestUser: true },
       // services can be customized, but must extend DeploymentAgnosticCommonServices
       services: options.services || services,
+      servicesRequiredForTestAnalysis: options.servicesRequiredForTestAnalysis,
       junit: options.junit,
       suiteTags: options.suiteTags,
 

x-pack/test/api_integration/deployment_agnostic/services/apm_api.ts

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { format } from 'url';
+import request from 'superagent';
+import type {
+  APIReturnType,
+  APIClientRequestParamsOf,
+} from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
+import { APIEndpoint } from '@kbn/apm-plugin/server';
+import { formatRequest } from '@kbn/server-route-repository';
+import type { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context';
+
+function createApmApiClient({ getService }: DeploymentAgnosticFtrProviderContext, role: string) {
+  const supertestWithoutAuth = getService('supertestWithoutAuth');
+  const samlAuth = getService('samlAuth');
+
+  return async <TEndpoint extends APIEndpoint>(
+    options: {
+      type?: 'form-data';
+      endpoint: TEndpoint;
+      spaceId?: string;
+    } & APIClientRequestParamsOf<TEndpoint> & {
+        params?: { query?: { _inspect?: boolean } };
+      }
+  ): Promise<SupertestReturnType<TEndpoint>> => {
+    const { endpoint, type } = options;
+
+    const params = 'params' in options ? (options.params as Record<string, any>) : {};
+
+    const roleAuthc = await samlAuth.createM2mApiKeyWithRoleScope(role);
+
+    const headers: Record<string, string> = {
+      ...samlAuth.getInternalRequestHeader(),
+      ...roleAuthc.apiKeyHeader,
+    };
+
+    const { method, pathname, version } = formatRequest(endpoint, params.path);
+    const pathnameWithSpaceId = options.spaceId ? `/s/${options.spaceId}${pathname}` : pathname;
+    const url = format({ pathname: pathnameWithSpaceId, query: params?.query });
+
+    // eslint-disable-next-line no-console
+    console.debug(`Calling APM API: ${method.toUpperCase()} ${url}`);
+
+    if (version) {
+      headers['Elastic-Api-Version'] = version;
+    }
+
+    let res: request.Response;
+    if (type === 'form-data') {
+      const fields: Array<[string, any]> = Object.entries(params.body);
+      const formDataRequest = supertestWithoutAuth[method](url)
+        .set(headers)
+        .set('Content-type', 'multipart/form-data');
+
+      for (const field of fields) {
+        void formDataRequest.field(field[0], field[1]);
+      }
+
+      res = await formDataRequest;
+    } else if (params.body) {
+      res = await supertestWithoutAuth[method](url).send(params.body).set(headers);
+    } else {
+      res = await supertestWithoutAuth[method](url).set(headers);
+    }
+
+    // supertest doesn't throw on http errors
+    if (res?.status !== 200) {
+      throw new ApmApiError(res, endpoint);
+    }
+
+    return res;
+  };
+}
+
+type ApiErrorResponse = Omit<request.Response, 'body'> & {
+  body: {
+    statusCode: number;
+    error: string;
+    message: string;
+    attributes: object;
+  };
+};
+
+export type ApmApiSupertest = ReturnType<typeof createApmApiClient>;
+
+export class ApmApiError extends Error {
+  res: ApiErrorResponse;
+
+  constructor(res: request.Response, endpoint: string) {
+    super(
+      `Unhandled ApmApiError.
+Status: "${res.status}"
+Endpoint: "${endpoint}"
+Body: ${JSON.stringify(res.body)}`
+    );
+
+    this.res = res;
+  }
+}
+
+export interface SupertestReturnType<TEndpoint extends APIEndpoint> {
+  status: number;
+  body: APIReturnType<TEndpoint>;
+}
+
+export function ApmApiProvider(context: DeploymentAgnosticFtrProviderContext) {
+  return {
+    async createApmApiClient() {
+      return {
+        readUser: createApmApiClient(context, 'viewer'),
+        adminUser: createApmApiClient(context, 'admin'),
+        writeUser: createApmApiClient(context, 'editor'),
+      };
+    },
+  };
+}

x-pack/test/api_integration/deployment_agnostic/services/index.ts

@@ -13,6 +13,9 @@ import { PackageApiProvider } from './package_api';
 import { RoleScopedSupertestProvider, SupertestWithRoleScope } from './role_scoped_supertest';
 import { SloApiProvider } from './slo_api';
 import { LogsSynthtraceEsClientProvider } from './logs_synthtrace_es_client';
+import { SynthtraceProvider } from './synthtrace';
+import { RegistryProvider } from './registry';
+import { ApmApiProvider } from './apm_api';
 
 export type {
   InternalRequestHeader,
@@ -31,6 +34,9 @@ export const services = {
   roleScopedSupertest: RoleScopedSupertestProvider,
   logsSynthtraceEsClient: LogsSynthtraceEsClientProvider,
   // create a new deployment-agnostic service and load here
+  synthtrace: SynthtraceProvider,
+  apmApi: ApmApiProvider,
+  registry: RegistryProvider,
 };
 
 export type SupertestWithRoleScopeType = SupertestWithRoleScope;