[APM] conditionally waiting for index status (elastic#166438)

Closes elastic#166831.

Checking for cluster health is not available on serverless. (More
context in this [internal
conversation](https://elastic.slack.com/archives/C877NKGBY/p1694595738607269?thread_ts=1694006744.669319&cid=C877NKGBY))

<img width="888" alt="image"
src="https://github.com/elastic/kibana/assets/1313018/80b55f81-0a01-4b9c-a37d-1cfcff572041">

This Pr aims to conditionally waitForIndex status only if we are in a
non-serverless deployment

---------

Co-authored-by: kibanamachine <[email protected]>

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -13,6 +13,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { waitForIndexStatus } from '@kbn/core-saved-objects-migration-server-internal';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
+import { ElasticsearchCapabilities } from '@kbn/core-elasticsearch-server';
 import { ML_ERRORS } from '../../../common/anomaly_detection';
 import { METRICSET_NAME, PROCESSOR_EVENT } from '../../../common/es_fields/apm';
 import { Environment } from '../../../common/environment_rt';
@@ -30,12 +31,14 @@ export async function createAnomalyDetectionJobs({
   indices,
   environments,
   logger,
+  esCapabilities,
 }: {
   mlClient?: MlClient;
   esClient: ElasticsearchClient;
   indices: APMIndices;
   environments: Environment[];
   logger: Logger;
+  esCapabilities: ElasticsearchCapabilities;
 }) {
   if (!mlClient) {
     throw Boom.notImplemented(ML_ERRORS.ML_NOT_AVAILABLE);
@@ -68,6 +71,7 @@ export async function createAnomalyDetectionJobs({
             esClient,
             environment,
             apmMetricIndex,
+            esCapabilities,
           })
         );
       } catch (e) {
@@ -97,12 +101,16 @@ async function createAnomalyDetectionJob({
   esClient,
   environment,
   apmMetricIndex,
+  esCapabilities,
 }: {
   mlClient: Required<MlClient>;
   esClient: ElasticsearchClient;
   environment: string;
   apmMetricIndex: string;
+  esCapabilities: ElasticsearchCapabilities;
 }) {
+  const { serverless } = esCapabilities;
+
   return withApmSpan('create_anomaly_detection_job', async () => {
     const randomToken = uuidv4().substr(-4);
 
@@ -136,12 +144,16 @@ async function createAnomalyDetectionJob({
       ],
     });
 
-    await waitForIndexStatus({
-      client: esClient,
-      index: '.ml-*',
-      timeout: DEFAULT_TIMEOUT,
-      status: 'yellow',
-    })();
+    // Waiting for the index is not enabled in serverless, this could potentially cause
+    // problems when creating jobs in parallels
+    if (!serverless) {
+      await waitForIndexStatus({
+        client: esClient,
+        index: '.ml-*',
+        timeout: DEFAULT_TIMEOUT,
+        status: 'yellow',
+      })();
+    }
 
     return anomalyDetectionJob;
   });

x-pack/plugins/apm/server/lib/helpers/get_es_capabilities.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { APMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes';
+
+export async function getESCapabilities({ core }: APMRouteHandlerResources) {
+  const es = (await core.start()).elasticsearch;
+
+  return es.getCapabilities();
+}

x-pack/plugins/apm/server/routes/settings/anomaly_detection/route.ts

@@ -9,6 +9,7 @@ import * as t from 'io-ts';
 import Boom from '@hapi/boom';
 import { maxSuggestions } from '@kbn/observability-plugin/common';
 import { ElasticsearchClient } from '@kbn/core/server';
+import { getESCapabilities } from '../../../lib/helpers/get_es_capabilities';
 import { isActivePlatinumLicense } from '../../../../common/license_check';
 import { ML_ERRORS } from '../../../../common/anomaly_detection';
 import { createApmServerRoute } from '../../apm_routes/create_apm_server_route';
@@ -72,6 +73,8 @@ const createAnomalyDetectionJobsRoute = createApmServerRoute({
     const licensingContext = await context.licensing;
     const esClient = (await context.core).elasticsearch.client;
 
+    const esCapabilities = await getESCapabilities(resources);
+
     const [mlClient, indices] = await Promise.all([
       getMlClient(resources),
       getApmIndices(),
@@ -87,6 +90,7 @@ const createAnomalyDetectionJobsRoute = createApmServerRoute({
       indices,
       environments,
       logger,
+      esCapabilities,
     });
 
     notifyFeatureUsage({
@@ -149,10 +153,18 @@ const anomalyDetectionUpdateToV3Route = createApmServerRoute({
         ),
     ]);
 
+    const esCapabilities = await getESCapabilities(resources);
+
     const { logger } = resources;
 
     return {
-      update: await updateToV3({ mlClient, logger, indices, esClient }),
+      update: await updateToV3({
+        mlClient,
+        logger,
+        indices,
+        esClient,
+        esCapabilities,
+      }),
     };
   },
 });

x-pack/plugins/apm/server/routes/settings/anomaly_detection/update_to_v3.ts

@@ -10,6 +10,7 @@ import pLimit from 'p-limit';
 import { ElasticsearchClient } from '@kbn/core/server';
 import { JOB_STATE } from '@kbn/ml-plugin/common';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
+import { ElasticsearchCapabilities } from '@kbn/core-elasticsearch-server';
 import { createAnomalyDetectionJobs } from '../../../lib/anomaly_detection/create_anomaly_detection_jobs';
 import { getAnomalyDetectionJobs } from '../../../lib/anomaly_detection/get_anomaly_detection_jobs';
 import { MlClient } from '../../../lib/helpers/get_ml_client';
@@ -20,11 +21,13 @@ export async function updateToV3({
   indices,
   mlClient,
   esClient,
+  esCapabilities,
 }: {
   logger: Logger;
   mlClient?: MlClient;
   indices: APMIndices;
   esClient: ElasticsearchClient;
+  esCapabilities: ElasticsearchCapabilities;
 }) {
   const allJobs = await getAnomalyDetectionJobs(mlClient);
 
@@ -63,6 +66,7 @@ export async function updateToV3({
     indices,
     environments,
     logger,
+    esCapabilities,
   });
 
   return true;