[ObsAssistant] Add Insights component to alerts details (#178330)

Depends on: #179380

This PR adds contextual insights to the alert details page. This makes
it possible to get good explanation of why an error occurred.

<img width="1469" alt="image"
src="https://github.com/elastic/kibana/assets/209966/ec3faae6-d29a-437a-8e59-234457e152eb">

**Video**

https://github.com/elastic/kibana/assets/209966/2cbdae10-298b-4817-aa92-052bbb11e913

packages/kbn-alerts-as-data-utils/src/schemas/generated/observability_apm_schema.ts

@@ -72,8 +72,10 @@ const ObservabilityApmAlertRequired = rt.type({
 // prettier-ignore
 const ObservabilityApmAlertOptional = rt.partial({
   'agent.name': schemaString,
+  'container.id': schemaString,
   'error.grouping_key': schemaString,
   'error.grouping_name': schemaString,
+  'host.name': schemaString,
   'kibana.alert.context': schemaUnknown,
   'kibana.alert.evaluation.threshold': schemaStringOrNumber,
   'kibana.alert.evaluation.value': schemaStringOrNumber,

packages/kbn-apm-synthtrace-client/src/lib/apm/instance.ts

@@ -86,6 +86,11 @@ export class Instance extends Entity<ApmFields> {
     return this;
   }
 
+  hostName(hostName: string) {
+    this.fields['host.name'] = hostName;
+    return this;
+  }
+
   podId(podId: string) {
     this.fields['kubernetes.pod.uid'] = podId;
     return this;

packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts

@@ -23,6 +23,7 @@ export type LogDocument = Fields &
     'event.dataset': string;
     'log.level'?: string;
     'host.name'?: string;
+    'container.id'?: string;
     'trace.id'?: string;
     'agent.id'?: string;
     'agent.name'?: string;
@@ -47,6 +48,16 @@ class Log extends Serializable<LogDocument> {
     return this;
   }
 
+  hostName(name: string) {
+    this.fields['host.name'] = name;
+    return this;
+  }
+
+  containerId(id: string) {
+    this.fields['container.id'] = id;
+    return this;
+  }
+
   namespace(value: string) {
     this.fields['data_stream.namespace'] = value;
     return this;

packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts

@@ -19,7 +19,7 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
 
       // Logs Data logic
       const MESSAGE_LOG_LEVELS = [
-        { message: 'A simple log', level: 'info' },
+        { message: 'A simple log with something random <random> in the middle', level: 'info' },
         { message: 'Yet another debug log', level: 'debug' },
         { message: 'Error with certificate: "ca_trusted_fingerprint"', level: 'error' },
       ];
@@ -44,11 +44,14 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
             .fill(0)
             .map(() => {
               const index = Math.floor(Math.random() * 3);
+              const { message, level } = MESSAGE_LOG_LEVELS[index];
+              const serviceName = SERVICE_NAMES[index];
+
               return log
                 .create()
-                .message(MESSAGE_LOG_LEVELS[index].message)
-                .logLevel(MESSAGE_LOG_LEVELS[index].level)
-                .service(SERVICE_NAMES[index])
+                .message(message.replace('<random>', generateShortId()))
+                .logLevel(level)
+                .service(serviceName)
                 .defaults({
                   'trace.id': generateShortId(),
                   'agent.name': 'nodejs',

packages/kbn-apm-synthtrace/src/scenarios/spiked_latency.ts

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { random } from 'lodash';
+import {
+  apm,
+  log,
+  ApmFields,
+  generateLongId,
+  generateShortId,
+  Instance,
+} from '@kbn/apm-synthtrace-client';
+import { Scenario } from '../cli/scenario';
+import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
+import { withClient } from '../lib/utils/with_client';
+
+const ENVIRONMENT = getSynthtraceEnvironment(__filename);
+const alwaysSpikeTransactionName = 'GET /always-spike';
+const sometimesSpikeTransactionName = 'GET /sometimes-spike';
+
+const scenario: Scenario<ApmFields> = async ({ logger }) => {
+  return {
+    generate: ({ range, clients: { apmEsClient, logsEsClient } }) => {
+      const serviceNames = ['spikey-frontend', 'spikey-backend'];
+
+      const clusters = [
+        { provider: 'gcp', region: 'eu-central-1' },
+        { provider: 'aws', region: 'us-east-1' },
+        { provider: 'azure', region: 'area-51' },
+      ].map((cluster, index) => ({
+        clusterName: `synth-cluster-${index}`,
+        clusterId: generateShortId(),
+        projectId: generateShortId(),
+        ressourceId: generateShortId(),
+        instanceId: generateShortId(),
+        cloudProvider: cluster.provider,
+        cloudRegion: cluster.region,
+      }));
+
+      const containerId = `spiked-${generateShortId()}`;
+      const hostName = `spiked-${generateShortId()}`;
+
+      function buildLogs(serviceName: string) {
+        return range
+          .interval('1m')
+          .rate(100)
+          .generator((timestamp) => {
+            const clusterIndex = Math.floor(Math.random() * clusters.length);
+            const {
+              clusterId,
+              clusterName,
+              projectId,
+              ressourceId,
+              instanceId,
+              cloudRegion,
+              cloudProvider,
+            } = clusters[clusterIndex];
+
+            return log
+              .create()
+              .message(`Error message #${generateShortId()} from ${serviceName}`)
+              .logLevel('error')
+              .service(serviceName)
+              .containerId(containerId)
+              .hostName(hostName)
+              .defaults({
+                'trace.id': generateShortId(),
+                'agent.name': 'synth-agent',
+                'orchestrator.cluster.name': clusterName,
+                'orchestrator.cluster.id': clusterId,
+                'orchestrator.resource.id': ressourceId,
+                'cloud.provider': cloudProvider,
+                'cloud.region': cloudRegion,
+                'cloud.availability_zone': `${cloudRegion}`,
+                'cloud.project.id': projectId,
+                'cloud.instance.id': instanceId,
+                'log.file.path': `/logs/${generateLongId()}/error.txt`,
+              })
+              .timestamp(timestamp);
+          });
+      }
+
+      const serviceInstances = serviceNames.map((serviceName) =>
+        apm
+          .service({ name: serviceName, environment: ENVIRONMENT, agentName: 'go' })
+          .instance(`my-instance`)
+      );
+
+      const transactionNames = [
+        'GET /order',
+        'GET /product',
+        alwaysSpikeTransactionName,
+        sometimesSpikeTransactionName,
+      ];
+
+      const buildTransactions = (serviceInstance: Instance, transactionName: string) => {
+        const interval = random(1, 100, false);
+        const rangeWithInterval = range.interval(`${interval}s`);
+
+        return rangeWithInterval.generator((timestamp, i) => {
+          const duration = getDuration(transactionName);
+          return serviceInstance
+            .containerId(containerId)
+            .hostName(hostName)
+            .transaction({ transactionName })
+            .timestamp(timestamp)
+            .duration(duration)
+            .children(
+              serviceInstance
+                .span({
+                  spanName: 'GET apm-*/_search',
+                  spanType: 'db',
+                  spanSubtype: 'elasticsearch',
+                })
+                .duration(duration * 0.9)
+                .destination('elasticsearch')
+                .timestamp(timestamp)
+                .outcome('success')
+            )
+            .success();
+        });
+      };
+
+      return [
+        withClient(
+          logsEsClient,
+          logger.perf('generating_logs', () =>
+            serviceNames.map((serviceName) => buildLogs(serviceName))
+          )
+        ),
+        withClient(
+          apmEsClient,
+          logger.perf('generating_apm_events', () =>
+            serviceInstances.flatMap((serviceInstance) =>
+              transactionNames.flatMap((transactionName) =>
+                buildTransactions(serviceInstance, transactionName)
+              )
+            )
+          )
+        ),
+      ];
+    },
+  };
+};
+
+function getDuration(transactionName: string) {
+  const spikedDuration = random(40000, 41000, false);
+  const normalDuration = random(400, 500, false);
+
+  switch (transactionName) {
+    case alwaysSpikeTransactionName:
+      return spikedDuration;
+    case sometimesSpikeTransactionName:
+      return Math.random() > 0.01 ? spikedDuration : normalDuration;
+    default:
+      return normalDuration;
+  }
+}
+
+export default scenario;

packages/kbn-es-types/src/search.ts

@@ -421,6 +421,7 @@ export type AggregateOf<
           {
             doc_count: number;
             key: string[];
+            key_as_string: string;
           } & SubAggregateOf<TAggregationContainer, TDocument>
         >;
       };
@@ -563,6 +564,16 @@ export type AggregateOf<
           } & SubAggregateOf<TAggregationContainer, TDocument>
         >;
       };
+      categorize_text: {
+        buckets: Array<
+          {
+            doc_count: number;
+            key: string | number;
+            regex?: string;
+            max_matching_length: number;
+          } & SubAggregateOf<TAggregationContainer, TDocument>
+        >;
+      };
       top_hits: {
         hits: {
           total: {