[Synthtrace] Adding Entities support (#196258)

## known issue
```
- Transforms are not started by synthtrace. 
Because it duplicates data ingested by synthrace on signal indices. And it takes a long time to generate data.

- We are not able to open the Inventory page because of 👆🏻.
```
---

```
node scripts/synthtrace.js traces_logs_entities.ts --clean --live
```
or 
```
node scripts/synthtrace.js traces_logs_entities.ts --clean --from=2024-04-08T08:00:00.000Z --to=2024-04-08T08:15:00.000Z
```

docs produces by the new scenario:
```
{
  "took": 1,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 3,
      "relation": "eq"
    },
    "max_score": 1,
    "hits": [
      {
        "_index": ".entities.v1.latest.builtin_services_from_ecs_data",
        "_id": "2846700000000001",
        "_score": 1,
        "_source": {
          "service": {
            "name": "synth-node-trace-logs",
            "environment": "Synthtrace: traces_logs_entities"
          },
          "source_data_stream": {
            "type": [
              "traces",
              "logs"
            ]
          },
          "agent": {
            "name": [
              "nodejs"
            ]
          },
          "entity": {
            "id": "2846700000000001",
            "type": "service",
            "definitionId": "latest",
            "lastSeenTimestamp": "2024-10-15T08:56:20.562Z"
          },
          "event": {
            "ingested": "2024-10-15T08:56:20.562Z"
          }
        }
      },
      {
        "_index": ".entities.v1.latest.builtin_services_from_ecs_data",
        "_id": "2846700000000000",
        "_score": 1,
        "_source": {
          "service": {
            "name": "synth-java-trace",
            "environment": "Synthtrace: traces_logs_entities"
          },
          "source_data_stream": {
            "type": [
              "traces"
            ]
          },
          "agent": {
            "name": [
              "java"
            ]
          },
          "entity": {
            "id": "2846700000000000",
            "type": "service",
            "definitionId": "latest",
            "lastSeenTimestamp": "2024-10-15T08:56:20.562Z"
          },
          "event": {
            "ingested": "2024-10-15T08:56:20.562Z"
          }
        }
      },
      {
        "_index": ".entities.v1.latest.builtin_services_from_ecs_data",
        "_id": "2846700000000002",
        "_score": 1,
        "_source": {
          "service": {
            "name": "synth-go-logs",
            "environment": "Synthtrace: traces_logs_entities"
          },
          "source_data_stream": {
            "type": [
              "logs"
            ]
          },
          "agent": {
            "name": [
              "go"
            ]
          },
          "entity": {
            "id": "2846700000000002",
            "type": "service",
            "definitionId": "latest",
            "lastSeenTimestamp": "2024-10-15T08:56:20.562Z"
          },
          "event": {
            "ingested": "2024-10-15T08:56:20.562Z"
          }
        }
      }
    ]
  }
}
```

packages/kbn-apm-synthtrace-client/index.ts

@@ -35,6 +35,6 @@ export { generateLongId, generateShortId } from './src/lib/utils/generate_id';
 export { appendHash, hashKeysOf } from './src/lib/utils/hash';
 export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } from './src/types';
 export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs';
-export { type AssetDocument } from './src/lib/assets';
 export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics';
 export { otel, type OtelDocument } from './src/lib/otel';
+export { type EntityFields, entities } from './src/lib/entities';

packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ContainerEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'container',
+      'entity.definitionId': 'latest',
+    });
+  }
+}
+
+export function containerEntity({
+  agentName,
+  dataStreamType,
+  dataStreamDataset,
+  containerId,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  dataStreamDataset: string;
+  containerId: string;
+  entityId: string;
+}) {
+  return new ContainerEntity({
+    'source_data_stream.type': dataStreamType,
+    'source_data_stream.dataset': dataStreamDataset,
+    'agent.name': agentName,
+    'container.id': containerId,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class HostEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'host',
+      'entity.definitionId': 'latest',
+    });
+  }
+}
+
+export function hostEntity({
+  agentName,
+  dataStreamType,
+  dataStreamDataset,
+  hostName,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  dataStreamDataset: string;
+  hostName: string;
+  entityId: string;
+}) {
+  return new HostEntity({
+    'source_data_stream.type': dataStreamType,
+    'source_data_stream.dataset': dataStreamDataset,
+    'agent.name': agentName,
+    'host.name': hostName,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Fields } from '../entity';
+import { serviceEntity } from './service_entity';
+import { hostEntity } from './host_entity';
+import { containerEntity } from './container_entity';
+
+export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
+
+export type EntityFields = Fields &
+  Partial<{
+    'agent.name': string[];
+    'source_data_stream.type': string | string[];
+    'source_data_stream.dataset': string | string[];
+    'event.ingested': string;
+    sourceIndex: string;
+    'entity.lastSeenTimestamp': string;
+    'entity.schemaVersion': string;
+    'entity.definitionVersion': string;
+    'entity.displayName': string;
+    'entity.identityFields': string | string[];
+    'entity.id': string;
+    'entity.type': string;
+    'entity.definitionId': string;
+    [key: string]: any;
+  }>;
+
+export const entities = { serviceEntity, hostEntity, containerEntity };

packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ServiceEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'service',
+      'entity.definitionId': 'latest',
+    });
+  }
+}
+
+export function serviceEntity({
+  agentName,
+  dataStreamType,
+  serviceName,
+  environment,
+  entityId,
+}: {
+  agentName: string[];
+  serviceName: string;
+  dataStreamType: EntityDataStreamType[];
+  environment?: string;
+  entityId: string;
+}) {
+  return new ServiceEntity({
+    'service.name': serviceName,
+    'service.environment': environment,
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace/index.ts

@@ -15,7 +15,7 @@ export { InfraSynthtraceEsClient } from './src/lib/infra/infra_synthtrace_es_cli
 export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_kibana_client';
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
-export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client';
+export { EntitiesSynthtraceEsClient } from './src/lib/entities/entities_synthtrace_es_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client';
 export {

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -14,19 +14,24 @@ import {
   LogsSynthtraceEsClient,
   SyntheticsSynthtraceEsClient,
   OtelSynthtraceEsClient,
+  EntitiesSynthtraceEsClient,
 } from '../..';
-import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client';
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
 import { RunOptions } from './utils/parse_run_cli_flags';
+import { EntitiesSynthtraceKibanaClient } from '../lib/apm/client/entities_synthtrace_kibana_client';
 
 interface EsClients {
   apmEsClient: ApmSynthtraceEsClient;
   logsEsClient: LogsSynthtraceEsClient;
   infraEsClient: InfraSynthtraceEsClient;
-  assetsEsClient: AssetsSynthtraceEsClient;
   syntheticsEsClient: SyntheticsSynthtraceEsClient;
   otelEsClient: OtelSynthtraceEsClient;
+  entitiesEsClient: EntitiesSynthtraceEsClient;
+}
+
+interface KibanaClients {
+  entitiesKibanaClient: EntitiesSynthtraceKibanaClient;
 }
 
 type Generate<TFields> = (options: {
@@ -35,6 +40,6 @@ type Generate<TFields> = (options: {
 }) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;
 
 export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
-  bootstrap?: (options: EsClients) => Promise<void>;
+  bootstrap?: (options: EsClients & KibanaClients) => Promise<void>;
   generate: Generate<TFields>;
 }>;

packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts

@@ -14,9 +14,10 @@ import { getInfraEsClient } from './get_infra_es_client';
 import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
-import { getAssetsEsClient } from './get_assets_es_client';
 import { getSyntheticsEsClient } from './get_synthetics_es_client';
 import { getOtelSynthtraceEsClient } from './get_otel_es_client';
+import { getEntitiesEsClient } from './get_entities_es_client';
+import { getEntitiesKibanaClient } from './get_entites_kibana_client';
 
 export async function bootstrap(runOptions: RunOptions) {
   const logger = createLogger(runOptions.logLevel);
@@ -58,12 +59,17 @@ export async function bootstrap(runOptions: RunOptions) {
     concurrency: runOptions.concurrency,
   });
 
-  const assetsEsClient = getAssetsEsClient({
+  const entitiesEsClient = getEntitiesEsClient({
     target: esUrl,
     logger,
     concurrency: runOptions.concurrency,
   });
 
+  const entitiesKibanaClient = getEntitiesKibanaClient({
+    target: kibanaUrl,
+    logger,
+  });
+
   const syntheticsEsClient = getSyntheticsEsClient({
     target: esUrl,
     logger,
@@ -79,7 +85,7 @@ export async function bootstrap(runOptions: RunOptions) {
     await apmEsClient.clean();
     await logsEsClient.clean();
     await infraEsClient.clean();
-    await assetsEsClient.clean();
+    await entitiesEsClient.clean();
     await syntheticsEsClient.clean();
     await otelEsClient.clean();
   }
@@ -89,11 +95,12 @@ export async function bootstrap(runOptions: RunOptions) {
     apmEsClient,
     logsEsClient,
     infraEsClient,
-    assetsEsClient,
+    entitiesEsClient,
     syntheticsEsClient,
     otelEsClient,
     version,
     kibanaUrl,
     esUrl,
+    entitiesKibanaClient,
   };
 }

packages/kbn-apm-synthtrace/src/lib/assets/aggregators/create_logs_assets_aggregator.ts → packages/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts

@@ -7,7 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { LogDocument } from '@kbn/apm-synthtrace-client';
-import { createAssetsAggregatorFactory } from '../../utils/create_assets_aggregator_factory';
+import { EntitiesSynthtraceKibanaClient } from '../../lib/apm/client/entities_synthtrace_kibana_client';
+import { Logger } from '../../lib/utils/create_logger';
 
-export const createLogsAssetsAggregator = createAssetsAggregatorFactory<LogDocument>();
+export function getEntitiesKibanaClient({ target, logger }: { target: string; logger: Logger }) {
+  const kibanaClient = new EntitiesSynthtraceKibanaClient({
+    logger,
+    target,
+  });
+
+  return kibanaClient;
+}

packages/kbn-apm-synthtrace/src/cli/utils/get_assets_es_client.ts → packages/kbn-apm-synthtrace/src/cli/utils/get_entities_es_client.ts

@@ -8,12 +8,12 @@
  */
 
 import { Client } from '@elastic/elasticsearch';
-import { AssetsSynthtraceEsClient } from '../../lib/assets/assets_synthtrace_es_client';
+import { EntitiesSynthtraceEsClient } from '../../lib/entities/entities_synthtrace_es_client';
 import { Logger } from '../../lib/utils/create_logger';
 import { RunOptions } from './parse_run_cli_flags';
 import { getEsClientTlsSettings } from './ssl';
 
-export function getAssetsEsClient({
+export function getEntitiesEsClient({
   target,
   logger,
   concurrency,
@@ -26,7 +26,7 @@ export function getAssetsEsClient({
     tls: getEsClientTlsSettings(target),
   });
 
-  return new AssetsSynthtraceEsClient({
+  return new EntitiesSynthtraceEsClient({
     client,
     logger,
     concurrency,

packages/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts

@@ -26,7 +26,7 @@ export async function startHistoricalDataUpload({
   from: Date;
   to: Date;
 }) {
-  const { logger, esUrl, version } = await bootstrap(runOptions);
+  const { logger, esUrl, version, kibanaUrl } = await bootstrap(runOptions);
 
   const cores = cpus().length;
 
@@ -93,6 +93,7 @@ export async function startHistoricalDataUpload({
         workerId: workerIndex.toString(),
         esUrl,
         version,
+        kibanaUrl,
       };
       const worker = new Worker(Path.join(__dirname, './worker.js'), {
         workerData,