[Inventory] Adding initial e2e structure (#196560)

closes #193992

How to open cypress dashboard locally:
```
node x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js --open
```

How to run cypress tests:
```
node x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js
```

How to run cypress tests multiple times:
```
node x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js --server
node x-pack/plugins/observability_solution/inventory/scripts/test/e2e.js --runner --times=X
```

---------

Co-authored-by: kibanamachine <[email protected]>
(cherry picked from commit 3bfa7c0)

# Conflicts:
#	.github/CODEOWNERS
#	packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts
#	packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts
#	packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts
#	packages/kbn-apm-synthtrace/index.ts
#	packages/kbn-apm-synthtrace/src/cli/scenario.ts
#	packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts
#	packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts
#	packages/kbn-spec-to-console/lib/convert/parts.js

.buildkite/ftr_oblt_stateful_configs.yml

@@ -3,9 +3,10 @@ disabled:
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
+  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
   - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts

.buildkite/pipelines/flaky_tests/groups.json

@@ -23,7 +23,7 @@
     {
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
-    },   
+    },
     {
       "key": "cypress/security_solution_rule_management",
       "name": "Security Solution Rule Management - Cypress"
@@ -87,6 +87,10 @@
     {
       "key": "cypress/apm_cypress",
       "name": "APM - Cypress"
+    },
+    {
+      "key": "cypress/inventory_cypress",
+      "name": "Inventory - Cypress"
     }
   ]
-}
+}

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -80,3 +80,20 @@ steps:
           limit: 3
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/inventory_cypress.yml

@@ -0,0 +1,17 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/inventory_cypress.sh
+    label: 'Inventory Cypress Tests'
+    agents:
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on:
+      - build
+      - quick_checks
+    timeout_in_minutes: 120
+    parallelism: 1
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -77,6 +77,16 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/observability_solution\/inventory/,
+        /^packages\/kbn-apm-synthtrace/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/inventory_cypress.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^x-pack\/plugins\/observability_solution\/observability_onboarding/,

.buildkite/scripts/steps/functional/inventory_cypress.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+export JOB=kibana-inventory-onboarding-cypress
+
+echo "--- Observability Inventory Cypress Tests"
+
+cd "$XPACK_DIR"
+
+node plugins/observability_solution/inventory/scripts/test/e2e.js \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

package.json

@@ -1444,6 +1444,7 @@
     "@kbn/get-repo-files": "link:packages/kbn-get-repo-files",
     "@kbn/import-locator": "link:packages/kbn-import-locator",
     "@kbn/import-resolver": "link:packages/kbn-import-resolver",
+    "@kbn/inventory-e2e": "link:x-pack/plugins/observability_solution/inventory/e2e",
     "@kbn/jest-serializers": "link:packages/kbn-jest-serializers",
     "@kbn/journeys": "link:packages/kbn-journeys",
     "@kbn/json-ast": "link:packages/kbn-json-ast",

packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ContainerEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'container',
+      'entity.definitionId': 'builtin_containers_from_ecs_data',
+      'entity.identityFields': ['container.id'],
+    });
+  }
+}
+
+export function containerEntity({
+  agentName,
+  dataStreamType,
+  containerId,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  containerId: string;
+  entityId: string;
+}) {
+  return new ContainerEntity({
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'container.id': containerId,
+    'entity.displayName': containerId,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class HostEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'host',
+      'entity.definitionId': 'builtin_hosts_from_ecs_data',
+      'entity.identityFields': ['host.name'],
+    });
+  }
+}
+
+export function hostEntity({
+  agentName,
+  dataStreamType,
+  hostName,
+  entityId,
+}: {
+  agentName: string[];
+  dataStreamType: EntityDataStreamType[];
+  hostName: string;
+  entityId: string;
+}) {
+  return new HostEntity({
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'host.name': hostName,
+    'entity.displayName': hostName,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { EntityDataStreamType, EntityFields } from '.';
+import { Serializable } from '../serializable';
+
+class ServiceEntity extends Serializable<EntityFields> {
+  constructor(fields: EntityFields) {
+    super({
+      ...fields,
+      'entity.type': 'service',
+      'entity.definitionId': 'builtin_services_from_ecs_data',
+      'entity.identityFields': ['service.name'],
+    });
+  }
+}
+
+export function serviceEntity({
+  agentName,
+  dataStreamType,
+  serviceName,
+  environment,
+  entityId,
+}: {
+  agentName: string[];
+  serviceName: string;
+  dataStreamType: EntityDataStreamType[];
+  environment?: string;
+  entityId: string;
+}) {
+  return new ServiceEntity({
+    'service.name': serviceName,
+    'entity.displayName': serviceName,
+    'service.environment': environment,
+    'source_data_stream.type': dataStreamType,
+    'agent.name': agentName,
+    'entity.id': entityId,
+  });
+}

packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts

@@ -6,8 +6,6 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
-import { randomInt } from 'crypto';
 import { Fields } from '../entity';
 import { Serializable } from '../serializable';
 
@@ -180,3 +178,12 @@ export const log = {
   create,
   createMinimal,
 };
+
+function randomInt(min: number, max: number) {
+  if (min > max) {
+    throw new Error('Min value must be less than or equal to max value.');
+  }
+
+  const random = Math.floor(Math.random() * (max - min + 1)) + min;
+  return random;
+}

packages/kbn-apm-synthtrace/index.ts

@@ -15,7 +15,8 @@ export { InfraSynthtraceEsClient } from './src/lib/infra/infra_synthtrace_es_cli
 export { InfraSynthtraceKibanaClient } from './src/lib/infra/infra_synthtrace_kibana_client';
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
-export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client';
+export { EntitiesSynthtraceEsClient } from './src/lib/entities/entities_synthtrace_es_client';
+export { EntitiesSynthtraceKibanaClient } from './src/lib/entities/entities_synthtrace_kibana_client';
 export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client';
 export {
   addObserverVersionTransform,

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -18,6 +18,7 @@ import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_cli
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
 import { RunOptions } from './utils/parse_run_cli_flags';
+import { EntitiesSynthtraceKibanaClient } from '../lib/entities/entities_synthtrace_kibana_client';
 
 interface EsClients {
   apmEsClient: ApmSynthtraceEsClient;

packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { EntityFields, ESDocumentWithOperation } from '@kbn/apm-synthtrace-client';
+import { pipeline, Readable, Transform } from 'stream';
+import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client';
+import { getDedotTransform } from '../shared/get_dedot_transform';
+import { getSerializeTransform } from '../shared/get_serialize_transform';
+import { Logger } from '../utils/create_logger';
+
+export type EntitiesSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
+
+interface Pipeline {
+  includeSerialization?: boolean;
+}
+
+export class EntitiesSynthtraceEsClient extends SynthtraceEsClient<EntityFields> {
+  constructor(options: { client: Client; logger: Logger } & EntitiesSynthtraceEsClientOptions) {
+    super({
+      ...options,
+      pipeline: entitiesPipeline(),
+    });
+    this.indices = ['.entities.v1.latest.builtin*'];
+  }
+
+  getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+    return entitiesPipeline({ includeSerialization });
+  }
+}
+
+function entitiesPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
+  return (base: Readable) => {
+    const serializationTransform = includeSerialization ? [getSerializeTransform()] : [];
+
+    return pipeline(
+      // @ts-expect-error Some weird stuff here with the type definition for pipeline. We have tests!
+      base,
+      ...serializationTransform,
+      lastSeenTimestampTransform(),
+      getRoutingTransform(),
+      getDedotTransform(),
+      (err: unknown) => {
+        if (err) {
+          throw err;
+        }
+      }
+    );
+  };
+}
+
+function lastSeenTimestampTransform() {
+  return new Transform({
+    objectMode: true,
+    transform(document: ESDocumentWithOperation<EntityFields>, encoding, callback) {
+      const timestamp = document['@timestamp'];
+      if (timestamp) {
+        const isoString = new Date(timestamp).toISOString();
+        document['entity.lastSeenTimestamp'] = isoString;
+        document['event.ingested'] = isoString;
+        delete document['@timestamp'];
+      }
+      callback(null, document);
+    },
+  });
+}
+
+function getRoutingTransform() {
+  return new Transform({
+    objectMode: true,
+    transform(document: ESDocumentWithOperation<EntityFields>, encoding, callback) {
+      const entityType: string | undefined = document['entity.type'];
+      if (entityType === undefined) {
+        throw new Error(`entity.type was not defined: ${JSON.stringify(document)}`);
+      }
+      const entityIndexName = `${entityType}s`;
+      document._action = {
+        index: {
+          _index: `.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`,
+          _id: document['entity.id'],
+        },
+      };
+
+      callback(null, document);
+    },
+  });
+}