[Logs Explorer] Add support for log generation in synthtrace

.github/CODEOWNERS

@@ -39,8 +39,8 @@ packages/analytics/shippers/gainsight @elastic/kibana-core
 packages/kbn-apm-config-loader @elastic/kibana-core @vigneshshanmugam
 x-pack/plugins/apm_data_access @elastic/obs-knowledge-team @elastic/obs-ux-infra_services-team
 x-pack/plugins/apm @elastic/obs-ux-infra_services-team
-packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team
-packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team
+packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
+packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
 packages/kbn-apm-utils @elastic/obs-ux-infra_services-team
 test/plugin_functional/plugins/app_link_test @elastic/kibana-core
 x-pack/test/usage_collection/plugins/application_usage_test @elastic/kibana-core

packages/kbn-apm-synthtrace-client/index.ts

@@ -33,3 +33,4 @@ export { dedot } from './src/lib/utils/dedot';
 export { generateLongId, generateShortId } from './src/lib/utils/generate_id';
 export { appendHash, hashKeysOf } from './src/lib/utils/hash';
 export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } from './src/types';
+export { log, type LogDocument } from './src/lib/logs';

packages/kbn-apm-synthtrace-client/kibana.jsonc

@@ -2,5 +2,5 @@
   "type": "shared-common",
   "id": "@kbn/apm-synthtrace-client",
   "devOnly": true,
-  "owner": "@elastic/obs-ux-infra_services-team"
+  "owner": ["@elastic/obs-ux-infra_services-team", "@elastic/obs-ux-logs-team"]
 }

packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from '../entity';
+import { Serializable } from '../serializable';
+
+export type LogDocument = Fields &
+  Partial<{
+    'input.type': string;
+    'log.file.path'?: string;
+    'service.name'?: string;
+    'data_stream.namespace': string;
+    'data_stream.type': string;
+    'data_stream.dataset': string;
+    message?: string;
+    'event.dataset': string;
+    'log.level'?: string;
+    'host.name'?: string;
+    'trace.id'?: string;
+    'agent.name'?: string;
+    'orchestrator.cluster.name'?: string;
+    'orchestrator.cluster.id'?: string;
+    'orchestrator.resource.id'?: string;
+    'cloud.provider'?: string;
+    'cloud.region'?: string;
+    'cloud.availability_zone'?: string;
+    'cloud.project.id'?: string;
+    'cloud.instance.id'?: string;
+  }>;
+
+class Log extends Serializable<LogDocument> {
+  service(name: string) {
+    this.fields['service.name'] = name;
+    return this;
+  }
+
+  namespace(value: string) {
+    this.fields['data_stream.namespace'] = value;
+    return this;
+  }
+
+  dataset(value: string) {
+    this.fields['data_stream.dataset'] = value;
+    this.fields['event.dataset'] = value;
+    return this;
+  }
+
+  logLevel(level: string) {
+    this.fields['log.level'] = level;
+    return this;
+  }
+
+  message(message: string) {
+    this.fields.message = message;
+    return this;
+  }
+}
+
+function create(): Log {
+  return new Log({
+    'input.type': 'logs',
+    'data_stream.namespace': 'default',
+    'data_stream.type': 'logs',
+    'data_stream.dataset': 'synth',
+    'event.dataset': 'synth',
+    'host.name': 'synth-host',
+  });
+}
+
+export const log = {
+  create,
+};

packages/kbn-apm-synthtrace/README.md

@@ -21,6 +21,7 @@ This library can currently be used in two ways:
 - `Instance`: a single instance of a monitored service. E.g., the workload for a monitored service might be spread across multiple containers. An `Instance` object contains fields like `service.node.name` and `container.id`.
 - `Timerange`: an object that will return an array of timestamps based on an interval and a rate. These timestamps can be used to generate events/metricsets.
 - `Transaction`, `Span`, `APMError` and `Metricset`: events/metricsets that occur on an instance. For more background, see the [explanation of the APM data model](https://www.elastic.co/guide/en/apm/get-started/7.15/apm-data-model.html)
+- `Log`: An instance of Log generating Service which supports additional helpers to customise fields like `messages`, `logLevel`
 
 #### Example
 
@@ -109,12 +110,22 @@ node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost
 
 The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
 
+### Understanding Scenario Files
+
+Scenario files accept 3 arguments, 2 of them optional and 1 mandatory
+
+| Arguments   | Type      | Description                                                                                                                                          |
+|-------------|:----------|------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `generate`  | mandatory | This is the main function responsible for returning the events which will be indexed                                                                 |
+| `bootstrap`  | optional  | In case some setup needs to be done, before the data is generated, this function provides access to all available ES Clients to play with            |
+| `setClient` | optional  | By default the apmEsClient used to generate data. If anyother client like logsEsClient needs to be used instead, this is where it should be returned |
+
 The following options are supported:
 
 ### Connection options
 
 | Option              | Type     | Default | Description                                                                                |
-| ------------------- | -------- | :------ | ------------------------------------------------------------------------------------------ |
+|---------------------|----------|:--------|--------------------------------------------------------------------------------------------|
 | `--target`          | [string] |         | Elasticsearch target                                                                       |
 | `--kibana`          | [string] |         | Kibana target, used to bootstrap datastreams/mappings/templates/settings                   |
 | `--versionOverride` | [string] |         | String to be used for `observer.version`. Defauls to the version of the installed package. |
@@ -128,12 +139,11 @@ Note:
 
 ### Scenario options
 
-| Option           | Type      | Default | Description                          |
-| ---------------- | --------- | :------ | ------------------------------------ |
-| `--from`         | [date]    | `now()` | The start of the time window         |
-| `--to`           | [date]    |         | The end of the time window           |
-| `--live`         | [boolean] |         | Generate and index data continuously |
-| `--scenarioOpts` |           |         | Raw options specific to the scenario |
+| Option   | Type      | Default | Description                          |
+|----------|-----------|:--------|--------------------------------------|
+| `--from` | [date]    | `now()` | The start of the time window         |
+| `--to`   | [date]    |         | The end of the time window           |
+| `--live` | [boolean] |         | Generate and index data continuously |
 
 Note:
 
@@ -143,11 +153,12 @@ Note:
 
 ### Setup options
 
-| Option       | Type      | Default | Description                             |
-| ------------ | --------- | :------ | --------------------------------------- |
-| `--clean`    | [boolean] | `false` | Clean APM data before indexing new data |
-| `--workers`  | [number]  |         | Amount of Node.js worker threads        |
-| `--logLevel` | [enum]    | `info`  | Log level                               |
+| Option       | Type      | Default | Description                                                             |
+|--------------|-----------|:--------|-------------------------------------------------------------------------|
+| `--clean`    | [boolean] | `false` | Clean APM data before indexing new data                                 |
+| `--workers`  | [number]  |         | Amount of Node.js worker threads                                        |
+| `--logLevel` | [enum]    | `info`  | Log level                                                               |
+| `--type`     | [string]  | `apm`   | Type of data to be generated, `log` must be passed when generating logs |
 
 ## Testing
 

packages/kbn-apm-synthtrace/index.ts

@@ -17,6 +17,8 @@ export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_
 
 export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
 
+export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
+
 export {
   addObserverVersionTransform,
   deleteSummaryFieldTransform,

packages/kbn-apm-synthtrace/kibana.jsonc

@@ -2,5 +2,5 @@
   "type": "shared-server",
   "id": "@kbn/apm-synthtrace",
   "devOnly": true,
-  "owner": "@elastic/obs-ux-infra_services-team"
+  "owner": ["@elastic/obs-ux-infra_services-team", "@elastic/obs-ux-logs-team"]
 }

packages/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -6,17 +6,24 @@
  * Side Public License, v 1.
  */
 
-import { SynthtraceGenerator, Timerange } from '@kbn/apm-synthtrace-client';
-import { Readable } from 'stream';
-import { ApmSynthtraceEsClient } from '../lib/apm/client/apm_synthtrace_es_client';
+import { Timerange } from '@kbn/apm-synthtrace-client';
 import { Logger } from '../lib/utils/create_logger';
 import { RunOptions } from './utils/parse_run_cli_flags';
+import { ApmSynthtraceEsClient, LogsSynthtraceEsClient } from '../..';
+import { ScenarioReturnType } from '../lib/utils/with_client';
 
 type Generate<TFields> = (options: {
   range: Timerange;
-}) => SynthtraceGenerator<TFields> | Array<SynthtraceGenerator<TFields>> | Readable;
+  client: {
+    apmEsClient: ApmSynthtraceEsClient;
+    logsEsClient: LogsSynthtraceEsClient;
+  };
+}) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;
 
 export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
-  bootstrap?: (options: { apmEsClient: ApmSynthtraceEsClient }) => Promise<void>;
+  bootstrap?: (options: {
+    apmEsClient: ApmSynthtraceEsClient;
+    logsEsClient: LogsSynthtraceEsClient;
+  }) => Promise<void>;
   generate: Generate<TFields>;
 }>;

packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts

@@ -7,7 +7,8 @@
  */
 
 import { createLogger } from '../../lib/utils/create_logger';
-import { getEsClient } from './get_es_client';
+import { getApmEsClient } from './get_apm_es_client';
+import { getLogsEsClient } from './get_logs_es_client';
 import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
@@ -26,22 +27,30 @@ export async function bootstrap(runOptions: RunOptions) {
 
   const version = runOptions.versionOverride || latestPackageVersion;
 
-  const apmEsClient = getEsClient({
+  const apmEsClient = getApmEsClient({
     target: esUrl,
     logger,
     concurrency: runOptions.concurrency,
     version,
   });
 
+  const logsEsClient = getLogsEsClient({
+    target: esUrl,
+    logger,
+    concurrency: runOptions.concurrency,
+  });
+
   await kibanaClient.installApmPackage(latestPackageVersion);
 
   if (runOptions.clean) {
     await apmEsClient.clean();
+    await logsEsClient.clean();
   }
 
   return {
     logger,
     apmEsClient,
+    logsEsClient,
     version,
     kibanaUrl,
     esUrl,

packages/kbn-apm-synthtrace/src/cli/utils/get_es_client.ts → packages/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts

@@ -7,11 +7,11 @@
  */
 
 import { Client } from '@elastic/elasticsearch';
-import { ApmSynthtraceEsClient } from '../../lib/apm/client/apm_synthtrace_es_client';
+import { ApmSynthtraceEsClient } from '../../..';
 import { Logger } from '../../lib/utils/create_logger';
 import { RunOptions } from './parse_run_cli_flags';
 
-export function getEsClient({
+export function getApmEsClient({
   target,
   logger,
   version,

packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client';
+import { Logger } from '../../lib/utils/create_logger';
+import { RunOptions } from './parse_run_cli_flags';
+
+export function getLogsEsClient({
+  target,
+  logger,
+  concurrency,
+}: Pick<RunOptions, 'concurrency'> & {
+  target: string;
+  logger: Logger;
+}) {
+  const client = new Client({
+    node: target,
+  });
+
+  return new LogsSynthtraceEsClient({
+    client,
+    logger,
+    concurrency,
+  });
+}