Integration assistant generate readme (elastic#192887)

## Summary

- Create an integration `README` when using the Integration Assistant so
that it is displayed when opening the newly created integration.
- Increase code coverage for the Integration Assistant plugin.

(cherry picked from commit e360963)

x-pack/plugins/integration_assistant/server/integration_builder/agent.test.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as Utils from '../util';
+import { createAgentInput } from './agent';
+import { InputType } from '../../common';
+
+jest.mock('../util', () => ({
+  ...jest.requireActual('../util'),
+  createSync: jest.fn(),
+  ensureDirSync: jest.fn(),
+}));
+
+describe('createAgentInput', () => {
+  const dataStreamPath = 'path';
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+  });
+
+  it('Should create expected files', async () => {
+    const inputTypes: InputType[] = ['aws-s3', 'filestream'];
+
+    createAgentInput(dataStreamPath, inputTypes);
+
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/agent/stream`);
+
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${dataStreamPath}/agent/stream/aws-s3.yml.hbs`,
+      expect.any(String)
+    );
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${dataStreamPath}/agent/stream/filestream.yml.hbs`,
+      expect.any(String)
+    );
+  });
+
+  it('Should not create agent files if there are no input types', async () => {
+    createAgentInput(dataStreamPath, []);
+
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/agent/stream`);
+    expect(Utils.createSync).not.toHaveBeenCalled();
+  });
+});

x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts

@@ -5,10 +5,213 @@
  * 2.0.
  */
 
-import { Integration } from '../../common';
+import * as buildIntegrationModule from './build_integration';
+import { testIntegration } from '../../__jest__/fixtures/build_integration';
+import * as Utils from '../util';
+import * as DataStreamModule from './data_stream';
+import * as FieldsModule from './fields';
+import * as AgentModule from './agent';
+import * as PipelineModule from './pipeline';
+import { DataStream, Docs, InputType, Pipeline, Integration } from '../../common';
 import { renderPackageManifestYAML } from './build_integration';
 import yaml from 'js-yaml';
 
+const mockedDataPath = 'path';
+const mockedId = 123;
+
+jest.mock('../util');
+jest.mock('./data_stream');
+jest.mock('./fields');
+jest.mock('./agent');
+jest.mock('./pipeline');
+
+(Utils.generateUniqueId as jest.Mock).mockReturnValue(mockedId);
+
+jest.mock('@kbn/utils', () => ({
+  getDataPath: jest.fn(() => mockedDataPath),
+}));
+
+jest.mock('adm-zip', () => {
+  return jest.fn().mockImplementation(() => ({
+    addLocalFolder: jest.fn(),
+    toBuffer: jest.fn(),
+  }));
+});
+
+describe('buildPackage', () => {
+  const packagePath = `${mockedDataPath}/integration-assistant-${mockedId}`;
+  const integrationPath = `${packagePath}/integration-1.0.0`;
+
+  const firstDatastreamName = 'datastream_1';
+  const secondDatastreamName = 'datastream_2';
+
+  const firstDataStreamInputTypes: InputType[] = ['filestream', 'kafka'];
+  const secondDataStreamInputTypes: InputType[] = ['kafka'];
+
+  const firstDataStreamDocs: Docs = [
+    {
+      key: 'foo',
+      anotherKey: 'bar',
+    },
+  ];
+  const secondDataStreamDocs: Docs = [{}];
+
+  const firstDataStreamPipeline: Pipeline = {
+    processors: [
+      {
+        set: {
+          field: 'ecs.version',
+          value: '8.11.0',
+        },
+      },
+    ],
+  };
+  const secondDataStreamPipeline: Pipeline = { processors: [] };
+
+  const firstDataStream: DataStream = {
+    name: firstDatastreamName,
+    title: 'Datastream_1',
+    description: 'Datastream_1 description',
+    inputTypes: firstDataStreamInputTypes,
+    docs: firstDataStreamDocs,
+    rawSamples: ['{"test1": "test1"}'],
+    pipeline: firstDataStreamPipeline,
+    samplesFormat: { name: 'ndjson', multiline: false },
+  };
+
+  const secondDataStream: DataStream = {
+    name: secondDatastreamName,
+    title: 'Datastream_2',
+    description: 'Datastream_2 description',
+    inputTypes: secondDataStreamInputTypes,
+    docs: secondDataStreamDocs,
+    rawSamples: ['{"test1": "test1"}'],
+    pipeline: secondDataStreamPipeline,
+    samplesFormat: { name: 'ndjson', multiline: false },
+  };
+
+  const firstDatastreamPath = `${integrationPath}/data_stream/${firstDatastreamName}`;
+  const secondDatastreamPath = `${integrationPath}/data_stream/${secondDatastreamName}`;
+
+  testIntegration.dataStreams = [firstDataStream, secondDataStream];
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+    await buildIntegrationModule.buildPackage(testIntegration);
+  });
+
+  it('Should create expected directories and files', async () => {
+    // Package & integration folders
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(packagePath);
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(integrationPath);
+
+    // _dev files
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`);
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${integrationPath}/_dev/build/docs/README.md`,
+      expect.any(String)
+    );
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${integrationPath}/_dev/build/build.yml`,
+      expect.any(String)
+    );
+
+    // Docs files
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`);
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${integrationPath}/docs/README.md`,
+      expect.any(String)
+    );
+
+    // Changelog file
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${integrationPath}/changelog.yml`,
+      expect.any(String)
+    );
+
+    // Manifest files
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${integrationPath}/manifest.yml`,
+      expect.any(String)
+    );
+  });
+
+  it('Should create logo files if info is present in the integration', async () => {
+    testIntegration.logo = 'logo';
+
+    await buildIntegrationModule.buildPackage(testIntegration);
+
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`);
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${integrationPath}/img/logo.svg`,
+      expect.any(Buffer)
+    );
+  });
+
+  it('Should not create logo files if info is not present in the integration', async () => {
+    jest.clearAllMocks();
+    testIntegration.logo = undefined;
+
+    await buildIntegrationModule.buildPackage(testIntegration);
+
+    expect(Utils.ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`);
+    expect(Utils.createSync).not.toHaveBeenCalledWith(
+      `${integrationPath}/img/logo.svg`,
+      expect.any(Buffer)
+    );
+  });
+
+  it('Should call createDataStream for each datastream', async () => {
+    expect(DataStreamModule.createDataStream).toHaveBeenCalledWith(
+      'integration',
+      firstDatastreamPath,
+      firstDataStream
+    );
+    expect(DataStreamModule.createDataStream).toHaveBeenCalledWith(
+      'integration',
+      secondDatastreamPath,
+      secondDataStream
+    );
+  });
+
+  it('Should call createAgentInput for each datastream', async () => {
+    expect(AgentModule.createAgentInput).toHaveBeenCalledWith(
+      firstDatastreamPath,
+      firstDataStreamInputTypes
+    );
+    expect(AgentModule.createAgentInput).toHaveBeenCalledWith(
+      secondDatastreamPath,
+      secondDataStreamInputTypes
+    );
+  });
+
+  it('Should call createPipeline for each datastream', async () => {
+    expect(PipelineModule.createPipeline).toHaveBeenCalledWith(
+      firstDatastreamPath,
+      firstDataStreamPipeline
+    );
+    expect(PipelineModule.createPipeline).toHaveBeenCalledWith(
+      secondDatastreamPath,
+      secondDataStreamPipeline
+    );
+  });
+
+  it('Should call createFieldMapping for each datastream', async () => {
+    expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith(
+      'integration',
+      firstDatastreamName,
+      firstDatastreamPath,
+      firstDataStreamDocs
+    );
+    expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith(
+      'integration',
+      secondDatastreamName,
+      secondDatastreamPath,
+      secondDataStreamDocs
+    );
+  });
+});
+
 describe('renderPackageManifestYAML', () => {
   test('generates the package manifest correctly', () => {
     const integration: Integration = {

x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts

@@ -104,13 +104,16 @@ function createChangelog(packageDir: string): void {
 
 function createReadme(packageDir: string, integration: Integration) {
   const readmeDirPath = joinPath(packageDir, '_dev/build/docs/');
+  const mainReadmeDirPath = joinPath(packageDir, 'docs/');
+  ensureDirSync(mainReadmeDirPath);
   ensureDirSync(readmeDirPath);
   const readmeTemplate = nunjucks.render('package_readme.md.njk', {
     package_name: integration.name,
     data_streams: integration.dataStreams,
   });
 
   createSync(joinPath(readmeDirPath, 'README.md'), readmeTemplate);
+  createSync(joinPath(mainReadmeDirPath, 'README.md'), readmeTemplate);
 }
 
 async function createZipArchive(workingDir: string, packageDirectoryName: string): Promise<Buffer> {

x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as Utils from '../util';
+import { DataStream, Docs, InputType, Pipeline } from '../../common';
+import { createDataStream } from './data_stream';
+import * as nunjucks from 'nunjucks';
+
+jest.mock('nunjucks');
+
+jest.mock('../util', () => ({
+  ...jest.requireActual('../util'),
+  removeDirSync: jest.fn(),
+  ensureDirSync: jest.fn(),
+  createSync: jest.fn(),
+  copySync: jest.fn(),
+  generateUniqueId: jest.fn(),
+  generateFields: jest.fn(),
+}));
+
+describe('createDataStream', () => {
+  const packageName = 'package';
+  const dataStreamPath = 'path';
+  const firstDatastreamName = 'datastream_1';
+  const firstDataStreamInputTypes: InputType[] = ['filestream', 'azure-eventhub'];
+  const firstDataStreamDocs: Docs = [
+    {
+      key: 'foo',
+      anotherKey: 'bar',
+    },
+  ];
+  const firstDataStreamPipeline: Pipeline = {
+    processors: [
+      {
+        set: {
+          field: 'ecs.version',
+          value: '8.11.0',
+        },
+      },
+    ],
+  };
+  const samples = '{"test1": "test1"}';
+  const firstDataStream: DataStream = {
+    name: firstDatastreamName,
+    title: 'Datastream_1',
+    description: 'Datastream_1 description',
+    inputTypes: firstDataStreamInputTypes,
+    docs: firstDataStreamDocs,
+    rawSamples: [samples],
+    pipeline: firstDataStreamPipeline,
+    samplesFormat: { name: 'ndjson', multiline: false },
+  };
+
+  it('Should create expected directories and files', async () => {
+    createDataStream(packageName, dataStreamPath, firstDataStream);
+
+    // pipeline
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(dataStreamPath);
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(
+      `${dataStreamPath}/elasticsearch/ingest_pipeline`
+    );
+
+    // dataStream files
+    expect(Utils.copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`);
+
+    // test files
+    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`);
+    expect(Utils.copySync).toHaveBeenCalledWith(
+      expect.any(String),
+      `${dataStreamPath}/_dev/test/pipeline/test-common-config.yml`
+    );
+    expect(Utils.createSync).toHaveBeenCalledWith(
+      `${dataStreamPath}/_dev/test/pipeline/test-${packageName}-datastream-1.log`,
+      samples
+    );
+
+    // // Manifest files
+    expect(Utils.createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined);
+    expect(nunjucks.render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything());
+    expect(nunjucks.render).toHaveBeenCalledWith(
+      `azure_eventhub_manifest.yml.njk`,
+      expect.anything()
+    );
+  });
+});