From ce0b8db59775c817a6cd76c921771598526ffe1d Mon Sep 17 00:00:00 2001 From: Hanna Tamoudi Date: Tue, 17 Sep 2024 18:31:18 +0200 Subject: [PATCH] Integration assistant generate readme (#192887) ## Summary - Create an integration `README` when using the Integration Assistant so that it is displayed when opening the newly created integration. - Increase code coverage for the Integration Assistant plugin. (cherry picked from commit e360963065ab783c9cf265869fb43b2f73f74368) --- .../server/integration_builder/agent.test.ts | 48 ++++ .../build_integration.test.ts | 205 +++++++++++++++++- .../integration_builder/build_integration.ts | 3 + .../integration_builder/data_stream.test.ts | 89 ++++++++ .../server/integration_builder/fields.test.ts | 74 +++++++ .../integration_builder/pipeline.test.ts | 70 ++++++ 6 files changed, 488 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/agent.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/pipeline.test.ts diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/agent.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/agent.test.ts new file mode 100644 index 0000000000000..44a26e40fe780 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/agent.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as Utils from '../util'; +import { createAgentInput } from './agent'; +import { InputType } from '../../common'; + +jest.mock('../util', () => ({ + ...jest.requireActual('../util'), + createSync: jest.fn(), + ensureDirSync: jest.fn(), +})); + +describe('createAgentInput', () => { + const dataStreamPath = 'path'; + + beforeEach(async () => { + jest.clearAllMocks(); + }); + + it('Should create expected files', async () => { + const inputTypes: InputType[] = ['aws-s3', 'filestream']; + + createAgentInput(dataStreamPath, inputTypes); + + expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/agent/stream`); + + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/agent/stream/aws-s3.yml.hbs`, + expect.any(String) + ); + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/agent/stream/filestream.yml.hbs`, + expect.any(String) + ); + }); + + it('Should not create agent files if there are no input types', async () => { + createAgentInput(dataStreamPath, []); + + expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/agent/stream`); + expect(Utils.createSync).not.toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts index b77f1fa77a1bb..50ec954bcb118 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts @@ -5,10 +5,213 @@ * 2.0. */ -import { Integration } from '../../common'; +import * as buildIntegrationModule from './build_integration'; +import { testIntegration } from '../../__jest__/fixtures/build_integration'; +import * as Utils from '../util'; +import * as DataStreamModule from './data_stream'; +import * as FieldsModule from './fields'; +import * as AgentModule from './agent'; +import * as PipelineModule from './pipeline'; +import { DataStream, Docs, InputType, Pipeline, Integration } from '../../common'; import { renderPackageManifestYAML } from './build_integration'; import yaml from 'js-yaml'; +const mockedDataPath = 'path'; +const mockedId = 123; + +jest.mock('../util'); +jest.mock('./data_stream'); +jest.mock('./fields'); +jest.mock('./agent'); +jest.mock('./pipeline'); + +(Utils.generateUniqueId as jest.Mock).mockReturnValue(mockedId); + +jest.mock('@kbn/utils', () => ({ + getDataPath: jest.fn(() => mockedDataPath), +})); + +jest.mock('adm-zip', () => { + return jest.fn().mockImplementation(() => ({ + addLocalFolder: jest.fn(), + toBuffer: jest.fn(), + })); +}); + +describe('buildPackage', () => { + const packagePath = `${mockedDataPath}/integration-assistant-${mockedId}`; + const integrationPath = `${packagePath}/integration-1.0.0`; + + const firstDatastreamName = 'datastream_1'; + const secondDatastreamName = 'datastream_2'; + + const firstDataStreamInputTypes: InputType[] = ['filestream', 'kafka']; + const secondDataStreamInputTypes: InputType[] = ['kafka']; + + const firstDataStreamDocs: Docs = [ + { + key: 'foo', + anotherKey: 'bar', + }, + ]; + const secondDataStreamDocs: Docs = [{}]; + + const firstDataStreamPipeline: Pipeline = { + processors: [ + { + set: { + field: 'ecs.version', + value: '8.11.0', + }, + }, + ], + }; + const secondDataStreamPipeline: Pipeline = { processors: [] }; + + const firstDataStream: DataStream = { + name: firstDatastreamName, + title: 'Datastream_1', + description: 'Datastream_1 description', + inputTypes: firstDataStreamInputTypes, + docs: firstDataStreamDocs, + rawSamples: ['{"test1": "test1"}'], + pipeline: firstDataStreamPipeline, + samplesFormat: { name: 'ndjson', multiline: false }, + }; + + const secondDataStream: DataStream = { + name: secondDatastreamName, + title: 'Datastream_2', + description: 'Datastream_2 description', + inputTypes: secondDataStreamInputTypes, + docs: secondDataStreamDocs, + rawSamples: ['{"test1": "test1"}'], + pipeline: secondDataStreamPipeline, + samplesFormat: { name: 'ndjson', multiline: false }, + }; + + const firstDatastreamPath = `${integrationPath}/data_stream/${firstDatastreamName}`; + const secondDatastreamPath = `${integrationPath}/data_stream/${secondDatastreamName}`; + + testIntegration.dataStreams = [firstDataStream, secondDataStream]; + + beforeEach(async () => { + jest.clearAllMocks(); + await buildIntegrationModule.buildPackage(testIntegration); + }); + + it('Should create expected directories and files', async () => { + // Package & integration folders + expect(Utils.ensureDirSync).toHaveBeenCalledWith(packagePath); + expect(Utils.ensureDirSync).toHaveBeenCalledWith(integrationPath); + + // _dev files + expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`); + expect(Utils.createSync).toHaveBeenCalledWith( + `${integrationPath}/_dev/build/docs/README.md`, + expect.any(String) + ); + expect(Utils.createSync).toHaveBeenCalledWith( + `${integrationPath}/_dev/build/build.yml`, + expect.any(String) + ); + + // Docs files + expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`); + expect(Utils.createSync).toHaveBeenCalledWith( + `${integrationPath}/docs/README.md`, + expect.any(String) + ); + + // Changelog file + expect(Utils.createSync).toHaveBeenCalledWith( + `${integrationPath}/changelog.yml`, + expect.any(String) + ); + + // Manifest files + expect(Utils.createSync).toHaveBeenCalledWith( + `${integrationPath}/manifest.yml`, + expect.any(String) + ); + }); + + it('Should create logo files if info is present in the integration', async () => { + testIntegration.logo = 'logo'; + + await buildIntegrationModule.buildPackage(testIntegration); + + expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`); + expect(Utils.createSync).toHaveBeenCalledWith( + `${integrationPath}/img/logo.svg`, + expect.any(Buffer) + ); + }); + + it('Should not create logo files if info is not present in the integration', async () => { + jest.clearAllMocks(); + testIntegration.logo = undefined; + + await buildIntegrationModule.buildPackage(testIntegration); + + expect(Utils.ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`); + expect(Utils.createSync).not.toHaveBeenCalledWith( + `${integrationPath}/img/logo.svg`, + expect.any(Buffer) + ); + }); + + it('Should call createDataStream for each datastream', async () => { + expect(DataStreamModule.createDataStream).toHaveBeenCalledWith( + 'integration', + firstDatastreamPath, + firstDataStream + ); + expect(DataStreamModule.createDataStream).toHaveBeenCalledWith( + 'integration', + secondDatastreamPath, + secondDataStream + ); + }); + + it('Should call createAgentInput for each datastream', async () => { + expect(AgentModule.createAgentInput).toHaveBeenCalledWith( + firstDatastreamPath, + firstDataStreamInputTypes + ); + expect(AgentModule.createAgentInput).toHaveBeenCalledWith( + secondDatastreamPath, + secondDataStreamInputTypes + ); + }); + + it('Should call createPipeline for each datastream', async () => { + expect(PipelineModule.createPipeline).toHaveBeenCalledWith( + firstDatastreamPath, + firstDataStreamPipeline + ); + expect(PipelineModule.createPipeline).toHaveBeenCalledWith( + secondDatastreamPath, + secondDataStreamPipeline + ); + }); + + it('Should call createFieldMapping for each datastream', async () => { + expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith( + 'integration', + firstDatastreamName, + firstDatastreamPath, + firstDataStreamDocs + ); + expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith( + 'integration', + secondDatastreamName, + secondDatastreamPath, + secondDataStreamDocs + ); + }); +}); + describe('renderPackageManifestYAML', () => { test('generates the package manifest correctly', () => { const integration: Integration = { diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts index 0598ee3ba2cca..0a97977c653a5 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts @@ -104,6 +104,8 @@ function createChangelog(packageDir: string): void { function createReadme(packageDir: string, integration: Integration) { const readmeDirPath = joinPath(packageDir, '_dev/build/docs/'); + const mainReadmeDirPath = joinPath(packageDir, 'docs/'); + ensureDirSync(mainReadmeDirPath); ensureDirSync(readmeDirPath); const readmeTemplate = nunjucks.render('package_readme.md.njk', { package_name: integration.name, @@ -111,6 +113,7 @@ function createReadme(packageDir: string, integration: Integration) { }); createSync(joinPath(readmeDirPath, 'README.md'), readmeTemplate); + createSync(joinPath(mainReadmeDirPath, 'README.md'), readmeTemplate); } async function createZipArchive(workingDir: string, packageDirectoryName: string): Promise { diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts new file mode 100644 index 0000000000000..e5b00b85bf1d5 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as Utils from '../util'; +import { DataStream, Docs, InputType, Pipeline } from '../../common'; +import { createDataStream } from './data_stream'; +import * as nunjucks from 'nunjucks'; + +jest.mock('nunjucks'); + +jest.mock('../util', () => ({ + ...jest.requireActual('../util'), + removeDirSync: jest.fn(), + ensureDirSync: jest.fn(), + createSync: jest.fn(), + copySync: jest.fn(), + generateUniqueId: jest.fn(), + generateFields: jest.fn(), +})); + +describe('createDataStream', () => { + const packageName = 'package'; + const dataStreamPath = 'path'; + const firstDatastreamName = 'datastream_1'; + const firstDataStreamInputTypes: InputType[] = ['filestream', 'azure-eventhub']; + const firstDataStreamDocs: Docs = [ + { + key: 'foo', + anotherKey: 'bar', + }, + ]; + const firstDataStreamPipeline: Pipeline = { + processors: [ + { + set: { + field: 'ecs.version', + value: '8.11.0', + }, + }, + ], + }; + const samples = '{"test1": "test1"}'; + const firstDataStream: DataStream = { + name: firstDatastreamName, + title: 'Datastream_1', + description: 'Datastream_1 description', + inputTypes: firstDataStreamInputTypes, + docs: firstDataStreamDocs, + rawSamples: [samples], + pipeline: firstDataStreamPipeline, + samplesFormat: { name: 'ndjson', multiline: false }, + }; + + it('Should create expected directories and files', async () => { + createDataStream(packageName, dataStreamPath, firstDataStream); + + // pipeline + expect(Utils.ensureDirSync).toHaveBeenCalledWith(dataStreamPath); + expect(Utils.ensureDirSync).toHaveBeenCalledWith( + `${dataStreamPath}/elasticsearch/ingest_pipeline` + ); + + // dataStream files + expect(Utils.copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`); + + // test files + expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`); + expect(Utils.copySync).toHaveBeenCalledWith( + expect.any(String), + `${dataStreamPath}/_dev/test/pipeline/test-common-config.yml` + ); + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/_dev/test/pipeline/test-${packageName}-datastream-1.log`, + samples + ); + + // // Manifest files + expect(Utils.createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined); + expect(nunjucks.render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything()); + expect(nunjucks.render).toHaveBeenCalledWith( + `azure_eventhub_manifest.yml.njk`, + expect.anything() + ); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts new file mode 100644 index 0000000000000..9bd134b21b62e --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as Utils from '../util'; +import * as nunjucks from 'nunjucks'; +import { createFieldMapping } from './fields'; +import { Docs } from '../../common'; + +jest.mock('nunjucks'); + +jest.mock('../util', () => ({ + ...jest.requireActual('../util'), + createSync: jest.fn(), +})); + +const mockedTemplate = 'mocked template'; + +(nunjucks.render as jest.Mock).mockReturnValue(mockedTemplate); + +describe('createFieldMapping', () => { + const dataStreamPath = 'path'; + const packageName = 'package'; + const dataStreamName = 'datastream'; + + beforeEach(async () => { + jest.clearAllMocks(); + }); + + it('Should create fields files', async () => { + const docs: Docs = [ + { + key: 'foo', + anotherKey: 'bar', + }, + ]; + + createFieldMapping(packageName, dataStreamName, dataStreamPath, docs); + + const expectedFields = `- name: key + type: keyword +- name: anotherKey + type: keyword +`; + + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/base-fields.yml`, + mockedTemplate + ); + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/fields/fields.yml`, + expectedFields + ); + }); + + it('Should create fields files even if docs value is empty', async () => { + createFieldMapping(packageName, dataStreamName, dataStreamPath, []); + + const expectedFields = `[] +`; + + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/base-fields.yml`, + mockedTemplate + ); + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/fields/fields.yml`, + expectedFields + ); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/pipeline.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/pipeline.test.ts new file mode 100644 index 0000000000000..95d197ba2081b --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/pipeline.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Pipeline } from '../../common'; +import * as Utils from '../util'; +import { createPipeline } from './pipeline'; + +jest.mock('../util'); + +describe('createPipeline', () => { + const dataStreamPath = 'path'; + + beforeEach(async () => { + jest.clearAllMocks(); + }); + + it('Should call createSync with formatted pipeline', async () => { + const dataStreamPipeline: Pipeline = { + processors: [ + { + set: { + field: 'ecs.version', + value: '8.11.0', + }, + }, + { + rename: { + field: 'message', + target_field: 'event.original', + ignore_missing: true, + if: 'ctx.event?.original == null', + }, + }, + ], + }; + createPipeline(dataStreamPath, dataStreamPipeline); + + const expectYamlContent = `--- +processors: + - set: + field: ecs.version + value: 8.11.0 + - rename: + field: message + target_field: event.original + ignore_missing: true + if: ctx.event?.original == null +`; + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/elasticsearch/ingest_pipeline/default.yml`, + expectYamlContent + ); + }); + + it('Should call createSync even if pipeline is empty', async () => { + createPipeline(dataStreamPath, {}); + + const expectYamlContent = `--- +{} +`; + expect(Utils.createSync).toHaveBeenCalledWith( + `${dataStreamPath}/elasticsearch/ingest_pipeline/default.yml`, + expectYamlContent + ); + }); +});