From 1fb9e8635c0733390f20d2e0db5ed3901ae43242 Mon Sep 17 00:00:00 2001 From: Hanna Tamoudi Date: Mon, 30 Sep 2024 17:46:05 +0200 Subject: [PATCH 1/3] [Automatic Import] add fields mapping to readme (#193717) (cherry picked from commit 508141423e7d18ce87628d826628d161c3292418) # Conflicts: # x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts --- .../build_integration.test.ts | 171 ++++++++-------- .../integration_builder/build_integration.ts | 43 +++-- .../integration_builder/data_stream.test.ts | 12 ++ .../server/integration_builder/data_stream.ts | 37 +++- .../server/integration_builder/fields.test.ts | 34 ++++ .../server/integration_builder/fields.ts | 20 +- .../integration_builder/readme_files.test.ts | 182 ++++++++++++++++++ .../integration_builder/readme_files.ts | 42 ++++ .../server/templates/build_readme.md.njk | 8 + .../server/templates/package_readme.md.njk | 43 ++--- .../server/templates/readme.njk | 31 +++ .../server/util/samples.test.ts | 159 ++++++++++++++- .../server/util/samples.ts | 43 ++++- 13 files changed, 664 insertions(+), 161 deletions(-) create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts create mode 100644 x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk create mode 100644 x-pack/plugins/integration_assistant/server/templates/readme.njk diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts index 50ec954bcb118..a576f9b58a2a0 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts @@ -4,50 +4,47 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import * as buildIntegrationModule from './build_integration'; +import { buildPackage, renderPackageManifestYAML } from './build_integration'; import { testIntegration } from '../../__jest__/fixtures/build_integration'; -import * as Utils from '../util'; -import * as DataStreamModule from './data_stream'; -import * as FieldsModule from './fields'; -import * as AgentModule from './agent'; -import * as PipelineModule from './pipeline'; +import { generateUniqueId, ensureDirSync, createSync } from '../util'; +import { createDataStream } from './data_stream'; +import { createFieldMapping } from './fields'; +import { createAgentInput } from './agent'; +import { createPipeline } from './pipeline'; import { DataStream, Docs, InputType, Pipeline, Integration } from '../../common'; -import { renderPackageManifestYAML } from './build_integration'; import yaml from 'js-yaml'; +import { createReadme } from './readme_files'; const mockedDataPath = 'path'; const mockedId = 123; - jest.mock('../util'); jest.mock('./data_stream'); jest.mock('./fields'); jest.mock('./agent'); jest.mock('./pipeline'); +jest.mock('./readme_files'); + +(createFieldMapping as jest.Mock).mockReturnValue([]); +(createDataStream as jest.Mock).mockReturnValue([]); -(Utils.generateUniqueId as jest.Mock).mockReturnValue(mockedId); +(generateUniqueId as jest.Mock).mockReturnValue(mockedId); jest.mock('@kbn/utils', () => ({ getDataPath: jest.fn(() => mockedDataPath), })); - jest.mock('adm-zip', () => { return jest.fn().mockImplementation(() => ({ addLocalFolder: jest.fn(), toBuffer: jest.fn(), })); }); - describe('buildPackage', () => { const packagePath = `${mockedDataPath}/integration-assistant-${mockedId}`; const integrationPath = `${packagePath}/integration-1.0.0`; - const firstDatastreamName = 'datastream_1'; const secondDatastreamName = 'datastream_2'; - const firstDataStreamInputTypes: InputType[] = ['filestream', 'kafka']; const secondDataStreamInputTypes: InputType[] = ['kafka']; - const firstDataStreamDocs: Docs = [ { key: 'foo', @@ -55,7 +52,6 @@ describe('buildPackage', () => { }, ]; const secondDataStreamDocs: Docs = [{}]; - const firstDataStreamPipeline: Pipeline = { processors: [ { @@ -67,7 +63,6 @@ describe('buildPackage', () => { ], }; const secondDataStreamPipeline: Pipeline = { processors: [] }; - const firstDataStream: DataStream = { name: firstDatastreamName, title: 'Datastream_1', @@ -78,7 +73,6 @@ describe('buildPackage', () => { pipeline: firstDataStreamPipeline, samplesFormat: { name: 'ndjson', multiline: false }, }; - const secondDataStream: DataStream = { name: secondDatastreamName, title: 'Datastream_2', @@ -89,127 +83,127 @@ describe('buildPackage', () => { pipeline: secondDataStreamPipeline, samplesFormat: { name: 'ndjson', multiline: false }, }; - const firstDatastreamPath = `${integrationPath}/data_stream/${firstDatastreamName}`; const secondDatastreamPath = `${integrationPath}/data_stream/${secondDatastreamName}`; - testIntegration.dataStreams = [firstDataStream, secondDataStream]; - beforeEach(async () => { jest.clearAllMocks(); - await buildIntegrationModule.buildPackage(testIntegration); + await buildPackage(testIntegration); }); - it('Should create expected directories and files', async () => { // Package & integration folders - expect(Utils.ensureDirSync).toHaveBeenCalledWith(packagePath); - expect(Utils.ensureDirSync).toHaveBeenCalledWith(integrationPath); + expect(ensureDirSync).toHaveBeenCalledWith(packagePath); + expect(ensureDirSync).toHaveBeenCalledWith(integrationPath); // _dev files - expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`); - expect(Utils.createSync).toHaveBeenCalledWith( - `${integrationPath}/_dev/build/docs/README.md`, - expect.any(String) - ); - expect(Utils.createSync).toHaveBeenCalledWith( + expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`); + expect(createSync).toHaveBeenCalledWith( `${integrationPath}/_dev/build/build.yml`, expect.any(String) ); - // Docs files - expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`); - expect(Utils.createSync).toHaveBeenCalledWith( - `${integrationPath}/docs/README.md`, - expect.any(String) - ); - // Changelog file - expect(Utils.createSync).toHaveBeenCalledWith( - `${integrationPath}/changelog.yml`, - expect.any(String) - ); + expect(createSync).toHaveBeenCalledWith(`${integrationPath}/changelog.yml`, expect.any(String)); // Manifest files - expect(Utils.createSync).toHaveBeenCalledWith( - `${integrationPath}/manifest.yml`, - expect.any(String) - ); + expect(createSync).toHaveBeenCalledWith(`${integrationPath}/manifest.yml`, expect.any(String)); }); - it('Should create logo files if info is present in the integration', async () => { testIntegration.logo = 'logo'; - - await buildIntegrationModule.buildPackage(testIntegration); - - expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`); - expect(Utils.createSync).toHaveBeenCalledWith( - `${integrationPath}/img/logo.svg`, - expect.any(Buffer) - ); + await buildPackage(testIntegration); + expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`); + expect(createSync).toHaveBeenCalledWith(`${integrationPath}/img/logo.svg`, expect.any(Buffer)); }); - it('Should not create logo files if info is not present in the integration', async () => { jest.clearAllMocks(); testIntegration.logo = undefined; - - await buildIntegrationModule.buildPackage(testIntegration); - - expect(Utils.ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`); - expect(Utils.createSync).not.toHaveBeenCalledWith( + await buildPackage(testIntegration); + expect(ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`); + expect(createSync).not.toHaveBeenCalledWith( `${integrationPath}/img/logo.svg`, expect.any(Buffer) ); }); - it('Should call createDataStream for each datastream', async () => { - expect(DataStreamModule.createDataStream).toHaveBeenCalledWith( + expect(createDataStream).toHaveBeenCalledWith( 'integration', firstDatastreamPath, firstDataStream ); - expect(DataStreamModule.createDataStream).toHaveBeenCalledWith( + expect(createDataStream).toHaveBeenCalledWith( 'integration', secondDatastreamPath, secondDataStream ); }); - it('Should call createAgentInput for each datastream', async () => { - expect(AgentModule.createAgentInput).toHaveBeenCalledWith( - firstDatastreamPath, - firstDataStreamInputTypes - ); - expect(AgentModule.createAgentInput).toHaveBeenCalledWith( - secondDatastreamPath, - secondDataStreamInputTypes - ); + expect(createAgentInput).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamInputTypes); + expect(createAgentInput).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamInputTypes); }); - it('Should call createPipeline for each datastream', async () => { - expect(PipelineModule.createPipeline).toHaveBeenCalledWith( - firstDatastreamPath, - firstDataStreamPipeline - ); - expect(PipelineModule.createPipeline).toHaveBeenCalledWith( - secondDatastreamPath, - secondDataStreamPipeline - ); + expect(createPipeline).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamPipeline); + expect(createPipeline).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamPipeline); }); - it('Should call createFieldMapping for each datastream', async () => { - expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith( + expect(createFieldMapping).toHaveBeenCalledWith( 'integration', firstDatastreamName, firstDatastreamPath, firstDataStreamDocs ); - expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith( + expect(createFieldMapping).toHaveBeenCalledWith( 'integration', secondDatastreamName, secondDatastreamPath, secondDataStreamDocs ); }); + + it('Should call createReadme once with sorted fields', async () => { + jest.clearAllMocks(); + + const firstDSFieldsMapping = [{ name: 'name a', description: 'description 1', type: 'type 1' }]; + + const firstDataStreamFields = [ + { name: 'name b', description: 'description 1', type: 'type 1' }, + ]; + + const secondDSFieldsMapping = [ + { name: 'name c', description: 'description 2', type: 'type 2' }, + { name: 'name e', description: 'description 3', type: 'type 3' }, + ]; + + const secondDataStreamFields = [ + { name: 'name d', description: 'description 2', type: 'type 2' }, + ]; + + (createFieldMapping as jest.Mock).mockReturnValueOnce(firstDSFieldsMapping); + (createDataStream as jest.Mock).mockReturnValueOnce(firstDataStreamFields); + + (createFieldMapping as jest.Mock).mockReturnValueOnce(secondDSFieldsMapping); + (createDataStream as jest.Mock).mockReturnValueOnce(secondDataStreamFields); + + await buildPackage(testIntegration); + + expect(createReadme).toHaveBeenCalledWith(integrationPath, testIntegration.name, [ + { + datastream: firstDatastreamName, + fields: [ + { name: 'name a', description: 'description 1', type: 'type 1' }, + + { name: 'name b', description: 'description 1', type: 'type 1' }, + ], + }, + { + datastream: secondDatastreamName, + fields: [ + { name: 'name c', description: 'description 2', type: 'type 2' }, + { name: 'name d', description: 'description 2', type: 'type 2' }, + { name: 'name e', description: 'description 3', type: 'type 3' }, + ], + }, + ]); + }); }); describe('renderPackageManifestYAML', () => { @@ -248,12 +242,9 @@ describe('renderPackageManifestYAML', () => { }, ], }; - const manifestContent = renderPackageManifestYAML(integration); - // The manifest content must be parseable as YAML. - const manifest = yaml.safeLoad(manifestContent); - + const manifest = yaml.load(manifestContent) as Record; expect(manifest).toBeDefined(); expect(manifest.title).toBe(integration.title); expect(manifest.name).toBe(integration.name); @@ -261,4 +252,4 @@ describe('renderPackageManifestYAML', () => { expect(manifest.description).toBe(integration.description); expect(manifest.icons).toBeTruthy(); }); -}); +}); \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts index 0a97977c653a5..72a8e55d045ab 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts @@ -16,6 +16,8 @@ import { createAgentInput } from './agent'; import { createDataStream } from './data_stream'; import { createFieldMapping } from './fields'; import { createPipeline } from './pipeline'; +import { createReadme } from './readme_files'; +import { Field, flattenObjectsList } from '../util/samples'; const initialVersion = '1.0.0'; @@ -37,17 +39,27 @@ export async function buildPackage(integration: Integration): Promise { const packageDir = createDirectories(workingDir, integration, packageDirectoryName); const dataStreamsDir = joinPath(packageDir, 'data_stream'); - - for (const dataStream of integration.dataStreams) { + const fieldsPerDatastream = integration.dataStreams.map((dataStream) => { const dataStreamName = dataStream.name; const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName); - createDataStream(integration.name, specificDataStreamDir, dataStream); + const dataStreamFields = createDataStream(integration.name, specificDataStreamDir, dataStream); createAgentInput(specificDataStreamDir, dataStream.inputTypes); createPipeline(specificDataStreamDir, dataStream.pipeline); - createFieldMapping(integration.name, dataStreamName, specificDataStreamDir, dataStream.docs); - } + const fields = createFieldMapping( + integration.name, + dataStreamName, + specificDataStreamDir, + dataStream.docs + ); + + return { + datastream: dataStreamName, + fields: mergeAndSortFields(fields, dataStreamFields), + }; + }); + createReadme(packageDir, integration.name, fieldsPerDatastream); const zipBuffer = await createZipArchive(workingDir, packageDirectoryName); removeDirSync(workingDir); @@ -67,7 +79,6 @@ function createDirectories( } function createPackage(packageDir: string, integration: Integration): void { - createReadme(packageDir, integration); createChangelog(packageDir); createBuildFile(packageDir); createPackageManifest(packageDir, integration); @@ -102,20 +113,6 @@ function createChangelog(packageDir: string): void { createSync(joinPath(packageDir, 'changelog.yml'), changelogTemplate); } -function createReadme(packageDir: string, integration: Integration) { - const readmeDirPath = joinPath(packageDir, '_dev/build/docs/'); - const mainReadmeDirPath = joinPath(packageDir, 'docs/'); - ensureDirSync(mainReadmeDirPath); - ensureDirSync(readmeDirPath); - const readmeTemplate = nunjucks.render('package_readme.md.njk', { - package_name: integration.name, - data_streams: integration.dataStreams, - }); - - createSync(joinPath(readmeDirPath, 'README.md'), readmeTemplate); - createSync(joinPath(mainReadmeDirPath, 'README.md'), readmeTemplate); -} - async function createZipArchive(workingDir: string, packageDirectoryName: string): Promise { const tmpPackageDir = joinPath(workingDir, packageDirectoryName); const zip = new AdmZip(); @@ -124,6 +121,12 @@ async function createZipArchive(workingDir: string, packageDirectoryName: string return buffer; } +function mergeAndSortFields(fields: Field[], dataStreamFields: Field[]): Field[] { + const mergedFields = [...fields, ...dataStreamFields]; + + return flattenObjectsList(mergedFields); +} + /* eslint-disable @typescript-eslint/naming-convention */ /** * Creates a package manifest dictionary. diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts index e5b00b85bf1d5..1a0507d539ac0 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts @@ -86,4 +86,16 @@ describe('createDataStream', () => { expect.anything() ); }); + + it('Should return the list of fields', async () => { + const fields = createDataStream(packageName, dataStreamPath, firstDataStream); + + expect(Array.isArray(fields)).toBe(true); + fields.forEach((field) => { + expect(field).toMatchObject({ + name: expect.any(String), + type: expect.any(String), + }); + }); + }); }); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts index 02b3f12f53d68..d66ee1958b3ea 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts @@ -7,14 +7,16 @@ import nunjucks from 'nunjucks'; import { join as joinPath } from 'path'; +import { load } from 'js-yaml'; import type { DataStream } from '../../common'; -import { copySync, createSync, ensureDirSync, listDirSync } from '../util'; +import { copySync, createSync, ensureDirSync, listDirSync, readSync } from '../util'; +import { Field } from '../util/samples'; export function createDataStream( packageName: string, specificDataStreamDir: string, dataStream: DataStream -): void { +): Field[] { const dataStreamName = dataStream.name; const pipelineDir = joinPath(specificDataStreamDir, 'elasticsearch', 'ingest_pipeline'); const title = dataStream.title; @@ -23,7 +25,7 @@ export function createDataStream( const useMultilineNDJSON = samplesFormat.name === 'ndjson' && samplesFormat.multiline === true; ensureDirSync(specificDataStreamDir); - createDataStreamFolders(specificDataStreamDir, pipelineDir); + const fields = createDataStreamFolders(specificDataStreamDir, pipelineDir); createPipelineTests(specificDataStreamDir, dataStream.rawSamples, packageName, dataStreamName); const dataStreams: string[] = []; @@ -51,19 +53,34 @@ export function createDataStream( }); createSync(joinPath(specificDataStreamDir, 'manifest.yml'), finalManifest); + + return fields; +} + +function createDataStreamFolders(specificDataStreamDir: string, pipelineDir: string): Field[] { + ensureDirSync(pipelineDir); + return copyFilesFromTemplateDir(specificDataStreamDir); } -function createDataStreamFolders(specificDataStreamDir: string, pipelineDir: string): void { +function copyFilesFromTemplateDir(specificDataStreamDir: string): Field[] { const dataStreamTemplatesDir = joinPath(__dirname, '../templates/data_stream'); const items = listDirSync(dataStreamTemplatesDir); + return items.flatMap((item) => { + const sourcePath = joinPath(dataStreamTemplatesDir, item); + const destinationPath = joinPath(specificDataStreamDir, item); + copySync(sourcePath, destinationPath); + const files = listDirSync(sourcePath); - for (const item of items) { - const s = joinPath(dataStreamTemplatesDir, item); - const d = joinPath(specificDataStreamDir, item); - copySync(s, d); - } + return loadFieldsFromFiles(sourcePath, files); + }); +} - ensureDirSync(pipelineDir); +function loadFieldsFromFiles(sourcePath: string, files: string[]): Field[] { + return files.flatMap((file) => { + const filePath = joinPath(sourcePath, file); + const content = readSync(filePath); + return load(content) as Field[]; + }); } function createPipelineTests( diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts index dc37fef76161f..bf849f3b7f2bc 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts @@ -71,4 +71,38 @@ describe('createFieldMapping', () => { expectedFields ); }); + + it('Should return all fields flattened', async () => { + const docs: Docs = [ + { + key: 'foo', + anotherKey: 'bar', + }, + ]; + + const baseFields = `- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword +- name: "@timestamp" + type: date + description: Event timestamp. +`; + (render as jest.Mock).mockReturnValue(baseFields); + + const fieldsResult = createFieldMapping(packageName, dataStreamName, dataStreamPath, docs); + + expect(fieldsResult).toEqual([ + { + name: 'data_stream.type', + type: 'constant_keyword', + description: 'Data stream type.', + }, + { name: 'data_stream.dataset', type: 'constant_keyword' }, + { name: '@timestamp', type: 'date', description: 'Event timestamp.' }, + { name: 'key', type: 'keyword' }, + { name: 'anotherKey', type: 'keyword' }, + ]); + }); }); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts b/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts index 79977ef2f3927..476bc0e74d697 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts @@ -6,7 +6,8 @@ */ import nunjucks from 'nunjucks'; - +import { load } from 'js-yaml'; +import { Field } from '../util/samples'; import { createSync, generateFields, mergeSamples } from '../util'; export function createFieldMapping( @@ -14,28 +15,33 @@ export function createFieldMapping( dataStreamName: string, specificDataStreamDir: string, docs: object[] -): void { +): Field[] { const dataStreamFieldsDir = `${specificDataStreamDir}/fields`; - createBaseFields(dataStreamFieldsDir, packageName, dataStreamName); - createCustomFields(dataStreamFieldsDir, docs); + const baseFields = createBaseFields(dataStreamFieldsDir, packageName, dataStreamName); + const customFields = createCustomFields(dataStreamFieldsDir, docs); + + return [...baseFields, ...customFields]; } function createBaseFields( dataStreamFieldsDir: string, packageName: string, dataStreamName: string -): void { +): Field[] { const datasetName = `${packageName}.${dataStreamName}`; const baseFields = nunjucks.render('base_fields.yml.njk', { module: packageName, dataset: datasetName, }); - createSync(`${dataStreamFieldsDir}/base-fields.yml`, baseFields); + + return load(baseFields) as Field[]; } -function createCustomFields(dataStreamFieldsDir: string, pipelineResults: object[]): void { +function createCustomFields(dataStreamFieldsDir: string, pipelineResults: object[]): Field[] { const mergedResults = mergeSamples(pipelineResults); const fieldKeys = generateFields(mergedResults); createSync(`${dataStreamFieldsDir}/fields.yml`, fieldKeys); + + return load(fieldKeys) as Field[]; } diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts new file mode 100644 index 0000000000000..ae9080fff8a74 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { testIntegration } from '../../__jest__/fixtures/build_integration'; +import { ensureDirSync, createSync } from '../util'; +import { configure } from 'nunjucks'; +import { join as joinPath } from 'path'; +import { createReadme } from './readme_files'; + +jest.mock('../util', () => ({ + ...jest.requireActual('../util'), + createSync: jest.fn(), + ensureDirSync: jest.fn(), +})); + +describe('createReadme', () => { + const integrationPath = 'path'; + + const templateDir = joinPath(__dirname, '../templates'); + configure([templateDir], { + autoescape: false, + }); + + beforeEach(async () => { + jest.clearAllMocks(); + }); + + it('Should create expected files', async () => { + const fields = [ + { + datastream: 'data_stream_1', + fields: [ + { + name: 'data_stream.type', + type: 'constant_keyword', + description: 'Data stream type.', + }, + { + name: 'data_stream.dataset', + type: 'constant_keyword', + description: 'Data stream dataset name.', + }, + { + name: 'event.dataset', + type: 'constant_keyword', + description: 'Event dataset', + value: 'package.datastream', + }, + { name: '@timestamp', type: 'date', description: 'Event timestamp.' }, + ], + }, + { + datastream: 'data_stream_2', + fields: [{ name: '@timestamp', type: 'date', description: 'Event timestamp.' }], + }, + ]; + + createReadme(integrationPath, testIntegration.name, fields); + + expect(createSync).toHaveBeenCalledWith( + `${integrationPath}/_dev/build/docs/README.md`, + expect.any(String) + ); + + // Docs files + expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`); + expect(createSync).toHaveBeenCalledWith( + `${integrationPath}/docs/README.md`, + expect.any(String) + ); + }); + + it('Should render a table per datastream with fields mapping in package readme', async () => { + const fields = [ + { + datastream: 'data_stream_1', + fields: [ + { + name: 'data_stream.type', + type: 'constant_keyword', + description: 'Data stream type.', + }, + { + name: 'data_stream.dataset', + type: 'constant_keyword', + }, + { + name: 'event.dataset', + type: 'constant_keyword', + description: 'Event dataset', + value: 'package.datastream', + }, + { name: '@timestamp', type: 'date', description: 'Event timestamp.' }, + ], + }, + { + datastream: 'data_stream_2', + fields: [{ name: '@timestamp', type: 'date', description: 'Event timestamp.' }], + }, + ]; + + createReadme(integrationPath, testIntegration.name, fields); + + const firstDatastreamFieldsDisplayed = ` +| Field | Description | Type | +|---|---|---| +| data_stream.type | Data stream type. | constant_keyword | +| data_stream.dataset | | constant_keyword | +| event.dataset | Event dataset | constant_keyword | +| @timestamp | Event timestamp. | date | +`; + + const secondDatastreamFieldsDisplayed = ` +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +`; + + expect(createSync).toHaveBeenCalledWith( + `${integrationPath}/docs/README.md`, + expect.stringContaining(firstDatastreamFieldsDisplayed) + ); + + expect(createSync).toHaveBeenCalledWith( + `${integrationPath}/docs/README.md`, + expect.stringContaining(secondDatastreamFieldsDisplayed) + ); + }); + + it('Should not render a fields mapping table in build readme', async () => { + const fields = [ + { + datastream: 'data_stream_1', + fields: [{ name: '@timestamp', type: 'date', description: 'Event timestamp.' }], + }, + ]; + + createReadme(integrationPath, testIntegration.name, fields); + + expect(createSync).toHaveBeenCalledWith( + `${integrationPath}/_dev/build/docs/README.md`, + expect.stringContaining('{{fields "data_stream_1"}}') + ); + }); + + it('Should render a formatted table per datastream with fields mapping in package readme', async () => { + const fields = [ + { + datastream: 'data_stream_1', + fields: [ + { + name: 'data_stream.type', + type: 'constant_keyword', + description: 'Data stream type.\n', + }, + { + name: 'data_stream.dataset', + type: 'constant_keyword', + }, + ], + }, + ]; + + createReadme(integrationPath, testIntegration.name, fields); + + const firstDatastreamFieldsDisplayed = ` +| Field | Description | Type | +|---|---|---| +| data_stream.type | Data stream type. | constant_keyword | +| data_stream.dataset | | constant_keyword | +`; + + expect(createSync).toHaveBeenCalledWith( + `${integrationPath}/docs/README.md`, + expect.stringContaining(firstDatastreamFieldsDisplayed) + ); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts new file mode 100644 index 0000000000000..163b2b04b52f9 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import nunjucks from 'nunjucks'; + +import { join as joinPath } from 'path'; +import { createSync, ensureDirSync } from '../util'; + +export function createReadme(packageDir: string, integrationName: string, fields: object[]) { + createPackageReadme(packageDir, integrationName, fields); + createBuildReadme(packageDir, integrationName, fields); +} + +function createPackageReadme(packageDir: string, integrationName: string, fields: object[]) { + const dirPath = joinPath(packageDir, 'docs/'); + createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, fields); +} + +function createBuildReadme(packageDir: string, integrationName: string, fields: object[]) { + const dirPath = joinPath(packageDir, '_dev/build/docs/'); + createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, fields); +} + +function createReadmeFile( + targetDir: string, + templateName: string, + integrationName: string, + fields: object[] +) { + ensureDirSync(targetDir); + + const template = nunjucks.render(templateName, { + package_name: integrationName, + fields, + }); + + createSync(joinPath(targetDir, 'README.md'), template); +} diff --git a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk new file mode 100644 index 0000000000000..e23fa4af9efe8 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk @@ -0,0 +1,8 @@ +{% include "readme.njk" %} +{% for data_stream in fields %} +### {{ data_stream.datastream }} + +Insert a description of the datastream here. + +{% raw %}{{fields {% endraw %}"{{ data_stream.datastream }}"{% raw %}}}{% endraw %} +{% endfor %} \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk index 02bf606ab386a..b47e3491b5bc2 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk @@ -1,38 +1,17 @@ -# {{ package_name }} Integration +{% include "readme.njk" %} +{% for data_stream in fields %} +### {{ data_stream.datastream }} -## Overview - -Explain what the integration is, define the third-party product that is providing data, establish its relationship to the larger ecosystem of Elastic products, and help the reader understand how it can be used to solve a tangible problem. -Check the [overview guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-overview) for more information. - -## Datastreams - -Provide a high-level overview of the kind of data that is collected by the integration. -Check the [datastreams guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-datastreams) for more information. - -## Requirements - -The requirements section helps readers to confirm that the integration will work with their systems. -Check the [requirements guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-requirements) for more information. - -## Setup - -Point the reader to the [Observability Getting started guide](https://www.elastic.co/guide/en/observability/master/observability-get-started.html) for generic, step-by-step instructions. Include any additional setup instructions beyond what’s included in the guide, which may include instructions to update the configuration of a third-party service. -Check the [setup guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-setup) for more information. - -## Troubleshooting (optional) - -Provide information about special cases and exceptions that aren’t necessary for getting started or won’t be applicable to all users. Check the [troubleshooting guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-troubleshooting) for more information. - -## Reference +Insert a description of the datastream here. -Provide detailed information about the log or metric types we support within the integration. Check the [reference guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-reference) for more information. +**ECS Field Reference** -## Logs -{% for data_stream in data_streams %} -### {{ data_stream.name }} +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. -Insert a description of the datastream here. +**Exported fields** -{% raw %}{{fields {% endraw %}"{{ data_stream.name }}"{% raw %}}}{% endraw %} +| Field | Description | Type | +|---|---|---| +{% for field in data_stream.fields %}| {{ field.name }} | {{ field.description | default('') | replace('\n', ' ') | trim }} | {{ field.type }} | {% endfor %} +{% endfor %} \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/templates/readme.njk b/x-pack/plugins/integration_assistant/server/templates/readme.njk new file mode 100644 index 0000000000000..91c1bf6f1b40c --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/templates/readme.njk @@ -0,0 +1,31 @@ +# {{ package_name }} Integration + +## Overview + +Explain what the integration is, define the third-party product that is providing data, establish its relationship to the larger ecosystem of Elastic products, and help the reader understand how it can be used to solve a tangible problem. +Check the [overview guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-overview) for more information. + +## Datastreams + +Provide a high-level overview of the kind of data that is collected by the integration. +Check the [datastreams guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-datastreams) for more information. + +## Requirements + +The requirements section helps readers to confirm that the integration will work with their systems. +Check the [requirements guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-requirements) for more information. + +## Setup + +Point the reader to the [Observability Getting started guide](https://www.elastic.co/guide/en/observability/master/observability-get-started.html) for generic, step-by-step instructions. Include any additional setup instructions beyond what’s included in the guide, which may include instructions to update the configuration of a third-party service. +Check the [setup guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-setup) for more information. + +## Troubleshooting (optional) + +Provide information about special cases and exceptions that aren’t necessary for getting started or won’t be applicable to all users. Check the [troubleshooting guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-troubleshooting) for more information. + +## Reference + +Provide detailed information about the log or metric types we support within the integration. Check the [reference guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-reference) for more information. + +## Logs \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/util/samples.test.ts b/x-pack/plugins/integration_assistant/server/util/samples.test.ts index 131135e842334..f87f9a96ca2c0 100644 --- a/x-pack/plugins/integration_assistant/server/util/samples.test.ts +++ b/x-pack/plugins/integration_assistant/server/util/samples.test.ts @@ -5,7 +5,164 @@ * 2.0. */ -import { merge } from './samples'; +import { flattenObjectsList, merge } from './samples'; + +describe('flattenObjectsList', () => { + it('Should return a list with flattened key/value entries', async () => { + const result = flattenObjectsList([ + { + name: 'a', + type: 'group', + fields: [ + { + name: 'b', + type: 'keyword', + description: 'Some description for b', + }, + { + name: 'c', + type: 'group', + fields: [ + { + name: 'd', + type: 'keyword', + }, + { + name: 'e', + description: 'Some description for e', + type: 'keyword', + }, + ], + }, + ], + }, + ]); + + expect(result).toEqual([ + { + name: 'a.b', + type: 'keyword', + description: 'Some description for b', + }, + { + name: 'a.c.d', + type: 'keyword', + description: undefined, + }, + { + name: 'a.c.e', + type: 'keyword', + description: 'Some description for e', + }, + ]); + }); + + it('Should return an empty list if passed an empty list', async () => { + const result = flattenObjectsList([]); + + expect(result).toEqual([]); + }); + + it('Should return a list with key/value entries', async () => { + const result = flattenObjectsList([ + { + name: 'a', + type: 'keyword', + description: 'Some description for a', + }, + ]); + + expect(result).toEqual([ + { + name: 'a', + type: 'keyword', + description: 'Some description for a', + }, + ]); + }); + + it('Should return an sorted list of key/value entries', async () => { + const result = flattenObjectsList([ + { + name: 'c', + type: 'group', + fields: [ + { + name: 'b', + type: 'keyword', + description: 'Some description for b', + }, + { + name: 'a', + type: 'group', + fields: [ + { + name: 'e', + type: 'keyword', + description: 'Some description for e', + }, + { + name: 'd', + type: 'keyword', + }, + ], + }, + ], + }, + ]); + + expect(result).toEqual([ + { + name: 'c.a.d', + type: 'keyword', + description: undefined, + }, + { + name: 'c.a.e', + type: 'keyword', + description: 'Some description for e', + }, + { + name: 'c.b', + type: 'keyword', + description: 'Some description for b', + }, + ]); + }); + + it('Should not error if group type is not an array', async () => { + const result = flattenObjectsList([ + { + name: 'a', + type: 'group', + fields: [ + { + name: 'b', + type: 'keyword', + description: 'Some description for b', + }, + { + name: 'c', + type: 'group', + }, + ], + }, + ]); + + expect(result).toEqual([ + { + name: 'a.b', + type: 'keyword', + description: 'Some description for b', + }, + { + name: 'a.c', + type: 'group', + description: undefined, + }, + ]); + }); +}); describe('merge', () => { it('Should return source if target is empty', async () => { diff --git a/x-pack/plugins/integration_assistant/server/util/samples.ts b/x-pack/plugins/integration_assistant/server/util/samples.ts index 745c3d214095f..afe450f9b0f2a 100644 --- a/x-pack/plugins/integration_assistant/server/util/samples.ts +++ b/x-pack/plugins/integration_assistant/server/util/samples.ts @@ -18,9 +18,10 @@ interface NewObj { }; } -interface Field { +export interface Field { name: string; type: string; + description?: string; fields?: Field[]; } @@ -233,3 +234,43 @@ export function mergeSamples(objects: any[]): string { return JSON.stringify(result, null, 2); } + +export function flattenObjectsList( + obj: Field[] +): Array<{ name: string; type: string; description?: string }> { + const result: Array<{ name: string; type: string; description?: string }> = []; + flattenObject(obj, '', '.', result); + + return sortArrayOfObjects(result); +} + +function flattenObject( + obj: Field[], + parentKey: string = '', + separator: string = '.', + result: Array<{ name: string; type: string; description?: string }> +): void { + obj.forEach((element) => { + if (element.name) { + const newKey = parentKey ? `${parentKey}${separator}${element.name}` : element.name; + + if (element.fields && Array.isArray(element.fields)) { + flattenObject(element.fields, newKey, separator, result); + } else { + result.push({ + name: newKey, + type: element.type, + description: element.description, + }); + } + } + }); +} + +function sortArrayOfObjects( + objectsArray: Array<{ name: string; type: string; description?: string }> +): Array<{ name: string; type: string; description?: string }> { + return objectsArray.sort((a, b) => { + return a.name.localeCompare(b.name); + }); +} From b2f0f998bca7012382c58be85449f11ea55940f7 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Thu, 10 Oct 2024 13:36:00 +0200 Subject: [PATCH 2/3] fix checks --- .../server/integration_builder/build_integration.test.ts | 4 ++-- .../server/integration_builder/data_stream.ts | 4 ++-- .../server/integration_builder/fields.ts | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts index a576f9b58a2a0..e90c079502a1e 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts @@ -244,7 +244,7 @@ describe('renderPackageManifestYAML', () => { }; const manifestContent = renderPackageManifestYAML(integration); // The manifest content must be parseable as YAML. - const manifest = yaml.load(manifestContent) as Record; + const manifest = yaml.safeLoad(manifestContent) as Record; expect(manifest).toBeDefined(); expect(manifest.title).toBe(integration.title); expect(manifest.name).toBe(integration.name); @@ -252,4 +252,4 @@ describe('renderPackageManifestYAML', () => { expect(manifest.description).toBe(integration.description); expect(manifest.icons).toBeTruthy(); }); -}); \ No newline at end of file +}); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts index d66ee1958b3ea..6338474fae0ed 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts @@ -7,7 +7,7 @@ import nunjucks from 'nunjucks'; import { join as joinPath } from 'path'; -import { load } from 'js-yaml'; +import { safeLoad } from 'js-yaml'; import type { DataStream } from '../../common'; import { copySync, createSync, ensureDirSync, listDirSync, readSync } from '../util'; import { Field } from '../util/samples'; @@ -79,7 +79,7 @@ function loadFieldsFromFiles(sourcePath: string, files: string[]): Field[] { return files.flatMap((file) => { const filePath = joinPath(sourcePath, file); const content = readSync(filePath); - return load(content) as Field[]; + return safeLoad(content) as Field[]; }); } diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts b/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts index 476bc0e74d697..1ab64dd636cd1 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/fields.ts @@ -6,7 +6,7 @@ */ import nunjucks from 'nunjucks'; -import { load } from 'js-yaml'; +import { safeLoad } from 'js-yaml'; import { Field } from '../util/samples'; import { createSync, generateFields, mergeSamples } from '../util'; @@ -35,7 +35,7 @@ function createBaseFields( }); createSync(`${dataStreamFieldsDir}/base-fields.yml`, baseFields); - return load(baseFields) as Field[]; + return safeLoad(baseFields) as Field[]; } function createCustomFields(dataStreamFieldsDir: string, pipelineResults: object[]): Field[] { @@ -43,5 +43,5 @@ function createCustomFields(dataStreamFieldsDir: string, pipelineResults: object const fieldKeys = generateFields(mergedResults); createSync(`${dataStreamFieldsDir}/fields.yml`, fieldKeys); - return load(fieldKeys) as Field[]; + return safeLoad(fieldKeys) as Field[]; } From acfe0f0addd54ba23ebe360d69325b0709ef1972 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Thu, 10 Oct 2024 14:52:58 +0200 Subject: [PATCH 3/3] fix types --- .../build_integration.test.ts | 27 +++++++++++++++++++ .../integration_builder/data_stream.test.ts | 27 ++++++++----------- .../server/integration_builder/fields.test.ts | 20 +++++--------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts index e90c079502a1e..925e77308369d 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import { buildPackage, renderPackageManifestYAML } from './build_integration'; import { testIntegration } from '../../__jest__/fixtures/build_integration'; import { generateUniqueId, ensureDirSync, createSync } from '../util'; @@ -17,6 +18,7 @@ import { createReadme } from './readme_files'; const mockedDataPath = 'path'; const mockedId = 123; + jest.mock('../util'); jest.mock('./data_stream'); jest.mock('./fields'); @@ -32,19 +34,24 @@ jest.mock('./readme_files'); jest.mock('@kbn/utils', () => ({ getDataPath: jest.fn(() => mockedDataPath), })); + jest.mock('adm-zip', () => { return jest.fn().mockImplementation(() => ({ addLocalFolder: jest.fn(), toBuffer: jest.fn(), })); }); + describe('buildPackage', () => { const packagePath = `${mockedDataPath}/integration-assistant-${mockedId}`; const integrationPath = `${packagePath}/integration-1.0.0`; + const firstDatastreamName = 'datastream_1'; const secondDatastreamName = 'datastream_2'; + const firstDataStreamInputTypes: InputType[] = ['filestream', 'kafka']; const secondDataStreamInputTypes: InputType[] = ['kafka']; + const firstDataStreamDocs: Docs = [ { key: 'foo', @@ -52,6 +59,7 @@ describe('buildPackage', () => { }, ]; const secondDataStreamDocs: Docs = [{}]; + const firstDataStreamPipeline: Pipeline = { processors: [ { @@ -63,6 +71,7 @@ describe('buildPackage', () => { ], }; const secondDataStreamPipeline: Pipeline = { processors: [] }; + const firstDataStream: DataStream = { name: firstDatastreamName, title: 'Datastream_1', @@ -73,6 +82,7 @@ describe('buildPackage', () => { pipeline: firstDataStreamPipeline, samplesFormat: { name: 'ndjson', multiline: false }, }; + const secondDataStream: DataStream = { name: secondDatastreamName, title: 'Datastream_2', @@ -83,13 +93,17 @@ describe('buildPackage', () => { pipeline: secondDataStreamPipeline, samplesFormat: { name: 'ndjson', multiline: false }, }; + const firstDatastreamPath = `${integrationPath}/data_stream/${firstDatastreamName}`; const secondDatastreamPath = `${integrationPath}/data_stream/${secondDatastreamName}`; + testIntegration.dataStreams = [firstDataStream, secondDataStream]; + beforeEach(async () => { jest.clearAllMocks(); await buildPackage(testIntegration); }); + it('Should create expected directories and files', async () => { // Package & integration folders expect(ensureDirSync).toHaveBeenCalledWith(packagePath); @@ -108,22 +122,29 @@ describe('buildPackage', () => { // Manifest files expect(createSync).toHaveBeenCalledWith(`${integrationPath}/manifest.yml`, expect.any(String)); }); + it('Should create logo files if info is present in the integration', async () => { testIntegration.logo = 'logo'; + await buildPackage(testIntegration); + expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`); expect(createSync).toHaveBeenCalledWith(`${integrationPath}/img/logo.svg`, expect.any(Buffer)); }); + it('Should not create logo files if info is not present in the integration', async () => { jest.clearAllMocks(); testIntegration.logo = undefined; + await buildPackage(testIntegration); + expect(ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`); expect(createSync).not.toHaveBeenCalledWith( `${integrationPath}/img/logo.svg`, expect.any(Buffer) ); }); + it('Should call createDataStream for each datastream', async () => { expect(createDataStream).toHaveBeenCalledWith( 'integration', @@ -136,14 +157,17 @@ describe('buildPackage', () => { secondDataStream ); }); + it('Should call createAgentInput for each datastream', async () => { expect(createAgentInput).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamInputTypes); expect(createAgentInput).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamInputTypes); }); + it('Should call createPipeline for each datastream', async () => { expect(createPipeline).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamPipeline); expect(createPipeline).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamPipeline); }); + it('Should call createFieldMapping for each datastream', async () => { expect(createFieldMapping).toHaveBeenCalledWith( 'integration', @@ -242,9 +266,12 @@ describe('renderPackageManifestYAML', () => { }, ], }; + const manifestContent = renderPackageManifestYAML(integration); + // The manifest content must be parseable as YAML. const manifest = yaml.safeLoad(manifestContent) as Record; + expect(manifest).toBeDefined(); expect(manifest.title).toBe(integration.title); expect(manifest.name).toBe(integration.name); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts index 1a0507d539ac0..0a269fa07a1c8 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts @@ -5,10 +5,10 @@ * 2.0. */ -import * as Utils from '../util'; +import { ensureDirSync, createSync, copySync } from '../util'; import { DataStream, Docs, InputType, Pipeline } from '../../common'; import { createDataStream } from './data_stream'; -import * as nunjucks from 'nunjucks'; +import { render } from 'nunjucks'; jest.mock('nunjucks'); @@ -59,32 +59,27 @@ describe('createDataStream', () => { createDataStream(packageName, dataStreamPath, firstDataStream); // pipeline - expect(Utils.ensureDirSync).toHaveBeenCalledWith(dataStreamPath); - expect(Utils.ensureDirSync).toHaveBeenCalledWith( - `${dataStreamPath}/elasticsearch/ingest_pipeline` - ); + expect(ensureDirSync).toHaveBeenCalledWith(dataStreamPath); + expect(ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/elasticsearch/ingest_pipeline`); // dataStream files - expect(Utils.copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`); + expect(copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`); // test files - expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`); - expect(Utils.copySync).toHaveBeenCalledWith( + expect(ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`); + expect(copySync).toHaveBeenCalledWith( expect.any(String), `${dataStreamPath}/_dev/test/pipeline/test-common-config.yml` ); - expect(Utils.createSync).toHaveBeenCalledWith( + expect(createSync).toHaveBeenCalledWith( `${dataStreamPath}/_dev/test/pipeline/test-${packageName}-datastream-1.log`, samples ); // // Manifest files - expect(Utils.createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined); - expect(nunjucks.render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything()); - expect(nunjucks.render).toHaveBeenCalledWith( - `azure_eventhub_manifest.yml.njk`, - expect.anything() - ); + expect(createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined); + expect(render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything()); + expect(render).toHaveBeenCalledWith(`azure_eventhub_manifest.yml.njk`, expect.anything()); }); it('Should return the list of fields', async () => { diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts index bf849f3b7f2bc..bb76577d64fd3 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts @@ -5,8 +5,8 @@ * 2.0. */ -import * as Utils from '../util'; -import * as nunjucks from 'nunjucks'; +import { createSync } from '../util'; +import { render } from 'nunjucks'; import { createFieldMapping } from './fields'; import { Docs } from '../../common'; @@ -19,7 +19,7 @@ jest.mock('../util', () => ({ const mockedTemplate = 'mocked template'; -(nunjucks.render as jest.Mock).mockReturnValue(mockedTemplate); +(render as jest.Mock).mockReturnValue(mockedTemplate); describe('createFieldMapping', () => { const dataStreamPath = 'path'; @@ -46,14 +46,11 @@ describe('createFieldMapping', () => { type: keyword `; - expect(Utils.createSync).toHaveBeenCalledWith( + expect(createSync).toHaveBeenCalledWith( `${dataStreamPath}/fields/base-fields.yml`, mockedTemplate ); - expect(Utils.createSync).toHaveBeenCalledWith( - `${dataStreamPath}/fields/fields.yml`, - expectedFields - ); + expect(createSync).toHaveBeenCalledWith(`${dataStreamPath}/fields/fields.yml`, expectedFields); }); it('Should create fields files even if docs value is empty', async () => { @@ -62,14 +59,11 @@ describe('createFieldMapping', () => { const expectedFields = `[] `; - expect(Utils.createSync).toHaveBeenCalledWith( + expect(createSync).toHaveBeenCalledWith( `${dataStreamPath}/fields/base-fields.yml`, mockedTemplate ); - expect(Utils.createSync).toHaveBeenCalledWith( - `${dataStreamPath}/fields/fields.yml`, - expectedFields - ); + expect(createSync).toHaveBeenCalledWith(`${dataStreamPath}/fields/fields.yml`, expectedFields); }); it('Should return all fields flattened', async () => {