Skip to content

Commit

Permalink
[Automatic Import] add fields mapping to readme (#193717)
Browse files Browse the repository at this point in the history 
(cherry picked from commit 5081414)

# Conflicts:
#	x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts
  • Loading branch information
@haetamoudi
haetamoudi committed Oct 10, 2024
1 parent cbef50e commit 1fb9e86
Show file tree
Hide file tree
Showing 13 changed files with 664 additions and 161 deletions.
171 changes: 81 additions & 90 deletions x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,58 +4,54 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import * as buildIntegrationModule from './build_integration';
import { buildPackage, renderPackageManifestYAML } from './build_integration';
import { testIntegration } from '../../__jest__/fixtures/build_integration';
import * as Utils from '../util';
import * as DataStreamModule from './data_stream';
import * as FieldsModule from './fields';
import * as AgentModule from './agent';
import * as PipelineModule from './pipeline';
import { generateUniqueId, ensureDirSync, createSync } from '../util';
import { createDataStream } from './data_stream';
import { createFieldMapping } from './fields';
import { createAgentInput } from './agent';
import { createPipeline } from './pipeline';
import { DataStream, Docs, InputType, Pipeline, Integration } from '../../common';
import { renderPackageManifestYAML } from './build_integration';
import yaml from 'js-yaml';
import { createReadme } from './readme_files';

const mockedDataPath = 'path';
const mockedId = 123;

jest.mock('../util');
jest.mock('./data_stream');
jest.mock('./fields');
jest.mock('./agent');
jest.mock('./pipeline');
jest.mock('./readme_files');

(createFieldMapping as jest.Mock).mockReturnValue([]);
(createDataStream as jest.Mock).mockReturnValue([]);

(Utils.generateUniqueId as jest.Mock).mockReturnValue(mockedId);
(generateUniqueId as jest.Mock).mockReturnValue(mockedId);

jest.mock('@kbn/utils', () => ({
getDataPath: jest.fn(() => mockedDataPath),
}));

jest.mock('adm-zip', () => {
return jest.fn().mockImplementation(() => ({
addLocalFolder: jest.fn(),
toBuffer: jest.fn(),
}));
});

describe('buildPackage', () => {
const packagePath = `${mockedDataPath}/integration-assistant-${mockedId}`;
const integrationPath = `${packagePath}/integration-1.0.0`;

const firstDatastreamName = 'datastream_1';
const secondDatastreamName = 'datastream_2';

const firstDataStreamInputTypes: InputType[] = ['filestream', 'kafka'];
const secondDataStreamInputTypes: InputType[] = ['kafka'];

const firstDataStreamDocs: Docs = [
{
key: 'foo',
anotherKey: 'bar',
},
];
const secondDataStreamDocs: Docs = [{}];

const firstDataStreamPipeline: Pipeline = {
processors: [
{
Expand All @@ -67,7 +63,6 @@ describe('buildPackage', () => {
],
};
const secondDataStreamPipeline: Pipeline = { processors: [] };

const firstDataStream: DataStream = {
name: firstDatastreamName,
title: 'Datastream_1',
Expand All @@ -78,7 +73,6 @@ describe('buildPackage', () => {
pipeline: firstDataStreamPipeline,
samplesFormat: { name: 'ndjson', multiline: false },
};

const secondDataStream: DataStream = {
name: secondDatastreamName,
title: 'Datastream_2',
Expand All @@ -89,127 +83,127 @@ describe('buildPackage', () => {
pipeline: secondDataStreamPipeline,
samplesFormat: { name: 'ndjson', multiline: false },
};

const firstDatastreamPath = `${integrationPath}/data_stream/${firstDatastreamName}`;
const secondDatastreamPath = `${integrationPath}/data_stream/${secondDatastreamName}`;

testIntegration.dataStreams = [firstDataStream, secondDataStream];

beforeEach(async () => {
jest.clearAllMocks();
await buildIntegrationModule.buildPackage(testIntegration);
await buildPackage(testIntegration);
});

it('Should create expected directories and files', async () => {
// Package & integration folders
expect(Utils.ensureDirSync).toHaveBeenCalledWith(packagePath);
expect(Utils.ensureDirSync).toHaveBeenCalledWith(integrationPath);
expect(ensureDirSync).toHaveBeenCalledWith(packagePath);
expect(ensureDirSync).toHaveBeenCalledWith(integrationPath);

// _dev files
expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`);
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/_dev/build/docs/README.md`,
expect.any(String)
);
expect(Utils.createSync).toHaveBeenCalledWith(
expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`);
expect(createSync).toHaveBeenCalledWith(
`${integrationPath}/_dev/build/build.yml`,
expect.any(String)
);

// Docs files
expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`);
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/docs/README.md`,
expect.any(String)
);

// Changelog file
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/changelog.yml`,
expect.any(String)
);
expect(createSync).toHaveBeenCalledWith(`${integrationPath}/changelog.yml`, expect.any(String));

// Manifest files
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/manifest.yml`,
expect.any(String)
);
expect(createSync).toHaveBeenCalledWith(`${integrationPath}/manifest.yml`, expect.any(String));
});

it('Should create logo files if info is present in the integration', async () => {
testIntegration.logo = 'logo';

await buildIntegrationModule.buildPackage(testIntegration);

expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`);
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/img/logo.svg`,
expect.any(Buffer)
);
await buildPackage(testIntegration);
expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`);
expect(createSync).toHaveBeenCalledWith(`${integrationPath}/img/logo.svg`, expect.any(Buffer));
});

it('Should not create logo files if info is not present in the integration', async () => {
jest.clearAllMocks();
testIntegration.logo = undefined;

await buildIntegrationModule.buildPackage(testIntegration);

expect(Utils.ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`);
expect(Utils.createSync).not.toHaveBeenCalledWith(
await buildPackage(testIntegration);
expect(ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`);
expect(createSync).not.toHaveBeenCalledWith(
`${integrationPath}/img/logo.svg`,
expect.any(Buffer)
);
});

it('Should call createDataStream for each datastream', async () => {
expect(DataStreamModule.createDataStream).toHaveBeenCalledWith(
expect(createDataStream).toHaveBeenCalledWith(
'integration',
firstDatastreamPath,
firstDataStream
);
expect(DataStreamModule.createDataStream).toHaveBeenCalledWith(
expect(createDataStream).toHaveBeenCalledWith(
'integration',
secondDatastreamPath,
secondDataStream
);
});

it('Should call createAgentInput for each datastream', async () => {
expect(AgentModule.createAgentInput).toHaveBeenCalledWith(
firstDatastreamPath,
firstDataStreamInputTypes
);
expect(AgentModule.createAgentInput).toHaveBeenCalledWith(
secondDatastreamPath,
secondDataStreamInputTypes
);
expect(createAgentInput).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamInputTypes);
expect(createAgentInput).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamInputTypes);
});

it('Should call createPipeline for each datastream', async () => {
expect(PipelineModule.createPipeline).toHaveBeenCalledWith(
firstDatastreamPath,
firstDataStreamPipeline
);
expect(PipelineModule.createPipeline).toHaveBeenCalledWith(
secondDatastreamPath,
secondDataStreamPipeline
);
expect(createPipeline).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamPipeline);
expect(createPipeline).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamPipeline);
});

it('Should call createFieldMapping for each datastream', async () => {
expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith(
expect(createFieldMapping).toHaveBeenCalledWith(
'integration',
firstDatastreamName,
firstDatastreamPath,
firstDataStreamDocs
);
expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith(
expect(createFieldMapping).toHaveBeenCalledWith(
'integration',
secondDatastreamName,
secondDatastreamPath,
secondDataStreamDocs
);
});

it('Should call createReadme once with sorted fields', async () => {
jest.clearAllMocks();

const firstDSFieldsMapping = [{ name: 'name a', description: 'description 1', type: 'type 1' }];

const firstDataStreamFields = [
{ name: 'name b', description: 'description 1', type: 'type 1' },
];

const secondDSFieldsMapping = [
{ name: 'name c', description: 'description 2', type: 'type 2' },
{ name: 'name e', description: 'description 3', type: 'type 3' },
];

const secondDataStreamFields = [
{ name: 'name d', description: 'description 2', type: 'type 2' },
];

(createFieldMapping as jest.Mock).mockReturnValueOnce(firstDSFieldsMapping);
(createDataStream as jest.Mock).mockReturnValueOnce(firstDataStreamFields);

(createFieldMapping as jest.Mock).mockReturnValueOnce(secondDSFieldsMapping);
(createDataStream as jest.Mock).mockReturnValueOnce(secondDataStreamFields);

await buildPackage(testIntegration);

expect(createReadme).toHaveBeenCalledWith(integrationPath, testIntegration.name, [
{
datastream: firstDatastreamName,
fields: [
{ name: 'name a', description: 'description 1', type: 'type 1' },

{ name: 'name b', description: 'description 1', type: 'type 1' },
],
},
{
datastream: secondDatastreamName,
fields: [
{ name: 'name c', description: 'description 2', type: 'type 2' },
{ name: 'name d', description: 'description 2', type: 'type 2' },
{ name: 'name e', description: 'description 3', type: 'type 3' },
],
},
]);
});
});

describe('renderPackageManifestYAML', () => {
Expand Down Expand Up @@ -248,17 +242,14 @@ describe('renderPackageManifestYAML', () => {
},
],
};

const manifestContent = renderPackageManifestYAML(integration);

// The manifest content must be parseable as YAML.
const manifest = yaml.safeLoad(manifestContent);

const manifest = yaml.load(manifestContent) as Record<string, unknown>;
expect(manifest).toBeDefined();
expect(manifest.title).toBe(integration.title);
expect(manifest.name).toBe(integration.name);
expect(manifest.type).toBe('integration');
expect(manifest.description).toBe(integration.description);
expect(manifest.icons).toBeTruthy();
});
});
});
43 changes: 23 additions & 20 deletions x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import { createAgentInput } from './agent';
import { createDataStream } from './data_stream';
import { createFieldMapping } from './fields';
import { createPipeline } from './pipeline';
import { createReadme } from './readme_files';
import { Field, flattenObjectsList } from '../util/samples';

const initialVersion = '1.0.0';

Expand All @@ -37,17 +39,27 @@ export async function buildPackage(integration: Integration): Promise<Buffer> {
const packageDir = createDirectories(workingDir, integration, packageDirectoryName);

const dataStreamsDir = joinPath(packageDir, 'data_stream');

for (const dataStream of integration.dataStreams) {
const fieldsPerDatastream = integration.dataStreams.map((dataStream) => {
const dataStreamName = dataStream.name;
const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);

createDataStream(integration.name, specificDataStreamDir, dataStream);
const dataStreamFields = createDataStream(integration.name, specificDataStreamDir, dataStream);
createAgentInput(specificDataStreamDir, dataStream.inputTypes);
createPipeline(specificDataStreamDir, dataStream.pipeline);
createFieldMapping(integration.name, dataStreamName, specificDataStreamDir, dataStream.docs);
}
const fields = createFieldMapping(
integration.name,
dataStreamName,
specificDataStreamDir,
dataStream.docs
);

return {
datastream: dataStreamName,
fields: mergeAndSortFields(fields, dataStreamFields),
};
});

createReadme(packageDir, integration.name, fieldsPerDatastream);
const zipBuffer = await createZipArchive(workingDir, packageDirectoryName);

removeDirSync(workingDir);
Expand All @@ -67,7 +79,6 @@ function createDirectories(
}

function createPackage(packageDir: string, integration: Integration): void {
createReadme(packageDir, integration);
createChangelog(packageDir);
createBuildFile(packageDir);
createPackageManifest(packageDir, integration);
Expand Down Expand Up @@ -102,20 +113,6 @@ function createChangelog(packageDir: string): void {
createSync(joinPath(packageDir, 'changelog.yml'), changelogTemplate);
}

function createReadme(packageDir: string, integration: Integration) {
const readmeDirPath = joinPath(packageDir, '_dev/build/docs/');
const mainReadmeDirPath = joinPath(packageDir, 'docs/');
ensureDirSync(mainReadmeDirPath);
ensureDirSync(readmeDirPath);
const readmeTemplate = nunjucks.render('package_readme.md.njk', {
package_name: integration.name,
data_streams: integration.dataStreams,
});

createSync(joinPath(readmeDirPath, 'README.md'), readmeTemplate);
createSync(joinPath(mainReadmeDirPath, 'README.md'), readmeTemplate);
}

async function createZipArchive(workingDir: string, packageDirectoryName: string): Promise<Buffer> {
const tmpPackageDir = joinPath(workingDir, packageDirectoryName);
const zip = new AdmZip();
Expand All @@ -124,6 +121,12 @@ async function createZipArchive(workingDir: string, packageDirectoryName: string
return buffer;
}

function mergeAndSortFields(fields: Field[], dataStreamFields: Field[]): Field[] {
const mergedFields = [...fields, ...dataStreamFields];

return flattenObjectsList(mergedFields);
}

/* eslint-disable @typescript-eslint/naming-convention */
/**
* Creates a package manifest dictionary.
Expand Down
Loading

0 comments on commit 1fb9e86

Please sign in to comment.