Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.x] [Automatic Import] add fields mapping to readme (#193717) #195747

Merged
merged 5 commits into from
Oct 10, 2024
Merged

[8.x] [Automatic Import] add fields mapping to readme (#193717) #195747

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1fb9e86
[Automatic Import] add fields mapping to readme (#193717)
haetamoudi Sep 30, 2024
b2f0f99
fix checks
bhapas Oct 10, 2024
acfe0f0
fix types
bhapas Oct 10, 2024
f20bae7
Merge branch '8.x' into backport/8.x/pr-193717
bhapas Oct 10, 2024
1ac0976
Merge branch '8.x' into backport/8.x/pr-193717
bhapas Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 80 additions & 62 deletions x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@
* 2.0.
*/

import * as buildIntegrationModule from './build_integration';
import { buildPackage, renderPackageManifestYAML } from './build_integration';
import { testIntegration } from '../../__jest__/fixtures/build_integration';
import * as Utils from '../util';
import * as DataStreamModule from './data_stream';
import * as FieldsModule from './fields';
import * as AgentModule from './agent';
import * as PipelineModule from './pipeline';
import { generateUniqueId, ensureDirSync, createSync } from '../util';
import { createDataStream } from './data_stream';
import { createFieldMapping } from './fields';
import { createAgentInput } from './agent';
import { createPipeline } from './pipeline';
import { DataStream, Docs, InputType, Pipeline, Integration } from '../../common';
import { renderPackageManifestYAML } from './build_integration';
import yaml from 'js-yaml';
import { createReadme } from './readme_files';

const mockedDataPath = 'path';
const mockedId = 123;
Expand All @@ -24,8 +24,12 @@ jest.mock('./data_stream');
jest.mock('./fields');
jest.mock('./agent');
jest.mock('./pipeline');
jest.mock('./readme_files');

(Utils.generateUniqueId as jest.Mock).mockReturnValue(mockedId);
(createFieldMapping as jest.Mock).mockReturnValue([]);
(createDataStream as jest.Mock).mockReturnValue([]);

(generateUniqueId as jest.Mock).mockReturnValue(mockedId);

jest.mock('@kbn/utils', () => ({
getDataPath: jest.fn(() => mockedDataPath),
Expand Down Expand Up @@ -97,119 +101,133 @@ describe('buildPackage', () => {

beforeEach(async () => {
jest.clearAllMocks();
await buildIntegrationModule.buildPackage(testIntegration);
await buildPackage(testIntegration);
});

it('Should create expected directories and files', async () => {
// Package & integration folders
expect(Utils.ensureDirSync).toHaveBeenCalledWith(packagePath);
expect(Utils.ensureDirSync).toHaveBeenCalledWith(integrationPath);
expect(ensureDirSync).toHaveBeenCalledWith(packagePath);
expect(ensureDirSync).toHaveBeenCalledWith(integrationPath);

// _dev files
expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`);
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/_dev/build/docs/README.md`,
expect.any(String)
);
expect(Utils.createSync).toHaveBeenCalledWith(
expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`);
expect(createSync).toHaveBeenCalledWith(
`${integrationPath}/_dev/build/build.yml`,
expect.any(String)
);

// Docs files
expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`);
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/docs/README.md`,
expect.any(String)
);

// Changelog file
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/changelog.yml`,
expect.any(String)
);
expect(createSync).toHaveBeenCalledWith(`${integrationPath}/changelog.yml`, expect.any(String));

// Manifest files
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/manifest.yml`,
expect.any(String)
);
expect(createSync).toHaveBeenCalledWith(`${integrationPath}/manifest.yml`, expect.any(String));
});

it('Should create logo files if info is present in the integration', async () => {
testIntegration.logo = 'logo';

await buildIntegrationModule.buildPackage(testIntegration);
await buildPackage(testIntegration);

expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`);
expect(Utils.createSync).toHaveBeenCalledWith(
`${integrationPath}/img/logo.svg`,
expect.any(Buffer)
);
expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/img`);
expect(createSync).toHaveBeenCalledWith(`${integrationPath}/img/logo.svg`, expect.any(Buffer));
});

it('Should not create logo files if info is not present in the integration', async () => {
jest.clearAllMocks();
testIntegration.logo = undefined;

await buildIntegrationModule.buildPackage(testIntegration);
await buildPackage(testIntegration);

expect(Utils.ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`);
expect(Utils.createSync).not.toHaveBeenCalledWith(
expect(ensureDirSync).not.toHaveBeenCalledWith(`${integrationPath}/img`);
expect(createSync).not.toHaveBeenCalledWith(
`${integrationPath}/img/logo.svg`,
expect.any(Buffer)
);
});

it('Should call createDataStream for each datastream', async () => {
expect(DataStreamModule.createDataStream).toHaveBeenCalledWith(
expect(createDataStream).toHaveBeenCalledWith(
'integration',
firstDatastreamPath,
firstDataStream
);
expect(DataStreamModule.createDataStream).toHaveBeenCalledWith(
expect(createDataStream).toHaveBeenCalledWith(
'integration',
secondDatastreamPath,
secondDataStream
);
});

it('Should call createAgentInput for each datastream', async () => {
expect(AgentModule.createAgentInput).toHaveBeenCalledWith(
firstDatastreamPath,
firstDataStreamInputTypes
);
expect(AgentModule.createAgentInput).toHaveBeenCalledWith(
secondDatastreamPath,
secondDataStreamInputTypes
);
expect(createAgentInput).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamInputTypes);
expect(createAgentInput).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamInputTypes);
});

it('Should call createPipeline for each datastream', async () => {
expect(PipelineModule.createPipeline).toHaveBeenCalledWith(
firstDatastreamPath,
firstDataStreamPipeline
);
expect(PipelineModule.createPipeline).toHaveBeenCalledWith(
secondDatastreamPath,
secondDataStreamPipeline
);
expect(createPipeline).toHaveBeenCalledWith(firstDatastreamPath, firstDataStreamPipeline);
expect(createPipeline).toHaveBeenCalledWith(secondDatastreamPath, secondDataStreamPipeline);
});

it('Should call createFieldMapping for each datastream', async () => {
expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith(
expect(createFieldMapping).toHaveBeenCalledWith(
'integration',
firstDatastreamName,
firstDatastreamPath,
firstDataStreamDocs
);
expect(FieldsModule.createFieldMapping).toHaveBeenCalledWith(
expect(createFieldMapping).toHaveBeenCalledWith(
'integration',
secondDatastreamName,
secondDatastreamPath,
secondDataStreamDocs
);
});

it('Should call createReadme once with sorted fields', async () => {
jest.clearAllMocks();

const firstDSFieldsMapping = [{ name: 'name a', description: 'description 1', type: 'type 1' }];

const firstDataStreamFields = [
{ name: 'name b', description: 'description 1', type: 'type 1' },
];

const secondDSFieldsMapping = [
{ name: 'name c', description: 'description 2', type: 'type 2' },
{ name: 'name e', description: 'description 3', type: 'type 3' },
];

const secondDataStreamFields = [
{ name: 'name d', description: 'description 2', type: 'type 2' },
];

(createFieldMapping as jest.Mock).mockReturnValueOnce(firstDSFieldsMapping);
(createDataStream as jest.Mock).mockReturnValueOnce(firstDataStreamFields);

(createFieldMapping as jest.Mock).mockReturnValueOnce(secondDSFieldsMapping);
(createDataStream as jest.Mock).mockReturnValueOnce(secondDataStreamFields);

await buildPackage(testIntegration);

expect(createReadme).toHaveBeenCalledWith(integrationPath, testIntegration.name, [
{
datastream: firstDatastreamName,
fields: [
{ name: 'name a', description: 'description 1', type: 'type 1' },

{ name: 'name b', description: 'description 1', type: 'type 1' },
],
},
{
datastream: secondDatastreamName,
fields: [
{ name: 'name c', description: 'description 2', type: 'type 2' },
{ name: 'name d', description: 'description 2', type: 'type 2' },
{ name: 'name e', description: 'description 3', type: 'type 3' },
],
},
]);
});
});

describe('renderPackageManifestYAML', () => {
Expand Down Expand Up @@ -252,7 +270,7 @@ describe('renderPackageManifestYAML', () => {
const manifestContent = renderPackageManifestYAML(integration);

// The manifest content must be parseable as YAML.
const manifest = yaml.safeLoad(manifestContent);
const manifest = yaml.safeLoad(manifestContent) as Record<string, unknown>;

expect(manifest).toBeDefined();
expect(manifest.title).toBe(integration.title);
Expand Down
43 changes: 23 additions & 20 deletions x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import { createAgentInput } from './agent';
import { createDataStream } from './data_stream';
import { createFieldMapping } from './fields';
import { createPipeline } from './pipeline';
import { createReadme } from './readme_files';
import { Field, flattenObjectsList } from '../util/samples';

const initialVersion = '1.0.0';

Expand All @@ -37,17 +39,27 @@ export async function buildPackage(integration: Integration): Promise<Buffer> {
const packageDir = createDirectories(workingDir, integration, packageDirectoryName);

const dataStreamsDir = joinPath(packageDir, 'data_stream');

for (const dataStream of integration.dataStreams) {
const fieldsPerDatastream = integration.dataStreams.map((dataStream) => {
const dataStreamName = dataStream.name;
const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);

createDataStream(integration.name, specificDataStreamDir, dataStream);
const dataStreamFields = createDataStream(integration.name, specificDataStreamDir, dataStream);
createAgentInput(specificDataStreamDir, dataStream.inputTypes);
createPipeline(specificDataStreamDir, dataStream.pipeline);
createFieldMapping(integration.name, dataStreamName, specificDataStreamDir, dataStream.docs);
}
const fields = createFieldMapping(
integration.name,
dataStreamName,
specificDataStreamDir,
dataStream.docs
);

return {
datastream: dataStreamName,
fields: mergeAndSortFields(fields, dataStreamFields),
};
});

createReadme(packageDir, integration.name, fieldsPerDatastream);
const zipBuffer = await createZipArchive(workingDir, packageDirectoryName);

removeDirSync(workingDir);
Expand All @@ -67,7 +79,6 @@ function createDirectories(
}

function createPackage(packageDir: string, integration: Integration): void {
createReadme(packageDir, integration);
createChangelog(packageDir);
createBuildFile(packageDir);
createPackageManifest(packageDir, integration);
Expand Down Expand Up @@ -102,20 +113,6 @@ function createChangelog(packageDir: string): void {
createSync(joinPath(packageDir, 'changelog.yml'), changelogTemplate);
}

function createReadme(packageDir: string, integration: Integration) {
const readmeDirPath = joinPath(packageDir, '_dev/build/docs/');
const mainReadmeDirPath = joinPath(packageDir, 'docs/');
ensureDirSync(mainReadmeDirPath);
ensureDirSync(readmeDirPath);
const readmeTemplate = nunjucks.render('package_readme.md.njk', {
package_name: integration.name,
data_streams: integration.dataStreams,
});

createSync(joinPath(readmeDirPath, 'README.md'), readmeTemplate);
createSync(joinPath(mainReadmeDirPath, 'README.md'), readmeTemplate);
}

async function createZipArchive(workingDir: string, packageDirectoryName: string): Promise<Buffer> {
const tmpPackageDir = joinPath(workingDir, packageDirectoryName);
const zip = new AdmZip();
Expand All @@ -124,6 +121,12 @@ async function createZipArchive(workingDir: string, packageDirectoryName: string
return buffer;
}

function mergeAndSortFields(fields: Field[], dataStreamFields: Field[]): Field[] {
const mergedFields = [...fields, ...dataStreamFields];

return flattenObjectsList(mergedFields);
}

/* eslint-disable @typescript-eslint/naming-convention */
/**
* Creates a package manifest dictionary.
Expand Down
39 changes: 23 additions & 16 deletions x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
* 2.0.
*/

import * as Utils from '../util';
import { ensureDirSync, createSync, copySync } from '../util';
import { DataStream, Docs, InputType, Pipeline } from '../../common';
import { createDataStream } from './data_stream';
import * as nunjucks from 'nunjucks';
import { render } from 'nunjucks';

jest.mock('nunjucks');

Expand Down Expand Up @@ -59,31 +59,38 @@ describe('createDataStream', () => {
createDataStream(packageName, dataStreamPath, firstDataStream);

// pipeline
expect(Utils.ensureDirSync).toHaveBeenCalledWith(dataStreamPath);
expect(Utils.ensureDirSync).toHaveBeenCalledWith(
`${dataStreamPath}/elasticsearch/ingest_pipeline`
);
expect(ensureDirSync).toHaveBeenCalledWith(dataStreamPath);
expect(ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/elasticsearch/ingest_pipeline`);

// dataStream files
expect(Utils.copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`);
expect(copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`);

// test files
expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`);
expect(Utils.copySync).toHaveBeenCalledWith(
expect(ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`);
expect(copySync).toHaveBeenCalledWith(
expect.any(String),
`${dataStreamPath}/_dev/test/pipeline/test-common-config.yml`
);
expect(Utils.createSync).toHaveBeenCalledWith(
expect(createSync).toHaveBeenCalledWith(
`${dataStreamPath}/_dev/test/pipeline/test-${packageName}-datastream-1.log`,
samples
);

// // Manifest files
expect(Utils.createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined);
expect(nunjucks.render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything());
expect(nunjucks.render).toHaveBeenCalledWith(
`azure_eventhub_manifest.yml.njk`,
expect.anything()
);
expect(createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined);
expect(render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything());
expect(render).toHaveBeenCalledWith(`azure_eventhub_manifest.yml.njk`, expect.anything());
});

it('Should return the list of fields', async () => {
const fields = createDataStream(packageName, dataStreamPath, firstDataStream);

expect(Array.isArray(fields)).toBe(true);
fields.forEach((field) => {
expect(field).toMatchObject({
name: expect.any(String),
type: expect.any(String),
});
});
});
});
Loading