From a0017325e410850c8e4d2ac796165debde621b64 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Thu, 5 Dec 2024 19:18:52 +0100 Subject: [PATCH 1/4] Add datastream description to readme --- .../integration_builder/build_integration.ts | 2 +- .../integration_builder/readme_files.ts | 31 ++++++++++++++----- .../server/templates/build_readme.md.njk | 2 +- .../server/templates/description_readme.njk | 7 +++-- .../server/templates/package_readme.md.njk | 2 +- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts index bf2e9b6b9d5a7..785c11125afd9 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts @@ -59,7 +59,7 @@ export async function buildPackage(integration: Integration): Promise { }; }); - createReadme(packageDir, integration.name, fieldsPerDatastream); + createReadme(packageDir, integration.name, integration.dataStreams, fieldsPerDatastream); const zipBuffer = await createZipArchive(workingDir, packageDirectoryName); removeDirSync(workingDir); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index 5467a1549cea2..2c7ad0f3d99ec 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -8,29 +8,46 @@ import { Environment, FileSystemLoader } from 'nunjucks'; import { join as joinPath } from 'path'; +import { DataStream } from '../../common'; import { createSync, ensureDirSync } from '../util'; -export function createReadme(packageDir: string, integrationName: string, fields: object[]) { - createPackageReadme(packageDir, integrationName, fields); - createBuildReadme(packageDir, integrationName, fields); +export function createReadme( + packageDir: string, + integrationName: string, + datastreams: DataStream[], + fields: object[] +) { + createPackageReadme(packageDir, integrationName, datastreams, fields); + createBuildReadme(packageDir, integrationName, datastreams, fields); } -function createPackageReadme(packageDir: string, integrationName: string, fields: object[]) { +function createPackageReadme( + packageDir: string, + integrationName: string, + datastreams: DataStream[], + fields: object[] +) { const dirPath = joinPath(packageDir, 'docs/'); // The readme nunjucks template files should be named in the format `somename_readme.md.njk` and not just `readme.md.njk` // since any file with `readme.*` pattern is skipped in build process in buildkite. - createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, fields); + createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, datastreams, fields); } -function createBuildReadme(packageDir: string, integrationName: string, fields: object[]) { +function createBuildReadme( + packageDir: string, + integrationName: string, + datastreams: DataStream[], + fields: object[] +) { const dirPath = joinPath(packageDir, '_dev/build/docs/'); - createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, fields); + createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, datastreams, fields); } function createReadmeFile( targetDir: string, templateName: string, integrationName: string, + datastreams: DataStream[], fields: object[] ) { ensureDirSync(targetDir); diff --git a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk index 1b58e55aebd37..1633eaf4cc3b4 100644 --- a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk @@ -2,7 +2,7 @@ {% for data_stream in fields %} ### {{ data_stream.datastream }} -Insert a description of the datastream here. +{{ data_stream.description }} {% raw %}{{fields {% endraw %}"{{ data_stream.datastream }}"{% raw %}}}{% endraw %} {% endfor %} \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/templates/description_readme.njk b/x-pack/plugins/integration_assistant/server/templates/description_readme.njk index 91c1bf6f1b40c..fe775ab8c97fa 100644 --- a/x-pack/plugins/integration_assistant/server/templates/description_readme.njk +++ b/x-pack/plugins/integration_assistant/server/templates/description_readme.njk @@ -7,8 +7,11 @@ Check the [overview guidelines](https://www.elastic.co/guide/en/integrations-dev ## Datastreams -Provide a high-level overview of the kind of data that is collected by the integration. -Check the [datastreams guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-datastreams) for more information. +{% for data_stream in fields %} +### {{ data_stream.datastream }} + +{{ data_stream.description }} +{% endfor %} ## Requirements diff --git a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk index bd56aba5ac1e5..732ddfeb021a4 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk @@ -2,7 +2,7 @@ {% for data_stream in fields %} ### {{ data_stream.datastream }} -Insert a description of the datastream here. +{{ data_stream.description }} **ECS Field Reference** From d7bec4734fe2020b4b8f9c615a2ba07d4c6ea318 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Fri, 6 Dec 2024 13:52:38 +0100 Subject: [PATCH 2/4] fix readme to use ds description --- .../server/integration_builder/readme_files.ts | 1 + .../server/templates/build_readme.md.njk | 6 +++--- .../server/templates/description_readme.njk | 6 +++--- .../server/templates/package_readme.md.njk | 6 +++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts index 2c7ad0f3d99ec..9b2b38c171d0d 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts @@ -61,6 +61,7 @@ function createReadmeFile( const renderedTemplate = template.render({ package_name: integrationName, + datastreams, fields, }); diff --git a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk index 1633eaf4cc3b4..40b8c4a2d2f61 100644 --- a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk @@ -1,8 +1,8 @@ {% include "./description_readme.njk" %} -{% for data_stream in fields %} -### {{ data_stream.datastream }} +{% for datastream in datastreams %} +### {{ datastream.title }} -{{ data_stream.description }} +{{ datastream.description }} {% raw %}{{fields {% endraw %}"{{ data_stream.datastream }}"{% raw %}}}{% endraw %} {% endfor %} \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/templates/description_readme.njk b/x-pack/plugins/integration_assistant/server/templates/description_readme.njk index fe775ab8c97fa..9fe32dccb2b2b 100644 --- a/x-pack/plugins/integration_assistant/server/templates/description_readme.njk +++ b/x-pack/plugins/integration_assistant/server/templates/description_readme.njk @@ -7,10 +7,10 @@ Check the [overview guidelines](https://www.elastic.co/guide/en/integrations-dev ## Datastreams -{% for data_stream in fields %} -### {{ data_stream.datastream }} +{% for datastream in datastreams %} +### {{ datastream.title }} -{{ data_stream.description }} +{{ datastream.description }} {% endfor %} ## Requirements diff --git a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk index 732ddfeb021a4..d0a7d4ba81e87 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk @@ -1,8 +1,8 @@ {% include "./description_readme.njk" %} -{% for data_stream in fields %} -### {{ data_stream.datastream }} +{% for datastream in datastreams %} +### {{ datastream.title }} -{{ data_stream.description }} +{{ datastream.description }} **ECS Field Reference** From 2b1ef957e1c1b3ef16bbdb77f82d1f15d604ce24 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 9 Dec 2024 13:42:04 +0100 Subject: [PATCH 3/4] fix readme templates --- .../server/integration_builder/readme_files.test.ts | 8 ++++---- .../server/templates/build_readme.md.njk | 4 ++-- .../server/templates/package_readme.md.njk | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts index ae9080fff8a74..2cb350458378e 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts @@ -59,7 +59,7 @@ describe('createReadme', () => { }, ]; - createReadme(integrationPath, testIntegration.name, fields); + createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields); expect(createSync).toHaveBeenCalledWith( `${integrationPath}/_dev/build/docs/README.md`, @@ -103,7 +103,7 @@ describe('createReadme', () => { }, ]; - createReadme(integrationPath, testIntegration.name, fields); + createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields); const firstDatastreamFieldsDisplayed = ` | Field | Description | Type | @@ -139,7 +139,7 @@ describe('createReadme', () => { }, ]; - createReadme(integrationPath, testIntegration.name, fields); + createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields); expect(createSync).toHaveBeenCalledWith( `${integrationPath}/_dev/build/docs/README.md`, @@ -165,7 +165,7 @@ describe('createReadme', () => { }, ]; - createReadme(integrationPath, testIntegration.name, fields); + createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields); const firstDatastreamFieldsDisplayed = ` | Field | Description | Type | diff --git a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk index 40b8c4a2d2f61..92b03575b10fb 100644 --- a/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk @@ -2,7 +2,7 @@ {% for datastream in datastreams %} ### {{ datastream.title }} -{{ datastream.description }} - +{{ datastream.description }}{% endfor %} +{% for data_stream in fields %} {% raw %}{{fields {% endraw %}"{{ data_stream.datastream }}"{% raw %}}}{% endraw %} {% endfor %} \ No newline at end of file diff --git a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk index d0a7d4ba81e87..3131822e3f2ee 100644 --- a/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk +++ b/x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk @@ -2,14 +2,14 @@ {% for datastream in datastreams %} ### {{ datastream.title }} -{{ datastream.description }} +{{ datastream.description }}{% endfor %} **ECS Field Reference** Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. **Exported fields** - +{% for data_stream in fields %} | Field | Description | Type | |---|---|---| {% for field in data_stream.fields %}| {{ field.name }} | {{ field.description | default('') | replace('\n', ' ') | trim }} | {{ field.type }} | From 592ce8e087c6e736f44ac23ed1e5333be9907295 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 9 Dec 2024 15:28:20 +0100 Subject: [PATCH 4/4] fix test --- .../build_integration.test.ts | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts index d7ce89e2e8f63..eaa50e87d41b4 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts @@ -209,24 +209,29 @@ describe('buildPackage', () => { await buildPackage(testIntegration); - expect(createReadme).toHaveBeenCalledWith(integrationPath, testIntegration.name, [ - { - datastream: firstDatastreamName, - fields: [ - { name: 'name a', description: 'description 1', type: 'type 1' }, + expect(createReadme).toHaveBeenCalledWith( + integrationPath, + testIntegration.name, + testIntegration.dataStreams, + [ + { + datastream: firstDatastreamName, + fields: [ + { name: 'name a', description: 'description 1', type: 'type 1' }, - { name: 'name b', description: 'description 1', type: 'type 1' }, - ], - }, - { - datastream: secondDatastreamName, - fields: [ - { name: 'name c', description: 'description 2', type: 'type 2' }, - { name: 'name d', description: 'description 2', type: 'type 2' }, - { name: 'name e', description: 'description 3', type: 'type 3' }, - ], - }, - ]); + { name: 'name b', description: 'description 1', type: 'type 1' }, + ], + }, + { + datastream: secondDatastreamName, + fields: [ + { name: 'name c', description: 'description 2', type: 'type 2' }, + { name: 'name d', description: 'description 2', type: 'type 2' }, + { name: 'name e', description: 'description 3', type: 'type 3' }, + ], + }, + ] + ); }); });