[8.x] [Automatic Import] Use datastream description in readme (#203236)…

… (#203516)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Automatic Import] Use datastream description in readme
(#203236)](#203236)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Bharat
Pasupula","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-12-10T06:32:22Z","message":"[Automatic
Import] Use datastream description in readme (#203236)\n\n## Release
note\r\n\r\nUse provided data stream description in generated
README\r\n\r\n## Summary\r\n\r\nCloses
[#201294](https://github.com/elastic/kibana/issues/201294)\r\n\r\nThis
PR fixes use of `datastream description` that the user entered in\r\nthe
input
field.\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/b093c978-2102-4bdc-824e-d60aa3fd908b)","sha":"d3161abc38f47284e5aaa41590ef89f8b78f28f3","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","v9.0.0","backport:prev-minor","Team:Security-Scalability","backport:version","Feature:AutomaticImport"],"title":"[Automatic
Import] Use datastream description in
readme","number":203236,"url":"https://github.com/elastic/kibana/pull/203236","mergeCommit":{"message":"[Automatic
Import] Use datastream description in readme (#203236)\n\n## Release
note\r\n\r\nUse provided data stream description in generated
README\r\n\r\n## Summary\r\n\r\nCloses
[#201294](https://github.com/elastic/kibana/issues/201294)\r\n\r\nThis
PR fixes use of `datastream description` that the user entered in\r\nthe
input
field.\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/b093c978-2102-4bdc-824e-d60aa3fd908b)","sha":"d3161abc38f47284e5aaa41590ef89f8b78f28f3"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/203236","number":203236,"mergeCommit":{"message":"[Automatic
Import] Use datastream description in readme (#203236)\n\n## Release
note\r\n\r\nUse provided data stream description in generated
README\r\n\r\n## Summary\r\n\r\nCloses
[#201294](https://github.com/elastic/kibana/issues/201294)\r\n\r\nThis
PR fixes use of `datastream description` that the user entered in\r\nthe
input
field.\r\n\r\n\r\n![image](https://github.com/user-attachments/assets/b093c978-2102-4bdc-824e-d60aa3fd908b)","sha":"d3161abc38f47284e5aaa41590ef89f8b78f28f3"}}]}]
BACKPORT-->

Co-authored-by: Bharat Pasupula <[email protected]>

x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts

@@ -209,24 +209,29 @@ describe('buildPackage', () => {
 
     await buildPackage(testIntegration);
 
-    expect(createReadme).toHaveBeenCalledWith(integrationPath, testIntegration.name, [
-      {
-        datastream: firstDatastreamName,
-        fields: [
-          { name: 'name a', description: 'description 1', type: 'type 1' },
+    expect(createReadme).toHaveBeenCalledWith(
+      integrationPath,
+      testIntegration.name,
+      testIntegration.dataStreams,
+      [
+        {
+          datastream: firstDatastreamName,
+          fields: [
+            { name: 'name a', description: 'description 1', type: 'type 1' },
 
-          { name: 'name b', description: 'description 1', type: 'type 1' },
-        ],
-      },
-      {
-        datastream: secondDatastreamName,
-        fields: [
-          { name: 'name c', description: 'description 2', type: 'type 2' },
-          { name: 'name d', description: 'description 2', type: 'type 2' },
-          { name: 'name e', description: 'description 3', type: 'type 3' },
-        ],
-      },
-    ]);
+            { name: 'name b', description: 'description 1', type: 'type 1' },
+          ],
+        },
+        {
+          datastream: secondDatastreamName,
+          fields: [
+            { name: 'name c', description: 'description 2', type: 'type 2' },
+            { name: 'name d', description: 'description 2', type: 'type 2' },
+            { name: 'name e', description: 'description 3', type: 'type 3' },
+          ],
+        },
+      ]
+    );
   });
 });
 

x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts

@@ -59,7 +59,7 @@ export async function buildPackage(integration: Integration): Promise<Buffer> {
     };
   });
 
-  createReadme(packageDir, integration.name, fieldsPerDatastream);
+  createReadme(packageDir, integration.name, integration.dataStreams, fieldsPerDatastream);
   const zipBuffer = await createZipArchive(workingDir, packageDirectoryName);
 
   removeDirSync(workingDir);

x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts

@@ -59,7 +59,7 @@ describe('createReadme', () => {
       },
     ];
 
-    createReadme(integrationPath, testIntegration.name, fields);
+    createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields);
 
     expect(createSync).toHaveBeenCalledWith(
       `${integrationPath}/_dev/build/docs/README.md`,
@@ -103,7 +103,7 @@ describe('createReadme', () => {
       },
     ];
 
-    createReadme(integrationPath, testIntegration.name, fields);
+    createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields);
 
     const firstDatastreamFieldsDisplayed = `
 | Field | Description | Type |
@@ -139,7 +139,7 @@ describe('createReadme', () => {
       },
     ];
 
-    createReadme(integrationPath, testIntegration.name, fields);
+    createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields);
 
     expect(createSync).toHaveBeenCalledWith(
       `${integrationPath}/_dev/build/docs/README.md`,
@@ -165,7 +165,7 @@ describe('createReadme', () => {
       },
     ];
 
-    createReadme(integrationPath, testIntegration.name, fields);
+    createReadme(integrationPath, testIntegration.name, testIntegration.dataStreams, fields);
 
     const firstDatastreamFieldsDisplayed = `
 | Field | Description | Type |

x-pack/plugins/integration_assistant/server/integration_builder/readme_files.ts

@@ -8,29 +8,46 @@
 import { Environment, FileSystemLoader } from 'nunjucks';
 
 import { join as joinPath } from 'path';
+import { DataStream } from '../../common';
 import { createSync, ensureDirSync } from '../util';
 
-export function createReadme(packageDir: string, integrationName: string, fields: object[]) {
-  createPackageReadme(packageDir, integrationName, fields);
-  createBuildReadme(packageDir, integrationName, fields);
+export function createReadme(
+  packageDir: string,
+  integrationName: string,
+  datastreams: DataStream[],
+  fields: object[]
+) {
+  createPackageReadme(packageDir, integrationName, datastreams, fields);
+  createBuildReadme(packageDir, integrationName, datastreams, fields);
 }
 
-function createPackageReadme(packageDir: string, integrationName: string, fields: object[]) {
+function createPackageReadme(
+  packageDir: string,
+  integrationName: string,
+  datastreams: DataStream[],
+  fields: object[]
+) {
   const dirPath = joinPath(packageDir, 'docs/');
   // The readme nunjucks template files should be named in the format `somename_readme.md.njk` and not just `readme.md.njk`
   // since any file with `readme.*` pattern is skipped in build process in buildkite.
-  createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, fields);
+  createReadmeFile(dirPath, 'package_readme.md.njk', integrationName, datastreams, fields);
 }
 
-function createBuildReadme(packageDir: string, integrationName: string, fields: object[]) {
+function createBuildReadme(
+  packageDir: string,
+  integrationName: string,
+  datastreams: DataStream[],
+  fields: object[]
+) {
   const dirPath = joinPath(packageDir, '_dev/build/docs/');
-  createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, fields);
+  createReadmeFile(dirPath, 'build_readme.md.njk', integrationName, datastreams, fields);
 }
 
 function createReadmeFile(
   targetDir: string,
   templateName: string,
   integrationName: string,
+  datastreams: DataStream[],
   fields: object[]
 ) {
   ensureDirSync(targetDir);
@@ -43,6 +60,7 @@ function createReadmeFile(
   const template = env.getTemplate(templateName);
   const renderedTemplate = template.render({
     package_name: integrationName,
+    datastreams,
     fields,
   });
 

x-pack/plugins/integration_assistant/server/templates/build_readme.md.njk

@@ -1,8 +1,8 @@
 {% include "./description_readme.njk" %}
-{% for data_stream in fields %}
-### {{ data_stream.datastream }}
-
-Insert a description of the datastream here.
+{% for datastream in datastreams %}
+### {{ datastream.title }}
 
+{{ datastream.description }}{% endfor %}
+{% for data_stream in fields %}
 {% raw %}{{fields {% endraw %}"{{ data_stream.datastream }}"{% raw %}}}{% endraw %}
 {% endfor %}

x-pack/plugins/integration_assistant/server/templates/description_readme.njk

@@ -7,8 +7,11 @@ Check the [overview guidelines](https://www.elastic.co/guide/en/integrations-dev
 
 ## Datastreams
 
-Provide a high-level overview of the kind of data that is collected by the integration. 
-Check the [datastreams guidelines](https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html#idg-docs-guidelines-datastreams) for more information.
+{% for datastream in datastreams %}
+### {{ datastream.title }}
+
+{{ datastream.description }}
+{% endfor %}
 
 ## Requirements
 

x-pack/plugins/integration_assistant/server/templates/package_readme.md.njk

@@ -1,15 +1,15 @@
 {% include "./description_readme.njk" %}
-{% for data_stream in fields %}
-### {{ data_stream.datastream }}
+{% for datastream in datastreams %}
+### {{ datastream.title }}
 
-Insert a description of the datastream here.
+{{ datastream.description }}{% endfor %}
 
 **ECS Field Reference**
 
 Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
 
 **Exported fields**
-
+{% for data_stream in fields %}
 | Field | Description | Type |
 |---|---|---|
 {% for field in data_stream.fields %}| {{ field.name }} | {{ field.description | default('') | replace('\n', ' ') | trim }} | {{ field.type }} |