More fixes (#956)

* Generate mkdocs with mermaid plugin

* Use mermaid format by default

* MD warnings

* warning in tabbed markdown

* Avoid deprecation for mkdocs emojis plugin

* Fix table

* refacto + font color black

* link to mermaid.live

* Replace images in markdown

* jscpd

defaults/mkdocs-project-doc/mkdocs.yml

@@ -24,7 +24,11 @@ markdown_extensions:
       check_paths: true
   - mdx_truly_sane_lists
   - attr_list
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - pymdownx.tabbed:
       alternate_style: true
 extra_javascript:

defaults/mkdocs/mkdocs.yml

@@ -28,7 +28,11 @@ markdown_extensions:
       check_paths: true
   - mdx_truly_sane_lists
   - attr_list
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - pymdownx.tabbed:
       alternate_style: true
 extra_javascript:

mkdocs.yml

@@ -31,7 +31,11 @@ markdown_extensions:
       check_paths: true
   - mdx_truly_sane_lists
   - attr_list
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - pymdownx.tabbed:
       alternate_style: true
 extra_javascript:

src/commands/hardis/doc/plugin/generate.ts

@@ -11,6 +11,7 @@ import * as yaml from 'js-yaml';
 import { uxLog } from '../../../../common/utils/index.js';
 import { PACKAGE_ROOT_DIR } from '../../../../settings.js';
 import { Config } from '@oclif/core';
+import { readMkDocsFile, writeMkDocsFile } from '../../../../common/utils/docUtils.js';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
 const messages = Messages.loadMessages('sfdx-hardis', 'org');
@@ -111,24 +112,14 @@ At each merge into master/main branch, the GitHub Action build-deploy-docs will
     }
 
     // Update mkdocs nav items
-    const mkdocsYml: any = yaml.load(
-      fs
-        .readFileSync(mkdocsYmlFile, 'utf-8')
-        .replace('!!python/name:materialx.emoji.twemoji', "'!!python/name:materialx.emoji.twemoji'")
-        .replace('!!python/name:materialx.emoji.to_svg', "'!!python/name:materialx.emoji.to_svg'")
-    );
+    const mkdocsYml: any = readMkDocsFile(mkdocsYmlFile);
     mkdocsYml.nav = mkdocsYml.nav.map((navItem: any) => {
       if (navItem['Commands']) {
         navItem['Commands'] = commandsNav;
       }
       return navItem;
     });
-    const mkdocsYmlStr = yaml
-      .dump(mkdocsYml)
-      .replace("'!!python/name:materialx.emoji.twemoji'", '!!python/name:materialx.emoji.twemoji')
-      .replace("'!!python/name:materialx.emoji.to_svg'", '!!python/name:materialx.emoji.to_svg');
-    await fs.writeFile(mkdocsYmlFile, mkdocsYmlStr);
-    uxLog(this, c.cyan(`Updated ${c.green(mkdocsYmlFile)}`));
+    await writeMkDocsFile(mkdocsYmlFile, mkdocsYml);
 
     // Return an object to be displayed with --json
     return { outputString: `Generated documentation` };

src/commands/hardis/doc/project2markdown.ts

@@ -7,7 +7,7 @@ import sortArray from 'sort-array';
 import { Messages } from '@salesforce/core';
 import { AnyJson } from '@salesforce/ts-types';
 import { WebSocketClient } from '../../../common/websocketClient.js';
-import { generatePackageXmlMarkdown } from '../../../common/utils/docUtils.js';
+import { generatePackageXmlMarkdown, readMkDocsFile, writeMkDocsFile } from '../../../common/utils/docUtils.js';
 import { countPackageXmlItems, parseXmlFile } from '../../../common/utils/xmlUtils.js';
 import { bool2emoji, execSfdxJson, getCurrentGitBranch, getGitRepoName, uxLog } from '../../../common/utils/index.js';
 import { CONSTANTS, getConfig } from '../../../config/index.js';
@@ -17,7 +17,6 @@ import { listFlowFiles } from '../../../common/utils/projectUtils.js';
 import { generateFlowMarkdownFile, generateHistoryDiffMarkdown, generateMarkdownFileWithMermaid } from '../../../common/utils/mermaidUtils.js';
 import { MetadataUtils } from '../../../common/metadata-utils/index.js';
 import { PACKAGE_ROOT_DIR } from '../../../settings.js';
-import yaml from 'js-yaml';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
 const messages = Messages.loadMessages('sfdx-hardis', 'org');
@@ -206,15 +205,7 @@ ${Project2Markdown.htmlInstructions}
       );
     }
     // Update mkdocs nav items
-    const mkdocsYml: any = yaml.load(
-      fs
-        .readFileSync(mkdocsYmlFile, 'utf-8')
-        .replace('!!python/name:materialx.emoji.twemoji', "'!!python/name:materialx.emoji.twemoji'")
-        .replace('!!python/name:materialx.emoji.to_svg', "'!!python/name:materialx.emoji.to_svg'")
-    );
-    if (!mkdocsYml.nav) {
-      mkdocsYml.nav = {}
-    }
+    const mkdocsYml: any = readMkDocsFile(mkdocsYmlFile);
     for (const menuName of Object.keys(this.mkDocsNavNodes)) {
       let pos = 0;
       let found = false;
@@ -233,15 +224,8 @@ ${Project2Markdown.htmlInstructions}
         mkdocsYml.nav.push(navMenu);
       }
     }
-    /* jscpd:ignore-start */
-    const mkdocsYmlStr = yaml
-      .dump(mkdocsYml)
-      .replace("'!!python/name:materialx.emoji.twemoji'", '!!python/name:materialx.emoji.twemoji')
-      .replace("'!!python/name:materialx.emoji.to_svg'", '!!python/name:materialx.emoji.to_svg');
-    await fs.writeFile(mkdocsYmlFile, mkdocsYmlStr);
-    uxLog(this, c.cyan(`Updated ${c.green(mkdocsYmlFile)}`));
+    await writeMkDocsFile(mkdocsYmlFile, mkdocsYml);
     uxLog(this, c.cyan(`To generate a HTML WebSite with this documentation with a single command, see instructions at ${CONSTANTS.DOC_URL_ROOT}/hardis/doc/project2markdown/`));
-    /* jscpd:ignore-end */
   }
 
   private async generateFlowsDocumentation() {
@@ -353,7 +337,7 @@ ${Project2Markdown.htmlInstructions}
         `[${flow.name}](${prefix}${flow.name}.md) [🕒](${prefix}${flow.name}-history.md)` :
         `[${flow.name}](${prefix}${flow.name}.md)`;
       lines.push(...[
-        `| ${flow.object} | ${flowNameCell} | ${flow.type} | ${flow.description} |`
+        `| ${flow.object} | ${flowNameCell} | ${flow.type} | ${flow.description.replace(/\n/gm, "<br/>".replace(/\|/gm, ""))} |`
       ]);
     }
     lines.push("");
@@ -447,21 +431,26 @@ ${Project2Markdown.htmlInstructions}
         // Generate manifest from package folder
         const packageManifestFile = path.join("manifest", packageDir.name + '-package.xml');
         await fs.ensureDir(path.dirname(packageManifestFile));
-        await execSfdxJson("sf project generate manifest" +
-          ` --source-dir ${packageDir.path}` +
-          ` --name ${packageManifestFile}`, this,
-          {
-            fail: true,
-            output: true,
-            debug: this.debugMode,
-          }
-        );
-        // Add package in available packages list
-        this.packageXmlCandidates.push({
-          path: packageManifestFile,
-          name: packageDir.name,
-          description: `Package.xml generated from content of SFDX package ${packageDir.name} (folder ${packageDir.path})`
-        });
+        try {
+          await execSfdxJson("sf project generate manifest" +
+            ` --source-dir ${packageDir.path}` +
+            ` --name ${packageManifestFile}`, this,
+            {
+              fail: true,
+              output: true,
+              debug: this.debugMode,
+            }
+          );
+          // Add package in available packages list
+          this.packageXmlCandidates.push({
+            path: packageManifestFile,
+            name: packageDir.name,
+            description: `Package.xml generated from content of SFDX package ${packageDir.name} (folder ${packageDir.path})`
+          });
+        }
+        catch (e: any) {
+          uxLog(this, c.red(`Unable to generate manifest from ${packageDir.path}: it won't appear in the documentation\n${e.message}`))
+        }
       }
     }
   }

src/common/gitProvider/azureDevops.ts

@@ -1,12 +1,15 @@
 import { GitProviderRoot } from "./gitProviderRoot.js";
 import * as azdev from "azure-devops-node-api";
 import c from "chalk";
+import fs from 'fs-extra';
 import { getCurrentGitBranch, getGitRepoUrl, git, isGitRepo, uxLog } from "../utils/index.js";
+import * as path from "path";
 import { PullRequestMessageRequest, PullRequestMessageResult } from "./index.js";
 import { CommentThreadStatus, GitPullRequest, GitPullRequestCommentThread, GitPullRequestSearchCriteria, PullRequestAsyncStatus, PullRequestStatus } from "azure-devops-node-api/interfaces/GitInterfaces.js";
 import { CONSTANTS } from "../../config/index.js";
 import { SfError } from "@salesforce/core";
 import { prompts } from "../utils/prompts.js";
+import { extractImagesFromMarkdown, replaceImagesInMarkdown } from "./utilsMarkdown.js";
 
 export class AzureDevopsProvider extends GitProviderRoot {
   private azureApi: InstanceType<typeof azdev.WebApi>;
@@ -361,6 +364,8 @@ _Powered by [sfdx-hardis](${CONSTANTS.DOC_URL_ROOT}) from job [${azureJobName}](
     if (globalThis.pullRequestDeploymentId) {
       messageBody += `\n<!-- sfdx-hardis deployment-id ${globalThis.pullRequestDeploymentId} -->`;
     }
+    // Upload attached images if necessary
+    messageBody = await this.uploadAndReplaceImageReferences(messageBody);
     // Get Azure Git API
     const azureGitApi = await this.azureApi.getGitApi();
     // Check for existing threads from a previous run
@@ -508,4 +513,40 @@ _Powered by [sfdx-hardis](${CONSTANTS.DOC_URL_ROOT}) from job [${azureJobName}](
     // Return null if the URL doesn't match expected patterns
     return null;
   }
+
+  /* jscpd:ignore-start */
+  private async uploadAndReplaceImageReferences(markdownBody: string) {
+    const replacements: any = {};
+    const markdownImages = extractImagesFromMarkdown(markdownBody);
+    for (const image of markdownImages) {
+      const imageUrl = await this.uploadImage(image);
+      if (imageUrl) {
+        replacements[image] = imageUrl;
+      }
+    }
+    markdownBody = replaceImagesInMarkdown(markdownBody, replacements);
+    return markdownBody;
+  }
+  /* jscpd:ignore-end */
+
+  private async uploadImage(localImagePath: string): Promise<string | null> {
+    try {
+      // Upload the image to Azure DevOps
+      const imageName = path.basename(localImagePath);
+      const imageContent = fs.createReadStream(localImagePath);
+      const witApi = await this.azureApi.getWorkItemTrackingApi();
+      const attachment = await witApi.createAttachment(
+        null, // Custom headers (usually null)
+        imageContent, // File content
+        process.env.SYSTEM_TEAMPROJECT, // Project name
+        imageName // File name
+      );
+      if (attachment && attachment.url) {
+        return attachment.url;
+      }
+    } catch (e) {
+      uxLog(this, c.yellow(`[Azure Integration] Error while uploading image ${localImagePath}\n${(e as Error).message}`));
+    }
+    return null;
+  }
 }

src/common/gitProvider/bitbucket.ts

@@ -1,9 +1,12 @@
 import { GitProviderRoot } from './gitProviderRoot.js';
 import c from 'chalk';
+import fs from "fs-extra";
 import { PullRequestMessageRequest, PullRequestMessageResult } from './index.js';
 import { git, uxLog } from '../utils/index.js';
 import bbPkg, { Schema } from 'bitbucket';
 import { CONSTANTS } from '../../config/index.js';
+import * as path from 'path';
+import { extractImagesFromMarkdown, replaceImagesInMarkdown } from './utilsMarkdown.js';
 const { Bitbucket } = bbPkg;
 
 export class BitbucketProvider extends GitProviderRoot {
@@ -207,6 +210,8 @@ export class BitbucketProvider extends GitProviderRoot {
       messageBody += `\n<!-- sfdx-hardis deployment-id ${globalThis.pullRequestDeploymentId} -->`;
     }
 
+    messageBody = await this.uploadAndReplaceImageReferences(messageBody);
+
     const commentBody: any = {
       content: {
         raw: messageBody,
@@ -272,4 +277,41 @@ export class BitbucketProvider extends GitProviderRoot {
     prInfo.targetBranch = prData?.destination?.branch?.name || '';
     return prInfo;
   }
+
+  private async uploadAndReplaceImageReferences(markdownBody: string) {
+    const replacements: any = {};
+    const markdownImages = extractImagesFromMarkdown(markdownBody);
+    for (const image of markdownImages) {
+      const imageUrl = await this.uploadImage(image);
+      if (imageUrl) {
+        replacements[image] = imageUrl;
+      }
+    }
+    markdownBody = replaceImagesInMarkdown(markdownBody, replacements);
+    return markdownBody;
+  }
+
+  // Upload the image to Bitbucket
+  private async uploadImage(localImagePath: string): Promise<string | null> {
+    try {
+      const imageBuffer = fs.readFileSync(localImagePath);
+      const imageBlob = new Blob([imageBuffer]);
+      const imageName = path.basename(localImagePath);
+      const filesForm = new FormData();
+      filesForm.append('files', imageBlob, imageName);
+      const attachmentResponse = await this.bitbucket.repositories.createIssueAttachments({
+        workspace: process.env.BITBUCKET_WORKSPACE || "",
+        repo_slug: process.env.BITBUCKET_REPO_SLUG || "",
+        issue_id: process.env.BITBUCKET_PR_ID || "", // Attach to the pull request as an "issue"
+        _body: filesForm,
+      });
+      if (attachmentResponse?.data?.links?.self?.href) {
+        return attachmentResponse.data.links.self.href;
+      }
+    } catch (e) {
+      uxLog(this, c.yellow(`[Bitbucket Integration] Error while uploading image ${localImagePath}\n${(e as Error).message}`));
+    }
+    return null;
+  }
+
 }

src/common/gitProvider/utilsMarkdown.ts

@@ -88,10 +88,10 @@ export async function flowDiffToMarkdownForPullRequest(flowNames: string[], from
     const fileMetadata = await MetadataUtils.findMetaFileFromTypeAndName("Flow", flowName);
     try {
       if (supportsMermaidInPrMarkdown) {
-        await generateMarkdownWithMermaid(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName);
+        await generateDiffMarkdownWithMermaid(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName);
       }
       else {
-        await generateMarkdownWithSvg(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName);
+        await generateDiffMarkdownWithSvg(fileMetadata, fromCommit, toCommit, flowDiffMarkdownList, flowName);
       }
     } catch (e: any) {
       uxLog(this, c.yellow(`[FlowGitDiff] Unable to generate Flow diff: ${e.message}`));
@@ -108,15 +108,15 @@ Error while generating Flows visual git diff
   }
 }
 
-async function generateMarkdownWithMermaid(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) {
+async function generateDiffMarkdownWithMermaid(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) {
   const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: false, debug: false });
   if (outputDiffMdFile) {
     const flowDiffMarkdownMermaid = await fs.readFile(outputDiffMdFile.replace(".md", ".mermaid.md"), "utf8");
     flowDiffMarkdownList.push({ name: flowName, markdown: flowDiffMarkdownMermaid });
   }
 }
 
-async function generateMarkdownWithSvg(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) {
+async function generateDiffMarkdownWithSvg(fileMetadata: string | null, fromCommit: string, toCommit: string, flowDiffMarkdownList: any, flowName: string) {
   const { outputDiffMdFile } = await generateFlowVisualGitDiff(fileMetadata, fromCommit, toCommit, { mermaidMd: true, svgMd: true, debug: false });
   flowDiffMarkdownList.push({ name: flowName, markdown: outputDiffMdFile });
 }
@@ -142,3 +142,16 @@ ${message.replace(/:\n-/gm, `:\n\n-`)}
 <br/>
 `;
 }
+
+export function extractImagesFromMarkdown(markdown: string): string[] {
+  const imageRegex = /!\[.*?\]\((.*?)\)/g;
+  const matches = Array.from(markdown.matchAll(imageRegex));
+  return matches.map((match) => match[1]).filter(file => fs.existsSync(file));
+}
+
+export function replaceImagesInMarkdown(markdown: string, replacements: any): string {
+  for (const replacedImage of Object.keys(replacements)) {
+    markdown = markdown.replaceAll(replacedImage, replacements[replacedImage]);
+  }
+  return markdown;
+}

src/common/utils/docUtils.ts

@@ -2,6 +2,7 @@ import * as path from "path";
 import c from 'chalk';
 import fs from 'fs-extra';
 import { uxLog } from "./index.js";
+import * as yaml from 'js-yaml';
 import { countPackageXmlItems, parsePackageXmlFile } from "./xmlUtils.js";
 import { CONSTANTS } from "../../config/index.js";
 import { SfError } from "@salesforce/core";
@@ -76,4 +77,32 @@ export async function generatePackageXmlMarkdown(inputFile: string | null, outpu
   uxLog(this, c.green(`Successfully generated ${path.basename(inputFile)} documentation into ${outputFile}`));
 
   return outputFile;
+}
+
+export function readMkDocsFile(mkdocsYmlFile: string): any {
+  const mkdocsYml: any = yaml.load(
+    fs
+      .readFileSync(mkdocsYmlFile, 'utf-8')
+      .replace('!!python/name:materialx.emoji.twemoji', "!!python/name:material.extensions.emoji.twemoji")
+      .replace('!!python/name:materialx.emoji.to_svg', "!!python/name:material.extensions.emoji.to_svg")
+      .replace('!!python/name:material.extensions.emoji.twemoji', "'!!python/name:material.extensions.emoji.twemoji'")
+      .replace('!!python/name:material.extensions.emoji.to_svg', "'!!python/name:material.extensions.emoji.to_svg'")
+      .replace('!!python/name:pymdownx.superfences.fence_code_format', "'!!python/name:pymdownx.superfences.fence_code_format'")
+  );
+  if (!mkdocsYml.nav) {
+    mkdocsYml.nav = {}
+  }
+  return mkdocsYml;
+}
+
+export async function writeMkDocsFile(mkdocsYmlFile: string, mkdocsYml: any) {
+  const mkdocsYmlStr = yaml
+    .dump(mkdocsYml)
+    .replace("!!python/name:materialx.emoji.twemoji", '!!python/name:material.extensions.emoji.twemoji')
+    .replace("!!python/name:materialx.emoji.to_svg", '!!python/name:material.extensions.emoji.to_svg')
+    .replace("'!!python/name:material.extensions.emoji.twemoji'", '!!python/name:material.extensions.emoji.twemoji')
+    .replace("'!!python/name:material.extensions.emoji.to_svg'", '!!python/name:material.extensions.emoji.to_svg')
+    .replace("'!!python/name:pymdownx.superfences.fence_code_format'", '!!python/name:pymdownx.superfences.fence_code_format');
+  await fs.writeFile(mkdocsYmlFile, mkdocsYmlStr);
+  uxLog(this, c.cyan(`Updated mkdocs-material config file at ${c.green(mkdocsYmlFile)}`));
 }