feat(gatsby-remark-prismjs): add support for language extensions (gat…

…sbyjs#11932)

* Extended 'gatsby-remark-prismjs' plugin to support adding new language definitions and extend current languages.

packages/gatsby-remark-prismjs/README.md

@@ -50,6 +50,24 @@ plugins: [
             // If setting this to true, the parser won't handle and highlight inline
             // code used in markdown i.e. single backtick code like `this`.
             noInlineHighlight: false,
+            // This adds a new language definition to Prism or extend an already
+            // existing language definition. More details on this option can be
+            // found under the header "Add new language definition or extend an
+            // existing language" below.
+            languageExtensions: [
+              {
+                language: "superscript",
+                extend: "javascript",
+                definition: {
+                  superscript_types: /(SuperType)/,
+                },
+                insertBefore: {
+                  function: {
+                    superscript_keywords: /(superif|superelse)/,
+                  },
+                },
+              },
+            ],
           },
         },
       ],
@@ -341,6 +359,61 @@ highlighted) text of `.some-class { background-color: red }`
 If you need to prevent any escaping or highlighting, you can use the `none`
 language; the inner contents will not be changed at all.
 
+### Add new language definition or extend an existing language
+
+You can provide a language extension by giving a single object or an array of
+language extension objects as the 'languageExtensions' option.
+
+A language extension object looks like this:
+
+```javascript
+languageExtensions: [
+  {
+    language: "superscript",
+    extend: "javascript",
+    definition: {
+      superscript_types: /(SuperType)/,
+    },
+    insertBefore: {
+      function: {
+        superscript_keywords: /(superif|superelse)/,
+      },
+    },
+  },
+]
+```
+
+'language' - (optional) The name of the new language.
+'extend' - (optional) The language you wish to extend.
+'definition' - (optional) This is the Prism language definition.
+'insertBefore' - (optional) Is used to define where in the language definition we want to insert our extension.
+More information of the format can be found here:
+https://prismjs.com/extending.html
+
+One of the parameters 'language' and 'extend' is needed. If only 'language'
+is given, a new language will be defined from scratch. If only 'extend' is
+given, an extension will be made to the given language. If both 'language'
+and 'extend' is given, a new language that extends the 'extend' language will
+be defined.
+
+In case a language is extended, note that the definitions will not be merged.
+If the extended language defintion and the given definition contains the same
+token, the original pattern will be overwritten.
+
+One of the parameters 'definition' and 'insertBefore' needs to be defined.
+'insertBefore' needs to be combined with 'definition' or 'extend' (otherwise
+there will not be any language definition tokens to insert before).
+
+In addition to this extension parameters the css also needs to be updated to
+get a style for the new tokens. Prism will wrap the matched tokens with a
+'span' element and give it the classes 'token' and the token name you defined.
+In the example above we would match 'superif' and 'superelse'. In the html
+it would result in the following when a match is found:
+
+```html
+<span class="token superscript_keywords">superif</span>
+```
+
 ## Implementation notes
 
 ### Line highlighting

packages/gatsby-remark-prismjs/src/__tests__/__snapshots__/index.js.snap

@@ -631,3 +631,4 @@ color<span class=\\"token punctuation\\">:</span> red<span class=\\"token punctu
   "type": "root",
 }
 `;
+

packages/gatsby-remark-prismjs/src/__tests__/index.js

@@ -105,6 +105,41 @@ describe(`remark prism plugin`, () => {
       plugin({ markdownAST })
       expect(markdownAST).toMatchSnapshot()
     })
+    it(`should not wrap keywords with <span class="token extensionTokenName"> if no extension given`, () => {
+      const code = `\`\`\`c\naRandomTypeKeyword var = 32\n\``
+      const markdownAST = remark.parse(code)
+
+      plugin({ markdownAST })
+
+      expect(markdownAST.children).toBeDefined()
+      expect(markdownAST.children).toHaveLength(1)
+
+      const htmlResult = markdownAST.children[0].value
+
+      expect(htmlResult).not.toMatch(/<span class="token extended_keywords">/)
+    })
+    it(`should wrap keywords with <span class="token extensionTokenName"> based on given extension`, () => {
+      const code = `\`\`\`c\naRandomTypeKeyword var = 32\n\``
+      const markdownAST = remark.parse(code)
+
+      const config = {
+        languageExtensions: {
+          extend: `c`,
+          definition: {
+            extended_keywords: /(aRandomTypeKeyword)/,
+          },
+        },
+      }
+
+      plugin({ markdownAST }, config)
+
+      expect(markdownAST.children).toBeDefined()
+      expect(markdownAST.children).toHaveLength(1)
+
+      const htmlResult = markdownAST.children[0].value
+
+      expect(htmlResult).toMatch(/<span class="token extended_keywords">/)
+    })
   })
 
   describe(`warnings`, () => {

packages/gatsby-remark-prismjs/src/__tests__/load-prism-language-extension.js

@@ -0,0 +1,143 @@
+const loadLanguageExtension = require(`../load-prism-language-extension`)
+const Prism = require(`prismjs`)
+
+describe(`extend/add prism language`, () => {
+  it(`should throw an error if the request is not an array or an object`, () => {
+    let request = 4
+
+    expect(() => loadLanguageExtension(request)).toThrow()
+
+    request = `A weird string value, instead of an object or array`
+
+    expect(() => loadLanguageExtension(request)).toThrow()
+  })
+  it(`should not throw an error if the request is an array`, () => {
+    let request = []
+
+    expect(() => loadLanguageExtension(request)).not.toThrow()
+  })
+  it(`should not throw an error if the request is a valid object (containing 'language' and 'definition')`, () => {
+    let request = {
+      language: `aTypicalLanguage`,
+      definition: /aRegexp/,
+    }
+
+    expect(() => loadLanguageExtension(request)).not.toThrow()
+  })
+  it(`should throw an error if the request is not a valid object (containing 'language' and 'definition')`, () => {
+    let request = {}
+
+    expect(() => loadLanguageExtension(request)).toThrow()
+  })
+  it(`should throw an error if the request is an array containing an invalid object (not containing 'language' and 'definition')`, () => {
+    let request = [
+      {
+        language: `aTypicalLanguage`,
+        definition: /aRegexp/,
+      },
+      {},
+    ]
+
+    expect(() => loadLanguageExtension(request)).toThrow()
+  })
+  it(`should extend pre-loaded language`, () => {
+    const request = {
+      extend: `clike`,
+      definition: {
+        flexc_keyword: `(__cm|__circ|_lpp_indirect|__accum|__size_t|__ptrdiff_t|__wchar_t|__fixed|__abscall|__extcall|__stkcall|__sat|__i64_t|__i32_t|__i16_t|__r32_t|__r16_t|__u64_t|__u32_t|__u16_t|__a40_t|__a24_t)`,
+      },
+    }
+
+    loadLanguageExtension(request)
+
+    expect(Prism.languages[request.extend]).toBeDefined()
+    expect(Prism.languages[request.extend]).toHaveProperty(`flexc_keyword`)
+    expect(Prism.languages[request.extend][`flexc_keyword`]).toEqual(
+      request.definition.flexc_keyword
+    )
+  })
+  it(`should extend not pre-loaded language`, () => {
+    const request = {
+      extend: `c`,
+      definition: {
+        flexc_keyword: `(__cm|__circ|_lpp_indirect|__accum|__size_t|__ptrdiff_t|__wchar_t|__fixed|__abscall|__extcall|__stkcall|__sat|__i64_t|__i32_t|__i16_t|__r32_t|__r16_t|__u64_t|__u32_t|__u16_t|__a40_t|__a24_t)`,
+      },
+    }
+
+    loadLanguageExtension(request)
+
+    expect(Prism.languages[request.extend]).toBeDefined()
+    expect(Prism.languages[request.extend]).toHaveProperty(`flexc_keyword`)
+    expect(Prism.languages[request.extend][`flexc_keyword`]).toEqual(
+      request.definition.flexc_keyword
+    )
+  })
+  it(`should add new language from existing language`, () => {
+    const request = {
+      language: `flexc`,
+      extend: `c`,
+      definition: {
+        flexc_keyword: `(__cm|__circ|_lpp_indirect|__accum|__size_t|__ptrdiff_t|__wchar_t|__fixed|__abscall|__extcall|__stkcall|__sat|__i64_t|__i32_t|__i16_t|__r32_t|__r16_t|__u64_t|__u32_t|__u16_t|__a40_t|__a24_t)`,
+      },
+    }
+
+    const languagesBeforeLoaded = Object.keys(Prism.languages)
+    expect(Prism.languages).not.toHaveProperty(request.language)
+
+    loadLanguageExtension(request)
+
+    let languagesAfterLoaded = Object.keys(Prism.languages)
+    expect(Prism.languages).toHaveProperty(request.language)
+    expect(languagesAfterLoaded.length).toBe(languagesBeforeLoaded.length + 1)
+    expect(Prism.languages[request.language][`flexc_keyword`]).toEqual(
+      request.definition.flexc_keyword
+    )
+  })
+  it(`should add new language`, () => {
+    const request = {
+      language: `flexc2`, //Check if it is possible to reset scope somehow, instead of giving a new name.
+      definition: {
+        flexc_keyword: `(__cm|__circ|_lpp_indirect|__accum|__size_t|__ptrdiff_t|__wchar_t|__fixed|__abscall|__extcall|__stkcall|__sat|__i64_t|__i32_t|__i16_t|__r32_t|__r16_t|__u64_t|__u32_t|__u16_t|__a40_t|__a24_t)`,
+      },
+    }
+
+    const languagesBeforeLoaded = Object.keys(Prism.languages)
+    expect(Prism.languages).not.toHaveProperty(request.language)
+
+    loadLanguageExtension(request)
+
+    let languagesAfterLoaded = Object.keys(Prism.languages)
+    expect(Prism.languages).toHaveProperty(request.language)
+    expect(languagesAfterLoaded.length).toBe(languagesBeforeLoaded.length + 1)
+    expect(Prism.languages[request.language][`flexc_keyword`]).toEqual(
+      request.definition.flexc_keyword
+    )
+  })
+  it(`should work to make two requests by sending an array`, () => {
+    const request = [
+      {
+        language: `flexc3`, //Check if it is possible to reset scope somehow, instead of giving a new name.
+        definition: {
+          flexc_keyword: `(__cm|__circ|_lpp_indirect|__accum|__size_t|__ptrdiff_t|__wchar_t|__fixed|__abscall|__extcall|__stkcall|__sat|__i64_t|__i32_t|__i16_t|__r32_t|__r16_t|__u64_t|__u32_t|__u16_t|__a40_t|__a24_t)`,
+        },
+      },
+      {
+        extend: `c`,
+        definition: {
+          new_token: `(__cm|__circ|_lpp_indirect|__accum|__size_t|__ptrdiff_t|__wchar_t|__fixed|__abscall|__extcall|__stkcall|__sat|__i64_t|__i32_t|__i16_t|__r32_t|__r16_t|__u64_t|__u32_t|__u16_t|__a40_t|__a24_t)`,
+        },
+      },
+    ]
+
+    const languagesBeforeLoaded = Object.keys(Prism.languages)
+    expect(Prism.languages).not.toHaveProperty(`flexc3`)
+    expect(Prism.languages[`c`]).not.toHaveProperty(`new_token`)
+
+    loadLanguageExtension(request)
+
+    let languagesAfterLoaded = Object.keys(Prism.languages)
+    expect(Prism.languages).toHaveProperty(`flexc3`)
+    expect(languagesAfterLoaded.length).toBe(languagesBeforeLoaded.length + 1)
+    expect(Prism.languages[`c`]).toHaveProperty(`new_token`)
+  })
+})

packages/gatsby-remark-prismjs/src/highlight-code.js

@@ -1,6 +1,5 @@
 const Prism = require(`prismjs`)
 const _ = require(`lodash`)
-
 const loadPrismLanguage = require(`./load-prism-language`)
 const handleDirectives = require(`./directives`)
 const unsupportedLanguages = new Set()
@@ -40,20 +39,16 @@ module.exports = (
   }
 
   const grammar = Prism.languages[language]
-
   const highlighted = Prism.highlight(code, grammar, language)
   const codeSplits = handleDirectives(highlighted, lineNumbersHighlight)
-
   let finalCode = ``
-
-  const lastIdx = codeSplits.length - 1
-  // Don't add back the new line character after highlighted lines
+  const lastIdx = codeSplits.length - 1 // Don't add back the new line character after highlighted lines
   // as they need to be display: block and full-width.
+
   codeSplits.forEach((split, idx) => {
     finalCode += split.highlight
       ? split.code
       : `${split.code}${idx == lastIdx ? `` : `\n`}`
   })
-
   return finalCode
 }

packages/gatsby-remark-prismjs/src/index.js

@@ -1,6 +1,7 @@
 const visit = require(`unist-util-visit`)
 
 const parseLineNumberRange = require(`./parse-line-number-range`)
+const loadLanguageExtension = require(`./load-prism-language-extension`)
 const highlightCode = require(`./highlight-code`)
 const addLineNumbers = require(`./add-line-numbers`)
 
@@ -12,13 +13,17 @@ module.exports = (
     aliases = {},
     noInlineHighlight = false,
     showLineNumbers: showLineNumbersGlobal = false,
+    languageExtensions = [],
   } = {}
 ) => {
   const normalizeLanguage = lang => {
     const lower = lang.toLowerCase()
     return aliases[lower] || lower
   }
 
+  //Load language extension if defined
+  loadLanguageExtension(languageExtensions)
+
   visit(markdownAST, `code`, node => {
     let language = node.lang
     let {