[add] highlighting of control characters

Untitled-1.js

@@ -0,0 +1,85 @@
+/* We've started Quokka for you automatically on this file.
+ *
+ * To open a new Quokka file:
+ *   - Press `Ctrl K, J` to create a new JavaScript File
+ *   - Press `Ctrl K, T` to create a new TypeScript File
+ *   - Press `Ctrl K, L` to open an interactive sample from:
+ *     https://github.com/wallabyjs/interactive-examples
+ *
+ * To start/restart Quokka on an existing file:
+ *   - Press `Ctrl K, Q`
+*/
+
+// See the output of console.log right next to your code
+const quokka = { isAwesome: false };
+
+console.log(quokka);
+
+// See the value of a variable simply by typing its name
+quokka;
+
+// Use sequence expression to compare objects
+const wallaby = { "is Quokka's BigBrother": true };
+
+(quokka, wallaby)
+
+// Gutter indicators show what code was executed (code coverage)
+
+// Orange indicators means only part of the line was executed
+// because JavaScript stops processing after first false value
+console.log('partialCoverage', false && true);
+
+// Green indicators means that Quokka executed all statements
+// on a line of code
+if (false) {
+  // White indicators means that a line of code was never
+  // executed by Quokka
+  console.log('noCoverage', true);
+}
+
+// Red indicators show where an error occurred. The error message
+// is also shown beside the error
+// throw new Error('Something went wrong');
+
+// There's a lot more Quokka can do! Visit our docs to learn more:
+// - https://quokkajs.com/docs/
+
+
+const moo = require('moo')
+
+let lexer = moo.compile({
+  WS:      /[ \t]+/,
+  comment: /\/\/.*?$/,
+  number:  /0|[1-9][0-9]*/,
+  string:  /"(?:\\["\\]|[^\n"\\])*"/,
+  letter:  /[a-zA-Z]*"/,
+  // lparen:  '(',
+  // rparen:  ')',
+  keyword: ['while', 'if', 'else', 'moo', 'cows'],
+  NL:      { match: /\n/, lineBreaks: true },
+})
+
+// k
+lexer.reset('aaa333\n')
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+console.log(lexer.next())
+
+screen;

package.json

@@ -5,7 +5,8 @@
   "description": "Monaco Editor for React",
   "scripts": {
     "start": "webpack serve --open --config webpack.config.js --hot --inline --progress --port 3000 ",
-    "build": "webpack build --mode production --config webpack.config.js"
+    "build": "webpack build --mode production --config webpack.config.js",
+    "test": "jest"
   },
   "authors": [
     "Leon Shi <[email protected]>",
@@ -19,6 +20,7 @@
     "@babel/preset-react": "^7.14.5",
     "babel-loader": "^8.2.2",
     "css-loader": "^6.3.0",
+    "jest": "^27.3.1",
     "monaco-editor-webpack-plugin": "^4.2.0",
     "style-loader": "^3.3.0",
     "webpack": "^5.55.1",
@@ -32,6 +34,9 @@
     "@mui/icons-material": "^5.0.5",
     "@mui/material": "^5.0.6",
     "file-loader": "^6.2.0",
+    "itt": "^0.7.3",
+    "moo": "^0.5.1",
+    "pprint": "^0.0.1",
     "randexp": "^0.5.3",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",

src/Dashboard.js

@@ -10,6 +10,8 @@ import RegexReplace from "./RegexReplace"
 import PatternSelector from './PatternSelector';
 
 import MonacoEditor from "react-monaco-editor";
+import { monaco } from 'react-monaco-editor';
+
 import GitHubForkRibbon from 'react-github-fork-ribbon';
 
 import { mainListItems, secondaryListItems } from './listItems';
@@ -112,6 +114,34 @@ function DashboardContent() {
       return
     }
   }
+
+
+// Register a new language
+monaco.languages.register({ id: 'mySpecialLanguage' });
+
+// Register a tokens provider for the language
+monaco.languages.setMonarchTokensProvider('mySpecialLanguage', {
+	tokenizer: {
+		root: [
+			[ /[\s\t\r]+/, "CONTROL"],
+			[/0|[1-9][0-9]*/, "NUMBERS"],
+			[/[a-zA-Z]+/, "LETTERS"],
+			[/[^\s\t\r\-0-9a-zA-Z]+/, "SYMBOLS"],
+		]
+	}
+});
+
+// Define a new theme that contains only rules that match this language
+monaco.editor.defineTheme('myCoolTheme', {
+	base: 'vs-dark',
+	inherit: true,
+	rules: [
+		{ token: 'CONTROL', foreground: '#F50B03', fontStyle: 'bold' },
+		{ token: 'NUMBERS', foreground: '#0BF503' },
+		{ token: 'LETTERS', foreground: '#4811F1',background:"#ffffff" },
+		{ token: 'SYMBOLS', foreground: '#F1ED11' },
+	]
+});
   return (
 
     <ThemeProvider theme={mdTheme}>
@@ -170,12 +200,12 @@ function DashboardContent() {
                     <MonacoEditor
                       // height="400"
                       // width="1000"
-                      language="javascript"
+                      language="mySpecialLanguage"
                       defaultValue={localStorage.editorSavedText}
                       options={options}
                       onChange={onChange}
                       editorDidMount={monacoEditorLoaded}
-                      theme={theme ? "vs-light" : "vs-dark"}
+                      theme="myCoolTheme"
                     />
                   </React.Fragment>
                 </Paper>

src/lib/patternParse.js

@@ -0,0 +1,58 @@
+const moo = require('moo')
+
+// https://stackoverflow.com/a/27845224/16309718
+String.prototype.count=function(c) { 
+    var result = 0, i = 0;
+    for(i;i<this.length;i++)if(this[i]==c)result++;
+    return result;
+  };
+
+// grammer
+let lexer = moo.compile({
+    // CONTROL: /[ \t]+/,
+    CONTROL: { match: /[ \t\n\r]+/, lineBreaks: true },
+    // comment: /\/\/.*?$/,
+    numbers: /0|[1-9][0-9]*/,
+    letters: /[a-zA-Z]+/,
+    // punctuation: /[\.,'";&:!?]+/,
+    symbols: /[\.,'";&:!?`~@#$%^*|\}\]\(\)]+/,
+    // string: /"(?:\\["\\]|[^\n"\\])*"/,
+    // lparen: '(',
+    // rparen: ')',
+    keyword: [`'while'`, 'if', 'else', 'moo', 'cows'],
+    // newline: { match: /\n/, lineBreaks: false },
+})
+
+
+const patternParse =(input,include_range=false)=>{
+    lexer.reset(input)
+    return Array.from(lexer)
+    .map((value,index)=>{
+        // im formating the output to match a monaco editor range format
+        // https://microsoft.github.io/monaco-editor/api/classes/monaco.range.html
+        result = 
+         {
+            token: value.type.toUpperCase()
+        }
+        if(include_range){
+            result["range"]={
+                startLineNumber:value.line,
+                // this is adds that number to the current line number to get the range from line to line
+                endLineNumber: value.line+value.lineBreaks,
+
+                startColumn: value.col,
+                endColumn:  ( value.text.length - value.text.lastIndexOf("\n"))
+                // value.lineBreaks === 0 ? value.text.length: value.text.length - value.text.lastIndexOf("\n") 
+            }}
+        return result
+    })   
+}
+
+console.log(patternParse("aaaa1111"))
+console.log(patternParse("while4"))
+
+input_example = "\n3333bbbbb"
+// patternParse(input_example)[2]
+console.log(patternParse(input_example,include_range=true))
+
+module.exports = patternParse

src/lib/patternParse.test.js

@@ -0,0 +1,54 @@
+const patternParse = require("./patternParse")
+
+test("'aaaaa44444' to be <letter><number>", () => {
+    input_example = "aaaaa44444"
+    output_expected = [
+        { token: "LETTERS" },
+        { token: "NUMBERS" }
+    ]
+    expect(patternParse(input_example)).toStrictEqual(output_expected)
+
+})
+
+test("'4.2' to be <num><punctuation><num>", () => {
+
+    input_example = "4.5"
+    output_expected = [
+        { token: "NUMBERS" },
+        { token: "SYMBOLS" },
+        { token: "NUMBERS" }
+    ]
+    expect(patternParse(input_example)).toStrictEqual(output_expected)
+})
+
+test(`"hello","world"-><Symbol><letter><Symbol><letter><Symbol>`, () => {
+    input_example = `"hello","world"`
+    output_expected = [
+        { token: "SYMBOLS" },
+        { token: "LETTERS" },
+        { token: "SYMBOLS" },
+        { token: "LETTERS" },
+        { token: "SYMBOLS" },
+    ]
+    expect(patternParse(input_example)).toStrictEqual(output_expected)
+})
+
+test("range output should be col 5, line 2 ", () => {
+    input_example = "\n1b"
+    output_expected = { token: "LETTERS", range: { startLineNumber: 2, endLineNumber: 2,  startColumn: 2, endColumn:3 } }
+    expect(patternParse(input_example,include_range=true)[2]).toStrictEqual(output_expected)
+
+})
+
+
+test("multiline token should start line number and end number should have be +1 differnce", () => {
+    input_example = 
+    `1\t
+    \t1`
+    // #cnc#
+    output_expected = { token: "CONTROL", range: { startLineNumber: 1, endLineNumber: 2,  startColumn: 2, endColumn:1  } }
+    // console.log(patternParse(input_example,include_range=true))
+
+    expect(patternParse(input_example,include_range=true)[1]).toStrictEqual(output_expected)
+
+})