feat(MathJson): WIP for arnog#500: Next Gen MathJSON

CHANGELOG.md

@@ -1,3 +1,20 @@
+## [Unreleased]
+
+### New Features
+
+-   **#500** Next Generation MathJSON. This is a breaking change: the new
+    MathJSON format is not backward compatible with the previour version.
+    For more details on the format, see `src/math-json/README.md`.
+
+    The `astToLatex()` and `latexToAST()` functions can still be used to
+    convert between Latex and MathJSON, but they now handle the new MathJSON
+    format. Note that the options to these functions have changed.
+
+    In particular, there is greatly improved customization of the parsing
+    possible. It is possible to define the syntax of custom notations, including
+    requesting that no parsing be done, at which point a string of Latex
+    tokens is returned.
+
 ## 0.54.0 (2020-06-24)
 
 ### Bug Fixes

src/editor/mathfield-class.ts

@@ -75,7 +75,6 @@ import {
 } from './keyboard-layout';
 
 import { atomToSpeakableText } from './atom-to-speakable-text';
-import { atomtoMathJson } from '../addons/math-json';
 // import { parseLatex } from '../math-json/math-json';
 import { atomsToMathML } from '../addons/math-ml';
 import { updateUndoRedoButtons } from './virtual-keyboard';
@@ -85,6 +84,7 @@ import coreStylesheet from '../../css/core.less';
 
 import popoverStylesheet from '../../css/popover.less';
 import keystrokeCaptionStylesheet from '../../css/keystroke-caption.less';
+import { parseLatex } from '../math-json/math-json';
 
 export class MathfieldPrivate implements Mathfield {
     model: ModelPrivate;
@@ -686,13 +686,15 @@ export class MathfieldPrivate implements Mathfield {
             this.config.textToSpeechMarkup = saveTextToSpeechMarkup;
             // this.config.atomIdsSettings = savedAtomIdsSettings;      // @revisit
         } else if (format === 'json') {
-            const json = atomtoMathJson(root);
+            const json = parseLatex(root.toLatex(true), {
+                form: 'canonical',
+            });
             result = JSON.stringify(json);
-            // result = JSON.stringify(parseLatex(root.toLatex()));
         } else if (format === 'json-2') {
-            const json = atomtoMathJson(root);
+            const json = parseLatex(root.toLatex(true), {
+                form: 'canonical',
+            });
             result = JSON.stringify(json, null, 2);
-            // result = JSON.stringify(parseLatex(root.toLatex()), null, 2);
         } else if (format === 'ASCIIMath') {
             result = atomToAsciiMath(root);
         } else {

src/math-json/forms.ts

@@ -596,6 +596,10 @@ export function fullForm(
             if (isSymbolObject(expr)) {
                 return expr.sym;
             }
+        } else {
+            if (isFunctionObject(expr)) {
+                expr.fn = expr.fn.map((x) => fullForm(dic, x));
+            }
         }
     }
     return expr;

src/math-json/latex/parse.ts

@@ -1035,6 +1035,7 @@ export class Scanner implements Scanner {
      */
     matchPrimary(): Expression | null {
         let result: Expression | null = null;
+        const originalIndex = this.index;
 
         //
         // 1. Is it a number?
@@ -1077,7 +1078,7 @@ export class Scanner implements Scanner {
             result = postfix ?? result;
         } while (postfix);
 
-        return result;
+        return this.decorate(result, originalIndex);
     }
 
     /**
@@ -1092,6 +1093,7 @@ export class Scanner implements Scanner {
      */
     matchExpression(minPrec = 0): Expression | null {
         let lhs: Expression = null;
+        const originalIndex = this.index;
 
         this.skipSpace();
 
@@ -1137,6 +1139,25 @@ export class Scanner implements Scanner {
                 done = true;
             }
         }
-        return lhs;
+        return this.decorate(lhs, originalIndex);
+    }
+
+    /**
+     * Add latex or other requested metadata to the expression
+     */
+    decorate(expr: Expression, start: number): Expression {
+        if (this.options.preserveLatex) {
+            const latex = this.latex(start, this.index);
+            if (Array.isArray(expr)) {
+                expr = { latex: latex, fn: expr };
+            } else if (typeof expr === 'number') {
+                expr = { latex: latex, num: Number(expr).toString() };
+            } else if (typeof expr === 'string') {
+                expr = { latex: latex, sym: expr };
+            } else {
+                expr.latex = latex;
+            }
+        }
+        return expr;
     }
 }

src/math-json/latex/public.ts

@@ -273,6 +273,12 @@ export type ParseLatexOptions = LatexNumberOptions & {
      */
     promoteUnknownFunctions?: RegExp;
 
+    /**
+     * If true, the expression will be decorated with the Latex
+     * fragments correspondind to each elements of the expression
+     */
+    preserveLatex?: boolean;
+
     dictionary?: LatexDictionary;
 };
 

src/math-json/latex/utils.ts

@@ -32,6 +32,7 @@ export const DEFAULT_PARSE_LATEX_OPTIONS: Required<ParseLatexOptions> = {
     promoteUnknownSymbols: /^[a-zA-Z]$/,
     promoteUnknownFunctions: /^[f|g]$/,
     invisiblePlusOperator: ADD,
+    preserveLatex: false,
     dictionary: [],
 };
 

src/mathlive.ts

@@ -12,12 +12,6 @@ import { coalesce, makeSpan, makeStruts } from './core/span';
 import { MACROS, MacroDictionary } from './core/definitions';
 import { MathfieldPrivate } from './editor/mathfield-class';
 import AutoRender from './addons/auto-render';
-import {
-    MathJsonLatexOptions,
-    MathJson,
-    atomtoMathJson,
-    jsonToLatex,
-} from './addons/math-json';
 import MathLiveDebug from './addons/debug';
 import { MATHSTYLES } from './core/mathstyle';
 import { defaultSpeakHook } from './editor/speech';
@@ -33,8 +27,9 @@ import { atomsToMathML } from './addons/math-ml';
 
 import './addons/definitions-metadata';
 import { AutoRenderOptionsPrivate } from './addons/auto-render';
-import { ErrorCode as MathJsonErrorCode } from './math-json/public';
-// import { parseLatex, emitLatex } from './math-json/math-json';
+import { ErrorCode, Form, Expression, Dictionary } from './math-json/public';
+import { parseLatex, emitLatex } from './math-json/math-json';
+import { ParseLatexOptions, EmitLatexOptions } from './math-json/latex/public';
 
 function latexToMarkup(
     text: string,
@@ -126,35 +121,26 @@ function latexToMathML(
 
 function latexToAST(
     latex: string,
-    options?: MathJsonLatexOptions & {
+    options?: ParseLatexOptions & {
         macros?: MacroDictionary;
-        onError?: ErrorListener<ParserErrorCode | MathJsonErrorCode>;
+        onError?: ErrorListener<ErrorCode>;
+        form?: Form | Form[];
     }
-): MathJson {
+): Expression {
     options = options ?? {};
     options.macros = { ...MACROS, ...(options.macros ?? {}) };
 
-    // return parseLatex(latex, options);
-
-    return atomtoMathJson(
-        parseString(
-            latex,
-            'math',
-            null,
-            options.macros,
-            false,
-            options.onError
-        ),
-        options
-    );
+    return parseLatex(latex, options);
 }
 
-function astToLatex(expr: MathJson, options: MathJsonLatexOptions): string {
-    return jsonToLatex(
-        typeof expr === 'string' ? JSON.parse(expr) : expr,
-        options
-    );
-    // return emitLatex(expr, options);
+function astToLatex(
+    expr: Expression,
+    options: EmitLatexOptions & {
+        dictionary?: Dictionary;
+        onError?: ErrorListener<ErrorCode>;
+    }
+): string {
+    return emitLatex(expr, options);
 }
 
 function latexToSpeakableText(