refactor: improve path processing (#6)

* refactor: improve path processing

Improve path processing with to array expressions.
Split translator code into more functions.

* fix comments logic

Also return undefined if path doesn't exists.

* fix logic for end of the statement

* docs: update readme (#7)

* Update readme intro and overview

* Update readme with arrays and objects docs

.eslintignore

@@ -1 +1,2 @@
-build
+build
+*.jt

docs/syntax.md

@@ -0,0 +1,40 @@
+Template is a set of statements and result the last statement is the output of the template.
+
+### Variables
+```js
+const a = 1
+let b = a + 2
+a + b
+```
+
+### Input and Bindings
+Input refers to the JSON document we would like to process using a template. Bindings refer to additional data or functions we would provide to process the data efficiently.
+
+Example:
+* Template: `"Hello " + (.name ?? $.defaultName)`
+* Evaluation: `engine.evaluate({name: 'World'}, {defaultName: 'World'});`
+* `{name: 'World'}` is input.
+  * `^.name` refers to "name" property of the input. We can also use `.name` to refer the same. `^` always refers to the root of the input and `.` refers to current context. Refer the [example](../test/scenarios/selectors/context_variables.jt) for more clarity.
+* `{defaultName: 'World'}` is bindings.
+  * `$.defaultName` refers to "defaultName" property of the bindings. Refer the [example](../test/scenarios/bindings/template.jt) for more clarity.
+
+### Arrays
+```js
+let arr = [1, 2, 3, 4]
+let a = arr[1, 2] // [2, 3]
+let b = arr[0:2] // [1, 2]
+let c = arr[-2:] // [3, 4]
+```
+Refer the [example](../test/scenarios/arrays/template.jt) for more clarity.
+
+### Objects
+```js
+let key = "some key"
+// { "a": 1, "b": 2, "c": 3, "some key": 4 }
+let obj = {a: 1, b: 2, c: 3, [key]: 4 } 
+let a = obj["a"] // 1
+let b = obj.a // 1
+let c = obj{["a", "b"]} // { "a": 1, "b": 2}
+let d = obj{~["a", "b"]} // { "c": 3, "some key": 4}
+```
+Refer the [example](../test/scenarios/objects/template.jt) for more clarity.

readme.md

@@ -19,24 +19,45 @@
 ---
 
 # rudder-json-template-engine
-
-A library for evaluating JSON template expressions is an extension to [jspath](https://github.com/dfilatov/jspath).
+A library to process JSON data using a custom syntax based on javascript and [jspath](https://github.com/dfilatov/jspath). We thank the jspath authors for their excellent work, as our library is an extension of the original library. We also want to thank [IBM](https://www.ibm.com/) team for their work on [jsonata](https://github.com/jsonata-js/jsonata), as we have taken several ideas from the library. You can also consider our library as an alternative to [jsonata](https://github.com/jsonata-js/jsonata).
 
 ## Overview
+This library generates a javascript function code from the template and then uses the function to evaluate the JSON data. It outputs the javascript code in the following stages:
+1. [Lexing](src/lexer.ts) (Tokenization)
+1. [Parsing](src/parser.ts) (AST Creation)
+1. [Translation](src/translator.ts) (Code generation)
 
-TODO
+[Engine](src/engine.ts) class abstracts the above steps and provides a convenient way to use the json templates to evaluate the inputs.
 
 ## Features
-
-TODO
+1. [Variables](test/scenarios/assignments/template.jt)
+1. [Arrays](test/scenarios//arrays/template.jt)
+1. [Objects](test/scenarios/objects/template.jt)
+1. [Functions](test/scenarios/functions/template.jt)
+1. [Bindings](test/scenarios/bindings/template.jt)
+1. [Paths](test/scenarios/paths/template.jt)
+    * [Filters](test/scenarios/filters/template.jt)
+    * [Selectors](test/scenarios/selectors/template.jt)
+    * [Context Variables](test/scenarios/selectors/context_variables.jt)
+1. [Conditions](test/scenarios/conditions/template.jt)
+    * [Comparisons](test/scenarios/comparisons/template.jt)
+1. [Math operations](test/scenarios/math/template.jt)
+1. [Logical operations](test/scenarios/logics/template.jt)
+
+For more examples, refer [Scenarios](test/scenarios)
+
+## [Syntax](docs/syntax.md)
 
 ## Getting started
+`npm install rudder-json-template-engine`
 
-TODO
+```ts
+const engine = new JsonTemplateEngine(`'Hello ' + .name`);
+engine.evaluate({name: 'World'});
+```
 
 ## Testing
-
-TODO
+`npm test`
 
 ## Contribute
 

src/index.ts

@@ -6,3 +6,4 @@ export * from './operators';
 export * from './parser';
 export * from './translator';
 export * from './types';
+export * from './utils';

src/lexer.ts

@@ -9,26 +9,29 @@ const MESSAGES = {
   UNEXP_EOT: 'Unexpected end of template',
 };
 
-const BLOCK_COMMENT_REGEX = /\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\//g;
-const SINGLE_LINE_COMMENT_REGEX = /\/\/[^\n\r]+?(?:\*\)|[\n\r])/g;
 
 export class JsonTemplateLexer {
   private readonly codeChars: string[];
   private buf: Token[];
   private idx = 0;
   constructor(template: string) {
     this.buf = [];
-    this.codeChars = template
-      .replace(BLOCK_COMMENT_REGEX, '')
-      .replace(SINGLE_LINE_COMMENT_REGEX, '')
-      .split('');
+    this.codeChars = template.split('');
   }
 
   init() {
     this.idx = 0;
     this.buf = [];
   }
 
+  currentIndex(): number {
+    return this.idx;
+  }
+
+  getCodeChars(start: number, end: number): string[] {
+    return this.codeChars.slice(start, end);
+  }
+
   match(value?: string, steps = 0): boolean {
     if (!value) {
       return false;
@@ -146,10 +149,62 @@ export class JsonTemplateLexer {
     return this.buf[steps];
   }
 
-  private advance(): Token {
-    while (JsonTemplateLexer.isWhiteSpace(this.codeChars[this.idx])) {
+  private isLineCommentStart(): boolean {
+    return this.codeChars[this.idx] === '/' && this.codeChars[this.idx + 1] === '/';
+  }
+
+  private isLineCommentEnd(): boolean {
+    return this.codeChars[this.idx] === '\n';
+  }
+
+  private isBlockCommentStart(): boolean {
+    return this.codeChars[this.idx] === '/' && this.codeChars[this.idx + 1] === '*';
+  }
+
+  private isBlockCommentEnd(): boolean {
+    return this.codeChars[this.idx] === '*' && this.codeChars[this.idx + 1] === '/';
+  }
+
+  private skipLineComment() {
+    if (!this.isLineCommentStart()) {
+      return;
+    }
+    while (!this.isLineCommentEnd()) {
       ++this.idx;
     }
+    ++this.idx;
+  }
+
+  private skipBlockComment() {
+    if (!this.isBlockCommentStart()) {
+      return;
+    }
+    while (!this.isBlockCommentEnd()) {
+      ++this.idx;
+    }
+    this.idx = this.idx + 2;
+  }
+
+  private isWhiteSpace() {
+    return ' \r\n\t'.includes(this.codeChars[this.idx]);
+  }
+
+  private skipWhitespace() {
+    while (this.isWhiteSpace()) {
+      ++this.idx;
+    }
+  }
+
+  private skipInput() {
+    while (this.isWhiteSpace() || this.isBlockCommentStart() || this.isLineCommentStart()) {
+      this.skipWhitespace();
+      this.skipLineComment();
+      this.skipBlockComment();
+    }
+  }
+
+  private advance(): Token {
+    this.skipInput();
 
     if (this.idx >= this.codeChars.length) {
       return {
@@ -183,18 +238,6 @@ export class JsonTemplateLexer {
     return this.advance();
   }
 
-  nextChar(): string {
-    return this.codeChars[this.idx];
-  }
-
-  ignoreNextChar() {
-    this.idx++;
-  }
-
-  matchNextChar(ch: string): boolean {
-    return this.nextChar() === ch;
-  }
-
   static isLiteralToken(token: Token) {
     return (
       token.type === TokenType.BOOL ||
@@ -229,10 +272,6 @@ export class JsonTemplateLexer {
     return '0123456789'.indexOf(ch) >= 0;
   }
 
-  private static isWhiteSpace(ch: string) {
-    return ' \r\n\t'.indexOf(ch) > -1;
-  }
-
   private static isIdStart(ch: string) {
     return ch === '$' || ch === '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
   }