From 5adf623c4b34b34c90c7530bc93368156dffe797 Mon Sep 17 00:00:00 2001 From: Dilip Kola <33080863+koladilip@users.noreply.github.com> Date: Tue, 22 Nov 2022 17:11:26 +0530 Subject: [PATCH] refactor: improve path processing (#6) * refactor: improve path processing Improve path processing with to array expressions. Split translator code into more functions. * fix comments logic Also return undefined if path doesn't exists. * fix logic for end of the statement * docs: update readme (#7) * Update readme intro and overview * Update readme with arrays and objects docs --- .eslintignore | 3 +- docs/syntax.md | 40 ++++ package-lock.json | 12 +- readme.md | 37 ++- src/index.ts | 1 + src/lexer.ts | 87 +++++-- src/parser.ts | 183 ++++++++------- src/translator.ts | 213 ++++++++++-------- src/types.ts | 13 +- src/utils.ts | 4 +- test/scenarios/bad_templates/data.ts | 7 +- .../invalid_variable_assignment6.jt | 2 + test/scenarios/comments/data.ts | 7 + test/scenarios/comments/template.jt | 6 + test/scenarios/filters/array_filters.jt | 2 +- test/scenarios/logics/template.jt | 6 +- test/scenarios/paths/template.jt | 10 +- test/scenarios/selectors/context_variables.jt | 3 +- test/scenarios/selectors/data.ts | 4 + test/test_engine.ts | 3 +- 20 files changed, 405 insertions(+), 238 deletions(-) create mode 100644 docs/syntax.md create mode 100644 test/scenarios/bad_templates/invalid_variable_assignment6.jt create mode 100644 test/scenarios/comments/data.ts create mode 100644 test/scenarios/comments/template.jt diff --git a/.eslintignore b/.eslintignore index c795b05..05da92e 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1,2 @@ -build \ No newline at end of file +build +*.jt \ No newline at end of file diff --git a/docs/syntax.md b/docs/syntax.md new file mode 100644 index 0000000..a41479e --- /dev/null +++ b/docs/syntax.md @@ -0,0 +1,40 @@ +Template is a set of statements and result the last statement is the output of the template. + +### Variables +```js +const a = 1 +let b = a + 2 +a + b +``` + +### Input and Bindings +Input refers to the JSON document we would like to process using a template. Bindings refer to additional data or functions we would provide to process the data efficiently. + +Example: +* Template: `"Hello " + (.name ?? $.defaultName)` +* Evaluation: `engine.evaluate({name: 'World'}, {defaultName: 'World'});` +* `{name: 'World'}` is input. + * `^.name` refers to "name" property of the input. We can also use `.name` to refer the same. `^` always refers to the root of the input and `.` refers to current context. Refer the [example](../test/scenarios/selectors/context_variables.jt) for more clarity. +* `{defaultName: 'World'}` is bindings. + * `$.defaultName` refers to "defaultName" property of the bindings. Refer the [example](../test/scenarios/bindings/template.jt) for more clarity. + +### Arrays +```js +let arr = [1, 2, 3, 4] +let a = arr[1, 2] // [2, 3] +let b = arr[0:2] // [1, 2] +let c = arr[-2:] // [3, 4] +``` +Refer the [example](../test/scenarios/arrays/template.jt) for more clarity. + +### Objects +```js +let key = "some key" +// { "a": 1, "b": 2, "c": 3, "some key": 4 } +let obj = {a: 1, b: 2, c: 3, [key]: 4 } +let a = obj["a"] // 1 +let b = obj.a // 1 +let c = obj{["a", "b"]} // { "a": 1, "b": 2} +let d = obj{~["a", "b"]} // { "c": 3, "some key": 4} +``` +Refer the [example](../test/scenarios/objects/template.jt) for more clarity. diff --git a/package-lock.json b/package-lock.json index 1fa90f7..0c0ab58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9084,9 +9084,9 @@ } }, "node_modules/typescript": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", - "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "version": "4.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", + "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -16070,9 +16070,9 @@ "dev": true }, "typescript": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", - "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "version": "4.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", + "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", "dev": true }, "unbox-primitive": { diff --git a/readme.md b/readme.md index 4ce0837..d21efb0 100644 --- a/readme.md +++ b/readme.md @@ -19,24 +19,45 @@ --- # rudder-json-template-engine - -A library for evaluating JSON template expressions is an extension to [jspath](https://github.com/dfilatov/jspath). +A library to process JSON data using a custom syntax based on javascript and [jspath](https://github.com/dfilatov/jspath). We thank the jspath authors for their excellent work, as our library is an extension of the original library. We also want to thank [IBM](https://www.ibm.com/) team for their work on [jsonata](https://github.com/jsonata-js/jsonata), as we have taken several ideas from the library. You can also consider our library as an alternative to [jsonata](https://github.com/jsonata-js/jsonata). ## Overview +This library generates a javascript function code from the template and then uses the function to evaluate the JSON data. It outputs the javascript code in the following stages: +1. [Lexing](src/lexer.ts) (Tokenization) +1. [Parsing](src/parser.ts) (AST Creation) +1. [Translation](src/translator.ts) (Code generation) -TODO +[Engine](src/engine.ts) class abstracts the above steps and provides a convenient way to use the json templates to evaluate the inputs. ## Features - -TODO +1. [Variables](test/scenarios/assignments/template.jt) +1. [Arrays](test/scenarios//arrays/template.jt) +1. [Objects](test/scenarios/objects/template.jt) +1. [Functions](test/scenarios/functions/template.jt) +1. [Bindings](test/scenarios/bindings/template.jt) +1. [Paths](test/scenarios/paths/template.jt) + * [Filters](test/scenarios/filters/template.jt) + * [Selectors](test/scenarios/selectors/template.jt) + * [Context Variables](test/scenarios/selectors/context_variables.jt) +1. [Conditions](test/scenarios/conditions/template.jt) + * [Comparisons](test/scenarios/comparisons/template.jt) +1. [Math operations](test/scenarios/math/template.jt) +1. [Logical operations](test/scenarios/logics/template.jt) + +For more examples, refer [Scenarios](test/scenarios) + +## [Syntax](docs/syntax.md) ## Getting started +`npm install rudder-json-template-engine` -TODO +```ts +const engine = new JsonTemplateEngine(`'Hello ' + .name`); +engine.evaluate({name: 'World'}); +``` ## Testing - -TODO +`npm test` ## Contribute diff --git a/src/index.ts b/src/index.ts index aee2b02..1198776 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,3 +6,4 @@ export * from './operators'; export * from './parser'; export * from './translator'; export * from './types'; +export * from './utils'; diff --git a/src/lexer.ts b/src/lexer.ts index e0bc57f..c860a32 100644 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -9,8 +9,6 @@ const MESSAGES = { UNEXP_EOT: 'Unexpected end of template', }; -const BLOCK_COMMENT_REGEX = /\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\//g; -const SINGLE_LINE_COMMENT_REGEX = /\/\/[^\n\r]+?(?:\*\)|[\n\r])/g; export class JsonTemplateLexer { private readonly codeChars: string[]; @@ -18,10 +16,7 @@ export class JsonTemplateLexer { private idx = 0; constructor(template: string) { this.buf = []; - this.codeChars = template - .replace(BLOCK_COMMENT_REGEX, '') - .replace(SINGLE_LINE_COMMENT_REGEX, '') - .split(''); + this.codeChars = template.split(''); } init() { @@ -29,6 +24,14 @@ export class JsonTemplateLexer { this.buf = []; } + currentIndex(): number { + return this.idx; + } + + getCodeChars(start: number, end: number): string[] { + return this.codeChars.slice(start, end); + } + match(value?: string, steps = 0): boolean { if (!value) { return false; @@ -146,10 +149,62 @@ export class JsonTemplateLexer { return this.buf[steps]; } - private advance(): Token { - while (JsonTemplateLexer.isWhiteSpace(this.codeChars[this.idx])) { + private isLineCommentStart(): boolean { + return this.codeChars[this.idx] === '/' && this.codeChars[this.idx + 1] === '/'; + } + + private isLineCommentEnd(): boolean { + return this.codeChars[this.idx] === '\n'; + } + + private isBlockCommentStart(): boolean { + return this.codeChars[this.idx] === '/' && this.codeChars[this.idx + 1] === '*'; + } + + private isBlockCommentEnd(): boolean { + return this.codeChars[this.idx] === '*' && this.codeChars[this.idx + 1] === '/'; + } + + private skipLineComment() { + if (!this.isLineCommentStart()) { + return; + } + while (!this.isLineCommentEnd()) { ++this.idx; } + ++this.idx; + } + + private skipBlockComment() { + if (!this.isBlockCommentStart()) { + return; + } + while (!this.isBlockCommentEnd()) { + ++this.idx; + } + this.idx = this.idx + 2; + } + + private isWhiteSpace() { + return ' \r\n\t'.includes(this.codeChars[this.idx]); + } + + private skipWhitespace() { + while (this.isWhiteSpace()) { + ++this.idx; + } + } + + private skipInput() { + while (this.isWhiteSpace() || this.isBlockCommentStart() || this.isLineCommentStart()) { + this.skipWhitespace(); + this.skipLineComment(); + this.skipBlockComment(); + } + } + + private advance(): Token { + this.skipInput(); if (this.idx >= this.codeChars.length) { return { @@ -183,18 +238,6 @@ export class JsonTemplateLexer { return this.advance(); } - nextChar(): string { - return this.codeChars[this.idx]; - } - - ignoreNextChar() { - this.idx++; - } - - matchNextChar(ch: string): boolean { - return this.nextChar() === ch; - } - static isLiteralToken(token: Token) { return ( token.type === TokenType.BOOL || @@ -229,10 +272,6 @@ export class JsonTemplateLexer { return '0123456789'.indexOf(ch) >= 0; } - private static isWhiteSpace(ch: string) { - return ' \r\n\t'.indexOf(ch) > -1; - } - private static isIdStart(ch: string) { return ch === '$' || ch === '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); } diff --git a/src/parser.ts b/src/parser.ts index a2be68d..4f664a9 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -13,17 +13,17 @@ import { SyntaxType, TokenType, UnaryExpression, - FilterExpression, + ObjectFilterExpression, RangeFilterExpression, Token, IndexFilterExpression, DefinitionExpression, SpreadExpression, ObjectPropExpression, - ToArrayExpression, ContextVariable, ConditionalExpression, OperatorType, + ArrayFilterExpression, } from './types'; import { JsosTemplateParserError } from './errors'; import { DATA_PARAM_KEY } from './constants'; @@ -48,11 +48,15 @@ export class JsonTemplateParser { return; } - if (this.lexer.match(';')) { + if(this.lexer.match(';')) { this.lexer.lex(); - } else if (this.lexer.matchNextChar('\n')) { - this.lexer.ignoreNextChar(); - } else { + return; + } + + const currIdx = this.lexer.currentIndex(); + const nextTokenStart = this.lexer.lookahead().range[0]; + const code = this.lexer.getCodeChars(currIdx, nextTokenStart); + if(!code.includes('\n')) { this.lexer.throwUnexpectedToken(); } } @@ -125,6 +129,12 @@ export class JsonTemplateParser { } } + private parseToArrayExpr(): Expression { + this.lexer.lex(); + this.lexer.lex(); + return { type: SyntaxType.TO_ARRAY }; + } + private parsePathPart(): Expression | Expression[] | undefined { if (this.lexer.match('.') && this.lexer.match('(', 1)) { this.lexer.lex(); @@ -133,7 +143,9 @@ export class JsonTemplateParser { return this.parseFunctionCallExpr(); } else if (this.lexer.matchPathPartSelector()) { return this.parseSelector(); - } else if (this.lexer.match('[') && !this.lexer.match(']', 1)) { + } else if (this.lexer.match('[') && this.lexer.match(']', 1)) { + return this.parseToArrayExpr(); + } else if (this.lexer.match('[')) { return this.parseArrayFiltersExpr(); } else if (this.lexer.match('{')) { return this.parseObjectFiltersExpr(); @@ -164,19 +176,24 @@ export class JsonTemplateParser { let newParts: Expression[] = []; for (let i = 0; i < parts.length; i++) { let expr = parts[i]; - if (expr.type === SyntaxType.SELECTOR && expr.selector === '.') { - const selectorExpr = expr as SelectorExpression; - if (!selectorExpr.prop) { + if (i !== parts.length - 1) { + if (expr.type === SyntaxType.TO_ARRAY) { continue; - } else if ( - !selectorExpr.contextVar && - selectorExpr.prop?.type === TokenType.ID && - parts[i + 1]?.type === SyntaxType.FUNCTION_CALL_EXPR - ) { - expr = parts[i + 1] as FunctionCallExpression; - expr.id = this.prependFunctionID(selectorExpr.prop.value, expr.id); - expr.dot = true; - i++; + } + if (expr.type === SyntaxType.SELECTOR && expr.selector === '.') { + const selectorExpr = expr as SelectorExpression; + if (!selectorExpr.prop) { + continue; + } else if ( + !selectorExpr.contextVar && + selectorExpr.prop?.type === TokenType.ID && + parts[i + 1].type === SyntaxType.FUNCTION_CALL_EXPR + ) { + expr = parts[i + 1] as FunctionCallExpression; + expr.id = this.prependFunctionID(selectorExpr.prop.value, expr.id); + expr.dot = true; + i++; + } } } newParts.push(expr); @@ -189,21 +206,15 @@ export class JsonTemplateParser { private static convertToFunctionCallExpr( expr: PathExpression, - ): FunctionCallExpression | PathExpression | FunctionExpression { - if (expr.parts[0]?.type === SyntaxType.FUNCTION_CALL_EXPR && typeof expr.root !== 'object') { - const fnExpr = expr.parts.shift() as FunctionCallExpression; - if (expr.root) { - fnExpr.id = this.prependFunctionID(expr.root, fnExpr.id); - fnExpr.dot = false; - } - return fnExpr; - } - if (CommonUtils.getLastElement(expr.parts)?.type === SyntaxType.FUNCTION_CALL_EXPR) { - const fnExpr = expr.parts.pop() as FunctionCallExpression; + ): FunctionCallExpression | PathExpression { + const fnExpr = expr.parts.pop() as FunctionCallExpression; + if(!expr.parts.length && expr.root && typeof expr.root !== 'object') { + fnExpr.id = this.prependFunctionID(expr.root, fnExpr.id); + fnExpr.dot = false; + } else { fnExpr.object = expr; - return fnExpr; } - return expr; + return fnExpr; } private parsePathRoot(root?: Expression): Expression | string | undefined { @@ -221,16 +232,24 @@ export class JsonTemplateParser { private parsePath( root?: Expression, ): PathExpression | FunctionCallExpression | FunctionExpression { - const pathExpr = { + let expr: PathExpression | FunctionCallExpression = { type: SyntaxType.PATH, root: this.parsePathRoot(root), parts: this.parsePathParts(), }; + if (!expr.parts.length) { + return expr; + } - JsonTemplateParser.setSubpath(pathExpr.parts); - - const shouldConvertAsBlock = JsonTemplateParser.pathContainsVariables(pathExpr.parts); - const expr = JsonTemplateParser.convertToFunctionCallExpr(pathExpr); + JsonTemplateParser.setSubpath(expr.parts); + const shouldConvertAsBlock = JsonTemplateParser.pathContainsVariables(expr.parts); + const lastPart = CommonUtils.getLastElement(expr.parts) as Expression; + if (lastPart.type === SyntaxType.TO_ARRAY) { + expr.parts.pop(); + expr.toArray = true; + } else if (lastPart.type === SyntaxType.FUNCTION_CALL_EXPR) { + expr = JsonTemplateParser.convertToFunctionCallExpr(expr); + } return shouldConvertAsBlock ? JsonTemplateParser.convertToBlockExpr(expr) : expr; } @@ -267,10 +286,7 @@ export class JsonTemplateParser { }; } - private parsePositionFilterExpr(): - | RangeFilterExpression - | IndexFilterExpression - | FilterExpression { + private parseRangeFilterExpr(): RangeFilterExpression | Expression { if (this.lexer.match(':')) { this.lexer.lex(); return { @@ -295,23 +311,41 @@ export class JsonTemplateParser { toIdx: this.parseBaseExpr(), }; } + return fromExpr; + } - if (!this.lexer.match(']')) { - this.lexer.expect(','); + private parseArrayIndexFilterExpr(expr?: Expression): IndexFilterExpression { + const parts: Expression[] = []; + if (expr) { + parts.push(expr); + if (!this.lexer.match(']')) { + this.lexer.expect(','); + } } return { type: SyntaxType.ARRAY_INDEX_FILTER_EXPR, indexes: { type: SyntaxType.ARRAY_EXPR, elements: [ - fromExpr, + ...parts, ...this.parseCommaSeparatedElements(']', () => this.parseSpreadExpr()), ], }, }; } - private parseObjectFilter(): IndexFilterExpression | FilterExpression { + private parseArrayFilterExpr(): Expression { + if (this.lexer.matchSpread()) { + return this.parseArrayIndexFilterExpr(); + } + const expr = this.parseRangeFilterExpr(); + if (expr.type === SyntaxType.RANGE_FILTER_EXPR) { + return expr; + } + return this.parseArrayIndexFilterExpr(expr); + } + + private parseObjectFilter(): IndexFilterExpression | ObjectFilterExpression { let exclude = false; if (this.lexer.match('~')) { this.lexer.lex(); @@ -331,11 +365,11 @@ export class JsonTemplateParser { }; } - private combineObjectFilters(objectFilters: FilterExpression[]): FilterExpression[] { + private combineObjectFilters(objectFilters: ObjectFilterExpression[]): ObjectFilterExpression[] { if (objectFilters.length <= 1) { return objectFilters; } - const expr1 = objectFilters.shift() as FilterExpression; + const expr1 = objectFilters.shift() as ObjectFilterExpression; const expr2 = this.combineObjectFilters(objectFilters); return [ { @@ -349,8 +383,8 @@ export class JsonTemplateParser { ]; } - private parseObjectFiltersExpr(): (FilterExpression | IndexFilterExpression)[] { - const objectFilters: FilterExpression[] = []; + private parseObjectFiltersExpr(): (ObjectFilterExpression | IndexFilterExpression)[] { + const objectFilters: ObjectFilterExpression[] = []; const indexFilters: IndexFilterExpression[] = []; while (this.lexer.match('{')) { @@ -359,7 +393,7 @@ export class JsonTemplateParser { if (expr.type === SyntaxType.OBJECT_INDEX_FILTER_EXPR) { indexFilters.push(expr as IndexFilterExpression); } else { - objectFilters.push(expr as FilterExpression); + objectFilters.push(expr as ObjectFilterExpression); } this.lexer.expect('}'); if (this.lexer.match('.') && this.lexer.match('{', 1)) { @@ -400,14 +434,20 @@ export class JsonTemplateParser { return ifExpr; } - private parseArrayFiltersExpr(): - | RangeFilterExpression - | IndexFilterExpression - | FilterExpression { - this.lexer.expect('['); - const expr = this.parsePositionFilterExpr(); - this.lexer.expect(']'); - return expr; + private parseArrayFiltersExpr(): ArrayFilterExpression { + const filters: Expression[] = []; + while (this.lexer.match('[') && !this.lexer.match(']', 1)) { + this.lexer.expect('['); + filters.push(this.parseArrayFilterExpr()); + this.lexer.expect(']'); + if (this.lexer.match('.') && this.lexer.match('[', 1)) { + this.lexer.lex(); + } + } + return { + type: SyntaxType.ARRAY_FILTER_EXPR, + filters, + }; } private parseCoalescingExpr(): BinaryExpression | Expression { @@ -576,16 +616,6 @@ export class JsonTemplateParser { return this.parseNextExpr(OperatorType.UNARY); } - private isToArrayExpr(): boolean { - let toArray = false; - while (this.lexer.match('[') && this.lexer.match(']', 1)) { - this.lexer.lex(); - this.lexer.lex(); - toArray = true; - } - return toArray; - } - private shouldSkipPathParsing(expr: Expression): boolean { switch (expr.type) { case SyntaxType.DEFINTION_EXPR: @@ -595,6 +625,8 @@ export class JsonTemplateParser { case SyntaxType.LITERAL: case SyntaxType.MATH_EXPR: case SyntaxType.COMPARISON_EXPR: + case SyntaxType.ARRAY_EXPR: + case SyntaxType.OBJECT_EXPR: if (this.lexer.match('(')) { return true; } @@ -604,29 +636,16 @@ export class JsonTemplateParser { return true; } break; - case SyntaxType.ARRAY_EXPR: - case SyntaxType.OBJECT_EXPR: - if (this.lexer.match('(')) { - return true; - } - break; } return false; } - private parsePathAfterExpr(): PathExpression | ToArrayExpression | Expression { + private parsePathAfterExpr(): PathExpression | Expression { let expr = this.parsePrimaryExpr(); if (this.shouldSkipPathParsing(expr)) { return expr; } while (this.lexer.matchPathPartSelector() || this.lexer.match('[') || this.lexer.match('(')) { - if (this.isToArrayExpr()) { - expr = { - type: SyntaxType.TO_ARRAY_EXPR, - value: expr, - }; - continue; - } expr = this.parsePath(expr); } return expr; diff --git a/src/translator.ts b/src/translator.ts index db0f099..fa4b2d8 100644 --- a/src/translator.ts +++ b/src/translator.ts @@ -21,10 +21,10 @@ import { DefinitionExpression, SpreadExpression, LambdaArgExpression, - ToArrayExpression, ContextVariable, - FilterExpression, ConditionalExpression, + ObjectFilterExpression, + ArrayFilterExpression, } from './types'; import { CommonUtils } from './utils'; @@ -133,21 +133,17 @@ export class JsonTemplateTranslator { return this.translateAssignmentExpr(expr as AssignmentExpression, dest, ctx); case SyntaxType.OBJECT_FILTER_EXPR: - return this.translateObjectFilterExpr(expr as FilterExpression, dest, ctx); + return this.translateObjectFilterExpr(expr as ObjectFilterExpression, dest, ctx); - case SyntaxType.RANGE_FILTER_EXPR: - return this.translateRangeFilterExpr(expr as RangeFilterExpression, dest, ctx); + case SyntaxType.ARRAY_FILTER_EXPR: + return this.translateArrayFilterExpr(expr as ArrayFilterExpression, dest, ctx); - case SyntaxType.ARRAY_INDEX_FILTER_EXPR: case SyntaxType.OBJECT_INDEX_FILTER_EXPR: return this.translateIndexFilterExpr(expr as IndexFilterExpression, dest, ctx); case SyntaxType.SELECTOR: return this.translateSelector(expr as SelectorExpression, dest, ctx); - case SyntaxType.TO_ARRAY_EXPR: - return this.translateToArrayExpr(expr as ToArrayExpression, dest, ctx); - case SyntaxType.CONDITIONAL_EXPR: return this.translateConditionalExpr(expr as ConditionalExpression, dest, ctx); @@ -164,6 +160,7 @@ export class JsonTemplateTranslator { code.push(this.translateExpr(expr.if, ifVar, ctx)); code.push(`if(${ifVar}){`); code.push(this.translateExpr(expr.then, thenVar, ctx)); + code.push(`${dest} = ${thenVar};`); code.push('} else {'); code.push(this.translateExpr(expr.else, elseVar, ctx)); @@ -179,13 +176,6 @@ export class JsonTemplateTranslator { return `${dest} = args[${expr.index}];`; } - private translateToArrayExpr(expr: ToArrayExpression, dest: string, ctx: string): string { - const code: string[] = []; - code.push(this.translateExpr(expr.value, dest, ctx)); - code.push(JsonTemplateTranslator.covertToArrayValue(dest)); - return code.join(''); - } - private translateSpreadExpr(expr: SpreadExpression, dest: string, ctx: string): string { return this.translateExpr(expr.value, dest, ctx); } @@ -193,16 +183,13 @@ export class JsonTemplateTranslator { private translatePathRoot(path: PathExpression, dest: string, ctx: string): string { if (typeof path.root === 'object') { return this.translateExpr(path.root, dest, ctx); - } else if(path.subPath && path.parts.length) { - if(JsonTemplateTranslator.isSinglePropSelection(path.parts[0])) { - const part = path.parts.shift() as SelectorExpression; - const propStr = CommonUtils.escapeStr(part.prop?.value); - const code: string[] = []; - code.push(`if(!${ctx}[${propStr}]) {continue;}`); - code.push(`${dest} = ${ctx}[${propStr}];`); - return code.join(''); - } - } + } else if (path.subPath && path.parts.length) { + if (JsonTemplateTranslator.isSinglePropSelection(path.parts[0])) { + const part = path.parts.shift() as SelectorExpression; + const propStr = CommonUtils.escapeStr(part.prop?.value); + return `${dest} = ${ctx}[${propStr}];`; + } + } return `${dest} = ${path.root || ctx};`; } @@ -232,37 +219,37 @@ export class JsonTemplateTranslator { return code.join(''); } - private translatePath(expr: PathExpression, dest: string, baseCtx: string): string { - const rootCode = this.translatePathRoot(expr, dest, baseCtx); + private translatePathParts(expr: PathExpression, dest: string): string { if (!expr.parts.length) { - return rootCode; + return ''; } - let code: string[] = [rootCode]; - const numParts = expr.parts.length; + const parts = expr.parts; + const code: string[] = []; + const numParts = parts.length; const dataVars = this.acquireVars(numParts); const indexVars = this.acquireVars(numParts); const itemVars = this.acquireVars(numParts); - const resultVar = this.acquireVar(); - code.push(resultVar, '= [];'); - code.push(dataVars[0], '=', dest, ';'); + const result = this.acquireVar(); + code.push(JsonTemplateTranslator.generateAssignmentCode(result, '[]')); + code.push(JsonTemplateTranslator.generateAssignmentCode(dataVars[0], dest)); for (let i = 0; i < numParts; i++) { - const part = expr.parts[i]; + const part = parts[i]; const idx = indexVars[i]; const item = itemVars[i]; const data = dataVars[i]; code.push(this.prepareDataForPathPart(part, data)); code.push(`for(${idx}=0; ${idx}<${data}.length; ${idx}++) {`); code.push(`${item} = ${data}[${idx}];`); - if (i > 0 && expr.parts[i - 1].context) { - code.push(this.translatePathContext(expr.parts[i - 1].context, item, idx)); + if (i > 0 && parts[i - 1].context) { + code.push(this.translatePathContext(parts[i - 1].context, item, idx)); } code.push(this.translateExpr(part, item, item)); code.push(`if(!${item}) { continue; }`); if (i < numParts - 1) { - code.push(dataVars[i + 1], '=', item, ';'); + code.push(JsonTemplateTranslator.generateAssignmentCode(dataVars[i+1], item)); } else { code.push(JsonTemplateTranslator.covertToArrayValue(item)); - code.push(`${resultVar} = ${resultVar}.concat(${item});`); + code.push(`${result} = ${result}.concat(${item});`); } } for (let i = 0; i < numParts; i++) { @@ -271,8 +258,21 @@ export class JsonTemplateTranslator { this.releaseVars(...indexVars); this.releaseVars(...itemVars); this.releaseVars(...dataVars); - this.releaseVars(resultVar); - code.push(dest, '=', JsonTemplateTranslator.returnSingleValueIfSafe(resultVar), ';'); + this.releaseVars(result); + if(!expr.toArray) { + code.push(result, '=', JsonTemplateTranslator.returnSingleValueIfSafe(result), ';'); + } + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, result)); + return code.join(''); + } + + private translatePath(expr: PathExpression, dest: string, baseCtx: string): string { + const code: string[] = []; + code.push(this.translatePathRoot(expr, dest, baseCtx)); + code.push(this.translatePathParts(expr, dest)); + if(expr.toArray && !expr.parts.length) { + code.push(JsonTemplateTranslator.covertToArrayValue(dest)); + } return code.join(''); } @@ -309,8 +309,7 @@ export class JsonTemplateTranslator { const ctxs = this.acquireVar(); const currCtx = this.acquireVar(); const result = this.acquireVar(); - - code.push(`${result} = [];`); + code.push(JsonTemplateTranslator.generateAssignmentCode(result, '[]')); const { prop } = expr; const propStr = CommonUtils.escapeStr(prop?.value); code.push(`${ctxs}=[${baseCtx}];`); @@ -366,16 +365,16 @@ export class JsonTemplateTranslator { ctx: string, ): string { let code: string[] = []; - const resultVar = this.acquireVar(); - code.push(resultVar, '=', ctx, ';'); + const result = this.acquireVar(); + code.push(JsonTemplateTranslator.generateAssignmentCode(result, ctx)); if (expr.object) { - code.push(this.translateExpr(expr.object, resultVar, ctx)); + code.push(this.translateExpr(expr.object, result, ctx)); } if (!expr.id) { - code.push(JsonTemplateTranslator.convertToSingleValue(resultVar)); + code.push(JsonTemplateTranslator.convertToSingleValue(result)); } - const functionArgsStr = this.translateSpreadableExpressions(expr.args, resultVar, code); - code.push(dest, '=', this.getFunctionName(expr, resultVar), '(', functionArgsStr, ');'); + const functionArgsStr = this.translateSpreadableExpressions(expr.args, result, code); + code.push(dest, '=', this.getFunctionName(expr, result), '(', functionArgsStr, ');'); return code.join(''); } @@ -432,7 +431,7 @@ export class JsonTemplateTranslator { private translateLiteralExpr(expr: LiteralExpression, dest: string, _ctx: string): string { const literalCode = this.translateLiteral(expr.tokenType, expr.value); - return `${dest} = ${literalCode};`; + return JsonTemplateTranslator.generateAssignmentCode(dest, literalCode); } private getSelectorAssignmentPart(expr: SelectorExpression): string { @@ -446,17 +445,24 @@ export class JsonTemplateTranslator { } } private getArrayIndexAssignmentPart( - expr: IndexFilterExpression, + expr: ArrayFilterExpression, code: string[], ctx: string, ): string { - if (expr.indexes.elements.length > 1) { - throw new JsosTemplateTranslatorError('Invalid assignment path'); + const parts: string[] = []; + for (const filter of expr.filters) { + if (filter.type === SyntaxType.RANGE_FILTER_EXPR) { + throw new JsosTemplateTranslatorError('Invalid assignment path'); + } + if (filter.indexes.elements.length > 1) { + throw new JsosTemplateTranslatorError('Invalid assignment path'); + } + const keyVar = this.acquireVar(); + code.push(this.translateExpr(filter.indexes.elements[0], keyVar, ctx)); + this.releaseVars(keyVar); + parts.push(`[${keyVar}]`); } - const keyVar = this.acquireVar(); - code.push(this.translateExpr(expr.indexes.elements[0], keyVar, ctx)); - this.releaseVars(keyVar); - return `[${keyVar}]`; + return parts.join(''); } private translateAssignmentExpr(expr: AssignmentExpression, dest: string, ctx: string): string { @@ -474,9 +480,9 @@ export class JsonTemplateTranslator { case SyntaxType.SELECTOR: assignmentPathParts.push(this.getSelectorAssignmentPart(part as SelectorExpression)); break; - case SyntaxType.ARRAY_INDEX_FILTER_EXPR: + case SyntaxType.ARRAY_FILTER_EXPR: assignmentPathParts.push( - this.getArrayIndexAssignmentPart(part as IndexFilterExpression, code, ctx), + this.getArrayIndexAssignmentPart(part as ArrayFilterExpression, code, ctx), ); break; default: @@ -484,8 +490,8 @@ export class JsonTemplateTranslator { } } const assignmentPath = assignmentPathParts.join(''); - code.push(`${assignmentPath}=${valueVar};`); - code.push(`${dest} = ${valueVar};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(assignmentPath, valueVar)); + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, valueVar)); this.releaseVars(valueVar); return code.join(''); } @@ -501,12 +507,12 @@ export class JsonTemplateTranslator { private translateDefinitionExpr(expr: DefinitionExpression, dest: string, ctx: string): string { const code: string[] = []; - const valueVar = this.acquireVar(); - code.push(this.translateExpr(expr.value, valueVar, ctx)); + const value = this.acquireVar(); + code.push(this.translateExpr(expr.value, value, ctx)); const defVars = this.translateDefinitionVars(expr); - code.push(`${expr.definition} ${defVars}=${valueVar};`); - code.push(`${dest} = ${valueVar};`); - this.releaseVars(valueVar); + code.push(`${expr.definition} ${defVars}=${value};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, value)); + this.releaseVars(value); return code.join(''); } @@ -535,11 +541,11 @@ export class JsonTemplateTranslator { code.push(this.translateExpr(expr.args[0], val1, ctx)); const condition = this.getLogicalConditionCode(expr.type, val1); code.push(`if(${condition}) {`); - code.push(`${dest} = ${val1};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, val1)); code.push('} else {'); const val2 = this.acquireVar(); code.push(this.translateExpr(expr.args[1], val2, ctx)); - code.push(`${dest} = ${val2};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, val2)); code.push('}'); this.releaseVars(val1, val2); return code.join(''); @@ -553,8 +559,8 @@ export class JsonTemplateTranslator { code.push(this.translateExpr(expr.args[0], val1, ctx)); code.push(this.translateExpr(expr.args[1], val2, ctx)); const inCode = `(Array.isArray(${val2}) ? ${val2}.includes(${val1}) : ${val1} in ${val2})`; - code.push(`${resultVar} = ${inCode};`); - code.push(`${dest} = ${resultVar};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(resultVar, inCode)); + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, resultVar)); return code.join(''); } @@ -575,9 +581,7 @@ export class JsonTemplateTranslator { } private static isArrayFilterExpr(expr: Expression): boolean { - return ( - expr.type === SyntaxType.ARRAY_INDEX_FILTER_EXPR || expr.type === SyntaxType.RANGE_FILTER_EXPR - ); + return expr.type === SyntaxType.ARRAY_FILTER_EXPR; } private static isValidSelectorForAssignment(expr: SelectorExpression): boolean { @@ -587,9 +591,10 @@ export class JsonTemplateTranslator { private static isSinglePropSelection(expr: Expression): boolean { if (expr.type === SyntaxType.SELECTOR) { const part = expr as SelectorExpression; - return part.selector === '.' && - (part.prop?.type === TokenType.ID || - part.prop?.type === TokenType.STR); + return ( + part.selector === '.' && + (part.prop?.type === TokenType.ID || part.prop?.type === TokenType.STR) + ); } return false; } @@ -610,14 +615,35 @@ export class JsonTemplateTranslator { } private static returnSingleValueIfSafe(varName: string): string { - return `(${varName}.length === 1 ? ${varName}[0] : ${varName})`; + return `(${varName}.length < 2 ? ${varName}[0] : ${varName})`; } private static covertToArrayValue(varName: string) { return `${varName} = Array.isArray(${varName}) ? ${varName} : [${varName}];`; } - private translateObjectFilterExpr(expr: FilterExpression, dest: string, ctx: string): string { + private static generateAssignmentCode(key: string, val: string): string { + return `${key}=${val};`; + } + + private translateArrayFilterExpr(expr: ArrayFilterExpression, dest: string, ctx: string): string { + const code: string[] = []; + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, ctx)); + for (const filter of expr.filters) { + if (filter.type === SyntaxType.ARRAY_INDEX_FILTER_EXPR) { + code.push(this.translateIndexFilterExpr(filter as IndexFilterExpression, dest, dest)); + } else { + code.push(this.translateRangeFilterExpr(filter as RangeFilterExpression, dest, dest)); + } + } + return code.join(''); + } + + private translateObjectFilterExpr( + expr: ObjectFilterExpression, + dest: string, + ctx: string, + ): string { const code: string[] = []; const condition = this.acquireVar(); code.push(this.translateExpr(expr.filter, condition, ctx)); @@ -636,7 +662,7 @@ export class JsonTemplateTranslator { if (shouldExclude) { code.push(`${allKeys}=Object.keys(${ctx}).filter(key => !${allKeys}.includes(key));`); } - code.push(`${resultVar} = {};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(resultVar, '{}')); code.push(`for(let key of ${allKeys}){`); code.push( `if(Object.prototype.hasOwnProperty.call(${ctx}, key)){${resultVar}[key] = ${ctx}[key];}`, @@ -647,7 +673,7 @@ export class JsonTemplateTranslator { private translateArrayIndexFilterExpr(ctx: string, allKeys: string, resultVar: string): string { const code: string[] = []; - code.push(`${resultVar} = [];`); + code.push(JsonTemplateTranslator.generateAssignmentCode(resultVar, '[]')); code.push(`for(let key of ${allKeys}){`); code.push(`if(typeof key === 'string'){`); code.push(`for(let childCtx of ${ctx}){`); @@ -655,12 +681,12 @@ export class JsonTemplateTranslator { code.push(`${resultVar}.push(childCtx[key]);`); code.push('}'); code.push('}'); - code.push('continue;'); - code.push('}'); + code.push('} else {'); code.push(`if(key < 0){key = ${ctx}.length + key;}`); - code.push( - `if(Object.prototype.hasOwnProperty.call(${ctx}, key)){${resultVar}.push(${ctx}[key]);}`, - ); + code.push(`if(Object.prototype.hasOwnProperty.call(${ctx}, key)){`); + code.push(`${resultVar}.push(${ctx}[key]);`); + code.push('}'); + code.push('}'); code.push('}'); code.push(`if(${allKeys}.length === 1) {${resultVar} = ${resultVar}[0];}`); return code.join(''); @@ -677,7 +703,7 @@ export class JsonTemplateTranslator { } else { code.push(this.translateArrayIndexFilterExpr(ctx, allKeys, resultVar)); } - code.push(`${dest}=${resultVar};`); + code.push(JsonTemplateTranslator.generateAssignmentCode(dest, resultVar)); this.releaseVars(allKeys); this.releaseVars(resultVar); return code.join(''); @@ -685,23 +711,22 @@ export class JsonTemplateTranslator { private translateRangeFilterExpr(expr: RangeFilterExpression, dest: string, ctx: string): string { const code: string[] = []; - let fromIdx, toIdx; + let fromIdx = this.acquireVar(); + let toIdx = this.acquireVar(); if (expr.fromIdx) { if (expr.toIdx) { - code.push(this.translateExpr(expr.fromIdx, (fromIdx = this.acquireVar()), ctx)); - code.push(this.translateExpr(expr.toIdx, (toIdx = this.acquireVar()), ctx)); + code.push(this.translateExpr(expr.fromIdx, fromIdx, ctx)); + code.push(this.translateExpr(expr.toIdx, toIdx, ctx)); code.push(dest, '=', ctx, '.slice(', fromIdx, ',', toIdx, ');'); - this.releaseVars(fromIdx, toIdx); } else { - code.push(this.translateExpr(expr.fromIdx, (fromIdx = this.acquireVar()), ctx)); + code.push(this.translateExpr(expr.fromIdx, fromIdx, ctx)); code.push(dest, '=', ctx, '.slice(', fromIdx, ');'); - this.releaseVars(fromIdx); } } else if (expr.toIdx) { - code.push(this.translateExpr(expr.toIdx, (toIdx = this.acquireVar()), ctx)); + code.push(this.translateExpr(expr.toIdx, toIdx, ctx)); code.push(dest, '=', ctx, '.slice(0,', toIdx, ');'); - this.releaseVars(toIdx); } + this.releaseVars(fromIdx, toIdx); return code.join(''); } diff --git a/src/types.ts b/src/types.ts index 9675080..73c89ce 100644 --- a/src/types.ts +++ b/src/types.ts @@ -62,11 +62,12 @@ export enum SyntaxType { OBJECT_INDEX_FILTER_EXPR, RANGE_FILTER_EXPR, OBJECT_FILTER_EXPR, + ARRAY_FILTER_EXPR, DEFINTION_EXPR, ASSIGNMENT_EXPR, OBJECT_PROP_EXPR, OBJECT_EXPR, - TO_ARRAY_EXPR, + TO_ARRAY, ARRAY_EXPR, FUNCTION_EXPR, FUNCTION_CALL_ARG, @@ -148,10 +149,14 @@ export interface IndexFilterExpression extends Expression { indexes: ArrayExpression; exclude?: boolean; } -export interface FilterExpression extends Expression { + +export interface ObjectFilterExpression extends Expression { filter: Expression; } +export interface ArrayFilterExpression extends Expression { + filters: (RangeFilterExpression | IndexFilterExpression)[]; +} export interface LiteralExpression extends Expression { value: string | number | boolean | null | undefined; tokenType: TokenType; @@ -159,6 +164,7 @@ export interface LiteralExpression extends Expression { export interface PathExpression extends Expression { parts: Expression[]; root?: Expression | string; + toArray?: boolean // Used in a part of another Path subPath?: boolean; } @@ -175,9 +181,6 @@ export interface SelectorExpression extends Expression { export interface SpreadExpression extends Expression { value: Expression; } -export interface ToArrayExpression extends Expression { - value: Expression; -} export interface FunctionCallExpression extends Expression { args: Expression[]; diff --git a/src/utils.ts b/src/utils.ts index 3f36fb2..90ccb36 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -6,11 +6,9 @@ export class CommonUtils { } static getLastElement(arr: T[]): T | undefined { - if (!arr.length) { - return undefined; - } return arr[arr.length - 1]; } + static convertToStatementsExpr(...expressions: Expression[]): StatementsExpression { return { type: SyntaxType.STATEMENTS_EXPR, diff --git a/test/scenarios/bad_templates/data.ts b/test/scenarios/bad_templates/data.ts index 0e98ce5..a7fe8ee 100644 --- a/test/scenarios/bad_templates/data.ts +++ b/test/scenarios/bad_templates/data.ts @@ -57,11 +57,14 @@ export const data: Sceanario[] = [ templatePath: 'invalid_variable_assignment4.jt', error: 'Invalid assignment path', }, - { templatePath: 'invalid_variable_assignment5.jt', error: 'Invalid assignment path', }, + { + templatePath: 'invalid_variable_assignment6.jt', + error: 'Invalid assignment path', + }, { templatePath: 'invalid_variable_definition.jt', error: 'Invalid normal vars', @@ -91,3 +94,5 @@ export const data: Sceanario[] = [ error: 'Unexpected token', }, ]; + +console.log(data[17]); \ No newline at end of file diff --git a/test/scenarios/bad_templates/invalid_variable_assignment6.jt b/test/scenarios/bad_templates/invalid_variable_assignment6.jt new file mode 100644 index 0000000..01ea07e --- /dev/null +++ b/test/scenarios/bad_templates/invalid_variable_assignment6.jt @@ -0,0 +1,2 @@ +let a = [{a: [1,2,3,4], b: 2}]; +a[1:3].b = 3; \ No newline at end of file diff --git a/test/scenarios/comments/data.ts b/test/scenarios/comments/data.ts new file mode 100644 index 0000000..ec02d3b --- /dev/null +++ b/test/scenarios/comments/data.ts @@ -0,0 +1,7 @@ +import { Sceanario } from '../../types'; + +export const data: Sceanario[] = [ + { + output: ["////", "/*** /// */"], + }, +]; diff --git a/test/scenarios/comments/template.jt b/test/scenarios/comments/template.jt new file mode 100644 index 0000000..ed2cf85 --- /dev/null +++ b/test/scenarios/comments/template.jt @@ -0,0 +1,6 @@ +// line comment +/** + //////////////// + * block comment + */ +["////", "/*** /// */"] \ No newline at end of file diff --git a/test/scenarios/filters/array_filters.jt b/test/scenarios/filters/array_filters.jt index d3746b9..cef006d 100644 --- a/test/scenarios/filters/array_filters.jt +++ b/test/scenarios/filters/array_filters.jt @@ -1,2 +1,2 @@ let a = [1, 2, 3, 4, 5]; -[a[2:], a[:3], a[3:5], a[1, 3], a[-1], a[:-1], a[-2:]] \ No newline at end of file +[a[2:], a[:3], a[3:5], a[...[1, 3]].[0, 1], a[-1], a[:-1], a[-2:]] \ No newline at end of file diff --git a/test/scenarios/logics/template.jt b/test/scenarios/logics/template.jt index 2895a09..996ee4c 100644 --- a/test/scenarios/logics/template.jt +++ b/test/scenarios/logics/template.jt @@ -1,5 +1 @@ -[ - 2 && 3, 0 && 3, 2 || 3, 0 || 3, - 0 ?? 3, null ?? undefined ?? 3, - !false, !!true -] +[2 && 3, 0 && 3, 2 || 3, 0 || 3, 0 ?? 3, null ?? undefined ?? 3, !false, !!true]; diff --git a/test/scenarios/paths/template.jt b/test/scenarios/paths/template.jt index 85f31f2..b8e6c28 100644 --- a/test/scenarios/paths/template.jt +++ b/test/scenarios/paths/template.jt @@ -1,7 +1,7 @@ [ - 3[0][][], - "aa"[][0], - ^.a, - .c.c.d, - .b[1].d + 3[], + "aa"[0][][0][][0], + ^.d ?? ^.a, + ^.c.d ?? .c.c.d, + .b[1].d ?? .b[0].d ] \ No newline at end of file diff --git a/test/scenarios/selectors/context_variables.jt b/test/scenarios/selectors/context_variables.jt index c77403f..66f5a55 100644 --- a/test/scenarios/selectors/context_variables.jt +++ b/test/scenarios/selectors/context_variables.jt @@ -2,5 +2,6 @@ cid: .id, cidx: ci, bid: b.id, - bidx: bi + bidx: bi, + a: ^.a }) \ No newline at end of file diff --git a/test/scenarios/selectors/data.ts b/test/scenarios/selectors/data.ts index 744485d..c03fc58 100644 --- a/test/scenarios/selectors/data.ts +++ b/test/scenarios/selectors/data.ts @@ -36,24 +36,28 @@ export const data: Sceanario[] = [ cidx: 0, bid: 1, bidx: 0, + a: 10, }, { cid: 2, cidx: 1, bid: 1, bidx: 0, + a: 10, }, { cid: 3, cidx: 0, bid: 2, bidx: 1, + a: 10, }, { cid: 4, cidx: 1, bid: 2, bidx: 1, + a: 10, }, ], }, diff --git a/test/test_engine.ts b/test/test_engine.ts index 461e0df..b3c14e2 100644 --- a/test/test_engine.ts +++ b/test/test_engine.ts @@ -217,8 +217,7 @@ const address = { // ); new JsonTemplateEngine(` -let a = [{a: [1, 2], b: "1"}, {a: [3, 4], b: 2}, {a:[5], b: 3}, {b: 4}] -a{.a.length > 1} +.b ?? .a `) .evaluate({ a: 1 }) .then((a) => console.log(JSON.stringify(a)));