feat: add support for context var index

rudderlabs · Jun 28, 2024 · 32299b3 · 32299b3
1 parent 048f031
commit 32299b3
Show file tree

Hide file tree

Showing 11 changed files with 257 additions and 67 deletions.
diff --git a/src/engine.ts b/src/engine.ts
@@ -120,9 +120,13 @@ export class JsonTemplateEngine {
     return JsonTemplateEngine.translateExpression(JsonTemplateEngine.parse(template, options));
   }
 
-  static reverseTranslate(expr: Expression, options?: EngineOptions): string {
+  static reverseTranslate(expr: Expression | FlatMappingPaths[], options?: EngineOptions): string {
     const translator = new JsonTemplateReverseTranslator(options);
-    return translator.translate(expr);
+    let newExpr = expr;
+    if (Array.isArray(expr)) {
+      newExpr = JsonTemplateEngine.parseMappingPaths(expr as FlatMappingPaths[], options);
+    }
+    return translator.translate(newExpr as Expression);
   }
 
   static convertMappingsToTemplate(mappings: FlatMappingPaths[], options?: EngineOptions): string {

diff --git a/src/parser.ts b/src/parser.ts
@@ -1443,9 +1443,9 @@ export class JsonTemplateParser {
     return this.lexer.throwUnexpectedToken();
   }
 
-  private static pathContainsVariables(parts: Expression[]): boolean {
+  private static shouldPathBeConvertedAsBlock(parts: Expression[]): boolean {
     return parts
-      .filter((part) => part.type === SyntaxType.PATH_OPTIONS)
+      .filter((part, index) => part.type === SyntaxType.PATH_OPTIONS && index < parts.length - 1)
       .some((part) => part.options?.index ?? part.options?.item);
   }
 
@@ -1550,17 +1550,21 @@ export class JsonTemplateParser {
       newPathExpr.parts.shift();
     }
 
-    const shouldConvertAsBlock = JsonTemplateParser.pathContainsVariables(newPathExpr.parts);
+    const shouldConvertAsBlock = JsonTemplateParser.shouldPathBeConvertedAsBlock(newPathExpr.parts);
     let lastPart = getLastElement(newPathExpr.parts);
     let fnExpr: FunctionCallExpression | undefined;
     if (lastPart?.type === SyntaxType.FUNCTION_CALL_EXPR) {
       fnExpr = newPathExpr.parts.pop() as FunctionCallExpression;
     }
 
     lastPart = getLastElement(newPathExpr.parts);
-    if (lastPart?.type === SyntaxType.PATH_OPTIONS) {
-      newPathExpr.parts.pop();
+    if (lastPart?.type === SyntaxType.PATH_OPTIONS && lastPart.options?.toArray) {
       newPathExpr.returnAsArray = lastPart.options?.toArray;
+      if (!lastPart.options.item && !lastPart.options.index) {
+        newPathExpr.parts.pop();
+      } else {
+        lastPart.options.toArray = false;
+      }
     }
     newPathExpr.parts = JsonTemplateParser.combinePathOptionParts(newPathExpr.parts);
 

diff --git a/src/reverse_translator.test.ts b/src/reverse_translator.test.ts
@@ -1,4 +1,5 @@
 import { JsonTemplateEngine } from './engine';
+import { PathType } from './types';
 
 describe('reverse_translator', () => {
   it('should reverse translate with indentation', () => {
@@ -12,7 +13,7 @@ describe('reverse_translator', () => {
 
   it('should reverse translate json mappings', () => {
     const template = JsonTemplateEngine.reverseTranslate(
-      JsonTemplateEngine.parse([
+      [
         {
           input: '$.userId',
           output: '$.user.id',
@@ -41,10 +42,11 @@ describe('reverse_translator', () => {
           input: '$.products[?(@.category)].(@.price * @.quantity * (1 - $.discount / 100)).sum()',
           output: '$.events[0].revenue',
         },
-      ]),
+      ],
+      { defaultPathType: PathType.JSON },
     );
     expect(template).toEqual(
-      '{\n    "user": {\n        "id": $.userId\n    },\n    "events": [{\n        "items": $.products{.category}.({\n            "discount": $.discount,\n            "product_id": .id,\n            "options": .variations[*].({\n                "s": .size\n            })[],\n            "value": .price * .quantity * (1 - $.discount / 100)\n        })[],\n        "name": $.events[0],\n        "revenue": $.products{.category}.(.price * .quantity * (1 - $.discount / 100)).sum()\n    }]\n}',
+      '{\n    "user": {\n        "id": $.userId\n    },\n    "events": [{\n        "items": $.products[?(@.category)].({\n            "discount": $.discount,\n            "product_id": @.id,\n            "options": @.variations[*].({\n                "s": @.size\n            })[],\n            "value": @.price * @.quantity * (1 - $.discount / 100)\n        })[],\n        "name": $.events[0],\n        "revenue": $.products[?(@.category)].(@.price * @.quantity * (1 - $.discount / 100)).sum()\n    }]\n}',
     );
   });
 });
diff --git a/src/translator.ts b/src/translator.ts
@@ -310,17 +310,21 @@ export class JsonTemplateTranslator {
     code.push(JsonTemplateTranslator.covertToArrayValue(data));
     if (
       JsonTemplateTranslator.isArrayFilterExpr(expr.parts[partNum]) ||
+      JsonTemplateTranslator.isAllFilterExpr(expr.parts[partNum]) ||
       JsonTemplateTranslator.isToArray(expr, partNum)
     ) {
       code.push(`${data} = [${data}];`);
     }
     return code.join('');
   }
 
+  static isAllFilterExpr(expr: Expression): boolean {
+    return (
+      expr.type === SyntaxType.OBJECT_FILTER_EXPR && expr.filter.type === SyntaxType.ALL_FILTER_EXPR
+    );
+  }
+
   private translatePathParts(expr: PathExpression, dest: string): string {
-    if (!expr.parts.length) {
-      return '';
-    }
     const { parts } = expr;
     const code: string[] = [];
     const numParts = parts.length;
@@ -384,8 +388,9 @@ export class JsonTemplateTranslator {
     }
     const code: string[] = [];
     code.push(this.translatePathRoot(expr, dest, ctx));
-    code.push(this.translatePathParts(expr, dest));
-    if (expr.returnAsArray && expr.parts.length === 0) {
+    if (expr.parts.length > 0) {
+      code.push(this.translatePathParts(expr, dest));
+    } else if (expr.returnAsArray) {
       code.push(JsonTemplateTranslator.covertToArrayValue(dest));
     }
     return code.join('');
@@ -786,11 +791,13 @@ export class JsonTemplateTranslator {
     ctx: string,
   ): string {
     const code: string[] = [];
-    const condition = this.acquireVar();
-    code.push(JsonTemplateTranslator.generateAssignmentCode(condition, 'true'));
-    code.push(this.translateExpr(expr.filter, condition, ctx));
-    code.push(`if(!${condition}) {${dest} = undefined;}`);
-    this.releaseVars(condition);
+    if (expr.filter.type !== SyntaxType.ALL_FILTER_EXPR) {
+      const condition = this.acquireVar();
+      code.push(JsonTemplateTranslator.generateAssignmentCode(condition, 'true'));
+      code.push(this.translateExpr(expr.filter, condition, ctx));
+      code.push(`if(!${condition}) {${dest} = undefined;}`);
+      this.releaseVars(condition);
+    }
     return code.join('');
   }
 

diff --git a/src/types.ts b/src/types.ts
@@ -284,7 +284,7 @@ export type FlatMappingPaths = {
 };
 
 export type FlatMappingAST = FlatMappingPaths & {
-  inputExpr: PathExpression;
+  inputExpr: Expression;
   outputExpr: PathExpression;
 };
 

diff --git a/src/utils/converter.ts b/src/utils/converter.ts
@@ -1,6 +1,5 @@
 /* eslint-disable no-param-reassign */
 import { JsonTemplateMappingError } from '../errors/mapping';
-import { EMPTY_EXPR } from '../constants';
 import {
   SyntaxType,
   PathExpression,
@@ -13,6 +12,7 @@ import {
   BlockExpression,
   TokenType,
   BinaryExpression,
+  PathType,
 } from '../types';
 import { createBlockExpression, getLastElement } from './common';
 
@@ -40,20 +40,23 @@ function findOrCreateObjectPropExpression(
 }
 
 function processArrayIndexFilter(
+  flatMapping: FlatMappingAST,
   currrentOutputPropAST: ObjectPropExpression,
   filter: IndexFilterExpression,
+  isLastPart: boolean,
 ): ObjectExpression {
   const filterIndex = filter.indexes.elements[0].value;
   if (currrentOutputPropAST.value.type !== SyntaxType.ARRAY_EXPR) {
     const elements: Expression[] = [];
-    elements[filterIndex] = currrentOutputPropAST.value;
+    elements[filterIndex] = isLastPart ? flatMapping.inputExpr : currrentOutputPropAST.value;
     currrentOutputPropAST.value = {
       type: SyntaxType.ARRAY_EXPR,
       elements,
     };
   } else if (!currrentOutputPropAST.value.elements[filterIndex]) {
-    (currrentOutputPropAST.value as ArrayExpression).elements[filterIndex] =
-      createObjectExpression();
+    (currrentOutputPropAST.value as ArrayExpression).elements[filterIndex] = isLastPart
+      ? flatMapping.inputExpr
+      : createObjectExpression();
   }
   return currrentOutputPropAST.value.elements[filterIndex];
 }
@@ -67,24 +70,27 @@ function isPathWithEmptyPartsAndObjectRoot(expr: Expression) {
 }
 
 function getPathExpressionForAllFilter(
-  currentInputAST: PathExpression,
+  currentInputAST: Expression,
   root: any,
   parts: Expression[] = [],
 ): PathExpression {
   return {
     type: SyntaxType.PATH,
     root,
-    pathType: currentInputAST.pathType,
-    inferredPathType: currentInputAST.inferredPathType,
+    pathType: currentInputAST.pathType || PathType.UNKNOWN,
+    inferredPathType: currentInputAST.inferredPathType || PathType.UNKNOWN,
     parts,
     returnAsArray: true,
   } as PathExpression;
 }
 
-function validateResultOfAllFilter(objectExpr: Expression, flatMapping: FlatMappingAST) {
+function validateResultOfAllFilter(
+  objectExpr: Expression | undefined,
+  flatMapping: FlatMappingAST,
+) {
   if (
+    !objectExpr?.props ||
     objectExpr.type !== SyntaxType.OBJECT_EXPR ||
-    !objectExpr.props ||
     !Array.isArray(objectExpr.props)
   ) {
     throw new JsonTemplateMappingError(
@@ -95,45 +101,95 @@ function validateResultOfAllFilter(objectExpr: Expression, flatMapping: FlatMapp
   }
 }
 
+function addToArrayToExpression(expr: Expression) {
+  return {
+    type: SyntaxType.PATH,
+    root: expr,
+    returnAsArray: true,
+    parts: [],
+  };
+}
+
+function handleAllFilterIndexFound(
+  currentInputAST: Expression,
+  currentOutputPropAST: ObjectPropExpression,
+  filterIndex: number,
+  isLastPart: boolean,
+) {
+  const matchedInputParts = currentInputAST.parts.splice(0, filterIndex + 1);
+  if (isPathWithEmptyPartsAndObjectRoot(currentOutputPropAST.value)) {
+    currentOutputPropAST.value = currentOutputPropAST.value.root;
+  }
+
+  if (currentOutputPropAST.value.type !== SyntaxType.PATH) {
+    matchedInputParts.push(
+      createBlockExpression(isLastPart ? currentInputAST : currentOutputPropAST.value),
+    );
+    currentOutputPropAST.value = getPathExpressionForAllFilter(
+      currentInputAST,
+      currentInputAST.root,
+      matchedInputParts,
+    );
+  }
+  currentInputAST.root = undefined;
+}
+
+function findAllFilterIndex(expr: Expression): number {
+  let filterIndex = -1;
+  if (expr.type === SyntaxType.PATH) {
+    filterIndex = expr.parts.findIndex((part) => part.type === SyntaxType.OBJECT_FILTER_EXPR);
+  }
+  return filterIndex;
+}
+
+function handleAllFilterIndexNotFound(
+  currentInputAST: Expression,
+  currentOutputPropAST: ObjectPropExpression,
+  isLastPart: boolean,
+): ObjectExpression | undefined {
+  if (currentOutputPropAST.value.type === SyntaxType.OBJECT_EXPR) {
+    const currObjectExpr = currentOutputPropAST.value as ObjectExpression;
+    currentOutputPropAST.value = isLastPart
+      ? addToArrayToExpression(currentInputAST)
+      : getPathExpressionForAllFilter(currentInputAST, currObjectExpr);
+    return currObjectExpr;
+  }
+  if (isPathWithEmptyPartsAndObjectRoot(currentOutputPropAST.value)) {
+    return currentOutputPropAST.value.root as ObjectExpression;
+  }
+}
+
+function getNextObjectExpressionForAllFilter(
+  flatMapping: FlatMappingAST,
+  currentOutputPropAST: ObjectPropExpression,
+  isLastPart: boolean,
+) {
+  const blockExpr = getLastElement(currentOutputPropAST.value.parts) as Expression;
+  const objectExpr = isLastPart ? createObjectExpression() : blockExpr?.statements?.[0];
+  validateResultOfAllFilter(objectExpr, flatMapping);
+  return objectExpr;
+}
+
 function processAllFilter(
   flatMapping: FlatMappingAST,
   currentOutputPropAST: ObjectPropExpression,
+  isLastPart: boolean,
 ): ObjectExpression {
   const { inputExpr: currentInputAST } = flatMapping;
-  const filterIndex = currentInputAST.parts.findIndex(
-    (part) => part.type === SyntaxType.OBJECT_FILTER_EXPR,
-  );
-
+  const filterIndex = findAllFilterIndex(currentInputAST);
   if (filterIndex === -1) {
-    if (currentOutputPropAST.value.type === SyntaxType.OBJECT_EXPR) {
-      const currObjectExpr = currentOutputPropAST.value as ObjectExpression;
-      currentOutputPropAST.value = getPathExpressionForAllFilter(currentInputAST, currObjectExpr);
-      return currObjectExpr;
-    }
-    if (isPathWithEmptyPartsAndObjectRoot(currentOutputPropAST.value)) {
-      return currentOutputPropAST.value.root as ObjectExpression;
+    const objectExpr = handleAllFilterIndexNotFound(
+      currentInputAST,
+      currentOutputPropAST,
+      isLastPart,
+    );
+    if (objectExpr) {
+      return objectExpr;
     }
   } else {
-    const matchedInputParts = currentInputAST.parts.splice(0, filterIndex + 1);
-    if (isPathWithEmptyPartsAndObjectRoot(currentOutputPropAST.value)) {
-      currentOutputPropAST.value = currentOutputPropAST.value.root;
-    }
-
-    if (currentOutputPropAST.value.type !== SyntaxType.PATH) {
-      matchedInputParts.push(createBlockExpression(currentOutputPropAST.value));
-      currentOutputPropAST.value = getPathExpressionForAllFilter(
-        currentInputAST,
-        currentInputAST.root,
-        matchedInputParts,
-      );
-    }
-    currentInputAST.root = undefined;
+    handleAllFilterIndexFound(currentInputAST, currentOutputPropAST, filterIndex, isLastPart);
   }
-
-  const blockExpr = getLastElement(currentOutputPropAST.value.parts) as Expression;
-  const objectExpr = blockExpr?.statements?.[0] || EMPTY_EXPR;
-  validateResultOfAllFilter(objectExpr, flatMapping);
-  return objectExpr;
+  return getNextObjectExpressionForAllFilter(flatMapping, currentOutputPropAST, isLastPart);
 }
 
 function isWildcardSelector(expr: Expression): boolean {
@@ -201,12 +257,30 @@ function handleNextPart(
 ): ObjectExpression | undefined {
   const nextOutputPart = flatMapping.outputExpr.parts[partNum];
   if (nextOutputPart.filter?.type === SyntaxType.ALL_FILTER_EXPR) {
-    return processAllFilter(flatMapping, currentOutputPropAST);
+    const objectExpr = processAllFilter(
+      flatMapping,
+      currentOutputPropAST,
+      partNum === flatMapping.outputExpr.parts.length - 1 && !nextOutputPart.options?.index,
+    );
+    if (nextOutputPart.options?.index) {
+      objectExpr.props.push({
+        type: SyntaxType.OBJECT_PROP_EXPR,
+        key: nextOutputPart.options?.index,
+        value: {
+          type: SyntaxType.PATH,
+          root: nextOutputPart.options?.index,
+          parts: [],
+        },
+      });
+    }
+    return objectExpr;
   }
   if (nextOutputPart.filter?.type === SyntaxType.ARRAY_INDEX_FILTER_EXPR) {
     return processArrayIndexFilter(
+      flatMapping,
       currentOutputPropAST,
       nextOutputPart.filter as IndexFilterExpression,
+      partNum === flatMapping.outputExpr.parts.length - 1,
     );
   }
   if (isWildcardSelector(nextOutputPart)) {