Improve handling of PlanVarPlaceHolders in platform executions (#2822)

finos · May 2, 2024 · cad726c · cad726c
1 parent 1787aed
commit cad726c
Show file tree

Hide file tree

Showing 8 changed files with 170 additions and 30 deletions.
diff --git a/...ine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/metaExtension.pure b/...ine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/metaExtension.pure
@@ -474,6 +474,25 @@ function meta::pure::functions::meta::findVariableExpressionsInValueSpecificatio
    findVariableExpressionsInValueSpecification($vs, true);
 }
 
+function meta::pure::functions::meta::findPlanVarPlaceHoldersInValueSpecification(vs: ValueSpecification[1]): meta::pure::executionPlan::PlanVarPlaceHolder[*]
+{
+   $vs->match([
+      f : FunctionExpression[1] | $f.parametersValues->evaluateAndDeactivate()->map(v | $v->findPlanVarPlaceHoldersInValueSpecification()),
+      i : InstanceValue[1] | $i.values->evaluateAndDeactivate()->map({x |
+         $x->match([
+            v : ValueSpecification[1] | $v->findPlanVarPlaceHoldersInValueSpecification(),
+            k : KeyExpression[1] | $k.expression->evaluateAndDeactivate()->findPlanVarPlaceHoldersInValueSpecification(),
+            l : LambdaFunction<Any>[1] | $l.expressionSequence->evaluateAndDeactivate()->map(e | $e->findPlanVarPlaceHoldersInValueSpecification()),
+            v : meta::pure::executionPlan::PlanVarPlaceHolder[1] | $v,
+            a : Any[1] | []
+         ])
+      }),
+      c : ClusteredValueSpecification[1] | $c.val->evaluateAndDeactivate()->findPlanVarPlaceHoldersInValueSpecification(),
+      r : RoutedValueSpecification[1] | $r.value->evaluateAndDeactivate()->findPlanVarPlaceHoldersInValueSpecification(),
+      a : Any[1] | []
+   ])
+}
+
 function meta::pure::functions::meta::findPropertyPathsInFunctionDefinition(func:FunctionDefinition<Any>[1]):Path<Nil,Any|*>[*]
 {
    $func.expressionSequence->evaluateAndDeactivate()->map(vs | $vs->collectPropertyPathsInValueSpecification([]));
@@ -652,17 +671,24 @@ function meta::pure::functions::meta::resolve(v:VariableExpression[1], vars:Map<
            if ($open->isEmpty(),
                  |[];,
                  |let res = $open.values;
-                  if($res->isEmpty(),
-                    |let multVal = ^MultiplicityValue(value=0);
-                     ^InstanceValue(multiplicity=PureZero, genericType=^GenericType(rawType=Nil), values=[])->evaluateAndDeactivate();,
-                    |let size = $res->size();
-                     let mult = if($size == 1,
-                                  |PureOne,
-                                  |let multVal = ^MultiplicityValue(value=$size);
-                                   ^Multiplicity(lowerBound=$multVal, upperBound=$multVal);
-                                );
-                     ^InstanceValue(multiplicity=$mult, genericType=$res->genericType(), values=$res)->evaluateAndDeactivate();
-                  );
+                  $res->match([
+                    {pv: meta::pure::executionPlan::PlanVarPlaceHolder[1] |
+                      ^InstanceValue(values=$pv, genericType=^GenericType(rawType=$pv.type), multiplicity=$pv.multiplicity->defaultIfEmpty(ZeroMany)->toOne())->evaluateAndDeactivate();
+                    },
+                    {a: Any[*] | 
+                      if($res->isEmpty(),
+                        |let multVal = ^MultiplicityValue(value=0);
+                         ^InstanceValue(multiplicity=PureZero, genericType=^GenericType(rawType=Nil), values=[])->evaluateAndDeactivate();,
+                        |let size = $res->size();
+                         let mult = if($size == 1,
+                                      |PureOne,
+                                      |let multVal = ^MultiplicityValue(value=$size);
+                                       ^Multiplicity(lowerBound=$multVal, upperBound=$multVal);
+                                    );
+                         ^InstanceValue(multiplicity=$mult, genericType=$res->genericType(), values=$res)->evaluateAndDeactivate();
+                      );
+                    }
+                  ]);
            );,
          | $val->toOne()
    );

diff --git a/...ed-core/src/main/resources/core/pure/platform/executionPlan/executionPlan_generation.pure b/...ed-core/src/main/resources/core/pure/platform/executionPlan/executionPlan_generation.pure
@@ -46,14 +46,16 @@ function meta::pure::platform::executionPlan::generation::processValueSpecificat
 function <<access.private>> meta::pure::platform::executionPlan::generation::defaultFunctionProcessor(fe:FunctionExpression[1], state:PlatformPlanGenerationState[1], extensions : Extension[*], debug:DebugContext[1]):ExecutionNode[1]
 {
    let children = $fe.parametersValues->evaluateAndDeactivate()->map(v|$v->recursivelyFetchClusteredValueSpecification(false))->map(v|$v->processValueSpecification($state, $extensions, $debug));
-   let funcParams = $fe->findVariableExpressionsInValueSpecification(false)->removeDuplicatesBy(v | $v.name);
-   let varInputs = $funcParams->map(v | ^VariableInput(name = $v.name->toOne(), type = $v.genericType->cast(@GenericType).rawType->toOne(), multiplicity = $v.multiplicity->toOne()));
+   let variableExpressions = $fe->findVariableExpressionsInValueSpecification(false)->removeDuplicatesBy(v | $v.name);
+   let variableExpressionInputs = $variableExpressions->map(v | ^VariableInput(name = $v.name->toOne(), type = $v.genericType->cast(@GenericType).rawType->toOne(), multiplicity = $v.multiplicity->toOne()));
+   let varPlaceHolders = $fe->findPlanVarPlaceHoldersInValueSpecification()->removeDuplicatesBy(v | $v.name);
+   let varPlaceHolderInputs = $varPlaceHolders->map(v | ^VariableInput(name = $v.name, type = $v.type, multiplicity = $v.multiplicity->toOne('Multiplicity not available for PlanVarPlaceHolder - \'' + $v.name + '\'')));
 
    ^PureExpressionPlatformExecutionNode
    (
       expression = $fe,
       resultType = ^ResultType(type=$fe.genericType.rawType->toOne()),
-      requiredVariableInputs = $varInputs,
+      requiredVariableInputs = $variableExpressionInputs->concatenate($varPlaceHolderInputs),
       resultSizeRange = $fe.multiplicity,
       executionNodes = $children,
       fromCluster = generatePlatformClusterForFunction($fe, $state.inScopeVars)

diff --git a/...d-engine-pure-code-compiled-core/src/main/resources/core/pure/router/printer/printer.pure b/...d-engine-pure-code-compiled-core/src/main/resources/core/pure/router/printer/printer.pure
@@ -79,7 +79,6 @@ function meta::pure::router::printer::asString(f:ValueSpecification[1], pref:Pre
                                                                      a:meta::pure::graphFetch::GraphFetchTree[1]|meta::pure::graphFetch::routing::asString($a, false),
                                                                      b:KeyExpression[1]|'^KeyExpression(key = ' + $b.key->asString($pref) + ', value = '+ $b.expression->asString($pref) + ')',
                                                                      a:BasicColumnSpecification<Any>[1]| '^BasicColumnSpecification<' + $a->genericType().typeArguments.rawType.name->makeString() + '>(name = \'' + $a.name + '\', func = ' + $a.func->asString($pref) + ')';,
-                                                                     r:RoutedVariablePlaceHolder[1]|'$'+$r.name,
                                                                      p:meta::pure::executionPlan::PlanVarPlaceHolder[1]|'$'+$p.name,
                                                                      a:Class<Any>[1]|if (efq($pref),|$a->elementToPath(),|$a->genericType().rawType.name->toOne()+' '+$a->toString()),
                                                                      a:Any[1]|$a->genericType().rawType.name->toOne()+' '+$a->toString();

diff --git a/...egend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/router_main.pure b/...egend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/router_main.pure
@@ -26,10 +26,6 @@ import meta::pure::router::printer::*;
 import meta::pure::router::routing::*;
 import meta::pure::router::utils::*;
 import meta::core::runtime::*;
-Class meta::pure::router::RoutedVariablePlaceHolder
-{
-   name : String[1];
-}
 
 function meta::pure::router::routeFunction(f:FunctionDefinition<Any>[1], extensions:Extension[*]):FunctionDefinition<Any>[1]
 {
@@ -53,16 +49,14 @@ function meta::pure::router::routeFunction(f:FunctionDefinition<Any>[1], exeCtx:
       assert($unavailableVars->isEmpty(), 'Unable to resolve var(s): ' + $unavailableVars->joinStrings(','));
 
       let unResolvedVars  = $reqVars->filter(v|let resolved = $a.second->get($v.name);
-                                               $resolved.values->isEmpty() || $resolved.values->at(0)->instanceOf(RoutedVariablePlaceHolder););
+                                               $resolved.values->isEmpty() || $resolved.values->at(0)->instanceOf(meta::pure::executionPlan::PlanVarPlaceHolder););
 
       if($unResolvedVars->isNotEmpty(),
         {|
           let unRoutedFunction = ^$l(expressionSequence = $vs, openVariables=$a.second->keys());
           let routedFunction   = routeFunction($unRoutedFunction, getPlatformRoutingStrategy(), $exeCtx, $a.second, $extensions, $debug);
           let deactivatedEs = $routedFunction.expressionSequence->evaluateAndDeactivate();
-          let vars             = $a.second->put($varName->toOne(), ^List<meta::pure::executionPlan::PlanVarPlaceHolder>(values=^meta::pure::executionPlan::PlanVarPlaceHolder(type = $deactivatedEs->last().genericType.rawType->toOne(), 
-                                                                                                                        name=$varName->toOne(), multiplicity = $deactivatedEs->last()->toOne().multiplicity, 
-                                                                                                                        supportsStream = $deactivatedEs->last()->toOne().multiplicity->isToMany())));
+          let vars             = $a.second->put($varName->toOne(), createPlanVarPlaceHolderInScopeVar($varName->toOne(), $deactivatedEs->last()->toOne()));
 
           ^$a
           (
@@ -81,7 +75,8 @@ function meta::pure::router::routeFunction(f:FunctionDefinition<Any>[1], exeCtx:
                 |let re   = $vs->reactivate($a.second);
                  let vars = $a.second->put($varName->toOne(), ^List<Any>(values=$re));
                  ^$a(first=$a.first, second = $vars);,
-                |let vars   = $a.second->put($varName->toOne(), ^List<RoutedVariablePlaceHolder>(values=^RoutedVariablePlaceHolder(name=$varName->toOne())));
+                |let deactivatedEs = $routed.expressionSequence->evaluateAndDeactivate();
+                 let vars   = $a.second->put($varName->toOne(), createPlanVarPlaceHolderInScopeVar($varName->toOne(), $deactivatedEs->last()->toOne()));
                  ^$a
                  (
                    first = ^$routed(expressionSequence = $a.first.expressionSequence->concatenate($routed.expressionSequence)->evaluateAndDeactivate()->toOneMany())->cast(@FunctionDefinition<Any>),
@@ -103,7 +98,7 @@ function meta::pure::router::routeFunction(f:FunctionDefinition<Any>[1], exeCtx:
   let vs      = $expressions->last()->toOne();
   let varName = $vs->extractLetVariableName();
   let vars    = if($varName->isNotEmpty(),
-                   | $initStatements.second->put($varName->toOne(), ^List<RoutedVariablePlaceHolder>(values=^RoutedVariablePlaceHolder(name=$varName->toOne()))),
+                   | $initStatements.second->put($varName->toOne(), createPlanVarPlaceHolderInScopeVar($varName->toOne(), $vs->cast(@FunctionExpression).parametersValues->evaluateAndDeactivate()->at(1))), // Since its a let function, use ->at(1) parameter
                    | $initStatements.second);
   let newF    = ^$l(expressionSequence    = $vs,
                     openVariables         = $vars->keys(),
@@ -113,6 +108,18 @@ function meta::pure::router::routeFunction(f:FunctionDefinition<Any>[1], exeCtx:
   ^$routed(expressionSequence = $initStatements.first.expressionSequence->tail()->concatenate($routed.expressionSequence)->evaluateAndDeactivate()->toOneMany())->cast(@FunctionDefinition<Any>);
 }
 
+function <<access.private>> meta::pure::router::createPlanVarPlaceHolderInScopeVar(varName: String[1], vs: ValueSpecification[1]): List<meta::pure::executionPlan::PlanVarPlaceHolder>[1]
+{
+   ^List<meta::pure::executionPlan::PlanVarPlaceHolder>(
+      values = ^meta::pure::executionPlan::PlanVarPlaceHolder(
+         name = $varName,
+         type = $vs.genericType.rawType->toOne(), 
+         multiplicity = $vs.multiplicity,
+         supportsStream = $vs.multiplicity->isToMany()
+      )
+   )
+}
+
 function meta::pure::router::routeFunction(f:FunctionDefinition<Any>[1], routingStrategy:RoutingStrategy[1], exeCtx: meta::pure::runtime::ExecutionContext[1], inScopeVars:Map<String, List<Any>>[0..1], extensions:meta::pure::extension::Extension[*], debug:DebugContext[1]):FunctionDefinition<Any>[1]
 {
   routeFunction($f, $routingStrategy, $exeCtx, $inScopeVars, false, $extensions, $debug);

diff --git a/legend-engine-core/legend-engine-core-query-pure-http-api/pom.xml b/legend-engine-core/legend-engine-core-query-pure-http-api/pom.xml
@@ -247,6 +247,11 @@
             <artifactId>legend-engine-xt-relationalStore-javaPlatformBinding-pure</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-code-core-extension</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>