From 6c5d72e916d39c5d22ba25cfa2ab41341d496e8a Mon Sep 17 00:00:00 2001 From: AFine-gs <69924417+AFine-gs@users.noreply.github.com> Date: Wed, 20 Nov 2024 07:02:46 -0500 Subject: [PATCH] Fix Col lineage with derivedProperties (#3251) fix col lineage with let expressions --- .../core/pure/lineage/scanProject.pure | 30 +++++++++++-------- .../tests/lineageTests.pure | 21 +++++++++++++ .../lineage/scanRelations/scanRelations.pure | 10 +++++-- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure index d31d8dda1f8..f7480485728 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure @@ -53,19 +53,11 @@ function <> meta::pure::lineage::scanProject::scanProjectRecursi ); ), pair('meta::pure::tds::project_T_MANY__ColumnSpecification_MANY__TabularDataSet_1_', - |let ins = $fe.parametersValues->at(1)->evaluateAndDeactivate(); - $ins->match( - [ - i:InstanceValue[1] | - let cols = $i.values->evaluateAndDeactivate()->cast(@SimpleFunctionExpression); - let colSpecs = $cols->map(c | $c->reactivate($vars)->cast(@BasicColumnSpecification)); - ^Project(projectfuncEntryPoint = $fe, - columns = zip($colSpecs.name, $colSpecs.func->cast(@FunctionDefinition)));, - a:Any[*]|[] - ] - ); - - ), + |let cols = $fe.parametersValues->at(1)->evaluateAndDeactivate(); + let columns = $cols->map(c|$c->meta::pure::lineage::scanProject::extractCol($vars)); + ^Project(projectfuncEntryPoint = $fe, + columns = zip($columns.name, $columns.func->cast(@FunctionDefinition))); + ), pair('meta::pure::tds::groupBy_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__TabularDataSet_1_', | let funcs = $fe.parametersValues->at(1)->match([s:InstanceValue[1]|$s.values, a:Any[*]|^Unknown()]); let aggregateFuncs = $fe.parametersValues->at(2)->scanAggregateValue(); @@ -109,6 +101,18 @@ function <> meta::pure::lineage::scanProject::scanProjectRecursi ); } +function <> meta::pure::lineage::scanProject::extractCol(vs:ValueSpecification[1],vars:Map>[1]):BasicColumnSpecification[*] +{ + $vs->match( + [ + i:InstanceValue[1] | let cols = $i.values->evaluateAndDeactivate()->cast(@SimpleFunctionExpression); + let colSpecs = $cols->map(c | $c->reactivate($vars)->cast(@BasicColumnSpecification));, + s:SimpleFunctionExpression[1] | $s->reactivate($vars)->cast(@BasicColumnSpecification);, + a:Any[*]| [] ; + ] + ); +} + function <> meta::pure::lineage::scanProject::scanRootGraphFetchTree(root:meta::pure::graphFetch::RootGraphFetchTree[1]):Pair>[*] { let rootGetAllExpression = meta::pure::lineage::scanProject::createGetAllApplicationForRootGraphFetchTree($root); diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure index 99466f9f4e7..474d3de929a 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure @@ -541,4 +541,25 @@ function <], firmName: [FirmTable.ID , FirmTable.LEGAL_NAME , PersonTable.PERSON_DETAILS ]]', $lineage); +} + +###Pure +import meta::analytics::lineage::*; +import meta::relational::extension::*; +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; +function <> meta::analytics::lineage::tests::relational::testColWithPrimitiveLet():Boolean[1] +{ + let fn= {businessDate:Date[1], type:String[1]| + let adjustedType = $type+ 'test'; + meta::relational::tests::milestoning::Product.all($businessDate)->project(col(p|$p.classification($businessDate).type,'type'))->filter(f|$f.getString('type')=='adjustedType');}; + let mapping = meta::relational::tests::milestoning::milestoningmap; + let runtime = meta::external::store::relational::tests::testRuntime(); + + let lineage = computeLineage($fn, $mapping, $runtime, relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'tb_dbdefaultProductClassificationTable', 'tb_dbdefaultProductTable'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::milestoning::Product', 'meta::relational::tests::milestoning::ProductClassification', 'pack_meta::pure::tds', 'pack_meta::relational::tests::milestoning'], + '[type: [ProductClassificationTable.type , ProductClassificationTable.type , ProductTable.type ]]', + $lineage); } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure index 31dd40ebd57..2f51573523f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure @@ -393,7 +393,7 @@ Class meta::pure::lineage::scanRelations::RelationalTreeAndVars } function <> meta::pure::lineage::scanRelations::generatRelationalTrees(f:FunctionDefinition[1], m:Mapping[1], r:Runtime[1], vars:Pair>[*], debug:DebugContext[1], extensions:Extension[*]):RelationalTreeAndVars[1] -{ +{ let routed = $f->routeFunction($m, $r, $extensions, $debug); let routedFunction = $routed->evaluateAndDeactivate()->toOne(); let inScopeVars = $f.expressionSequence->evaluateAndDeactivate()->fold({vs, a | if ($vs->isLetFunction() , @@ -402,8 +402,12 @@ function <> meta::pure::lineage::scanRelations::generatRelationa let inScopeVars = $a->keyValues(); let unavailableVars = $varExprs.name->forAll(var | $var->in($inScopeVars.first)); assert($unavailableVars, 'Unable to resolve var(s): '+ $varExprs.name->filter(var | !$var->in($inScopeVars.first))->joinStrings()); - let re = $vs->reactivate($a); - $a->put($varName->toOne(), ^List(values=$re));, + + let value = if($vs.genericType.rawType->toOne()->instanceOf(PrimitiveType) && !$vs->cast(@SimpleFunctionExpression).parametersValues->at(1)->instanceOf(InstanceValue), + | ^PlanVarPlaceHolder(name=$varName->toOne(), type = $vs.genericType.rawType->toOne(), multiplicity=$vs.multiplicity), + | $vs->reactivate($a); + ); + $a->put($varName->toOne(), ^List(values=$value));, | $a; );