From 70f9f13d344de18fb135aa80ad79ba8e18d40ad5 Mon Sep 17 00:00:00 2001 From: AFine-gs <69924417+AFine-gs@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:56:14 -0400 Subject: [PATCH] support_for_multi_expressions_and_tds_joins_with_N_2_columns_support_for_multi_expressions.patch (#3141) --- .../core/pure/lineage/scanProperties.pure | 2 +- .../core_analytics_lineage/fullAnalytics.pure | 21 +-- .../tests/lineageTests.pure | 120 +++++++++++++++--- .../lineage/scanRelations/scanRelations.pure | 63 ++++----- 4 files changed, 138 insertions(+), 68 deletions(-) diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure index fc176eadc77..54d02a6e732 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure @@ -85,7 +85,7 @@ function <> meta::pure::lineage::scanProperties::dummyNodeForCla ^PropertyPathNode( class = $c, property= getDummyProperty()); } -function <> meta::pure::lineage::scanProperties::findAndAddChildrenAtTheEnd(tree: PropertyPathTree[1], path: PropertyPathNode[*], toAdd: PropertyPathTree[*]):PropertyPathTree[1] +function meta::pure::lineage::scanProperties::findAndAddChildrenAtTheEnd(tree: PropertyPathTree[1], path: PropertyPathNode[*], toAdd: PropertyPathTree[*]):PropertyPathTree[1] { if($path->isEmpty(), | ^$tree(children += $toAdd), diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure index a365ac266c2..27b62739e47 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure @@ -86,27 +86,28 @@ function meta::analytics::lineage::computeLineage(f:FunctionDefinition[1], let modelToModelMappings = $mappings->init(); let sourceMapping = $mappings->last()->toOne(); - let funcBody = $f.expressionSequence->at(0)->evaluateAndDeactivate(); - let updatedFuncBody = $funcBody->meta::pure::lineage::analytics::inlineQualifiedProperties(newMap([]->cast(@Pair), VariableExpression->classPropertyByName('name')->cast(@Property)), $f->openVariableValues(), $extensions); - let propertyTrees = $updatedFuncBody->buildMultiLevelPropertyTrees($modelToModelMappings, $extensions); + let funcBody = $f.expressionSequence->evaluateAndDeactivate(); + let updatedFuncBody = $funcBody->map(e|$e->meta::pure::lineage::analytics::inlineQualifiedProperties(newMap([]->cast(@Pair), VariableExpression->classPropertyByName('name')->cast(@Property)), $f->openVariableValues(), $extensions)); + let propertyTrees = $updatedFuncBody->map(e|$e->buildMultiLevelPropertyTrees($modelToModelMappings, $extensions)); + + let combinedTrees = if( $propertyTrees->size()>1,|$propertyTrees->first()->toOne()->findAndAddChildrenAtTheEnd([],$propertyTrees->tail()),|$propertyTrees->toOne()); let reprocessedFuncBody = if($modelToModelMappings->isEmpty() && $mappings->size() == 1, |$updatedFuncBody, - |$updatedFuncBody->cast(@FunctionExpression)->meta::pure::mapping::modelToModel::chain::allReprocess([], $modelToModelMappings, $extensions, noDebug()).res); + |$updatedFuncBody->map(e|$e->cast(@FunctionExpression)->meta::pure::mapping::modelToModel::chain::allReprocess([], $modelToModelMappings, $extensions, noDebug()).res)); let vars = $f->functionType().parameters->evaluateAndDeactivate()->map(p| pair($p.name, ^List(values = ^PlanVarPlaceHolder(name=$p.name, type = $p.genericType.rawType->toOne(), multiplicity=$p.multiplicity)))); let relationTree = if($f->functionReturnType().rawType->toOne()->_subTypeOf(TabularDataSet) && $r->isNotEmpty() && $modelToModelMappings->isEmpty(), | scanRelations(^LambdaFunction<{->Any[*]}>(expressionSequence = $funcBody), $sourceMapping, $r->toOne(), $vars, noDebug(), $extensions), - | scanRelations($propertyTrees->last()->toOne(), $sourceMapping)); - + | scanRelations($combinedTrees->last()->toOne(), $sourceMapping)); let classLineageMapping = if($modelToModelMappings->isEmpty() && $mappings->size() == 1, | $sourceMapping, | $modelToModelMappings); ^FunctionAnalytics ( - databaseLineage = $f->toFlowDatabase($sourceMapping, $propertyTrees->last()->toOne(), $r)->toGraph(), - classLineage = $f->toFlowClass($propertyTrees->at(0), $classLineageMapping)->toGraph(), - functionTrees = $propertyTrees, + databaseLineage = $f->toFlowDatabase($sourceMapping, $combinedTrees, $r)->toGraph(), + classLineage = $f->toFlowClass($combinedTrees, $classLineageMapping)->toGraph(), + functionTrees = $combinedTrees, relationTree = $relationTree, - reportLineage = buildReportLineage($reprocessedFuncBody, $sourceMapping) + reportLineage = buildReportLineage($reprocessedFuncBody->last()->toOne(), $sourceMapping) ); } diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure index 53af3b6ec2f..dad8cb95911 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure @@ -19,7 +19,7 @@ import meta::relational::tests::functions::pureToSqlQuery::calendarAggregations: function <> meta::analytics::lineage::tests::relational::calendarAggregations():Boolean[1] { - let r = meta::analytics::lineage::computeLineage(| Employee.all() + let lineage = meta::analytics::lineage::computeLineage(| Employee.all() ->groupBy( [p|$p.hireDate], [ agg(p | annualized($p.hireDate, 'NY', %2022-11-16, $p.fteFactor), y | $y->sum()) ], @@ -27,8 +27,10 @@ function <> meta::analytics::lineage::tests::re ->filter(x | isNotEmpty($x.getNumber('annualized'))) ->restrict(['includedDate']) ,EmployeeMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); - assertSameElements(['Lambda','tb_EmployeeDatabasedefaultEmployeeTable','db_EmployeeDatabase'], $r.databaseLineage.nodes.data.id); - + meta::analytics::lineage::assertLineage(['Lambda', 'db_EmployeeDatabase', 'tb_EmployeeDatabasedefaultEmployeeTable'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::functions::pureToSqlQuery::calendarAggregations::Employee', 'pack_meta::pure::tds', 'pack_meta::relational::tests::functions::pureToSqlQuery::calendarAggregations'], + '[annualized: [EmployeeTable.fteFactor , EmployeeTable.hireDate ], includedDate: [EmployeeTable.hireDate ]]', + $lineage); } @@ -47,8 +49,10 @@ function <> meta::analytics::lineage::tests::re let lineage = computeLineage($fn,meta::relational::tests::tds::tdsJoin::testJoinTDSMappingTwoDatabaseWithColumnsMappedViaJoinsAndDynaFunction, meta::relational::tests::tds::tdsJoin::twoDBRunTime(), meta::relational::extension::relationalExtensions()); - assertSameElements(['Lambda', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultfirmTable', 'tb_database2defaultfirmTable', 'db_dbInc', 'db_database2'],$lineage.databaseLineage.nodes.data.id); - + meta::analytics::lineage::assertLineage(['Lambda', 'db_database2', 'db_dbInc', 'tb_database2defaultfirmTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Firm', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::tds::tdsJoin'], + '[eID: [firmTable.ID , firmTable.ID , personTable.FIRMID ], fID: [firmTable.ID ], firstName: [personTable.FIRSTNAME ], legalName: [firmTable.LEGALNAME ], managerID: [personTable.MANAGERID ]]', + $lineage); } @@ -78,8 +82,33 @@ function <> meta::analytics::lineage::tests::re let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); - assertSameElements(['Lambda', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultaddressTable', 'db_dbInc'],$lineage.databaseLineage.nodes.data.id); + meta::analytics::lineage::assertLineage(['Lambda', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'db_dbInc'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::model::simple'], + '[addressName2: [addressTable.ID , addressTable.NAME , firmTable.ID , personTable.ADDRESSID , personTable.FIRMID ], addressName: [addressTable.ID , addressTable.NAME , firmTable.ID , personTable.ADDRESSID , personTable.FIRMID ]]', + $lineage); +} + +###Pure +import meta::pure::lineage::scanRelations::*; +import meta::relational::extension::*; +import meta::relational::metamodel::join::*; +import meta::analytics::lineage::*; +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; +function <> meta::analytics::lineage::tests::relational::testTDSJoinOnQualifiedProperty():Boolean[1] +{ + let fn= {|Person.all()->meta::pure::tds::project([col(p|$p.name, 'name')]) + ->join(Firm.all()->project([col(p|$p.legalName, 'eName')]), JoinType.INNER, {a,b|$a.getString('name') == $b.getString('eName');}); + }; + + + + let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbInc', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::model::simple'], + '[eName: [firmTable.LEGALNAME ], name: [personTable.FIRSTNAME , personTable.LASTNAME ]]', + $lineage); } @@ -97,8 +126,10 @@ function <> meta::analytics::lineage::tests::re 'addressName' ])}; let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); - assertSameElements(['Lambda', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultaddressTable', 'db_dbInc'],$lineage.databaseLineage.nodes.data.id); - + meta::analytics::lineage::assertLineage(['Lambda', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'db_dbInc'], + ['Lambda', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[addressName: [addressTable.ID , addressTable.NAME , firmTable.ID , firmTable.LEGALNAME , personTable.ADDRESSID , personTable.FIRMID ], legalName: [firmTable.LEGALNAME ]]', + $lineage); } ###Pure @@ -121,10 +152,13 @@ function <> meta::analytics::lineage::tests::re 'addressName2' ]) , JoinType.INNER, {a,b|$a.getString('legalName') == $b.getString('firstname') && $a.getString('addressName') == $b.getString('addressName2') ;});}; let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); - assertSameElements(['Lambda', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'db_dbInc'],$lineage.databaseLineage.nodes.data.id); + meta::analytics::lineage::assertLineage(['Lambda', 'tb_dbIncdefaultpersonTable', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'db_dbInc'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::model::simple'], + '[addressName2: [addressTable.ID , addressTable.NAME , personTable.ADDRESSID ], addressName: [addressTable.ID , addressTable.NAME , firmTable.ID , personTable.ADDRESSID , personTable.FIRMID ], firstname: [personTable.FIRSTNAME ], legalName: [firmTable.LEGALNAME ]]', + $lineage); +} -} ###Pure import meta::analytics::lineage::*; @@ -135,14 +169,64 @@ function <> meta::analytics::lineage::tests::re let fn = { |Order.all()-> filter( p| $p.pnlContact->exists(l|$l.address.name == 'Main St' && $l.firm.legalName == 'Co' )) ->project([ f|$f.quantity ], ['quantity'])}; let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); - assertSameElements(['Lambda', 'db_db', 'db_dbInc', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable', 'tb_dbdefaultaccountTable', 'tb_dbdefaultorderPnlTable', 'tb_dbdefaultorderTable', 'tb_dbdefaultsalesPersonTable'],$lineage.databaseLineage.nodes.data.id); + meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'db_dbInc', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable', 'tb_dbdefaultaccountTable', 'tb_dbdefaultorderPnlTable', 'tb_dbdefaultorderTable', 'tb_dbdefaultsalesPersonTable'], + ['Lambda', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Order', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[quantity: [orderTable.quantity ]]', + $lineage); +} + +###Pure +import meta::relational::metamodel::join::*; +import meta::analytics::lineage::*; +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; +function <> meta::analytics::lineage::tests::relational::testFunctionWithLet():Boolean[1] +{ + let fn= {| let co = 'co'; + let foo = Firm.all()->filter(f|$f.legalName==$co)->project([ f|$f.legalName, + f|$f.employees.address.name + ], + [ + 'legalName', + 'addressName' + ]);}; + let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbInc', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[addressName: [addressTable.ID , addressTable.NAME , firmTable.ID , personTable.ADDRESSID , personTable.FIRMID ], legalName: [firmTable.LEGALNAME ]]', + $lineage); +} + + + +###Pure +import meta::pure::executionPlan::*; +import meta::relational::metamodel::join::*; +import meta::analytics::lineage::*; +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; +function <> meta::analytics::lineage::tests::relational::testFunctionWithMultiExpression():Boolean[1] +{ + let fn= {| + let people = Person.all().firstName; + Firm.all()->filter(f|$f.legalName->in($people))->project([ f|$f.legalName + + ], + [ + 'legalName' + ]);}; + let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbInc', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[legalName: [firmTable.LEGALNAME ]]', + $lineage); } ###Pure import meta::pure::graphFetch::execution::*; import meta::relational::tests::model::simple::*; -function <> meta::analytics::lineage::fullAnalyticasTest::testForSimpleRelationalGraphFetch():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForSimpleRelationalGraphFetch():Boolean[1] { let tree = #{ Person { @@ -164,7 +248,7 @@ function <> meta::analytics::lineage::fullAnalyticasTest::testForSimp } -function <> meta::analytics::lineage::fullAnalyticasTest::testForComplextRelationalGraphFetch():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForComplextRelationalGraphFetch():Boolean[1] { let tree = #{ Person { @@ -193,7 +277,7 @@ function <> meta::analytics::lineage::fullAnalyticasTest::testForComp ###Pure import meta::relational::tests::milestoning::*; import meta::pure::graphFetch::execution::*; -function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithMilestoning():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForRelationalGraphFetchWithMilestoning():Boolean[1] { let tree = #{ Order { @@ -220,7 +304,7 @@ function <> meta::analytics::lineage::fullAnalyticasTest::testForRela ###Pure import meta::pure::graphFetch::tests::XStore::inMemoryAndRelational::*; import meta::pure::graphFetch::execution::*; -function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithCrossStore():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForRelationalGraphFetchWithCrossStore():Boolean[1] { let tree = #{ Trade { @@ -246,7 +330,7 @@ function <> meta::analytics::lineage::fullAnalyticasTest::testForRela ###Pure import meta::pure::graphFetch::execution::*; import meta::relational::graphFetch::tests::union::rootLevel::*; -function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithRootUnionSameStore():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForRelationalGraphFetchWithRootUnionSameStore():Boolean[1] { let tree = #{ Trade { @@ -273,7 +357,7 @@ function <> meta::analytics::lineage::fullAnalyticasTest::testForRela $lineage); } -function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithRootUnionCrossStore():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForRelationalGraphFetchWithRootUnionCrossStore():Boolean[1] { let tree = #{ Trade { @@ -302,7 +386,7 @@ function <> meta::analytics::lineage::fullAnalyticasTest::testForRela ###Pure import meta::relational::tests::model::simple::*; import meta::pure::graphFetch::execution::*; -function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithPropertyUnion():Boolean[1] +function <> meta::analytics::lineage::tests::relational::graphFetch::testForRelationalGraphFetchWithPropertyUnion():Boolean[1] { let tree = #{ Firm{ diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure index 105d83cb5fe..bbaefb1a0c2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::functions::pureToSqlQuery::metamodel::*; import meta::pure::executionPlan::*; import meta::pure::router::externalFormat::metamodel::clustering::*; import meta::pure::router::metamodel::clustering::*; @@ -355,41 +356,30 @@ function meta::pure::lineage::scanRelations::scanRelations(f:FunctionDefinition< let childTrees = meta::pure::lineage::scanRelations::generatRelationalTrees($f, $m, $r, $vars, $debug, $extensions); - let rootTree = ^RelationTree ( root = true, children = $childTrees->sortBy(x:RelationTree[1] | if($x.join->isEmpty(), | $x.relation->cast(@NamedRelation).name->toOne() + '->' + $x.columns.name->joinStrings('[',',',']'), | $x.relation->cast(@NamedRelation).name->toOne() + '->' + $x.columns.name->joinStrings('[',',',']') + '->' + $x.join.name->toOne())) ); - if($debug.debug, | print('Generated Root Tree - \n\n' + $rootTree->relationTreeAsString($debug.space) + '\n\n'), |[]); $rootTree; } -function meta::pure::lineage::scanRelations::collectStoreMappingCluster(cluster:ClusteredValueSpecification[1], inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*]):ClusterAndVars[*] +function meta::pure::lineage::scanRelations::collectStoreMappingCluster(cluster:ValueSpecification[1], inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*]):ClusterAndVars[*] { $cluster->match([ sc:StoreMappingClusteredValueSpecification[1] | let DBCluster = $sc->filter(cvs | $cvs.store->instanceOf(Database)); //TODO: Refactor to support additional store types if($DBCluster->isNotEmpty() ,| ^ClusterAndVars(cluster=$DBCluster->toOne() ),|[]);, - - pl:PlatformClusteredValueSpecification[1] | $pl.val->processPlatformValueSpecification($inScopeVars,$extensions); , - a:Any[*]| fail('Found unsupported cluster type '+ $a->type().name->toOne()) ; []; - ]); - -} - -function meta::pure::lineage::scanRelations::processPlatformValueSpecification(val:ValueSpecification[1], inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*]):ClusterAndVars[*] -{ - $val->match([ + pl:PlatformClusteredValueSpecification[1] | $pl.val->collectStoreMappingCluster($inScopeVars,$extensions); , + s:meta::pure::router::store::metamodel::StoreMappingRoutedValueSpecification[1] | $s.value-> collectStoreMappingCluster($inScopeVars,$extensions), fn:SimpleFunctionExpression[1] | if ($fn.func ==letFunction_String_1__T_m__T_m_ && $fn.parametersValues->at(1)->instanceOf(ClusteredValueSpecification), //TODO: This might need a more robust function processor for different functions if they become relevent | let varName = $fn.parametersValues->at(0)->cast(@InstanceValue).values->toOne()->toString(); let value = $fn.parametersValues->at(1)->cast(@ClusteredValueSpecification)->collectStoreMappingCluster($inScopeVars,$extensions); ^ClusterAndVars(var=$varName,cluster=$value.cluster->last()); , |[] - );, - - a:Any[*]| []; + );, + a:Any[*]| fail('Found unsupported cluster type '+ $a->type().name->toOne()) ; []; ]); } @@ -411,16 +401,12 @@ function <> meta::pure::lineage::scanRelations::generatRelationa { let routed = $f->routeFunction($m, $r, $extensions, $debug); let routedFunction = $routed->evaluateAndDeactivate()->toOne(); - let inScopeVars = $f.expressionSequence->evaluateAndDeactivate()->fold({vs, a | - - if ($vs->isLetFunction(), - | let varName = $vs->meta::pure::router::utils::extractLetVariableName(); - let varExprs = $vs->findVariableExpressionsInValueSpecification(); - + let inScopeVars = $f.expressionSequence->evaluateAndDeactivate()->fold({vs, a | if ($vs->isLetFunction() , + | let varName = $vs->meta::pure::router::utils::extractLetVariableName(); + let varExprs = $vs->findVariableExpressionsInValueSpecification(false); let inScopeVars = $a->keyValues(); let unavailableVars = $varExprs.name->forAll(var | $var->in($inScopeVars.first)); - assert($unavailableVars, 'Unable to resolve var(s): '+ $varExprs.name->filter(var | !$var->in($inScopeVars.first))->joinStrings()); - + assert($unavailableVars, 'Unable to resolve var(s): '+ $varExprs.name->filter(var | !$var->in($inScopeVars.first))->joinStrings()); let re = $vs->reactivate($a); $a->put($varName->toOne(), ^List(values=$re));, @@ -428,22 +414,25 @@ function <> meta::pure::lineage::scanRelations::generatRelationa ); }, $f->openVariableValues()->putAll($vars)); - - let allClusters = $routedFunction.expressionSequence->evaluateAndDeactivate()->cast(@ClusteredValueSpecification)->map(c|$c-> meta::pure::lineage::scanRelations::collectStoreMappingCluster($inScopeVars,$extensions) ); + let allClusters = $routedFunction.expressionSequence->evaluateAndDeactivate()->cast(@ClusteredValueSpecification)->map(c|$c-> meta::pure::lineage::scanRelations::collectStoreMappingCluster($inScopeVars,$extensions) ); let AllTrees = $allClusters->fold({ v, a| let fn = $v.cluster->toOne().val->byPassValueSpecificationWrapper()->cast(@FunctionExpression); let context = ^RelationalExecutionContext(); - let SQL = $v.cluster->toOne().val->byPassValueSpecificationWrapper()->cast(@FunctionExpression)->toSQLQuery($m, $a.vars, $context, noDebug(), $extensions) ; + let SQL = $v.cluster->toOne().val->byPassValueSpecificationWrapper()->cast(@FunctionExpression)->toSQLQuery($m, $a.vars, $context, $debug, $extensions) ; if($debug.debug, | print('Generated SQL Query - ' + sqlQueryToStringPretty($SQL, DatabaseType.H2, 'GMT', [], $extensions) + '\n\n'), |[]); let possibleClass = $fn.genericType.rawType->toOne(); - let updatedVars = if($v.var->isNotEmpty(), + let updatedVars = if($v.var->isNotEmpty() && $SQL ->instanceOf(TdsSelectSqlQuery), | let class = if ($possibleClass == Any, | let getAllClass = findMainClassInGetAllExpression($fn); if($getAllClass->isEmpty(), | $possibleClass, | $getAllClass);, | $possibleClass ); let tdsType = $class->meta::relational::mapping::TDSSelectQueryToTDSResultType($SQL->cast(@TdsSelectSqlQuery),$context); let varName = $v.var->toOne(); let varset = ^PlanSetPlaceHolder(name=$varName,tdsColumns = $tdsType->match([x:TDSResultType[1]|$x.tdsColumns, a:Any[1]|[]])); $inScopeVars->put($varName,^List(values=$varset));, - |$a.vars); - let updatedTree= generateRelationTreeFromRelationalOperationElement($SQL, noDebug(), $extensions)->concatenate($a.tree); + | if( $v.var->isNotEmpty() && $fn->instanceOf(DataType), + | let varName = $v.var->toOne(); + $inScopeVars->put($varName,^List(values= ^VarSetPlaceHolder(varName= $v.var->toOne())));, + |$a.vars)); + + let updatedTree= generateRelationTreeFromRelationalOperationElement($SQL, $debug, $extensions)->concatenate($a.tree); ^RelationalTreeAndVars(tree= $updatedTree, vars=$updatedVars); } , ^RelationalTreeAndVars(vars=$inScopeVars)).tree; @@ -611,8 +600,7 @@ function <> meta::pure::lineage::scanRelations::processJoinFromT let alias = if($tac.alias.name == $sourceAliasName, | $sourceOperation, |$targetOperation); let tree = if($tac.alias.name == $sourceAliasName, | $sourceTree, |$targetTree); let sourceColumn = $tac->extractSourceColumn($alias, $relationIndentifiers, $tree); - - if($sourceColumn->isEmpty(), + if($sourceColumn->isEmpty() || ($sourceColumn->size() > 1), |pair($tac, ^SQLNull()), | assert( $sourceColumn.owner->cast(@NamedRelation)->isNotEmpty(), 'Expecting column to have an owner ' +$sourceColumn.name->makeString() + 'in ' + $j.joinName ); let namedRelation = $sourceColumn.owner->cast(@NamedRelation)->toOne(); @@ -635,7 +623,7 @@ function <> meta::pure::lineage::scanRelations::processJoinFromT |$nonNullPairs); let updatedAliases = $updatedNonNullPairs.second.alias->removeDuplicatesBy(ta | $ta.name + '@' + $ta.relation->getRelationName()); - if($updatedAliases->size() == 0, + if($updatedAliases->size() == 0 , |if($debug.debug, | print($debug.space + 'Expect this case to arise when join does not reference any columns. Example join operation - true, false, 1 = 1\n');, |[]); let sourceRelation = $sourceOperation->extractSourceRelation($relationIndentifiers); let orgJoin = $j.join; @@ -700,7 +688,7 @@ function <> meta::pure::lineage::scanRelations::isNull(d:DynaFun |false); } -function <> meta::pure::lineage::scanRelations::extractSourceColumn(tableAliasColumn:TableAliasColumn[1], sourceOperation:RelationalOperationElement[1], relationIdentifiers:String[*], sourceTree:RelationTree[1]):Column[0..1] +function <> meta::pure::lineage::scanRelations::extractSourceColumn(tableAliasColumn:TableAliasColumn[1], sourceOperation:RelationalOperationElement[1], relationIdentifiers:String[*], sourceTree:RelationTree[1]):Column[*] { $sourceOperation->match([ v:ViewSelectSQLQuery[1]| if(getRelationName($v.view)->in($relationIdentifiers), |$tableAliasColumn.column, |[]), @@ -714,20 +702,17 @@ function <> meta::pure::lineage::scanRelations::extractSourceCol ]); } -function <> meta::pure::lineage::scanRelations::extractSourceColumnFromSelectSQLQuery(select : SelectSQLQuery[1], tableAliasColumn:TableAliasColumn[1], relationIdentifiers:String[*], sourceTree:RelationTree[1]):Column[0..1] +function <> meta::pure::lineage::scanRelations::extractSourceColumnFromSelectSQLQuery(select : SelectSQLQuery[1], tableAliasColumn:TableAliasColumn[1], relationIdentifiers:String[*], sourceTree:RelationTree[1]):Column[*] { let reqTableAliasColumn = $select.columns->filter(col | let colName = $col->match([a:Alias[1] | $a.name, tac:TableAliasColumn[1] | $tac.column.name]); $colName == $tableAliasColumn.column.name;)->extractTableAliasColumns()->removeDuplicates(); - assert($reqTableAliasColumn->size() <= 1 ,|'Expected max 1 column. Found : ' + $reqTableAliasColumn->size()->toString() +' in ' +$reqTableAliasColumn.column.name->joinStrings()); - if($reqTableAliasColumn->isEmpty() || $select.data->isEmpty(), |[], |let rootTree = $select.data->toOne(); let childTrees = $rootTree->children()->map(child | $child->extractJoinChildren()); - let sourceOperation = $rootTree->concatenate($childTrees)->filter(tree | $tree.alias.name == $reqTableAliasColumn.alias.name).alias->toOne(); - $reqTableAliasColumn->toOne()->extractSourceColumn($sourceOperation, $relationIdentifiers, $sourceTree); + $reqTableAliasColumn->map(c|$c->extractSourceColumn( $rootTree->concatenate($childTrees)->filter(tree | $tree.alias.name == $c.alias.name).alias->toOne(), $relationIdentifiers, $sourceTree)); ); }