Skip to content

Commit

Permalink
Fix Col lineage with derivedProperties (#3251)
Browse files Browse the repository at this point in the history 
 fix col lineage with let expressions
  • Loading branch information
@AFine-gs
AFine-gs authored Nov 20, 2024
1 parent a9292a2 commit 6c5d72e
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 16 deletions.
30 changes: 17 additions & 13 deletions ...gend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,11 @@ function <<access.private>> meta::pure::lineage::scanProject::scanProjectRecursi
);
),
pair('meta::pure::tds::project_T_MANY__ColumnSpecification_MANY__TabularDataSet_1_',
|let ins = $fe.parametersValues->at(1)->evaluateAndDeactivate();
$ins->match(
[
i:InstanceValue[1] |
let cols = $i.values->evaluateAndDeactivate()->cast(@SimpleFunctionExpression);
let colSpecs = $cols->map(c | $c->reactivate($vars)->cast(@BasicColumnSpecification<Any>));
^Project(projectfuncEntryPoint = $fe,
columns = zip($colSpecs.name, $colSpecs.func->cast(@FunctionDefinition<Any>)));,
a:Any[*]|[]
]
);

),
|let cols = $fe.parametersValues->at(1)->evaluateAndDeactivate();
let columns = $cols->map(c|$c->meta::pure::lineage::scanProject::extractCol($vars));
^Project(projectfuncEntryPoint = $fe,
columns = zip($columns.name, $columns.func->cast(@FunctionDefinition<Any>)));
),
pair('meta::pure::tds::groupBy_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__TabularDataSet_1_',
| let funcs = $fe.parametersValues->at(1)->match([s:InstanceValue[1]|$s.values, a:Any[*]|^Unknown()]);
let aggregateFuncs = $fe.parametersValues->at(2)->scanAggregateValue();
Expand Down Expand Up @@ -109,6 +101,18 @@ function <<access.private>> meta::pure::lineage::scanProject::scanProjectRecursi
);
}

function <<access.private>> meta::pure::lineage::scanProject::extractCol(vs:ValueSpecification[1],vars:Map<String, List<Any>>[1]):BasicColumnSpecification<Any>[*]
{
$vs->match(
[
i:InstanceValue[1] | let cols = $i.values->evaluateAndDeactivate()->cast(@SimpleFunctionExpression);
let colSpecs = $cols->map(c | $c->reactivate($vars)->cast(@BasicColumnSpecification<Any>));,
s:SimpleFunctionExpression[1] | $s->reactivate($vars)->cast(@BasicColumnSpecification<Any>);,
a:Any[*]| [] ;
]
);
}

function <<access.private>> meta::pure::lineage::scanProject::scanRootGraphFetchTree(root:meta::pure::graphFetch::RootGraphFetchTree<Any>[1]):Pair<String, FunctionDefinition<Any>>[*]
{
let rootGetAllExpression = meta::pure::lineage::scanProject::createGetAllApplicationForRootGraphFetchTree($root);
Expand Down
21 changes: 21 additions & 0 deletions ...-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -541,4 +541,25 @@ function <<meta::pure::profiles::test.Test, meta::pure::profiles::test.AlloyOnly
['my::test::testSemiStructuredMappingQuery__TabularDataSet_1_','my::model::Person','my::model::Firm','pack_my::model'],
'[employeeName: [PersonTable.PERSON_DETAILS <DynaFunction>], firmName: [FirmTable.ID <JoinTreeNode>, FirmTable.LEGAL_NAME <TableAliasColumn>, PersonTable.PERSON_DETAILS <JoinTreeNode>]]',
$lineage);
}

###Pure
import meta::analytics::lineage::*;
import meta::relational::extension::*;
import meta::relational::tests::*;
import meta::relational::tests::model::simple::*;
function <<meta::pure::profiles::test.Test>> meta::analytics::lineage::tests::relational::testColWithPrimitiveLet():Boolean[1]
{
let fn= {businessDate:Date[1], type:String[1]|
let adjustedType = $type+ 'test';
meta::relational::tests::milestoning::Product.all($businessDate)->project(col(p|$p.classification($businessDate).type,'type'))->filter(f|$f.getString('type')=='adjustedType');};
let mapping = meta::relational::tests::milestoning::milestoningmap;
let runtime = meta::external::store::relational::tests::testRuntime();

let lineage = computeLineage($fn, $mapping, $runtime, relationalExtensions());

meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'tb_dbdefaultProductClassificationTable', 'tb_dbdefaultProductTable'],
['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::milestoning::Product', 'meta::relational::tests::milestoning::ProductClassification', 'pack_meta::pure::tds', 'pack_meta::relational::tests::milestoning'],
'[type: [ProductClassificationTable.type <JoinTreeNode>, ProductClassificationTable.type <TableAliasColumn>, ProductTable.type <JoinTreeNode>]]',
$lineage);
}
10 changes: 7 additions & 3 deletions ...re/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ Class meta::pure::lineage::scanRelations::RelationalTreeAndVars
}

function <<access.private>> meta::pure::lineage::scanRelations::generatRelationalTrees(f:FunctionDefinition<Any>[1], m:Mapping[1], r:Runtime[1], vars:Pair<String, List<Any>>[*], debug:DebugContext[1], extensions:Extension[*]):RelationalTreeAndVars[1]
{
{
let routed = $f->routeFunction($m, $r, $extensions, $debug);
let routedFunction = $routed->evaluateAndDeactivate()->toOne();
let inScopeVars = $f.expressionSequence->evaluateAndDeactivate()->fold({vs, a | if ($vs->isLetFunction() ,
Expand All @@ -402,8 +402,12 @@ function <<access.private>> meta::pure::lineage::scanRelations::generatRelationa
let inScopeVars = $a->keyValues();
let unavailableVars = $varExprs.name->forAll(var | $var->in($inScopeVars.first));
assert($unavailableVars, 'Unable to resolve var(s): '+ $varExprs.name->filter(var | !$var->in($inScopeVars.first))->joinStrings());
let re = $vs->reactivate($a);
$a->put($varName->toOne(), ^List<Any>(values=$re));,

let value = if($vs.genericType.rawType->toOne()->instanceOf(PrimitiveType) && !$vs->cast(@SimpleFunctionExpression).parametersValues->at(1)->instanceOf(InstanceValue),
| ^PlanVarPlaceHolder(name=$varName->toOne(), type = $vs.genericType.rawType->toOne(), multiplicity=$vs.multiplicity),
| $vs->reactivate($a);
);
$a->put($varName->toOne(), ^List<Any>(values=$value));,

| $a;
);
Expand Down

0 comments on commit 6c5d72e

Please sign in to comment.