fixing report lineage for lambdas with only groupBys (#2865)

finos · May 27, 2024 · 94da376 · 94da376
1 parent 8951b23
commit 94da376
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 0 deletions.
diff --git a/...gend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure b/...gend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure
@@ -59,6 +59,16 @@ function <<access.private>> meta::pure::lineage::scanProject::scanProjectRecursi
                                                                ]
                                                             );
 
+                                                      ),
+                                                      pair('meta::pure::tds::groupBy_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__TabularDataSet_1_',
+                                                        | let funcs = $fe.parametersValues->at(1)->match([s:InstanceValue[1]|$s.values, a:Any[*]|^Unknown()]);
+                                                          let aggregateFuncs = $fe.parametersValues->at(2)->scanAggregateValue();
+                                                          let allFuncs = $funcs->concatenate($aggregateFuncs);
+                                                          let names = $fe.parametersValues->at(3)->match([s:InstanceValue[1]|$s.values,a:Any[*]|^Unknown()]);
+                                                          ^Project(projectfuncEntryPoint = $fe,
+                                                                    columns = zip($names->match([s:String[*]|$s, a:Unknown[*]|range(1, $allFuncs->size(),1)->map(z|'unknown_'+$z->toString())]),
+                                                                                  $allFuncs->match([s:FunctionDefinition<Any>[*]|$s, a:Unknown[*]|[]]))
+                                                          );
                                                       )
                                                     ];
                                      let funcFullPath = $fe.func->toOne()->elementToPath();
@@ -87,3 +97,12 @@ function <<access.private>> meta::pure::lineage::scanProject::scanProjectRecursi
    ]
   );
 }
+
+function <<access.private>> meta::pure::lineage::scanProject::scanAggregateValue(val:Any[*]):FunctionDefinition<Any>[*]
+{
+  $val->match([
+    s:InstanceValue[*]|$s.values->scanAggregateValue(),
+    se: FunctionExpression[*]| $se->map(x| $x.parametersValues->at(0))->scanAggregateValue(),
+    f: FunctionDefinition<Any>[*]|$f,
+    a:Any[*]|[]]);
+}
diff --git a/...lytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/scanProjectTests.pure b/...lytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/scanProjectTests.pure
@@ -28,6 +28,17 @@ function <<access.private>> meta::pure::lineage::scanProject::test::executeWithP
                                 ]), simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions());
 }
 
+function <<access.private>> meta::pure::lineage::scanProject::test::executeWithGroupBy():Any[1]
+{
+   execute(|Trade.all()->groupBy([x|$x.date->adjust(0, DurationUnit.DAYS)],
+               [
+                  agg(x | $x.quantity, y | $y->sum()),
+                  agg(x | $x.id, y | $y->count())
+               ],
+               ['tradeDate', 'quantity', 'count']
+               ), simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions());
+}
+
 function <<access.private>> meta::pure::lineage::scanProject::test::executeWithProjectUsingCol():Any[1]
 {
    execute(|Firm.all()->project([
@@ -94,3 +105,10 @@ function <<meta::pure::profiles::test.Test>> meta::pure::lineage::scanProject::t
    let p = $x.funcEntryPoint->toOne()->evaluateAndDeactivate()->meta::pure::lineage::scanProject::scanProject();
    assertEquals(['a', 'b', 'c'], $p.columns.first);
 }
+
+function <<meta::pure::profiles::test.Test>> meta::pure::lineage::scanProject::test::testFindGroupBy():Boolean[1]
+{
+   let x = meta::pure::lineage::scanProject::test::executeWithGroupBy__Any_1_->scanExecutes();
+   let p = $x.funcEntryPoint->toOne()->evaluateAndDeactivate()->meta::pure::lineage::scanProject::scanProject();
+   assertEquals(['tradeDate', 'quantity', 'count'], $p.columns.first);
+}