update FCT testing

finos · Nov 21, 2024 · 0472968 · 0472968
1 parent 19f17f1
commit 0472968
Show file tree

Hide file tree

Showing 10 changed files with 150 additions and 105 deletions.
diff --git a/...ore-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure b/...ore-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure
@@ -21,13 +21,6 @@ Profile meta::pure::test::fct::FCT
     stereotypes : [test, adapter];
 }
 
-function
-
-   meta::pure::test::fct::executeWrapper<T|y>(f:FunctionDefinition<{->T[y]}>[1],holder:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::ExecuteResult<T>[1]
-{
-  ^ExecuteResult<T>(result=meta::pure::router::execute($f,$holder.mapping,$holder.runtime,$holder.context,$holder.extensions));
-}
-
 
 Class meta::pure::test::fct::TestResult
 {
@@ -40,28 +33,17 @@ Class meta::pure::test::fct::ExecuteResult<T> extends meta::pure::test::fct::Tes
 
 }
 
-function meta::pure::test::fct::assertTDSExecuteResult<T>(result:meta::pure::test::fct::TestResult[1],assertion:Function<{meta::pure::mapping::Result<T|*>[1]->Boolean[1]}>[1]):Boolean[1]
-{
 
-  if($result->instanceOf(ExecuteResult),  
-           |let tds = $result->cast(@ExecuteResult<TabularDataSet>).result;
-              $assertion->eval($tds);,
-          | true);
-}
 
 
-function meta::pure::test::fct::assertLineageResult(result:meta::pure::test::fct::TestResult[1], storeLineage:String[*], classLineage:String[*], reportLineage:String[1]):Boolean[1]
-{
 
-  if($result->instanceOf(meta::pure::test::fct::LineageResult),  
-           | let lineage = $result->cast(@meta::pure::test::fct::LineageResult).result;
-            meta::pure::lineage::test::assertLineage($storeLineage,$classLineage,$reportLineage,$lineage);,
-           | true);
-}
 
-Class meta::pure::test::fct::LineageResult extends  meta::pure::test::fct::TestResult
+Class  meta::pure::test::fct::TestParameters
 {
-    result:meta::pure::lineage::result::LineageResult[1];
+  mapping:meta::pure::mapping::Mapping[1];
+  runtime:meta::core::runtime::Runtime[1];
+  extensions:meta::pure::extension::Extension[*];
+  context:meta::pure::runtime::ExecutionContext[1]; 
 }
 
 
@@ -81,13 +63,35 @@ function meta::pure::test::fct::testParameters(mapping:meta::pure::mapping::Mapp
                                           context = $context);
 }
 
+function
 
+   meta::pure::test::fct::executeWrapper<T|y>(f:FunctionDefinition<{->T[y]}>[1],holder:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::ExecuteResult<T>[1]
+{
+  ^ExecuteResult<T>(result=meta::pure::router::execute($f,$holder.mapping,$holder.runtime,$holder.context,$holder.extensions));
+}
 
-Class  meta::pure::test::fct::TestParameters
+
+function meta::pure::test::fct::assertTDSExecuteResult<T>(result:meta::pure::test::fct::TestResult[1],assertion:Function<{meta::pure::mapping::Result<T|*>[1]->Boolean[1]}>[1]):Boolean[1]
 {
-  mapping:meta::pure::mapping::Mapping[1];
-  runtime:meta::core::runtime::Runtime[1];
-  extensions:meta::pure::extension::Extension[*];
-  context:meta::pure::runtime::ExecutionContext[1]; 
+
+  if($result->instanceOf(ExecuteResult),  
+           |let tds = $result->cast(@ExecuteResult<TabularDataSet>).result;
+              $assertion->eval($tds);,
+          | true);
+}
+
+
+Class meta::pure::test::fct::LineageResult extends  meta::pure::test::fct::TestResult
+{
+    result:meta::pure::lineage::result::LineageResult[1];
 }
 
+
+function meta::pure::test::fct::assertLineageResult( storeLineage:String[*], classLineage:String[*], reportLineage:String[1],result:meta::pure::test::fct::TestResult[1]):Boolean[1]
+{
+
+  if($result->instanceOf(meta::pure::test::fct::LineageResult),  
+           | let lineage = $result->cast(@meta::pure::test::fct::LineageResult).result;
+            meta::pure::lineage::test::assertLineage($storeLineage,$classLineage,$reportLineage,$lineage);,
+           | true);
+}
diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/pom.xml b/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine-core-testable</artifactId>
-        <version>4.65.4-SNAPSHOT</version>
+        <version>4.66.2-SNAPSHOT</version>
     </parent>
     <name>Legend Engine - Testable - FCT</name>
     <artifactId>legend-engine-test-fct</artifactId>

diff --git a/...ytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml b/...ytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml
@@ -255,6 +255,14 @@
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-test-fct</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-test-framework</artifactId>
+        </dependency>
         <!-- TEST -->
     </dependencies>
 

diff --git a/...ne-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure b/...ne-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import meta::pure::lineage::graph::*;
 import meta::pure::executionPlan::*;
 import meta::core::runtime::*;
 import meta::pure::lineage::scanRelations::*;
@@ -81,21 +82,6 @@ Class meta::analytics::lineage::PropertyElement
 }
 
 
-// function meta::analytics::lineage::computeTestLineage(f:FunctionDefinition<Any>[1], m:Mapping[1], r:Runtime[0..1], extensions:meta::pure::extension::Extension[*]):meta::pure::lineage::result::LineageResult[1]
-// {
-//       let lineage = meta::analytics::lineage::computeLineage($f,$m,$r,$extensions);
-//      ^meta::pure::lineage::result::LineageResult(storeLineage=$lineage.databaseLineage,
-//                                                  classLineage=$lineage.classLineage,
-//                                                  reportLineage= $lineage.reportLineage->reportLineageToResult() );
-
-// }
-
-// function meta::analytics::lineage::reportLineageToResult(report:ReportLineage[1]):meta::pure::lineage::result::ReportLineage[1]
-// {
-//    ^meta::pure::lineage::result::ReportLineage(properties=$report.columns->map(c | ^meta::pure::lineage::result::PropertyLineage(propertyName=$c.name, resultDetail=  $c.columns->map(t|$t.column.owner->cast(@Table).name->toOne()+'.'+$t.column.name->toOne()+' <'+$t.context+'>')->removeDuplicates()->sort()->makeString(': [', ', ', ']'))));
-// }
-
-
 function meta::analytics::lineage::computeLineage(f:FunctionDefinition<Any>[1], m:Mapping[1], r:Runtime[0..1], extensions:meta::pure::extension::Extension[*]):FunctionAnalytics[1]
 {
    let mappings = if($r->isEmpty(), |$m, |$m->concatenate(getMappingsFromRuntime($r->toOne())));

diff --git a/...end-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/graph.pure b/...end-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/graph.pure
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 
+import meta::pure::lineage::graph::*;
 import meta::analytics::lineage::flow::*;
 import meta::analytics::lineage::graph::*;
 import meta::json::*;
@@ -21,40 +22,6 @@ import meta::relational::metamodel::*;
 import meta::relational::metamodel::relation::*;
 
 
-Class meta::analytics::lineage::graph::Node
-{
-   data : NodeData[1];
-}
-
-Class meta::analytics::lineage::graph::NodeData
-{
-   id : String[1];
-   text : String[1];
-   type : String[1];
-   displayType : String[0..1];
-   parent : Node[0..1];
-}
-
-Class meta::analytics::lineage::graph::Edge
-{
-   data : EdgeData[1];
-}
-
-Class meta::analytics::lineage::graph::EdgeData
-{
-   id : String[1];
-   text : String[1];
-   type : String[1];
-   source : Node[1];
-   target : Node[1];
-}
-
-Class meta::analytics::lineage::graph::Graph
-{
-   nodes : Node[*];
-   edges : Edge[*];
-}
-
 function meta::analytics::lineage::graph::toJSON(graph : Graph[1]):JSONElement[1]
 {
    $graph->meta::json::toJSONElement([

diff --git a/...gine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/lineage_fct.pure b/...gine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/lineage_fct.pure
@@ -0,0 +1,31 @@
+import meta::relational::metamodel::relation::*;
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+//   meta::pure::test::fct::executeWrapper<T|y>(f:FunctionDefinition<{->T[y]}>[1],holder:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::ExecuteResult<T>[1]
+
+function meta::analytics::lineage::computeTestLineage<X|y>(f:FunctionDefinition<{->X[y]}>[1],parameters:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::LineageResult[1]
+{
+      let lineage = meta::analytics::lineage::computeLineage($f,$parameters.mapping,$parameters.runtime,$parameters.extensions);
+     ^meta::pure::test::fct::LineageResult(result=  ^meta::pure::lineage::result::LineageResult(storeLineage=$lineage.databaseLineage,
+                                                                                                classLineage=$lineage.classLineage,
+                                                                                                reportLineage= $lineage.reportLineage->meta::analytics::lineage::reportLineageToPureReportLineage() ));
+
+}
+
+function meta::analytics::lineage::reportLineageToPureReportLineage(report:meta::analytics::lineage::ReportLineage[1]):meta::pure::lineage::result::ReportLineage[1]
+{
+   ^meta::pure::lineage::result::ReportLineage(properties=$report.columns->map(c | ^meta::pure::lineage::result::PropertyLineage(propertyName=$c.name, resultDetail=  $c.columns->map(t|$t.column.owner->cast(@Table).name->toOne()+'.'+$t.column.name->toOne()+' <'+$t.context+'>')->removeDuplicates()->sort()->makeString(': [', ', ', ']'))));
+}
diff --git a/...-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/...-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import meta::pure::test::fct::*;
 import meta::analytics::lineage::graph::*;
 import meta::relational::extension::*;
 import meta::pure::functions::date::calendar::*;
 import meta::relational::tests::functions::pureToSqlQuery::calendarAggregations::*;
 
-function <<meta::pure::profiles::test.Test>> meta::analytics::lineage::tests::relational::calendarAggregations():Boolean[1]
+function <<test.Test>> meta::analytics::lineage::tests::relational::calendarAggregations():Boolean[1]
 {
  let lineage = meta::analytics::lineage::computeLineage(|  Employee.all()
      ->groupBy(
@@ -36,23 +37,27 @@ function <<meta::pure::profiles::test.Test>> meta::analytics::lineage::tests::re
 
 
 ###Pure 
+import meta::pure::test::fct::*;
 import meta::analytics::lineage::*;
 import meta::relational::extension::*;
 import meta::relational::metamodel::join::*;
 import meta::relational::tests::tds::tdsJoin::*;
-function <<meta::pure::profiles::test.Test>> meta::analytics::lineage::tests::relational::testMultipleDB():Boolean[1]
+import meta::relational::tests::*;
+import meta::relational::tests::model::simple::*;
+
+function <<FCT.test>> meta::analytics::lineage::tests::relational::testMultipleDB<X>(toEval:Function<{FunctionDefinition<X>[1],TestParameters[1]->TestResult[1]}>[1]):Boolean[1]
 {
 
     let fn= {|testJoinTDS_Person.all()->meta::pure::tds::project([col(p|$p.firstName, 'firstName'), col(p|$p.employerID, 'eID'), col(p|$p.managerID, 'managerID')])
                                      ->join(testJoinTDS_Firm.all()->project([col(p|$p.firmID, 'fID'), col(p|$p.legalName, 'legalName')]), JoinType.INNER, {a,b|$a.getInteger('eID') == $b.getInteger('fID');});};
-                       
-                        
-
-    let lineage = computeLineage($fn,meta::relational::tests::tds::tdsJoin::testJoinTDSMappingTwoDatabaseWithColumnsMappedViaJoinsAndDynaFunction, meta::relational::tests::tds::tdsJoin::twoDBRunTime(), meta::relational::extension::relationalExtensions());
-  meta::analytics::lineage::assertLineage(['Lambda', 'db_database2', 'db_dbInc', 'tb_database2defaultfirmTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'],
-                                          ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Firm', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::tds::tdsJoin'],
-                                          '[eID: [firmTable.ID <JoinTreeNode>, firmTable.ID <RelationalOperationElementWithJoin>, personTable.FIRMID <JoinTreeNode>], fID: [firmTable.ID <TableAliasColumn>], firstName: [personTable.FIRSTNAME <TableAliasColumn>], legalName: [firmTable.LEGALNAME <TableAliasColumn>], managerID: [personTable.MANAGERID <DynaFunction>]]',
-                                           $lineage);
+
+    let parameters  =  meta::pure::test::fct::testParameters(  meta::relational::tests::tds::tdsJoin::testJoinTDSMappingTwoDatabaseWithColumnsMappedViaJoinsAndDynaFunction, meta::relational::tests::tds::tdsJoin::twoDBRunTime(), meta::relational::extension::relationalExtensions());
+    let result = $toEval->eval($fn,$parameters);
+
+  meta::pure::test::fct::assertLineageResult(['Lambda', 'db_database2', 'db_dbInc', 'tb_database2defaultfirmTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'],
+                                            ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Firm', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::tds::tdsJoin'],
+                                            '[eID: [firmTable.ID <JoinTreeNode>, firmTable.ID <RelationalOperationElementWithJoin>, personTable.FIRMID <JoinTreeNode>], fID: [firmTable.ID <TableAliasColumn>], firstName: [personTable.FIRSTNAME <TableAliasColumn>], legalName: [firmTable.LEGALNAME <TableAliasColumn>], managerID: [personTable.MANAGERID <DynaFunction>]]',
+                                           $result);
 }
 
 

diff --git a/...ineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage_FCT.java b/...ineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage_FCT.java
@@ -0,0 +1,44 @@
+//  Copyright 2024 Goldman Sachs
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.finos.legend.pure.code.core;
+
+import junit.framework.Test;
+import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.map.MutableMap;
+import org.eclipse.collections.impl.factory.Maps;
+import org.finos.legend.engine.test.fct.FCTTestSuitBuilder;
+import org.finos.legend.pure.m3.execution.test.TestCollection;
+import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport;
+import static org.finos.legend.engine.test.fct.FCTTestSuitBuilder.buildFCTTestCollection;
+import static org.finos.legend.engine.test.shared.framework.PureTestHelperFramework.*;
+
+public class Test_Analytics_Lineage_FCT
+{
+    public static Test suite()
+    {
+
+        CompiledExecutionSupport support = getClassLoaderExecutionSupport();
+        MutableMap<String,String> exclusions = Maps.mutable.empty();
+
+        TestCollection collection = buildFCTTestCollection("meta::analytics::lineage::tests", support.getProcessorSupport());
+        return wrapSuite(
+                () -> true,
+                () -> FCTTestSuitBuilder.buildFCTTestSuiteWithExecutorFunction(collection, exclusions,"meta::analytics::lineage::computeTestLineage_FunctionDefinition_1__TestParameters_1__LineageResult_1_", support),
+                () -> false,
+                Lists.mutable.empty()
+        );
+    }
+
+}
diff --git a/...e-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure b/...e-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure
@@ -420,10 +420,10 @@ function meta::relational::mapping::assertTDS(result:meta::pure::test::fct::Test
 
   if($result->instanceOf(meta::pure::test::fct::ExecuteResult),  
            |let tds = $result->cast(@meta::pure::test::fct::ExecuteResult<TabularDataSet>).result;
-           $expected.csv->println();
-            $tds.values->toOne()->toCSV()->println();
-           // assertSameElements($expected.csv,    $tds.values->toOne()->toCSV());,
-              true;,
-
+
+            let expetedCols = $expected->columns();
+            let header = $expetedCols->map(c|$c.name)->joinStrings(',');
+            let data =   $expetedCols->map(c|$c.name)->joinStrings(',') + '\n'+  $expected->map(x|$expetedCols->map(c|s($c->eval($x)->first()))->joinStrings(','))->joinStrings('\n')+'\n';//->joinStrings('\n')+'\n';
+            assertEquals($data,    $tds.values->toOne()->toCSV());,
           | true);
 }
diff --git a/...sources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure b/...sources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure
@@ -32,24 +32,24 @@ function <<test.BeforePackage>> meta::relational::tests::mapping::association::e
 }
 
 
-function <<FCT.test>> meta::relational::tests::mapping::association::embedded::testEmbbeddedAssociation<T>(f:Function<{FunctionDefinition<T>[1],TestParameters[1]->TestResult[1]}>[1]):Boolean[1]
+function <<FCT.test>> meta::relational::tests::mapping::association::embedded::testEmbbeddedAssociation<X>(toEval:Function<{FunctionDefinition<X>[1],TestParameters[1]->TestResult[1]}>[1]):Boolean[1]
 {
    let fn = {|Person.all()->project([p|$p.lastName,p|$p.firm.legalName, p|$p.firm.organizations.name], ['name', 'firm', 'team'])};
    let parameters  =  meta::pure::test::fct::testParameters(  associationMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions());
-   let fctResult = $f->eval($fn,$parameters);
-    let expectedTDS = 
-#TDS
-name, firm, team
-Smith, Firm, X Team 1
-Smith, Firm X, Team 2
-Johnson, Firm X, Team 1
-Roberts, Firm A, Team 3#;
-                     
+   let fctResult = $toEval->eval($fn,$parameters);
+   let expectedTDS = #TDS
+                      name, firm, team
+                      Smith,Firm X,Team 1
+                      Smith,Firm X,Team 2
+                      Johnson,Firm X,Team 1
+                      Johnson,Firm X,Team 2
+                      Roberts,Firm A,Team 3#;
+
     assertTDS($fctResult,$expectedTDS);
 
-    assertLineageResult($fctResult,['Lambda', 'db_myDB', 'tb_myDBdefaultORGANIZATIONS', 'tb_myDBdefaultPERSON_FIRM_DENORM'],
+    assertLineageResult(['Lambda', 'db_myDB', 'tb_myDBdefaultORGANIZATIONS', 'tb_myDBdefaultPERSON_FIRM_DENORM'],
                                    ['Lambda', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Organization', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'],
-                                   '[firm: [PERSON_FIRM_DENORM.FIRM_LEGALNAME <TableAliasColumn>], name: [PERSON_FIRM_DENORM.PERSON_LASTNAME <TableAliasColumn>], team: [ORGANIZATIONS.FIRM_ID <JoinTreeNode>, ORGANIZATIONS.NAME <TableAliasColumn>, PERSON_FIRM_DENORM.FIRM_ID <JoinTreeNode>]]');
+                                   '[firm: [PERSON_FIRM_DENORM.FIRM_LEGALNAME <TableAliasColumn>], name: [PERSON_FIRM_DENORM.PERSON_LASTNAME <TableAliasColumn>], team: [ORGANIZATIONS.FIRM_ID <JoinTreeNode>, ORGANIZATIONS.NAME <TableAliasColumn>, PERSON_FIRM_DENORM.FIRM_ID <JoinTreeNode>]]',$fctResult);
 
 
 }