diff --git a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java index 52fddc10fa0..8ea535586e6 100644 --- a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java +++ b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/test/pureClient/stores/Test_Relational_UsingPureClientTestSuite.java @@ -16,75 +16,79 @@ import junit.framework.Test; import junit.framework.TestSuite; +import org.eclipse.collections.api.block.predicate.Predicate; import org.finos.legend.engine.server.test.shared.PureWithEngineHelper; import org.finos.legend.pure.m3.execution.test.TestCollection; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; - import static org.finos.legend.engine.server.test.shared.PureTestHelper.wrapSuite; import static org.finos.legend.engine.test.shared.framework.PureTestHelperFramework.*; +import static org.finos.legend.pure.m3.fct.shared.FCTTools.isFCTTest; public class Test_Relational_UsingPureClientTestSuite extends TestSuite { public static Test suite() throws Exception { + return wrapSuite( () -> PureWithEngineHelper.initClientVersionIfNotAlreadySet("vX_X_X"), () -> { CompiledExecutionSupport executionSupport = getClassLoaderExecutionSupport(); + Predicate filter = ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport()) && !isFCTTest(ci, executionSupport.getProcessorSupport()); TestSuite suite = new TestSuite(); - // suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::constraints", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - // suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::testDataGeneration::tests::alloy", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::advanced", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::groupBy", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::injection", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::map", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::association", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::boolean", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::dates", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::distinct", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::dynajoin", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::embedded", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::enumeration", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::extend", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::filter", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::groupBy", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::include", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::inheritance", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::innerjoin", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::join", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::merge", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::multigrain", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::propertyfunc", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::selfJoin", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::sqlFunction", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::subType", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::tree", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::union", executionSupport.getProcessorSupport(), ci -> !ci.getName().contains("testPksWithImportDataFlow") && satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mergerules", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::projection", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::query", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::tds", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::milestoning", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::objectReferenceIn", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::postProcessor::filterPushDown", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::simple", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::embedded", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::qualifier", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::milestoning", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::classMappingFilterWithInnerJoin", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::chain", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::resultSourcing", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::crossDatabase", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::subType", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::union::propertyLevel", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::aggregationAware", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::advanced::resultSourcing", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::inMemoryAndRelational", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStoreUnion::inMemoryAndRelational", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::ordered", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::pureToSqlQuery::calendarAggregations", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::platform", executionSupport.getProcessorSupport(), ci -> satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); + // suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::constraints", executionSupport.getProcessorSupport(), filter), executionSupport)); + // suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::testDataGeneration::tests::alloy", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::advanced", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::groupBy", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::injection", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::map", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::association", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::boolean", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::dates", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::distinct", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::dynajoin", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::embedded", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::enumeration", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::extend", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::filter", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::groupBy", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::include", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::inheritance", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::innerjoin", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::join", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::merge", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::multigrain", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::propertyfunc", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::selfJoin", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::sqlFunction", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::subType", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::tree", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mapping::union", executionSupport.getProcessorSupport(), ci -> !ci.getName().contains("testPksWithImportDataFlow") && satisfiesConditions(ci, executionSupport.getProcessorSupport()) && !isFCTTest(ci, executionSupport.getProcessorSupport())), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::mergerules", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::projection", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::query", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::tds", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::milestoning", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::objectReferenceIn", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::postProcessor::filterPushDown", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::simple", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::embedded", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::qualifier", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::milestoning", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::classMappingFilterWithInnerJoin", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::chain", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::resultSourcing", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::crossDatabase", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::subType", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::graphFetch::tests::union::propertyLevel", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::aggregationAware", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::advanced::resultSourcing", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::inMemoryAndRelational", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStoreUnion::inMemoryAndRelational", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::pure::graphFetch::tests::XStore::ordered", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::functions::pureToSqlQuery::calendarAggregations", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(buildSuite(TestCollection.collectTests("meta::relational::tests::platform", executionSupport.getProcessorSupport(), filter), executionSupport)); return suite; }, diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure index 63e11211bcb..fab36b9a7a0 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure @@ -39,4 +39,18 @@ function meta::pure::lineage::analytics::inlineQualifiedProperties(vs: ValueSpec ve : VariableExpression[1] | let resolved = $ve->resolve($vars, $openVars); if($resolved->isEmpty(), | $ve, | $resolved->toOne());, v : ValueSpecification[1] | $v ]); -} \ No newline at end of file +} + + +function meta::pure::lineage::analytics::reportLineageToString(report:meta::pure::lineage::result::ReportLineage[1]):String[1] +{ + $report.properties->map(c | $c.propertyName + $c.resultDetail)->sort()->joinStrings('[', ', ', ']'); +} + +function meta::pure::lineage::test::assertLineage(storeLineage:String[*], classLineage:String[*], reportLineage:String[1], result:meta::pure::lineage::result::LineageResult[1]):Boolean[1] +{ + + assertSameElements($storeLineage, $result.storeLineage.nodes.data.id) && assertSameElements($classLineage, $result.classLineage.nodes.data.id) && assertEquals($reportLineage, meta::pure::lineage::analytics::reportLineageToString($result.reportLineage)); +} + + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure new file mode 100644 index 00000000000..9644029434e --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure @@ -0,0 +1,80 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +Class meta::pure::lineage::report::Source +{ + <> context: String[1]; +} + +Class meta::pure::lineage::report::PropertySource extends meta::pure::lineage::report::Source +{ + <> classPath: String[1]; + <> propertyName: String[1]; +} + + +Class meta::pure::lineage::graph::Node +{ + data : meta::pure::lineage::graph::NodeData[1]; +} + +Class meta::pure::lineage::graph::NodeData +{ + id : String[1]; + text : String[1]; + type : String[1]; + displayType : String[0..1]; + parent : meta::pure::lineage::graph::Node[0..1]; +} + +Class meta::pure::lineage::graph::Edge +{ + data : meta::pure::lineage::graph::EdgeData[1]; +} + +Class meta::pure::lineage::graph::EdgeData +{ + id : String[1]; + text : String[1]; + type : String[1]; + source : meta::pure::lineage::graph::Node[1]; + target : meta::pure::lineage::graph::Node[1]; +} + +Class meta::pure::lineage::graph::Graph +{ + nodes : meta::pure::lineage::graph::Node[*]; + edges : meta::pure::lineage::graph::Edge[*]; +} + +Class meta::pure::lineage::result::PropertyLineage +{ + propertyName : String[1]; + resultDetail : String[1]; +} + +Class meta::pure::lineage::result::ReportLineage +{ + properties : meta::pure::lineage::result::PropertyLineage[*]; +} + + +Class meta::pure::lineage::result::LineageResult +{ + storeLineage: meta::pure::lineage::graph::Graph[1]; + classLineage: meta::pure::lineage::graph::Graph[1]; + reportLineage : meta::pure::lineage::result::ReportLineage[1]; +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure new file mode 100644 index 00000000000..2866be6913a --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure @@ -0,0 +1,92 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::core::runtime::*; +import meta::pure::mapping::*; +import meta::pure::test::fct::*; + + +Class meta::pure::test::fct::TestResult +{ + +} + +Class meta::pure::test::fct::ExecuteResult extends meta::pure::test::fct::TestResult +{ + result:meta::pure::mapping::Result[1]; + +} + + + + + + +Class meta::pure::test::fct::TestParameters +{ + mapping:meta::pure::mapping::Mapping[1]; + runtime:meta::core::runtime::Runtime[1]; + extensions:meta::pure::extension::Extension[*]; + context:meta::pure::runtime::ExecutionContext[1]; +} + + +function meta::pure::test::fct::testParameters(mapping:meta::pure::mapping::Mapping[1],runtime:meta::core::runtime::Runtime[1],extensions:meta::pure::extension::Extension[*]):meta::pure::test::fct::TestParameters[1] +{ + ^meta::pure::test::fct::TestParameters( mapping = $mapping, + runtime = $runtime, + extensions = $extensions, + context = ^meta::pure::runtime::ExecutionContext()); +} + +function meta::pure::test::fct::testParameters(mapping:meta::pure::mapping::Mapping[1],runtime:meta::core::runtime::Runtime[1],context:meta::pure::runtime::ExecutionContext[1], extensions:meta::pure::extension::Extension[*]):meta::pure::test::fct::TestParameters[1] +{ + ^meta::pure::test::fct::TestParameters( mapping = $mapping, + runtime = $runtime, + extensions = $extensions, + context = $context); +} + +function + + meta::pure::test::fct::executeWrapper(f:FunctionDefinition<{->T[y]}>[1],holder:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::ExecuteResult[1] +{ + ^ExecuteResult(result=meta::pure::router::execute($f,$holder.mapping,$holder.runtime,$holder.context,$holder.extensions)); +} + + +function meta::pure::test::fct::assertTDSExecuteResult(result:meta::pure::test::fct::TestResult[1],assertion:Function<{meta::pure::mapping::Result[1]->Boolean[1]}>[1]):Boolean[1] +{ + + if($result->instanceOf(ExecuteResult), + |let tds = $result->cast(@ExecuteResult).result; + $assertion->eval($tds);, + | true); +} + + +Class meta::pure::test::fct::LineageResult extends meta::pure::test::fct::TestResult +{ + result:meta::pure::lineage::result::LineageResult[1]; +} + + +function meta::pure::test::fct::assertLineageResult( storeLineage:String[*], classLineage:String[*], reportLineage:String[1],result:meta::pure::test::fct::TestResult[1]):Boolean[1] +{ + + if($result->instanceOf(meta::pure::test::fct::LineageResult), + | let lineage = $result->cast(@meta::pure::test::fct::LineageResult).result; + meta::pure::lineage::test::assertLineage($storeLineage,$classLineage,$reportLineage,$lineage);, + | true); +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/pom.xml b/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/pom.xml new file mode 100644 index 00000000000..71701614f27 --- /dev/null +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/pom.xml @@ -0,0 +1,70 @@ + + + + + 4.0.0 + + org.finos.legend.engine + legend-engine-core-testable + 4.67.3-SNAPSHOT + + Legend Engine - Testable - FCT + legend-engine-test-fct + + + 11 + 11 + UTF-8 + + + + org.finos.legend.pure + legend-pure-m4 + + + org.finos.legend.pure + legend-pure-m3-core + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + org.eclipse.collections + eclipse-collections-api + + + junit + junit + + + io.opentracing + opentracing-noop + + + io.opentracing + opentracing-util + + + org.eclipse.collections + eclipse-collections + + + + \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/src/main/java/org/finos/legend/engine/test/fct/FCTTestSuitBuilder.java b/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/src/main/java/org/finos/legend/engine/test/fct/FCTTestSuitBuilder.java new file mode 100644 index 00000000000..96d72385fa5 --- /dev/null +++ b/legend-engine-core/legend-engine-core-testable/legend-engine-test-fct/src/main/java/org/finos/legend/engine/test/fct/FCTTestSuitBuilder.java @@ -0,0 +1,241 @@ +/* + * // Copyright 2023 Goldman Sachs + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + */ + +package org.finos.legend.engine.test.fct; + +import io.opentracing.noop.NoopTracerFactory; +import io.opentracing.util.GlobalTracer; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Comparator; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.impl.utility.ArrayIterate; +import org.finos.legend.pure.m3.exception.PureAssertFailException; +import org.finos.legend.pure.m3.execution.ExecutionSupport; +import org.finos.legend.pure.m3.execution.test.PureTestBuilder; +import org.finos.legend.pure.m3.execution.test.TestCollection; +import org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation._package._Package; +import org.finos.legend.pure.m3.navigation.profile.Profile; +import org.finos.legend.pure.m3.pct.shared.PCTTools; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.FunctionProcessor; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.IdBuilder; +import org.junit.Assert; +import static org.finos.legend.pure.m3.execution.test.TestCollection.collectTests; +import static org.junit.Assert.fail; + +public class FCTTestSuitBuilder extends PureTestBuilder +{ + public static String LINEAGE_FUNCTION = "meta::analytics::lineage::computeTestLineage_FunctionDefinition_1__TestParameters_1__LineageResult_1_"; + public static String EXECUTE_FUNCTION = "meta::pure::test::fct::executeWrapper_FunctionDefinition_1__TestParameters_1__ExecuteResult_1_"; + + public static TestSuite buildFCTTestSuiteWithExecutorFunction(TestCollection collection, MutableMap exclusions, String function, boolean includeBeforeAndAfter, ExecutionSupport executionSupport) + { + PureTestBuilder.F2, Object> testExecutor = (a, b) -> fctExecuteFn(a, function, exclusions, executionSupport, b); + TestSuite suite = new TestSuite(); + suite.addTest(buildFCTSuite(collection, testExecutor,includeBeforeAndAfter, executionSupport)); + return suite; + } + + public static void addCollectionToSuite(TestSuite suite, TestCollection collection, MutableMap exclusions, String function, boolean includeBeforeAndAfter, ExecutionSupport executionSupport) + { + PureTestBuilder.F2, Object> testExecutor = (a, b) -> fctExecuteFn(a, function, exclusions, executionSupport, b); + suite.addTest(buildFCTSuite(collection, testExecutor, includeBeforeAndAfter,executionSupport)); + } + + private static Object fctExecuteFn(CoreInstance coreInstance, String FCTexecutor, MutableMap exclusions, ExecutionSupport executionSupport, MutableList paramList) throws Throwable + { + Class _class = Class.forName("org.finos.legend.pure.generated." + IdBuilder.sourceToId(coreInstance.getSourceInformation())); + if (isFCTTest(coreInstance, ((CompiledExecutionSupport) executionSupport).getProcessorSupport())) + { + CoreInstance executor = _Package.getByUserPath(FCTexecutor, ((CompiledExecutionSupport) executionSupport).getProcessorSupport()); + if (executor == null) + { + throw new RuntimeException(FCTexecutor + "can't be found in the graph"); + } + paramList.add(executor); + } + paramList = paramList.with(executionSupport); + Object[] params = paramList.toArray(); + String methodName = FunctionProcessor.functionNameToJava(coreInstance); + Method method = params.length == 1 ? _class.getMethod(methodName, ExecutionSupport.class) + : ArrayIterate.detect(_class.getMethods(), m -> methodName.equals(m.getName())); + + // NOTE: mock out the global tracer for test + // See https://github.com/opentracing/opentracing-java/issues/170 + // See https://github.com/opentracing/opentracing-java/issues/364 + GlobalTracer.registerIfAbsent(NoopTracerFactory.create()); + try + { + Object res = method.invoke(null, params); + // Ensure we didn't expect an error + String message = exclusions.get(PackageableElement.getUserPathForPackageableElement(coreInstance, "::")); + if (message != null) + { + PCTTools.displayExpectedErrorFailMessage(message, coreInstance, FCTexecutor); + } + return res; + } + catch (InvocationTargetException e) + { + // Check if the error was expected + String message = exclusions.get(PackageableElement.getUserPathForPackageableElement(coreInstance, "::")); + Throwable thrown = e.getCause().getMessage().contains("Unexpected error executing function with params") && e.getCause().getCause() != null ? e.getCause().getCause() : e.getCause(); + if (message != null && thrown.getMessage().contains(message)) + { + return null; + } + else + { + PCTTools.displayErrorMessage(message, coreInstance, FCTexecutor, ((CompiledExecutionSupport) executionSupport).getProcessorSupport(), thrown); + if (thrown instanceof PureAssertFailException) + { + fail(thrown.getMessage()); + } + throw thrown; + } + } + finally + { + try + { + // HACK since GlobalTracer api doesnt provide a way to reset the tracer which is needed for testing + Field tracerField = GlobalTracer.get().getClass().getDeclaredField("isRegistered"); + tracerField.setAccessible(true); + tracerField.set(GlobalTracer.get(), false); + Assert.assertFalse(GlobalTracer.isRegistered()); + } + catch (Exception ignored) + { + } + } + } + + + public static TestCollection buildFCTTestCollection(String path, ProcessorSupport processorSupport) + { + return collectTests(path, processorSupport, (node) -> isFCTTest(node, processorSupport)); + } + + public static boolean isFCTTest(CoreInstance node, ProcessorSupport processorSupport) + { + return Profile.hasStereotype(node, "meta::pure::test::fct::FCT", "test", processorSupport); + } + + public static TestSuite buildFCTSuite(TestCollection testCollection, F2, Object> executor, boolean includeBeforeAndAfter, ExecutionSupport executionSupport) + { + MutableList subSuites = Lists.mutable.empty(); + for (TestCollection collection : testCollection.getSubCollections().toSortedList(Comparator.comparing(a -> a.getPackage().getName()))) + { + subSuites.add(buildSuite(collection, executor, executionSupport)); + } + return buildFCTSuite(org.finos.legend.pure.m3.navigation.PackageableElement.PackageableElement.getUserPathForPackageableElement(testCollection.getPackage()), + testCollection.getBeforeFunctions(), + testCollection.getAfterFunctions(), + testCollection.getPureAndAlloyOnlyFunctions(), + testCollection.getTestFunctionParam(), + testCollection.getTestFunctionParamCustomizer(), + testCollection.getTestParameterizationId(), + subSuites, + executor, + executionSupport, + includeBeforeAndAfter + ); + } + + private static TestSuite buildFCTSuite(String packageName, RichIterable beforeFunctions, RichIterable afterFunctions, + RichIterable testFunctions, Object param, CoreInstance paramCustomizer, String parameterizationId, + ListIterable subSuites, F2, Object> executor, ExecutionSupport executionSupport, boolean includeBeforeAndAfter) + { + TestSuite suite = new TestSuite(); + suite.setName(packageName + (parameterizationId == null ? "" : "[" + parameterizationId + "]")); + if (includeBeforeAndAfter) + { + beforeFunctions.collect(fn -> new FCTPureTestCase(fn, param, paramCustomizer, parameterizationId, executor, executionSupport)).each(suite::addTest); + } + for (Test subSuite : subSuites.toSortedList(Comparator.comparing(TestSuite::getName))) + { + suite.addTest(subSuite); + } + for (CoreInstance testFunc : testFunctions.toSortedList(Comparator.comparing(CoreInstance::getName))) + { + Test theTest = new FCTPureTestCase(testFunc, param, paramCustomizer, parameterizationId, executor, executionSupport); + suite.addTest(theTest); + } + + if (includeBeforeAndAfter) + { + afterFunctions.collect(fn -> new FCTPureTestCase(fn, param, paramCustomizer, parameterizationId, executor, executionSupport)).each(suite::addTest); + } + return suite; + } + + public static class FCTPureTestCase extends TestCase + { + + CoreInstance coreInstance; + Object param; + CoreInstance paramCustomizer; + ExecutionSupport executionSupport; + F2, Object> executor; + + public FCTPureTestCase() + { + } + + FCTPureTestCase(CoreInstance coreInstance, Object param, CoreInstance paramCustomizer, String parameterizationId, F2, Object> executor, ExecutionSupport executionSupport) + { + super(coreInstance.getValueForMetaPropertyToOne("functionName").getName() + (parameterizationId == null ? "" : "[" + parameterizationId + "]")); + this.coreInstance = coreInstance; + this.param = param; + this.paramCustomizer = paramCustomizer; + this.executionSupport = executionSupport; + this.executor = executor; + } + + @Override + protected void runTest() throws Throwable + { + Object customizedParam = this.param; + if (this.param != null && this.paramCustomizer != null) + { + customizedParam = this.executor.value(this.paramCustomizer, Lists.mutable.with(this.coreInstance, this.param)); + } + if (customizedParam != null) + { + this.executor.value(this.coreInstance, Lists.mutable.with(customizedParam)); + } + else + { + this.executor.value(this.coreInstance, Lists.mutable.empty()); + } + } + } + + + +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-testable/pom.xml b/legend-engine-core/legend-engine-core-testable/pom.xml index 3f0fb18386a..bd803aa6ba3 100644 --- a/legend-engine-core/legend-engine-core-testable/pom.xml +++ b/legend-engine-core/legend-engine-core-testable/pom.xml @@ -36,5 +36,7 @@ legend-engine-testable-http-api legend-engine-execution-test-data-generation legend-engine-execution-test-data-generation-api + legend-engine-test-fct + \ No newline at end of file diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-http-api/src/main/java/org/finos/legend/engine/api/analytics/LineageAnalytics.java b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-http-api/src/main/java/org/finos/legend/engine/api/analytics/LineageAnalytics.java index 3ae40b06e3c..65d50c7198b 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-http-api/src/main/java/org/finos/legend/engine/api/analytics/LineageAnalytics.java +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-http-api/src/main/java/org/finos/legend/engine/api/analytics/LineageAnalytics.java @@ -35,8 +35,8 @@ import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType; import org.finos.legend.pure.generated.Root_meta_analytics_lineage_ReportLineage; import org.finos.legend.pure.generated.Root_meta_analytics_lineage_flow_Flow; -import org.finos.legend.pure.generated.Root_meta_analytics_lineage_graph_Graph; import org.finos.legend.pure.generated.Root_meta_pure_functions_collection_List_Impl; +import org.finos.legend.pure.generated.Root_meta_pure_lineage_graph_Graph; import org.finos.legend.pure.generated.Root_meta_pure_lineage_scanProperties_PropertyPathNode; import org.finos.legend.pure.generated.Root_meta_pure_lineage_scanProperties_propertyTree_PropertyPathTree; import org.finos.legend.pure.generated.Root_meta_pure_lineage_scanRelations_RelationTree; @@ -121,7 +121,7 @@ public Response classLineage(LineageAnalyticsInput lineageAnalyticsInput, @Pac4J org.eclipse.collections.api.RichIterable> res = core_pure_lineage_scanProperties.Root_meta_pure_lineage_scanProperties_scanProperties_ValueSpecification_1__List_1__Function_MANY__Map_$0_1$__Res_$0_1$_((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification) function._expressionSequence().getFirst(), new Root_meta_pure_functions_collection_List_Impl(""), Lists.fixedSize.empty(), new PureMap(Maps.fixedSize.empty()), pureModel.getExecutionSupport())._result(); Root_meta_pure_lineage_scanProperties_propertyTree_PropertyPathTree propertyTree = core_pure_lineage_scanProperties.Root_meta_pure_lineage_scanProperties_propertyTree_buildPropertyTree_List_MANY__PropertyPathTree_1_(res, pureModel.getExecutionSupport()); Root_meta_analytics_lineage_flow_Flow flow = core_analytics_lineage_fullAnalytics.Root_meta_analytics_lineage_flowClass_toFlowClass_FunctionDefinition_1__PropertyPathTree_1__Flow_1_(function, propertyTree, pureModel.getExecutionSupport()); - Root_meta_analytics_lineage_graph_Graph graph = core_analytics_lineage_graph.Root_meta_analytics_lineage_graph_toGraph_Flow_1__Graph_1_(flow, pureModel.getExecutionSupport()); + Root_meta_pure_lineage_graph_Graph graph = core_analytics_lineage_graph.Root_meta_analytics_lineage_graph_toGraph_Flow_1__Graph_1_(flow, pureModel.getExecutionSupport()); LOGGER.info(new LogInfo(identity.getName(), LoggingEventType.LINEAGE_CLASS_LINEAGE_STOP, System.currentTimeMillis() - start).toString()); return ManageConstantResult.manageResult(identity.getName(), org.finos.legend.pure.generated.core_pure_protocol_protocol.Root_meta_alloy_metadataServer_alloyToJSON_Any_1__String_1_(graph, pureModel.getExecutionSupport())); } @@ -154,7 +154,7 @@ public Response databaseLineage(LineageAnalyticsInput lineageAnalyticsInput, @Pa org.eclipse.collections.api.RichIterable> res = core_pure_lineage_scanProperties.Root_meta_pure_lineage_scanProperties_scanProperties_ValueSpecification_1__List_1__Function_MANY__Map_$0_1$__Res_$0_1$_((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification) function._expressionSequence().getFirst(), new Root_meta_pure_functions_collection_List_Impl(""), Lists.fixedSize.empty(), new PureMap(Maps.fixedSize.empty()), pureModel.getExecutionSupport())._result(); Root_meta_pure_lineage_scanProperties_propertyTree_PropertyPathTree propertyTree = core_pure_lineage_scanProperties.Root_meta_pure_lineage_scanProperties_propertyTree_buildPropertyTree_List_MANY__PropertyPathTree_1_(res, pureModel.getExecutionSupport()); Root_meta_analytics_lineage_flow_Flow flow = core_analytics_lineage_fullAnalytics.Root_meta_analytics_lineage_flowDatabase_toFlowDatabase_FunctionDefinition_1__Mapping_MANY__PropertyPathTree_1__Flow_1_(function, Lists.fixedSize.of(pureModel.getMapping(lineageAnalyticsInput.mapping)), propertyTree, pureModel.getExecutionSupport()); - Root_meta_analytics_lineage_graph_Graph graph = core_analytics_lineage_graph.Root_meta_analytics_lineage_graph_toGraph_Flow_1__Graph_1_(flow, pureModel.getExecutionSupport()); + Root_meta_pure_lineage_graph_Graph graph = core_analytics_lineage_graph.Root_meta_analytics_lineage_graph_toGraph_Flow_1__Graph_1_(flow, pureModel.getExecutionSupport()); LOGGER.info(new LogInfo(identity.getName(), LoggingEventType.LINEAGE_DATABASE_LINEAGE_STOP, System.currentTimeMillis() - start).toString()); return ManageConstantResult.manageResult(identity.getName(), org.finos.legend.pure.generated.core_pure_protocol_protocol.Root_meta_alloy_metadataServer_alloyToJSON_Any_1__String_1_(graph, pureModel.getExecutionSupport())); } diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml index dbbb930a11c..f60e0000cdf 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml @@ -153,6 +153,7 @@ legend-engine-pure-code-compiled-core ${project.version} + org.finos.legend.engine legend-engine-xt-relationalStore-core-pure @@ -239,6 +240,11 @@ org.finos.legend.engine legend-engine-xt-relationalStore-core-pure + + org.finos.legend.engine + legend-engine-xt-relationalStore-core-pure + test-jar + @@ -255,6 +261,14 @@ junit junit + + org.finos.legend.engine + legend-engine-test-fct + + + org.finos.legend.engine + legend-engine-test-framework + diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure index c99afa14ddb..d5e55edd37e 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::pure::lineage::graph::*; import meta::pure::executionPlan::*; import meta::core::runtime::*; import meta::pure::lineage::scanRelations::*; @@ -80,6 +81,7 @@ Class meta::analytics::lineage::PropertyElement type : String[0..1]; } + function meta::analytics::lineage::computeLineage(f:FunctionDefinition[1], m:Mapping[1], r:Runtime[0..1], extensions:meta::pure::extension::Extension[*]):FunctionAnalytics[1] { let mappings = if($r->isEmpty(), |$m, |$m->concatenate(getMappingsFromRuntime($r->toOne()))); diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/graph.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/graph.pure index e9398d6fae1..f5834943460 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/graph.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/graph.pure @@ -13,6 +13,7 @@ // limitations under the License. +import meta::pure::lineage::graph::*; import meta::analytics::lineage::flow::*; import meta::analytics::lineage::graph::*; import meta::json::*; @@ -21,40 +22,6 @@ import meta::relational::metamodel::*; import meta::relational::metamodel::relation::*; -Class meta::analytics::lineage::graph::Node -{ - data : NodeData[1]; -} - -Class meta::analytics::lineage::graph::NodeData -{ - id : String[1]; - text : String[1]; - type : String[1]; - displayType : String[0..1]; - parent : Node[0..1]; -} - -Class meta::analytics::lineage::graph::Edge -{ - data : EdgeData[1]; -} - -Class meta::analytics::lineage::graph::EdgeData -{ - id : String[1]; - text : String[1]; - type : String[1]; - source : Node[1]; - target : Node[1]; -} - -Class meta::analytics::lineage::graph::Graph -{ - nodes : Node[*]; - edges : Edge[*]; -} - function meta::analytics::lineage::graph::toJSON(graph : Graph[1]):JSONElement[1] { $graph->meta::json::toJSONElement([ diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/lineage_fct.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/lineage_fct.pure new file mode 100644 index 00000000000..667f12984d3 --- /dev/null +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/lineage_fct.pure @@ -0,0 +1,31 @@ +import meta::relational::metamodel::relation::*; +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// meta::pure::test::fct::executeWrapper(f:FunctionDefinition<{->T[y]}>[1],holder:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::ExecuteResult[1] + +function meta::analytics::lineage::computeTestLineage(f:FunctionDefinition<{->X[y]}>[1],parameters:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::LineageResult[1] +{ + let lineage = meta::analytics::lineage::computeLineage($f,$parameters.mapping,$parameters.runtime,$parameters.extensions); + ^meta::pure::test::fct::LineageResult(result= ^meta::pure::lineage::result::LineageResult(storeLineage=$lineage.databaseLineage, + classLineage=$lineage.classLineage, + reportLineage= $lineage.reportLineage->meta::analytics::lineage::reportLineageToPureReportLineage() )); + +} + +function meta::analytics::lineage::reportLineageToPureReportLineage(report:meta::analytics::lineage::ReportLineage[1]):meta::pure::lineage::result::ReportLineage[1] +{ + ^meta::pure::lineage::result::ReportLineage(properties=$report.columns->map(c | ^meta::pure::lineage::result::PropertyLineage(propertyName=$c.name, resultDetail= $c.columns->map(t|$t.column.owner->cast(@Table).name->toOne()+'.'+$t.column.name->toOne()+' <'+$t.context+'>')->removeDuplicates()->sort()->makeString(': [', ', ', ']')))); +} \ No newline at end of file diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure index 474d3de929a..e6ddc827097 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::pure::test::fct::*; import meta::analytics::lineage::graph::*; import meta::relational::extension::*; import meta::pure::functions::date::calendar::*; import meta::relational::tests::functions::pureToSqlQuery::calendarAggregations::*; -function <> meta::analytics::lineage::tests::relational::calendarAggregations():Boolean[1] +function <> meta::analytics::lineage::tests::relational::calendarAggregations():Boolean[1] { let lineage = meta::analytics::lineage::computeLineage(| Employee.all() ->groupBy( @@ -36,23 +37,27 @@ function <> meta::analytics::lineage::tests::re ###Pure +import meta::pure::test::fct::*; import meta::analytics::lineage::*; import meta::relational::extension::*; import meta::relational::metamodel::join::*; import meta::relational::tests::tds::tdsJoin::*; -function <> meta::analytics::lineage::tests::relational::testMultipleDB():Boolean[1] +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; + +function <> meta::analytics::lineage::tests::relational::testMultipleDB(toEval:Function<{FunctionDefinition[1],TestParameters[1]->TestResult[1]}>[1]):Boolean[1] { let fn= {|testJoinTDS_Person.all()->meta::pure::tds::project([col(p|$p.firstName, 'firstName'), col(p|$p.employerID, 'eID'), col(p|$p.managerID, 'managerID')]) ->join(testJoinTDS_Firm.all()->project([col(p|$p.firmID, 'fID'), col(p|$p.legalName, 'legalName')]), JoinType.INNER, {a,b|$a.getInteger('eID') == $b.getInteger('fID');});}; - - - - let lineage = computeLineage($fn,meta::relational::tests::tds::tdsJoin::testJoinTDSMappingTwoDatabaseWithColumnsMappedViaJoinsAndDynaFunction, meta::relational::tests::tds::tdsJoin::twoDBRunTime(), meta::relational::extension::relationalExtensions()); - meta::analytics::lineage::assertLineage(['Lambda', 'db_database2', 'db_dbInc', 'tb_database2defaultfirmTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], - ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Firm', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::tds::tdsJoin'], - '[eID: [firmTable.ID , firmTable.ID , personTable.FIRMID ], fID: [firmTable.ID ], firstName: [personTable.FIRSTNAME ], legalName: [firmTable.LEGALNAME ], managerID: [personTable.MANAGERID ]]', - $lineage); + + let parameters = meta::pure::test::fct::testParameters( meta::relational::tests::tds::tdsJoin::testJoinTDSMappingTwoDatabaseWithColumnsMappedViaJoinsAndDynaFunction, meta::relational::tests::tds::tdsJoin::twoDBRunTime(), meta::relational::extension::relationalExtensions()); + let result = $toEval->eval($fn,$parameters); + + meta::pure::test::fct::assertLineageResult(['Lambda', 'db_database2', 'db_dbInc', 'tb_database2defaultfirmTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Firm', 'meta::relational::tests::tds::tdsJoin::testJoinTDS_Person', 'pack_meta::pure::tds', 'pack_meta::relational::tests::tds::tdsJoin'], + '[eID: [firmTable.ID , firmTable.ID , personTable.FIRMID ], fID: [firmTable.ID ], firstName: [personTable.FIRSTNAME ], legalName: [firmTable.LEGALNAME ], managerID: [personTable.MANAGERID ]]', + $result); } @@ -543,6 +548,9 @@ function <> meta::analytics::lineage::tests::re { let fn= {businessDate:Date[1], type:String[1]| let adjustedType = $type+ 'test'; - meta::relational::tests::milestoning::Product.all($businessDate)->project(col(p|$p.classification($businessDate).type,'type'))->filter(f|$f.getString('type')=='adjustedType');}; + meta::relational::tests::milestoning::Product.all($businessDate)->project(col(p|$p.classification($businessDate).type,'type'))->filter(f|$f.getString('type')=='adjustedType');}; let mapping = meta::relational::tests::milestoning::milestoningmap; let runtime = meta::external::store::relational::tests::testRuntime(); let lineage = computeLineage($fn, $mapping, $runtime, relationalExtensions()); - + meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'tb_dbdefaultProductClassificationTable', 'tb_dbdefaultProductTable'], - ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::milestoning::Product', 'meta::relational::tests::milestoning::ProductClassification', 'pack_meta::pure::tds', 'pack_meta::relational::tests::milestoning'], + ['Lambda', 'meta::pure::tds::TDSRow', 'meta::relational::tests::milestoning::Product', 'meta::relational::tests::milestoning::ProductClassification', 'pack_meta::pure::tds', 'pack_meta::relational::tests::milestoning'], '[type: [ProductClassificationTable.type , ProductClassificationTable.type , ProductTable.type ]]', $lineage); } \ No newline at end of file diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage.java b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage.java index f1912f0b92e..151b4e6c5da 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage.java +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage.java @@ -15,20 +15,26 @@ package org.finos.legend.pure.code.core; import junit.framework.TestSuite; +import org.eclipse.collections.api.block.predicate.Predicate; import org.finos.legend.pure.m3.execution.test.PureTestBuilder; import org.finos.legend.pure.m3.execution.test.TestCollection; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; +import static org.finos.legend.pure.m3.fct.shared.FCTTools.isFCTTest; public class Test_Analytics_Lineage { public static TestSuite suite() { + CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); + Predicate filter = ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport()) && !isFCTTest(ci, executionSupport.getProcessorSupport()); + TestSuite suite = new TestSuite(); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::analytics::lineage::tests", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::lineage::scanExecutes::test", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::lineage::scanProject::test", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::analytics::lineage::tests", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::lineage::scanExecutes::test", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::lineage::scanProject::test", executionSupport.getProcessorSupport(), filter), executionSupport)); return suite; } } diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage_FCT.java b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage_FCT.java new file mode 100644 index 00000000000..9e1f3af6731 --- /dev/null +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/test/java/org/finos/legend/pure/code/core/Test_Analytics_Lineage_FCT.java @@ -0,0 +1,51 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core; + +import junit.framework.Test; +import junit.framework.TestSuite; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.impl.factory.Maps; +import org.finos.legend.engine.test.fct.FCTTestSuitBuilder; +import org.finos.legend.pure.code.core.relational.Test_Pure_Relational_FCT_Collection; +import org.finos.legend.pure.m3.execution.test.TestCollection; +import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; +import static org.finos.legend.engine.test.fct.FCTTestSuitBuilder.LINEAGE_FUNCTION; +import static org.finos.legend.engine.test.fct.FCTTestSuitBuilder.addCollectionToSuite; +import static org.finos.legend.engine.test.fct.FCTTestSuitBuilder.buildFCTTestCollection; +import static org.finos.legend.engine.test.shared.framework.PureTestHelperFramework.*; + +public class Test_Analytics_Lineage_FCT +{ + public static Test suite() + { + + CompiledExecutionSupport support = getClassLoaderExecutionSupport(); + MutableMap exclusions = Maps.mutable.empty(); + + TestCollection collectionRelational = Test_Pure_Relational_FCT_Collection.buildCollection(support); + TestCollection collection = buildFCTTestCollection("meta::analytics::lineage::tests", support.getProcessorSupport()); + TestSuite suite = FCTTestSuitBuilder.buildFCTTestSuiteWithExecutorFunction(collection, exclusions, LINEAGE_FUNCTION, false, support); + addCollectionToSuite(suite,collectionRelational,exclusions,LINEAGE_FUNCTION, false,support); + return wrapSuite( + () -> true, + () -> suite, + () -> false, + Lists.mutable.empty() + ); + } + +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml index 58666bc42d2..ac0185800f7 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/pom.xml @@ -178,11 +178,11 @@ legend-pure-m2-dsl-path-grammar ${legend.pure.version} - - - - - + + org.finos.legend.pure + legend-pure-m2-dsl-tds-pure + ${legend.pure.version} + org.finos.legend.pure @@ -195,7 +195,11 @@ legend-engine-pure-code-compiled-core ${project.version} - + + org.finos.legend.engine + legend-engine-test-fct + ${project.version} + org.finos.legend.engine legend-engine-language-pure-dsl-service-pure @@ -216,6 +220,17 @@ + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + @@ -280,10 +295,10 @@ legend-engine-pure-functions-relation-pure runtime - - - - + + org.finos.legend.pure + legend-pure-m2-dsl-tds-pure + org.finos.legend.engine @@ -436,6 +451,15 @@ legend-engine-pure-runtime-java-extension-shared-functions-planExecution test + + org.finos.legend.engine + legend-engine-test-framework + test + + + org.finos.legend.engine + legend-engine-test-fct + @@ -469,7 +493,7 @@ - + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure index 2e0e4fed023..ee69ee3e103 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure @@ -412,3 +412,18 @@ function meta::relational::mapping::findMainClassInGetAllExpression(vs:ValueSpec let getAllFe = $vs->findExpressionsForFunctionInValueSpecification([getAll_Class_1__T_MANY_, getAll_Class_1__Date_1__T_MANY_, getAll_Class_1__Date_1__Date_1__T_MANY_]); if($getAllFe->isEmpty(), | Any, | $getAllFe.parametersValues->at(0)->cast(@ExtendedRoutedValueSpecification)->byPassRouterInfo()->cast(@InstanceValue).values->toOne()->cast(@Class)); } + + + +function meta::relational::mapping::assertTDS(result:meta::pure::test::fct::TestResult[1],expected:meta::pure::metamodel::relation::TDS[1]):Boolean[1] +{ + + if($result->instanceOf(meta::pure::test::fct::ExecuteResult), + |let tds = $result->cast(@meta::pure::test::fct::ExecuteResult).result; + + let expetedCols = $expected->columns(); + let header = $expetedCols->map(c|$c.name)->joinStrings(','); + let data = $expetedCols->map(c|$c.name)->joinStrings(',') + '\n'+ $expected->map(x|$expetedCols->map(c|s($c->eval($x)->first()))->joinStrings(','))->joinStrings('\n')+'\n';//->joinStrings('\n')+'\n'; + assertEquals($data, $tds.values->toOne()->toCSV());, + | true); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure index a297279e530..89c6f829f33 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::mapping::*; +import meta::pure::test::fct::*; import meta::relational::tests::*; import meta::external::store::relational::tests::*; import meta::pure::profiles::*; @@ -30,13 +32,30 @@ function <> meta::relational::tests::mapping::association::e } - -function <> meta::relational::tests::mapping::association::embedded::testPersonToOrganisations():Boolean[1] +function <> meta::relational::tests::mapping::association::embedded::testEmbbeddedAssociation(toEval:Function<{FunctionDefinition[1],TestParameters[1]->TestResult[1]}>[1]):Boolean[1] { - let result1 = execute(|Person.all()->project([p|$p.lastName,p|$p.firm.legalName, p|$p.firm.organizations.name], ['name', 'firm', 'team']), associationMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()).values->at(0); - assertEquals(5, $result1.rows->size()); - assertSameElements(['Smith Firm X Team 1', 'Smith Firm X Team 2', 'Johnson Firm X Team 1', 'Johnson Firm X Team 2', 'Roberts Firm A Team 3'], $result1.rows->map(r|$r.getString('name')+' '+$r.getString('firm')->toString()+ ' ' +$r.getString('team'))); -} + let fn = {|Person.all()->project([p|$p.lastName,p|$p.firm.legalName, p|$p.firm.organizations.name], ['name', 'firm', 'team'])}; + let parameters = meta::pure::test::fct::testParameters( associationMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); + let fctResult = $toEval->eval($fn,$parameters); + let expectedTDS = #TDS + name, firm, team + Smith,Firm X,Team 1 + Smith,Firm X,Team 2 + Johnson,Firm X,Team 1 + Johnson,Firm X,Team 2 + Roberts,Firm A,Team 3#; + + assertTDS($fctResult,$expectedTDS); + + assertLineageResult(['Lambda', 'db_myDB', 'tb_myDBdefaultORGANIZATIONS', 'tb_myDBdefaultPERSON_FIRM_DENORM'], + ['Lambda', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Organization', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[firm: [PERSON_FIRM_DENORM.FIRM_LEGALNAME ], name: [PERSON_FIRM_DENORM.PERSON_LASTNAME ], team: [ORGANIZATIONS.FIRM_ID , ORGANIZATIONS.NAME , PERSON_FIRM_DENORM.FIRM_ID ]]',$fctResult); + + +} + + + function <> meta::relational::tests::mapping::association::embedded::testFirmToOrganisations():Boolean[1] { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational.java index 13e77d43f47..0e9816dc22a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational.java @@ -15,10 +15,13 @@ package org.finos.legend.pure.code.core.relational; import junit.framework.TestSuite; +import org.eclipse.collections.api.block.predicate.Predicate; import org.finos.legend.pure.m3.execution.test.PureTestBuilder; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; import org.finos.legend.pure.m3.execution.test.TestCollection; import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; +import static org.finos.legend.pure.m3.fct.shared.FCTTools.isFCTTest; public class Test_Pure_Relational { @@ -27,20 +30,21 @@ public static TestSuite suite() CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); executionSupport.getConsole().disable(); TestSuite suite = new TestSuite(); + Predicate filter = ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport()) && !isFCTTest(ci, executionSupport.getProcessorSupport()); //NOTE- we are not collecting parameterized test collection in meta::relational here - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::relational", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::alloy::objectReference", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::alloy::service::execution", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::lineage", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::graphFetch", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::router", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::alloy::connections", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::tds", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::executionPlan", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::mapping", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::milestoning", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::protocols::pure", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); - suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::relational::tests::csv", executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::relational", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::alloy::objectReference", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::alloy::service::execution", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::lineage", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::graphFetch", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::router", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::alloy::connections", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::tds", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::executionPlan", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::mapping", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::milestoning", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::protocols::pure", executionSupport.getProcessorSupport(), filter), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::relational::tests::csv", executionSupport.getProcessorSupport(), filter), executionSupport)); return suite; } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational_Execute_FCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational_Execute_FCT.java new file mode 100644 index 00000000000..79a09895528 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational_Execute_FCT.java @@ -0,0 +1,43 @@ +/* + * // Copyright 2023 Goldman Sachs + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + */ + +package org.finos.legend.pure.code.core.relational; + +import junit.framework.Test; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.impl.factory.Maps; +import org.finos.legend.engine.test.fct.FCTTestSuitBuilder; +import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; +import static org.finos.legend.engine.test.fct.FCTTestSuitBuilder.EXECUTE_FUNCTION; +import static org.finos.legend.engine.test.shared.framework.PureTestHelperFramework.*; + +public class Test_Pure_Relational_Execute_FCT +{ + public static Test suite() + { + CompiledExecutionSupport support = getClassLoaderExecutionSupport(); + MutableMap exclusions = Maps.mutable.empty(); + + return wrapSuite( + () -> true, + () -> FCTTestSuitBuilder.buildFCTTestSuiteWithExecutorFunction(Test_Pure_Relational_FCT_Collection.buildCollection(support), exclusions,EXECUTE_FUNCTION, true,support), + () -> false, + Lists.mutable.empty() + ); + } + +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational_FCT_Collection.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational_FCT_Collection.java new file mode 100644 index 00000000000..a36121b6f9e --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/test/java/org/finos/legend/pure/code/core/relational/Test_Pure_Relational_FCT_Collection.java @@ -0,0 +1,30 @@ +/* + * // Copyright 2023 Goldman Sachs + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + */ + +package org.finos.legend.pure.code.core.relational; + +import org.finos.legend.pure.m3.execution.test.TestCollection; +import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; +import static org.finos.legend.engine.test.fct.FCTTestSuitBuilder.buildFCTTestCollection; + +public class Test_Pure_Relational_FCT_Collection +{ + public static TestCollection buildCollection(CompiledExecutionSupport support) + { + return buildFCTTestCollection("meta::relational::tests::mapping::association::embedded", support.getProcessorSupport()); + } + +} diff --git a/pom.xml b/pom.xml index ff2d4f60d42..9a0addcfc68 100644 --- a/pom.xml +++ b/pom.xml @@ -1216,6 +1216,12 @@ legend-engine-xt-relationalStore-core-pure ${project.version} + + org.finos.legend.engine + legend-engine-xt-relationalStore-core-pure + ${project.version} + test-jar + org.finos.legend.engine legend-engine-xt-relationalStore-sqlPlanning-pure @@ -2377,6 +2383,11 @@ legend-engine-testable-http-api ${project.version} + + org.finos.legend.engine + legend-engine-test-fct + ${project.version} + org.finos.legend.engine legend-engine-test-framework