From c39700a23411595a9bc6f6ea973b4d5d07df615a Mon Sep 17 00:00:00 2001 From: afine-gs Date: Wed, 13 Nov 2024 12:35:01 -0500 Subject: [PATCH] Initial FCT --- .../core/pure/lineage/helperFunctions.pure | 16 +++- .../core/pure/lineage/lineageModel.pure | 80 +++++++++++++++++ .../main/resources/core/pure/test/fct.pure | 89 +++++++++++++++++++ .../core_analytics_lineage/fullAnalytics.pure | 16 ++++ .../tests/lineageTests.pure | 4 +- .../helperFunctions/helperFunctions.pure | 15 ++++ .../association/testAssociationEmbedded.pure | 31 +++++-- pom.xml | 2 +- 8 files changed, 244 insertions(+), 9 deletions(-) create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure index 63e11211bcb..fab36b9a7a0 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/helperFunctions.pure @@ -39,4 +39,18 @@ function meta::pure::lineage::analytics::inlineQualifiedProperties(vs: ValueSpec ve : VariableExpression[1] | let resolved = $ve->resolve($vars, $openVars); if($resolved->isEmpty(), | $ve, | $resolved->toOne());, v : ValueSpecification[1] | $v ]); -} \ No newline at end of file +} + + +function meta::pure::lineage::analytics::reportLineageToString(report:meta::pure::lineage::result::ReportLineage[1]):String[1] +{ + $report.properties->map(c | $c.propertyName + $c.resultDetail)->sort()->joinStrings('[', ', ', ']'); +} + +function meta::pure::lineage::test::assertLineage(storeLineage:String[*], classLineage:String[*], reportLineage:String[1], result:meta::pure::lineage::result::LineageResult[1]):Boolean[1] +{ + + assertSameElements($storeLineage, $result.storeLineage.nodes.data.id) && assertSameElements($classLineage, $result.classLineage.nodes.data.id) && assertEquals($reportLineage, meta::pure::lineage::analytics::reportLineageToString($result.reportLineage)); +} + + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure new file mode 100644 index 00000000000..9644029434e --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/lineageModel.pure @@ -0,0 +1,80 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +Class meta::pure::lineage::report::Source +{ + <> context: String[1]; +} + +Class meta::pure::lineage::report::PropertySource extends meta::pure::lineage::report::Source +{ + <> classPath: String[1]; + <> propertyName: String[1]; +} + + +Class meta::pure::lineage::graph::Node +{ + data : meta::pure::lineage::graph::NodeData[1]; +} + +Class meta::pure::lineage::graph::NodeData +{ + id : String[1]; + text : String[1]; + type : String[1]; + displayType : String[0..1]; + parent : meta::pure::lineage::graph::Node[0..1]; +} + +Class meta::pure::lineage::graph::Edge +{ + data : meta::pure::lineage::graph::EdgeData[1]; +} + +Class meta::pure::lineage::graph::EdgeData +{ + id : String[1]; + text : String[1]; + type : String[1]; + source : meta::pure::lineage::graph::Node[1]; + target : meta::pure::lineage::graph::Node[1]; +} + +Class meta::pure::lineage::graph::Graph +{ + nodes : meta::pure::lineage::graph::Node[*]; + edges : meta::pure::lineage::graph::Edge[*]; +} + +Class meta::pure::lineage::result::PropertyLineage +{ + propertyName : String[1]; + resultDetail : String[1]; +} + +Class meta::pure::lineage::result::ReportLineage +{ + properties : meta::pure::lineage::result::PropertyLineage[*]; +} + + +Class meta::pure::lineage::result::LineageResult +{ + storeLineage: meta::pure::lineage::graph::Graph[1]; + classLineage: meta::pure::lineage::graph::Graph[1]; + reportLineage : meta::pure::lineage::result::ReportLineage[1]; +} + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure new file mode 100644 index 00000000000..70dc4c633c5 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/test/fct.pure @@ -0,0 +1,89 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::core::runtime::*; +import meta::pure::mapping::*; +import meta::pure::test::fct::*; + + +function + + meta::pure::test::fct::executeWrapper(f:FunctionDefinition<{->T[y]}>[1],holder:meta::pure::test::fct::TestParameters[1]):meta::pure::test::fct::ExecuteResult[1] +{ + ^ExecuteResult(result=meta::pure::router::execute($f,$holder.mapping,$holder.runtime,$holder.context,$holder.extensions)); +} + + +Class meta::pure::test::fct::TestResult +{ + +} + +Class meta::pure::test::fct::ExecuteResult extends meta::pure::test::fct::TestResult +{ + result:meta::pure::mapping::Result[1]; + +} + +function meta::pure::test::fct::assertTDSExecuteResult(result:meta::pure::test::fct::TestResult[1],assertion:Function<{meta::pure::mapping::Result[1]->Boolean[1]}>[1]):Boolean[1] +{ + + if($result->instanceOf(ExecuteResult), + |let tds = $result->cast(@ExecuteResult).result; + $assertion->eval($tds);, + | true); +} + + +function meta::pure::test::fct::assertLineageResult(result:meta::pure::test::fct::TestResult[1], storeLineage:String[*], classLineage:String[*], reportLineage:String[1]):Boolean[1] +{ + + if($result->instanceOf(meta::pure::test::fct::LineageResult), + | let lineage = $result->cast(@meta::pure::test::fct::LineageResult).result; + meta::pure::lineage::test::assertLineage($storeLineage,$classLineage,$reportLineage,$lineage);, + | true); +} + +Class meta::pure::test::fct::LineageResult extends meta::pure::test::fct::TestResult +{ + result:meta::pure::lineage::result::LineageResult[1]; +} + + +function meta::pure::test::fct::testParameters(mapping:meta::pure::mapping::Mapping[1],runtime:meta::core::runtime::Runtime[1],extensions:meta::pure::extension::Extension[*]):meta::pure::test::fct::TestParameters[1] +{ + ^meta::pure::test::fct::TestParameters( mapping = $mapping, + runtime = $runtime, + extensions = $extensions, + context = ^meta::pure::runtime::ExecutionContext()); +} + +function meta::pure::test::fct::testParameters(mapping:meta::pure::mapping::Mapping[1],runtime:meta::core::runtime::Runtime[1],context:meta::pure::runtime::ExecutionContext[1], extensions:meta::pure::extension::Extension[*]):meta::pure::test::fct::TestParameters[1] +{ + ^meta::pure::test::fct::TestParameters( mapping = $mapping, + runtime = $runtime, + extensions = $extensions, + context = $context); +} + + + +Class meta::pure::test::fct::TestParameters +{ + mapping:meta::pure::mapping::Mapping[1]; + runtime:meta::core::runtime::Runtime[1]; + extensions:meta::pure::extension::Extension[*]; + context:meta::pure::runtime::ExecutionContext[1]; +} + diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure index c99afa14ddb..3c503ca2ba6 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure @@ -80,6 +80,22 @@ Class meta::analytics::lineage::PropertyElement type : String[0..1]; } + +// function meta::analytics::lineage::computeTestLineage(f:FunctionDefinition[1], m:Mapping[1], r:Runtime[0..1], extensions:meta::pure::extension::Extension[*]):meta::pure::lineage::result::LineageResult[1] +// { +// let lineage = meta::analytics::lineage::computeLineage($f,$m,$r,$extensions); +// ^meta::pure::lineage::result::LineageResult(storeLineage=$lineage.databaseLineage, +// classLineage=$lineage.classLineage, +// reportLineage= $lineage.reportLineage->reportLineageToResult() ); + +// } + +// function meta::analytics::lineage::reportLineageToResult(report:ReportLineage[1]):meta::pure::lineage::result::ReportLineage[1] +// { +// ^meta::pure::lineage::result::ReportLineage(properties=$report.columns->map(c | ^meta::pure::lineage::result::PropertyLineage(propertyName=$c.name, resultDetail= $c.columns->map(t|$t.column.owner->cast(@Table).name->toOne()+'.'+$t.column.name->toOne()+' <'+$t.context+'>')->removeDuplicates()->sort()->makeString(': [', ', ', ']')))); +// } + + function meta::analytics::lineage::computeLineage(f:FunctionDefinition[1], m:Mapping[1], r:Runtime[0..1], extensions:meta::pure::extension::Extension[*]):FunctionAnalytics[1] { let mappings = if($r->isEmpty(), |$m, |$m->concatenate(getMappingsFromRuntime($r->toOne()))); diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure index 99466f9f4e7..0f45bc29772 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure @@ -541,4 +541,6 @@ function <], firmName: [FirmTable.ID , FirmTable.LEGAL_NAME , PersonTable.PERSON_DETAILS ]]', $lineage); -} \ No newline at end of file +} + + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure index 2e0e4fed023..3b637b82097 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/helperFunctions/helperFunctions.pure @@ -412,3 +412,18 @@ function meta::relational::mapping::findMainClassInGetAllExpression(vs:ValueSpec let getAllFe = $vs->findExpressionsForFunctionInValueSpecification([getAll_Class_1__T_MANY_, getAll_Class_1__Date_1__T_MANY_, getAll_Class_1__Date_1__Date_1__T_MANY_]); if($getAllFe->isEmpty(), | Any, | $getAllFe.parametersValues->at(0)->cast(@ExtendedRoutedValueSpecification)->byPassRouterInfo()->cast(@InstanceValue).values->toOne()->cast(@Class)); } + + + +function meta::relational::mapping::assertTDS(result:meta::pure::test::fct::TestResult[1],expected:meta::pure::metamodel::relation::TDS[1]):Boolean[1] +{ + + if($result->instanceOf(meta::pure::test::fct::ExecuteResult), + |let tds = $result->cast(@meta::pure::test::fct::ExecuteResult).result; + $expected.csv->println(); + $tds.values->toOne()->toCSV()->println(); + // assertSameElements($expected.csv, $tds.values->toOne()->toCSV());, + true;, + + | true); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure index a297279e530..5eb2490514f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/tests/mapping/association/testAssociationEmbedded.pure @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::mapping::*; +import meta::pure::test::fct::*; import meta::relational::tests::*; import meta::external::store::relational::tests::*; import meta::pure::profiles::*; @@ -30,13 +32,30 @@ function <> meta::relational::tests::mapping::association::e } - -function <> meta::relational::tests::mapping::association::embedded::testPersonToOrganisations():Boolean[1] +function <> meta::relational::tests::mapping::association::embedded::testEmbbeddedAssociation(f:Function<{FunctionDefinition[1],TestParameters[1]->TestResult[1]}>[1]):Boolean[1] { - let result1 = execute(|Person.all()->project([p|$p.lastName,p|$p.firm.legalName, p|$p.firm.organizations.name], ['name', 'firm', 'team']), associationMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()).values->at(0); - assertEquals(5, $result1.rows->size()); - assertSameElements(['Smith Firm X Team 1', 'Smith Firm X Team 2', 'Johnson Firm X Team 1', 'Johnson Firm X Team 2', 'Roberts Firm A Team 3'], $result1.rows->map(r|$r.getString('name')+' '+$r.getString('firm')->toString()+ ' ' +$r.getString('team'))); -} + let fn = {|Person.all()->project([p|$p.lastName,p|$p.firm.legalName, p|$p.firm.organizations.name], ['name', 'firm', 'team'])}; + let parameters = meta::pure::test::fct::testParameters( associationMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); + let fctResult = $f->eval($fn,$parameters); + let expectedTDS = +#TDS +name, firm, team +Smith, Firm, X Team 1 +Smith, Firm X, Team 2 +Johnson, Firm X, Team 1 +Roberts, Firm A, Team 3#; + + assertTDS($fctResult,$expectedTDS); + + assertLineageResult($fctResult,['Lambda', 'db_myDB', 'tb_myDBdefaultORGANIZATIONS', 'tb_myDBdefaultPERSON_FIRM_DENORM'], + ['Lambda', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Organization', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[firm: [PERSON_FIRM_DENORM.FIRM_LEGALNAME ], name: [PERSON_FIRM_DENORM.PERSON_LASTNAME ], team: [ORGANIZATIONS.FIRM_ID , ORGANIZATIONS.NAME , PERSON_FIRM_DENORM.FIRM_ID ]]'); + + +} + + + function <> meta::relational::tests::mapping::association::embedded::testFirmToOrganisations():Boolean[1] { diff --git a/pom.xml b/pom.xml index 3fd700bfeb7..9d566916622 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ - 5.22.0 + 5.22.2-SNAPSHOT 0.25.7 12.59.0