diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/HelperModelBuilder.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/HelperModelBuilder.java index 07412f27e4d..b89f19d4120 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/HelperModelBuilder.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/HelperModelBuilder.java @@ -265,7 +265,7 @@ public static RichIterablehamcrest-core test + + commons-io + commons-io + test + - \ No newline at end of file + diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/GrammarParseTestUtils.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/GrammarParseTestUtils.java new file mode 100644 index 00000000000..60636a2bb94 --- /dev/null +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/GrammarParseTestUtils.java @@ -0,0 +1,64 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.pure.grammar.test; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.collections.impl.list.mutable.FastList; +import org.eclipse.collections.impl.utility.internal.IterableIterate; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; + +import java.io.IOException; +import java.util.Objects; + +public class GrammarParseTestUtils +{ + public static PureModelContextData loadPureModelContextFromResource(String resource, Class clazz) + { + return loadPureModelContextFromResource(resource, null, clazz); + } + + public static PureModelContextData loadPureModelContextFromResource(String resource, String extraCode, Class clazz) + { + return loadPureModelContextFromResources(FastList.newListWith(resource), extraCode, clazz); + } + + public static PureModelContextData loadPureModelContextFromResources(Class clazz, String... resources) + { + return loadPureModelContextFromResources(FastList.newListWith(resources), null, clazz); + } + + public static PureModelContextData loadPureModelContextFromResources(Iterable resources, String extraCode, Class clazz) + { + String resourceCode = StringUtils.join(IterableIterate.collect(resources, r -> getResource(r, clazz)), "\n"); + String code = extraCode != null ? resourceCode + "\n" + extraCode : resourceCode; + + return PureModelContextData.newBuilder().withPureModelContextData(PureGrammarParser.newInstance().parseModel(code)).build(); + } + + + public static String getResource(String resource, Class clazz) + { + try + { + return IOUtils.toString(Objects.requireNonNull(clazz.getClassLoader().getResourceAsStream(resource))); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure index 934c82329cc..d23c25a8705 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure @@ -719,7 +719,8 @@ function meta::pure::lineage::scanProperties::printPropertyNodeLists(properyList function <> meta::pure::lineage::scanProperties::propertyTree::recurseBuildTree(list:List[*]):PropertyPathTree[*] { let m = $list->filter(l|!$l.values->isEmpty())->groupBy(l|$l.values->at(0)); - $m->keys()->map( + + $m->keys()->sortBy(v | $v.property.name->toOne())->map( k|^PropertyPathTree(display = $k.property->match([p:Property[1]|'',q:QualifiedProperty[1]|'(Q)'])+$k.property.name->toOne(), value = $k, children = $m->get($k).values->map(l|^$l(values=$l.values->slice(1, $l.values->size())))->recurseBuildTree(), diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure index e8fcae27ff1..c5ae8242572 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure @@ -113,6 +113,7 @@ function meta::pure::metamodel::serialization::grammar::printInstanceValue(i:Ins f:FunctionExpression[1] | $f->printPropertyOrFunctionExpression($configuration, $space), x:InstanceValue[1] | $x->printInstanceValue($configuration, $space), z:KeyExpression[*] | $z->map(f|$f.key.values->toOne()->toString())->joinStrings(','), + v:VariableExpression[1] | $v->printValueSpecification($configuration, $space), r:Runtime[1]|'^meta::core::runtime::Runtime('+ 'connectionStores='+$r.connectionStores->map(s|'^meta::core::runtime::ConnectionStore(connection='+$s.connection->printConnection($configuration)+', element='+$s.element->match([x:String[1]|$x,x:Store[1]|$x->elementToPath()])+')')->makeString(',')+ ')', diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml index 107323486a3..323d02e1017 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml @@ -271,6 +271,12 @@ h2 test + + org.finos.legend.engine + legend-engine-language-pure-grammar + test + test-jar + diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java index 056a70608c2..2030175ce9d 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java @@ -118,7 +118,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, this.extensions); // 3. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString()); @@ -146,7 +146,7 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteT // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions); // 3. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context // 4. execute plan @@ -176,7 +176,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions); Lambda lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions); return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper); } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java index 48c3106bb85..599ed3c018f 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java @@ -18,6 +18,7 @@ import org.finos.legend.engine.language.pure.compiler.Compiler; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtensionLoader; +import org.finos.legend.engine.language.pure.grammar.test.GrammarParseTestUtils; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree; import org.finos.legend.engine.shared.core.ObjectMapperFactory; @@ -42,8 +43,7 @@ public class TestDataQualityApi @Test public void testDataQualityGetPropertyPathTree() throws IOException { - URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithDataQualityValidation.json")); - PureModelContextData pureModelContextData = objectMapper.readValue(url, PureModelContextData.class); + PureModelContextData pureModelContextData = GrammarParseTestUtils.loadPureModelContextFromResource("inputs/modelWithDataQualityValidation.pure", TestDataQualityApi.class); CompilerExtensionLoader.logExtensionList(); PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName()); org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation"); @@ -59,8 +59,7 @@ public void testDataQualityGetPropertyPathTree() throws IOException @Test public void testDataQualityGetPropertyPathTree_WithNullDataQualityValidation() throws IOException { - URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithNullDataQualityValidation.json")); - PureModelContextData pureModelContextData = objectMapper.readValue(url, PureModelContextData.class); + PureModelContextData pureModelContextData = GrammarParseTestUtils.loadPureModelContextFromResource("inputs/modelWithNullDataQualityValidation.pure", TestDataQualityApi.class); CompilerExtensionLoader.logExtensionList(); PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName()); org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation"); diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.json deleted file mode 100644 index ca5cafab1d9..00000000000 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.json +++ /dev/null @@ -1,1533 +0,0 @@ -{ - "_type": "data", - "elements": [ - { - "_type": "connection", - "connectionValue": { - "_type": "RelationalDatabaseConnection", - "authenticationStrategy": { - "_type": "h2Default", - "sourceInformation": { - "endColumn": 18, - "endLine": 10, - "sourceId": "", - "startColumn": 3, - "startLine": 10 - } - }, - "databaseType": "H2", - "datasourceSpecification": { - "_type": "h2Local", - "sourceInformation": { - "endColumn": 4, - "endLine": 9, - "sourceId": "", - "startColumn": 3, - "startLine": 6 - }, - "testDataSetupSqls": [] - }, - "element": "meta::dataquality::db", - "elementSourceInformation": { - "endColumn": 30, - "endLine": 4, - "sourceId": "", - "startColumn": 10, - "startLine": 4 - }, - "postProcessorWithParameter": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 11, - "sourceId": "", - "startColumn": 1, - "startLine": 2 - }, - "type": "H2" - }, - "name": "H2", - "package": "meta::dataquality", - "sourceInformation": { - "endColumn": 1, - "endLine": 11, - "sourceId": "", - "startColumn": 1, - "startLine": 2 - } - }, - { - "_type": "relational", - "filters": [], - "includedStores": [], - "joins": [ - { - "name": "Address_Person", - "operation": { - "_type": "dynaFunc", - "funcName": "equal", - "parameters": [ - { - "_type": "column", - "column": "ID", - "sourceInformation": { - "endColumn": 62, - "endLine": 20, - "sourceId": "", - "startColumn": 24, - "startLine": 20 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 35, - "endLine": 20, - "sourceId": "", - "startColumn": 24, - "startLine": 20 - }, - "table": "addressTable" - }, - "tableAlias": "addressTable" - }, - { - "_type": "column", - "column": "ADDRESSID", - "sourceInformation": { - "endColumn": 62, - "endLine": 20, - "sourceId": "", - "startColumn": 42, - "startLine": 20 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 52, - "endLine": 20, - "sourceId": "", - "startColumn": 42, - "startLine": 20 - }, - "table": "personTable" - }, - "tableAlias": "personTable" - } - ], - "sourceInformation": { - "endColumn": 62, - "endLine": 20, - "sourceId": "", - "startColumn": 40, - "startLine": 20 - } - }, - "sourceInformation": { - "endColumn": 63, - "endLine": 20, - "sourceId": "", - "startColumn": 4, - "startLine": 20 - } - }, - { - "name": "Address_Location", - "operation": { - "_type": "dynaFunc", - "funcName": "equal", - "parameters": [ - { - "_type": "column", - "column": "LOCATIONID", - "sourceInformation": { - "endColumn": 67, - "endLine": 21, - "sourceId": "", - "startColumn": 26, - "startLine": 21 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 37, - "endLine": 21, - "sourceId": "", - "startColumn": 26, - "startLine": 21 - }, - "table": "addressTable" - }, - "tableAlias": "addressTable" - }, - { - "_type": "column", - "column": "ID", - "sourceInformation": { - "endColumn": 67, - "endLine": 21, - "sourceId": "", - "startColumn": 52, - "startLine": 21 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 64, - "endLine": 21, - "sourceId": "", - "startColumn": 52, - "startLine": 21 - }, - "table": "locationTable" - }, - "tableAlias": "locationTable" - } - ], - "sourceInformation": { - "endColumn": 67, - "endLine": 21, - "sourceId": "", - "startColumn": 50, - "startLine": 21 - } - }, - "sourceInformation": { - "endColumn": 68, - "endLine": 21, - "sourceId": "", - "startColumn": 4, - "startLine": 21 - } - } - ], - "name": "db", - "package": "meta::dataquality", - "schemas": [ - { - "name": "default", - "sourceInformation": { - "endColumn": 1, - "endLine": 22, - "sourceId": "", - "startColumn": 1, - "startLine": 14 - }, - "tables": [ - { - "columns": [ - { - "name": "ID", - "nullable": false, - "sourceInformation": { - "endColumn": 40, - "endLine": 16, - "sourceId": "", - "startColumn": 23, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "FIRSTNAME", - "nullable": true, - "sourceInformation": { - "endColumn": 64, - "endLine": 16, - "sourceId": "", - "startColumn": 43, - "startLine": 16 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - }, - { - "name": "LASTNAME", - "nullable": true, - "sourceInformation": { - "endColumn": 87, - "endLine": 16, - "sourceId": "", - "startColumn": 67, - "startLine": 16 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - }, - { - "name": "AGE", - "nullable": true, - "sourceInformation": { - "endColumn": 96, - "endLine": 16, - "sourceId": "", - "startColumn": 90, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "ADDRESSID", - "nullable": true, - "sourceInformation": { - "endColumn": 111, - "endLine": 16, - "sourceId": "", - "startColumn": 99, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "FIRMID", - "nullable": true, - "sourceInformation": { - "endColumn": 123, - "endLine": 16, - "sourceId": "", - "startColumn": 114, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - } - ], - "milestoning": [], - "name": "personTable", - "primaryKey": [ - "ID" - ], - "sourceInformation": { - "endColumn": 124, - "endLine": 16, - "sourceId": "", - "startColumn": 4, - "startLine": 16 - } - }, - { - "columns": [ - { - "name": "ID", - "nullable": false, - "sourceInformation": { - "endColumn": 41, - "endLine": 17, - "sourceId": "", - "startColumn": 24, - "startLine": 17 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "LOCATIONID", - "nullable": true, - "sourceInformation": { - "endColumn": 57, - "endLine": 17, - "sourceId": "", - "startColumn": 44, - "startLine": 17 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "TYPE", - "nullable": true, - "sourceInformation": { - "endColumn": 67, - "endLine": 17, - "sourceId": "", - "startColumn": 60, - "startLine": 17 - }, - "type": { - "_type": "Integer" - } - } - ], - "milestoning": [], - "name": "addressTable", - "primaryKey": [ - "ID" - ], - "sourceInformation": { - "endColumn": 68, - "endLine": 17, - "sourceId": "", - "startColumn": 4, - "startLine": 17 - } - }, - { - "columns": [ - { - "name": "ID", - "nullable": false, - "sourceInformation": { - "endColumn": 42, - "endLine": 18, - "sourceId": "", - "startColumn": 25, - "startLine": 18 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "STREET", - "nullable": true, - "sourceInformation": { - "endColumn": 63, - "endLine": 18, - "sourceId": "", - "startColumn": 45, - "startLine": 18 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - }, - { - "name": "LOCALITY", - "nullable": true, - "sourceInformation": { - "endColumn": 86, - "endLine": 18, - "sourceId": "", - "startColumn": 66, - "startLine": 18 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - } - ], - "milestoning": [], - "name": "locationTable", - "primaryKey": [ - "ID" - ], - "sourceInformation": { - "endColumn": 87, - "endLine": 18, - "sourceId": "", - "startColumn": 4, - "startLine": 18 - } - } - ], - "views": [] - } - ], - "sourceInformation": { - "endColumn": 1, - "endLine": 22, - "sourceId": "", - "startColumn": 1, - "startLine": 14 - }, - "stereotypes": [] - }, - { - "_type": "mapping", - "associationMappings": [], - "classMappings": [ - { - "_type": "relational", - "class": "meta::dataquality::Person", - "classSourceInformation": { - "endColumn": 28, - "endLine": 29, - "sourceId": "", - "startColumn": 4, - "startLine": 29 - }, - "distinct": false, - "groupBy": [], - "primaryKey": [], - "propertyMappings": [ - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Person", - "property": "name", - "sourceInformation": { - "endColumn": 10, - "endLine": 31, - "sourceId": "", - "startColumn": 7, - "startLine": 31 - } - }, - "relationalOperation": { - "_type": "column", - "column": "FIRSTNAME", - "sourceInformation": { - "endColumn": 57, - "endLine": 31, - "sourceId": "", - "startColumn": 14, - "startLine": 31 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 47, - "endLine": 31, - "sourceId": "", - "startColumn": 37, - "startLine": 31 - }, - "table": "personTable" - }, - "tableAlias": "personTable" - }, - "sourceInformation": { - "endColumn": 57, - "endLine": 31, - "sourceId": "", - "startColumn": 12, - "startLine": 31 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Person", - "property": "age", - "sourceInformation": { - "endColumn": 9, - "endLine": 32, - "sourceId": "", - "startColumn": 7, - "startLine": 32 - } - }, - "relationalOperation": { - "_type": "column", - "column": "AGE", - "sourceInformation": { - "endColumn": 50, - "endLine": 32, - "sourceId": "", - "startColumn": 13, - "startLine": 32 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 46, - "endLine": 32, - "sourceId": "", - "startColumn": 36, - "startLine": 32 - }, - "table": "personTable" - }, - "tableAlias": "personTable" - }, - "sourceInformation": { - "endColumn": 50, - "endLine": 32, - "sourceId": "", - "startColumn": 11, - "startLine": 32 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Person", - "property": "addresses", - "sourceInformation": { - "endColumn": 15, - "endLine": 33, - "sourceId": "", - "startColumn": 7, - "startLine": 33 - } - }, - "relationalOperation": { - "_type": "elemtWithJoins", - "joins": [ - { - "db": "meta::dataquality::db", - "name": "Address_Person", - "sourceInformation": { - "endColumn": 56, - "endLine": 33, - "sourceId": "", - "startColumn": 42, - "startLine": 33 - } - } - ], - "sourceInformation": { - "endColumn": 56, - "endLine": 33, - "sourceId": "", - "startColumn": 19, - "startLine": 33 - } - }, - "sourceInformation": { - "endColumn": 56, - "endLine": 33, - "sourceId": "", - "startColumn": 17, - "startLine": 33 - } - } - ], - "root": false, - "sourceInformation": { - "endColumn": 4, - "endLine": 34, - "sourceId": "", - "startColumn": 4, - "startLine": 29 - } - }, - { - "_type": "relational", - "class": "meta::dataquality::Address", - "classSourceInformation": { - "endColumn": 29, - "endLine": 36, - "sourceId": "", - "startColumn": 4, - "startLine": 36 - }, - "distinct": false, - "groupBy": [], - "primaryKey": [], - "propertyMappings": [ - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Address", - "property": "addressId", - "sourceInformation": { - "endColumn": 15, - "endLine": 38, - "sourceId": "", - "startColumn": 7, - "startLine": 38 - } - }, - "relationalOperation": { - "_type": "column", - "column": "ID", - "sourceInformation": { - "endColumn": 56, - "endLine": 38, - "sourceId": "", - "startColumn": 19, - "startLine": 38 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 53, - "endLine": 38, - "sourceId": "", - "startColumn": 42, - "startLine": 38 - }, - "table": "addressTable" - }, - "tableAlias": "addressTable" - }, - "sourceInformation": { - "endColumn": 56, - "endLine": 38, - "sourceId": "", - "startColumn": 17, - "startLine": 38 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Address", - "property": "location", - "sourceInformation": { - "endColumn": 14, - "endLine": 39, - "sourceId": "", - "startColumn": 7, - "startLine": 39 - } - }, - "relationalOperation": { - "_type": "elemtWithJoins", - "joins": [ - { - "db": "meta::dataquality::db", - "name": "Address_Location", - "sourceInformation": { - "endColumn": 57, - "endLine": 39, - "sourceId": "", - "startColumn": 41, - "startLine": 39 - } - } - ], - "sourceInformation": { - "endColumn": 57, - "endLine": 39, - "sourceId": "", - "startColumn": 18, - "startLine": 39 - } - }, - "sourceInformation": { - "endColumn": 57, - "endLine": 39, - "sourceId": "", - "startColumn": 16, - "startLine": 39 - } - } - ], - "root": false, - "sourceInformation": { - "endColumn": 4, - "endLine": 40, - "sourceId": "", - "startColumn": 4, - "startLine": 36 - } - }, - { - "_type": "relational", - "class": "meta::dataquality::Location", - "classSourceInformation": { - "endColumn": 30, - "endLine": 42, - "sourceId": "", - "startColumn": 4, - "startLine": 42 - }, - "distinct": false, - "groupBy": [], - "primaryKey": [], - "propertyMappings": [ - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Location", - "property": "street", - "sourceInformation": { - "endColumn": 12, - "endLine": 44, - "sourceId": "", - "startColumn": 7, - "startLine": 44 - } - }, - "relationalOperation": { - "_type": "column", - "column": "STREET", - "sourceInformation": { - "endColumn": 58, - "endLine": 44, - "sourceId": "", - "startColumn": 16, - "startLine": 44 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 51, - "endLine": 44, - "sourceId": "", - "startColumn": 39, - "startLine": 44 - }, - "table": "locationTable" - }, - "tableAlias": "locationTable" - }, - "sourceInformation": { - "endColumn": 58, - "endLine": 44, - "sourceId": "", - "startColumn": 14, - "startLine": 44 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Location", - "property": "locality", - "sourceInformation": { - "endColumn": 14, - "endLine": 45, - "sourceId": "", - "startColumn": 7, - "startLine": 45 - } - }, - "relationalOperation": { - "_type": "column", - "column": "LOCALITY", - "sourceInformation": { - "endColumn": 62, - "endLine": 45, - "sourceId": "", - "startColumn": 18, - "startLine": 45 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 53, - "endLine": 45, - "sourceId": "", - "startColumn": 41, - "startLine": 45 - }, - "table": "locationTable" - }, - "tableAlias": "locationTable" - }, - "sourceInformation": { - "endColumn": 62, - "endLine": 45, - "sourceId": "", - "startColumn": 16, - "startLine": 45 - } - } - ], - "root": false, - "sourceInformation": { - "endColumn": 4, - "endLine": 46, - "sourceId": "", - "startColumn": 4, - "startLine": 42 - } - } - ], - "enumerationMappings": [], - "includedMappings": [], - "name": "dataqualitymappings", - "package": "meta::dataquality", - "sourceInformation": { - "endColumn": 1, - "endLine": 47, - "sourceId": "", - "startColumn": 1, - "startLine": 26 - }, - "tests": [] - }, - { - "_type": "runtime", - "name": "DataQualityRuntime", - "package": "meta::dataquality", - "runtimeValue": { - "_type": "engineRuntime", - "connectionStores": [], - "connections": [ - { - "sourceInformation": { - "endColumn": 5, - "endLine": 61, - "sourceId": "", - "startColumn": 5, - "startLine": 58 - }, - "store": { - "path": "meta::dataquality::db", - "sourceInformation": { - "endColumn": 25, - "endLine": 58, - "sourceId": "", - "startColumn": 5, - "startLine": 58 - }, - "type": "STORE" - }, - "storeConnections": [ - { - "connection": { - "_type": "connectionPointer", - "connection": "meta::dataquality::H2", - "sourceInformation": { - "endColumn": 41, - "endLine": 60, - "sourceId": "", - "startColumn": 21, - "startLine": 60 - } - }, - "id": "connection_1", - "sourceInformation": { - "endColumn": 41, - "endLine": 60, - "sourceId": "", - "startColumn": 7, - "startLine": 60 - } - } - ] - } - ], - "mappings": [ - { - "path": "meta::dataquality::dataqualitymappings", - "sourceInformation": { - "endColumn": 42, - "endLine": 54, - "sourceId": "", - "startColumn": 5, - "startLine": 54 - }, - "type": "MAPPING" - } - ], - "sourceInformation": { - "endColumn": 1, - "endLine": 63, - "sourceId": "", - "startColumn": 1, - "startLine": 50 - } - }, - "sourceInformation": { - "endColumn": 1, - "endLine": 63, - "sourceId": "", - "startColumn": 1, - "startLine": 50 - } - }, - { - "_type": "class", - "constraints": [ - { - "functionDefinition": { - "_type": "lambda", - "body": [ - { - "_type": "func", - "function": "greaterThanEqual", - "parameters": [ - { - "_type": "property", - "parameters": [ - { - "_type": "var", - "name": "this", - "sourceInformation": { - "endColumn": 25, - "endLine": 70, - "sourceId": "", - "startColumn": 21, - "startLine": 70 - } - } - ], - "property": "age", - "sourceInformation": { - "endColumn": 29, - "endLine": 70, - "sourceId": "", - "startColumn": 27, - "startLine": 70 - } - }, - { - "_type": "integer", - "sourceInformation": { - "endColumn": 35, - "endLine": 70, - "sourceId": "", - "startColumn": 34, - "startLine": 70 - }, - "value": 18 - } - ], - "sourceInformation": { - "endColumn": 35, - "endLine": 70, - "sourceId": "", - "startColumn": 31, - "startLine": 70 - } - } - ], - "parameters": [] - }, - "name": "mustBeOfLegalAge", - "sourceInformation": { - "endColumn": 35, - "endLine": 70, - "sourceId": "", - "startColumn": 3, - "startLine": 70 - } - } - ], - "name": "Person", - "originalMilestonedProperties": [], - "package": "meta::dataquality", - "properties": [ - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "name", - "propertyTypeSourceInformation": { - "endColumn": 16, - "endLine": 73, - "sourceId": "", - "startColumn": 11, - "startLine": 73 - }, - "sourceInformation": { - "endColumn": 20, - "endLine": 73, - "sourceId": "", - "startColumn": 4, - "startLine": 73 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "age", - "propertyTypeSourceInformation": { - "endColumn": 16, - "endLine": 74, - "sourceId": "", - "startColumn": 10, - "startLine": 74 - }, - "sourceInformation": { - "endColumn": 20, - "endLine": 74, - "sourceId": "", - "startColumn": 4, - "startLine": 74 - }, - "stereotypes": [], - "taggedValues": [], - "type": "Integer" - }, - { - "multiplicity": { - "lowerBound": 0 - }, - "name": "addresses", - "propertyTypeSourceInformation": { - "endColumn": 41, - "endLine": 75, - "sourceId": "", - "startColumn": 16, - "startLine": 75 - }, - "sourceInformation": { - "endColumn": 45, - "endLine": 75, - "sourceId": "", - "startColumn": 4, - "startLine": 75 - }, - "stereotypes": [], - "taggedValues": [], - "type": "meta::dataquality::Address" - } - ], - "qualifiedProperties": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 76, - "sourceId": "", - "startColumn": 1, - "startLine": 68 - }, - "stereotypes": [], - "superTypes": [], - "taggedValues": [] - }, - { - "_type": "class", - "constraints": [], - "name": "Address", - "originalMilestonedProperties": [], - "package": "meta::dataquality", - "properties": [ - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "location", - "propertyTypeSourceInformation": { - "endColumn": 40, - "endLine": 80, - "sourceId": "", - "startColumn": 14, - "startLine": 80 - }, - "sourceInformation": { - "endColumn": 44, - "endLine": 80, - "sourceId": "", - "startColumn": 4, - "startLine": 80 - }, - "stereotypes": [], - "taggedValues": [], - "type": "meta::dataquality::Location" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "locationStreet", - "propertyTypeSourceInformation": { - "endColumn": 25, - "endLine": 81, - "sourceId": "", - "startColumn": 20, - "startLine": 81 - }, - "sourceInformation": { - "endColumn": 29, - "endLine": 81, - "sourceId": "", - "startColumn": 4, - "startLine": 81 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "addressId", - "propertyTypeSourceInformation": { - "endColumn": 20, - "endLine": 82, - "sourceId": "", - "startColumn": 15, - "startLine": 82 - }, - "sourceInformation": { - "endColumn": 24, - "endLine": 82, - "sourceId": "", - "startColumn": 4, - "startLine": 82 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - } - ], - "qualifiedProperties": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 83, - "sourceId": "", - "startColumn": 1, - "startLine": 78 - }, - "stereotypes": [], - "superTypes": [], - "taggedValues": [] - }, - { - "_type": "class", - "constraints": [], - "name": "Location", - "originalMilestonedProperties": [], - "package": "meta::dataquality", - "properties": [ - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "street", - "propertyTypeSourceInformation": { - "endColumn": 17, - "endLine": 87, - "sourceId": "", - "startColumn": 12, - "startLine": 87 - }, - "sourceInformation": { - "endColumn": 21, - "endLine": 87, - "sourceId": "", - "startColumn": 4, - "startLine": 87 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "locality", - "propertyTypeSourceInformation": { - "endColumn": 19, - "endLine": 88, - "sourceId": "", - "startColumn": 14, - "startLine": 88 - }, - "sourceInformation": { - "endColumn": 23, - "endLine": 88, - "sourceId": "", - "startColumn": 4, - "startLine": 88 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - } - ], - "qualifiedProperties": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 89, - "sourceId": "", - "startColumn": 1, - "startLine": 85 - }, - "stereotypes": [], - "superTypes": [], - "taggedValues": [] - }, - { - "_type": "dataQualityValidation", - "context": { - "_type": "mappingAndRuntimeDataQualityExecutionContext", - "_type": "mappingAndRuntimeDataQualityExecutionContext", - "mapping": { - "path": "meta::dataquality::dataqualitymappings", - "sourceInformation": { - "endColumn": 73, - "endLine": 94, - "sourceId": "", - "startColumn": 36, - "startLine": 94 - }, - "type": "MAPPING" - }, - "runtime": { - "path": "meta::dataquality::DataQualityRuntime", - "sourceInformation": { - "endColumn": 112, - "endLine": 94, - "sourceId": "", - "startColumn": 76, - "startLine": 94 - }, - "type": "RUNTIME" - } - }, - "dataQualityRootGraphFetchTree": { - "_type": "dataQualityRootGraphFetchTree", - "_type": "dataQualityRootGraphFetchTree", - "class": "meta::dataquality::Person", - "constraints": [ - "mustBeOfLegalAge" - ], - "sourceInformation": { - "endColumn": 31, - "endLine": 97, - "sourceId": "", - "startColumn": 7, - "startLine": 97 - }, - "subTrees": [ - { - "_type": "dataQualityPropertyGraphFetchTree", - "_type": "dataQualityPropertyGraphFetchTree", - "constraints": [], - "parameters": [], - "property": "name", - "sourceInformation": { - "endColumn": 12, - "endLine": 98, - "sourceId": "", - "startColumn": 9, - "startLine": 98 - }, - "subTrees": [], - "subTypeTrees": [] - } - ], - "subTypeTrees": [] - }, - "filter": { - "_type": "lambda", - "body": [ - { - "_type": "func", - "function": "equal", - "parameters": [ - { - "_type": "property", - "parameters": [ - { - "_type": "var", - "name": "p", - "sourceInformation": { - "endColumn": 47, - "endLine": 95, - "sourceId": "", - "startColumn": 46, - "startLine": 95 - } - } - ], - "property": "name", - "sourceInformation": { - "endColumn": 52, - "endLine": 95, - "sourceId": "", - "startColumn": 49, - "startLine": 95 - } - }, - { - "_type": "string", - "sourceInformation": { - "endColumn": 60, - "endLine": 95, - "sourceId": "", - "startColumn": 55, - "startLine": 95 - }, - "value": "John" - } - ], - "sourceInformation": { - "endColumn": 54, - "endLine": 95, - "sourceId": "", - "startColumn": 53, - "startLine": 95 - } - } - ], - "parameters": [ - { - "_type": "var", - "class": "meta::dataquality::Person", - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "p", - "sourceInformation": { - "endColumn": 39, - "endLine": 95, - "sourceId": "", - "startColumn": 15, - "startLine": 95 - } - } - ], - "sourceInformation": { - "endColumn": 60, - "endLine": 95, - "sourceId": "", - "startColumn": 44, - "startLine": 95 - } - }, - "name": "PersonDataQualityValidation", - "package": "meta::dataquality", - "sourceInformation": { - "endColumn": 1, - "endLine": 101, - "sourceId": "", - "startColumn": 1, - "startLine": 92 - }, - "stereotypes": [], - "taggedValues": [] - }, - { - "_type": "sectionIndex", - "name": "SectionIndex", - "package": "__internal__", - "sections": [ - { - "_type": "importAware", - "elements": [], - "imports": [], - "parserName": "Pure", - "sourceInformation": { - "endColumn": 8, - "endLine": 1, - "sourceId": "", - "startColumn": 1, - "startLine": 1 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::H2" - ], - "imports": [], - "parserName": "Connection", - "sourceInformation": { - "endColumn": 2, - "endLine": 13, - "sourceId": "", - "startColumn": 8, - "startLine": 2 - } - }, - { - "_type": "default", - "elements": [ - "meta::dataquality::db" - ], - "parserName": "Relational", - "sourceInformation": { - "endColumn": 1, - "endLine": 25, - "sourceId": "", - "startColumn": 1, - "startLine": 14 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::dataqualitymappings" - ], - "imports": [], - "parserName": "Mapping", - "sourceInformation": { - "endColumn": 2, - "endLine": 49, - "sourceId": "", - "startColumn": 1, - "startLine": 26 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::DataQualityRuntime" - ], - "imports": [], - "parserName": "Runtime", - "sourceInformation": { - "endColumn": 1, - "endLine": 67, - "sourceId": "", - "startColumn": 1, - "startLine": 50 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::Person", - "meta::dataquality::Address", - "meta::dataquality::Location" - ], - "imports": [], - "parserName": "Pure", - "sourceInformation": { - "endColumn": 2, - "endLine": 91, - "sourceId": "", - "startColumn": 1, - "startLine": 68 - } - }, - { - "_type": "default", - "elements": [ - "meta::dataquality::PersonDataQualityValidation" - ], - "parserName": "DataQualityValidation", - "sourceInformation": { - "endColumn": 1, - "endLine": 103, - "sourceId": "", - "startColumn": 1, - "startLine": 92 - } - } - ] - } - ] -} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.pure new file mode 100644 index 00000000000..4881285c8ce --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.pure @@ -0,0 +1,117 @@ +###Connection +RelationalDatabaseConnection meta::dataquality::H2 +{ + store: meta::dataquality::db; + type: H2; + specification: LocalH2 + { + }; + auth: DefaultH2; +} + + +###Relational +Database meta::dataquality::db +( + Table personTable + ( + ID INTEGER PRIMARY KEY, + FIRSTNAME VARCHAR(200), + LASTNAME VARCHAR(200), + AGE INTEGER, + ADDRESSID INTEGER, + FIRMID INTEGER + ) + Table addressTable + ( + ID INTEGER PRIMARY KEY, + LOCATIONID INTEGER, + TYPE INTEGER + ) + Table locationTable + ( + ID INTEGER PRIMARY KEY, + STREET VARCHAR(200), + LOCALITY VARCHAR(200) + ) + + Join Address_Person(addressTable.ID = personTable.ADDRESSID) + Join Address_Location(addressTable.LOCATIONID = locationTable.ID) +) + + +###Mapping +Mapping meta::dataquality::dataqualitymappings +( + meta::dataquality::Person: Relational + { + name: [meta::dataquality::db]personTable.FIRSTNAME, + age: [meta::dataquality::db]personTable.AGE, + addresses: [meta::dataquality::db]@Address_Person + } + meta::dataquality::Address: Relational + { + addressId: [meta::dataquality::db]addressTable.ID, + location: [meta::dataquality::db]@Address_Location + } + meta::dataquality::Location: Relational + { + street: [meta::dataquality::db]locationTable.STREET, + locality: [meta::dataquality::db]locationTable.LOCALITY + } +) + + +###Runtime +Runtime meta::dataquality::DataQualityRuntime +{ + mappings: + [ + meta::dataquality::dataqualitymappings + ]; + connections: + [ + meta::dataquality::db: + [ + connection_1: meta::dataquality::H2 + ] + ]; +} + + +###Pure +Class meta::dataquality::Person +[ + mustBeOfLegalAge: $this.age >= 18 +] +{ + name: String[1]; + age: Integer[1]; + addresses: meta::dataquality::Address[*]; +} + +Class meta::dataquality::Address +{ + location: meta::dataquality::Location[1]; + locationStreet: String[1]; + addressId: String[1]; +} + +Class meta::dataquality::Location +{ + street: String[1]; + locality: String[1]; +} + + +###DataQualityValidation +DataQualityValidation meta::dataquality::PersonDataQualityValidation +{ + context: fromMappingAndRuntime(meta::dataquality::dataqualitymappings, meta::dataquality::DataQualityRuntime); + validationTree: $[ + meta::dataquality::Person{ + name + } + ]$; + filter: p: meta::dataquality::Person[1]|$p.name == 'John'; +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java index 877d0bc5110..92aba1ae064 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java @@ -79,6 +79,16 @@ public CompilerExtension build() return new DataQualityCompilerExtension(); } + + private GenericType genericType(CompileContext context, String _class) + { + GenericType gt = context.resolveGenericType("meta::external::dataquality::DataQuality"); + + return gt._typeArguments(org.eclipse.collections.impl.factory.Lists.fixedSize.of( + context.resolveGenericType(_class) + )); + } + @Override public Iterable> getExtraProcessors() { @@ -88,11 +98,11 @@ public Iterable> getExtraProcessors() org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class), (dataquality, compileContext) -> { - Root_meta_external_dataquality_DataQuality_Impl metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>( + Root_meta_external_dataquality_DataQuality metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>( dataquality.name, SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), compileContext.pureModel.getClass("meta::external::dataquality::DataQuality") - ); + )._classifierGenericType(genericType(compileContext, dataquality.dataQualityRootGraphFetchTree._class)); return metamodel; }, (dataquality, compileContext) -> @@ -213,6 +223,7 @@ private static Root_meta_external_dataquality_DataQualityRootGraphFetchTree buil GenericType genericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) ._rawType(classifier) ._typeArguments(org.eclipse.collections.impl.factory.Lists.fixedSize.of(context.pureModel.getGenericType(parentClass))); + return new Root_meta_external_dataquality_DataQualityRootGraphFetchTree_Impl<>("", SourceInformationHelper.toM3SourceInformation(rootGraphFetchTree.sourceInformation), classifier) ._class(parentClass) ._classifierGenericType(genericType) diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/pom.xml index 11d575f6419..094de935d29 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/pom.xml +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/pom.xml @@ -62,10 +62,6 @@ - - org.finos.legend.engine - legend-engine-language-pure-grammar - org.finos.legend.engine legend-engine-pure-code-core-extension @@ -162,6 +158,22 @@ json-unit test + + org.finos.legend.engine + legend-engine-xt-dataquality-pure-test + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test + test-jar + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-legendCompiler + runtime + diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java index 1f7d45850ca..e96fb0815de 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java @@ -39,18 +39,18 @@ public class DataQualityLambdaGenerator { public static final int DEFAULT_QUERY_LIMIT = 100; - public static LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath) + public static LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath, Function> extensions) { PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); - return generateLambda(pureModel, packageableElement); + return generateLambda(pureModel, packageableElement, extensions); } - public static LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement) + public static LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement, Function> extensions) { - return core_dataquality_generation_dataquality.Root_meta_external_dataquality_executeDataQualityValidation_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.empty(), pureModel.getExecutionSupport()); + return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality) packageableElement, Lists.immutable.empty(), pureModel.getExecutionSupport()); } - public static LambdaFunction generateLambdaForTrial(PureModel pureModel, String qualifiedPath, Integer queryLimit) + public static LambdaFunction generateLambdaForTrial(PureModel pureModel, String qualifiedPath, Integer queryLimit, Function> extensions) { PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); int trialQueryLimit = DEFAULT_QUERY_LIMIT; @@ -58,7 +58,8 @@ public static LambdaFunction generateLambdaForTrial(PureModel pureModel, String { trialQueryLimit = queryLimit; } - return core_dataquality_generation_dataquality.Root_meta_external_dataquality_executeDataQualityValidation_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.of((long)trialQueryLimit), pureModel.getExecutionSupport()); + + return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.of((long)trialQueryLimit), pureModel.getExecutionSupport()); } public static Lambda transformLambda(LambdaFunction lambda, PureModel pureModel, Function> extensions) diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java index 6b640b385cf..6d9e87aa439 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java @@ -80,7 +80,7 @@ public List generate(PackageableElement packageableElement, PureModel Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); // 1. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, routerExtensions); // 2. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java index b7305e84c1a..3973d45560c 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java @@ -18,9 +18,10 @@ import net.javacrumbs.jsonunit.core.Option; import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.impl.list.mutable.FastList; import org.finos.legend.engine.language.pure.compiler.Compiler; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; -import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; +import org.finos.legend.engine.language.pure.grammar.test.GrammarParseTestUtils; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.shared.core.deployment.DeploymentMode; @@ -33,19 +34,18 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +//NOTE: conversion tests can be found in the pure-test module. Tests here are to ensure the java->pure flow e2e. public class TestDataQualityLambdaGenerator { @Test - public void testAssertionForNestedConstraints() + public void testNestedConstraints() { - String validation = COMPILATION_PREREQUISITE_CODE + - "###DataQualityValidation\n" + - "DataQualityValidation meta::dataquality::PersonDataQualityValidation\n" + + String validation = "###DataQualityValidation\n" + + "DataQualityValidation meta::dataquality::Validation\n" + "{\n" + - " context: fromMappingAndRuntime(meta::dataquality::dataqualitymappings, meta::dataquality::DataQualityRuntime);\n" + - " filter: p:meta::dataquality::Person[1] | $p.name=='John';\n" + + " context: fromMappingAndRuntime(meta::external::dataquality::tests::domain::dataqualitymappings, meta::external::dataquality::tests::domain::DataQualityRuntime);\n" + " validationTree: $[\n" + - " meta::dataquality::Person{\n" + + " meta::external::dataquality::tests::domain::Person{\n" + " name,\n" + " addresses{\n" + " addressId\n" + @@ -53,132 +53,60 @@ public void testAssertionForNestedConstraints() " }\n" + " ]$;\n" + "}"; - PureModelContextData modelData = PureGrammarParser.newInstance().parseModel(validation); - PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); - assertEquals("Nested constraints are not currently supported!", - assertThrows(PureAssertFailException.class, () -> DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::PersonDataQualityValidation")).getInfo()); + assertLambdaException(validation, "Nested constraints are not currently supported!"); } @Test - public void testLambdaGeneration_multipleConstraints() + public void testLambdaGeneration() { - String validation = COMPILATION_PREREQUISITE_CODE + - "###DataQualityValidation\n" + - "DataQualityValidation meta::dataquality::PersonDataQualityValidation\n" + + String validation = "###DataQualityValidation\n" + + "DataQualityValidation meta::dataquality::Validation\n" + "{\n" + - " context: fromMappingAndRuntime(meta::dataquality::dataqualitymappings, meta::dataquality::DataQualityRuntime);\n" + + " context: fromMappingAndRuntime(meta::external::dataquality::tests::domain::dataqualitymappings, meta::external::dataquality::tests::domain::DataQualityRuntime);\n" + " validationTree: $[\n" + - " meta::dataquality::Person{\n" + - " name,\n" + - " addresses{\n" + - " addressId\n" + - " }\n" + + " meta::external::dataquality::tests::domain::Person{\n" + + " name" + " }\n" + " ]$;\n" + "}"; - PureModelContextData modelData = PureGrammarParser.newInstance().parseModel(validation); + + String expected = "{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"from_T_m__Mapping_1__Runtime_1__T_m_\",\"function\":\"from\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"serialize_Checked_MANY__RootGraphFetchTree_1__String_1_\",\"function\":\"serialize\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_\",\"function\":\"graphFetchChecked\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"filter_T_MANY__Function_1__T_MANY_\",\"function\":\"filter\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"getAll_Class_1__T_MANY_\",\"function\":\"getAll\",\"_type\":\"func\",\"parameters\":[{\"fullPath\":\"meta::external::dataquality::tests::domain::Person\",\"_type\":\"packageableElementPtr\"}]},{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"not_Boolean_1__Boolean_1_\",\"function\":\"not\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"greaterThanEqual_Number_1__Number_1__Boolean_1_\",\"function\":\"greaterThanEqual\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"age\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}]},{\"_type\":\"integer\",\"value\":18}]}]}],\"parameters\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"_type\":\"var\",\"name\":\"x\",\"class\":\"meta::external::dataquality::tests::domain::Person\"}]}]},{\"_type\":\"classInstance\",\"type\":\"rootGraphFetchTree\",\"value\":{\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"property\":\"name\"},{\"_type\":\"propertyGraphFetchTree\",\"property\":\"age\"}],\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::external::dataquality::tests::domain::Person\"}}]},{\"_type\":\"classInstance\",\"type\":\"rootGraphFetchTree\",\"value\":{\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"property\":\"name\"},{\"_type\":\"propertyGraphFetchTree\",\"property\":\"age\"}],\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::external::dataquality::tests::domain::Person\"}}]},{\"fullPath\":\"meta::external::dataquality::tests::domain::dataqualitymappings\",\"_type\":\"packageableElementPtr\"},{\"_type\":\"classInstance\",\"type\":\"runtimeInstance\",\"value\":{\"runtime\":{\"_type\":\"legacyRuntime\",\"connections\":[{\"_type\":\"RelationalDatabaseConnection\",\"authenticationStrategy\":{\"_type\":\"h2Default\"},\"type\":\"H2\",\"datasourceSpecification\":{\"_type\":\"h2Local\"},\"element\":\"meta::external::dataquality::tests::domain::db\"}]}}}]}]}"; + + assertLambda(validation, expected); + } + + private void assertLambda(String modelString, String expected) + { + String lambdaJson = generateLambda(modelString); + JsonAssert.assertJsonEquals(expected, lambdaJson, JsonAssert.when(Option.IGNORING_ARRAY_ORDER)); + } + + private void assertLambdaException(String modelString, String exception) + { + assertEquals(exception, assertThrows(PureAssertFailException.class, () -> generateLambda(modelString)).getInfo()); + } + + private String generateLambda(String modelString) + { + PureModelContextData modelData = loadWithModel(modelString); PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); - LambdaFunction lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::PersonDataQualityValidation"); - Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); - String lambdaJson = DataQualityLambdaGenerator.transformLambdaAsJson(lambdaFunction, model, routerExtensions); - JsonAssert.assertJsonEquals("{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"from_T_m__Mapping_1__Runtime_1__T_m_\",\"function\":\"from\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"serialize_T_MANY__RootGraphFetchTree_1__String_1_\",\"function\":\"serialize\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_\",\"function\":\"graphFetchChecked\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"filter_T_MANY__Function_1__T_MANY_\",\"function\":\"filter\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"getAll_Class_1__Date_1__T_MANY_\",\"function\":\"getAll\",\"_type\":\"func\",\"parameters\":[{\"fullPath\":\"meta::dataquality::Person\",\"_type\":\"packageableElementPtr\"},{\"_type\":\"var\",\"name\":\"businessDate\"}]},{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"or_Boolean_1__Boolean_1__Boolean_1_\",\"function\":\"or\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"not_Boolean_1__Boolean_1_\",\"function\":\"not\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"greaterThanEqual_Number_1__Number_1__Boolean_1_\",\"function\":\"greaterThanEqual\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"age\",\"parameters\":[{\"_type\":\"var\",\"name\":\"this\"}]},{\"_type\":\"integer\",\"value\":0}]}]},{\"fControl\":\"or_Boolean_1__Boolean_1__Boolean_1_\",\"function\":\"or\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"not_Boolean_1__Boolean_1_\",\"function\":\"not\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"greaterThanEqual_Number_1__Number_1__Boolean_1_\",\"function\":\"greaterThanEqual\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"age\",\"parameters\":[{\"_type\":\"var\",\"name\":\"this\"}]},{\"_type\":\"integer\",\"value\":18}]}]},{\"fControl\":\"not_Boolean_1__Boolean_1_\",\"function\":\"not\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"lessThan_Number_1__Number_1__Boolean_1_\",\"function\":\"lessThan\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"length_String_1__Integer_1_\",\"function\":\"length\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"name\",\"parameters\":[{\"_type\":\"var\",\"name\":\"this\"}]}]},{\"_type\":\"integer\",\"value\":1000}]}]}]}]}]}]},{\"_type\":\"classInstance\",\"type\":\"rootGraphFetchTree\",\"value\":{\"subTrees\":[{\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"property\":\"addressId\"}],\"_type\":\"propertyGraphFetchTree\",\"property\":\"addresses\"},{\"_type\":\"propertyGraphFetchTree\",\"property\":\"age\"},{\"_type\":\"propertyGraphFetchTree\",\"property\":\"name\"}],\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::dataquality::Person\"}}]},{\"_type\":\"classInstance\",\"type\":\"rootGraphFetchTree\",\"value\":{\"subTrees\":[{\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"property\":\"addressId\"}],\"_type\":\"propertyGraphFetchTree\",\"property\":\"addresses\"},{\"_type\":\"propertyGraphFetchTree\",\"property\":\"age\"},{\"_type\":\"propertyGraphFetchTree\",\"property\":\"name\"}],\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::dataquality::Person\"}}]},{\"fullPath\":\"meta::dataquality::dataqualitymappings\",\"_type\":\"packageableElementPtr\"},{\"_type\":\"classInstance\",\"type\":\"runtimeInstance\",\"value\":{\"runtime\":{\"_type\":\"legacyRuntime\",\"connections\":[{\"_type\":\"RelationalDatabaseConnection\",\"authenticationStrategy\":{\"_type\":\"h2Default\"},\"type\":\"H2\",\"datasourceSpecification\":{\"_type\":\"h2Local\"},\"element\":\"meta::dataquality::db\"}]}}}]}]}", - lambdaJson, JsonAssert.when(Option.IGNORING_ARRAY_ORDER)); + Function> routerExtensions = extensions(); + LambdaFunction lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", routerExtensions); + return DataQualityLambdaGenerator.transformLambdaAsJson(lambdaFunction, model, routerExtensions); } - private static final String COMPILATION_PREREQUISITE_CODE = "###Connection\n" + - "RelationalDatabaseConnection meta::dataquality::H2\n" + - "{\n" + - " store: meta::dataquality::db;\n" + - " type: H2;\n" + - " specification: LocalH2\n" + - " { \n" + - " testDataSetupSqls: [];\n" + - " };\n" + - " auth: DefaultH2;\n" + - "}\n" + - "\n" + - "###Relational\n" + - "Database meta::dataquality::db\n" + - "(\n" + - " Table personTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT)\n" + - " Table addressTable (ID INT PRIMARY KEY, LOCATIONID INT, TYPE INT)\n" + - " Table locationTable (ID INT PRIMARY KEY, STREET VARCHAR(200), LOCALITY VARCHAR(200))\n" + - "\n" + - " Join Address_Person(addressTable.ID = personTable.ADDRESSID)\n" + - " Join Address_Location(addressTable.LOCATIONID = locationTable.ID)\n" + - ")\n" + - "\n" + - "\n" + - "###Mapping\n" + - "Mapping meta::dataquality::dataqualitymappings\n" + - "(\n" + - "\n" + - " meta::dataquality::Person : Relational\n" + - " {\n" + - " name : [meta::dataquality::db]personTable.FIRSTNAME,\n" + - " age : [meta::dataquality::db]personTable.AGE,\n" + - " addresses : [meta::dataquality::db]@Address_Person\n" + - " }\n" + - "\n" + - " meta::dataquality::Address : Relational\n" + - " {\n" + - " addressId : [meta::dataquality::db]addressTable.ID,\n" + - " location : [meta::dataquality::db]@Address_Location\n" + - " }\n" + - "\n" + - " meta::dataquality::Location : Relational\n" + - " {\n" + - " street : [meta::dataquality::db]locationTable.STREET,\n" + - " locality : [meta::dataquality::db]locationTable.LOCALITY\n" + - " }\n" + - ")\n" + - "\n" + - "###Runtime\n" + - "Runtime meta::dataquality::DataQualityRuntime\n" + - "{\n" + - " mappings:\n" + - " [\n" + - " meta::dataquality::dataqualitymappings\n" + - " ];\n" + - " connections:\n" + - " [\n" + - " meta::dataquality::db:\n" + - " [\n" + - " connection_1: meta::dataquality::H2\n" + - " ]\n" + - " ];\n" + - "}\n" + - "\n" + - "\n" + - "\n" + - "###Pure\n" + - "Class meta::dataquality::Person\n" + - "[\n" + - " mustBeOfLegalAge: $this.age >= 18,\n" + - " validNameLength: $this.name->length() < 1000,\n" + - " ageMustBePositive: $this.age >= 0\n" + - "]\n" + - "{\n" + - " name : String[1];\n" + - " age : Integer[1];\n" + - " addresses : meta::dataquality::Address[*];\n" + - "}\n" + - "\n" + - "Class meta::dataquality::Address\n" + - "[\n" + - " validAddressId: $this.addressId->isNotEmpty()\n" + - "]\n" + - "{\n" + - " location: meta::dataquality::Location[1];\n" + - " locationStreet: String[1];\n" + - " addressId: String[1];\n" + - "}\n" + - "\n" + - "Class meta::dataquality::Location\n" + - "{\n" + - " street: String[1];\n" + - " locality: String[1];\n" + - "}\n"; + private static PureModelContextData loadWithModel(String code) + { + return GrammarParseTestUtils.loadPureModelContextFromResources( + FastList.newListWith( + "core_dataquality_test/dataquality_test_model.pure", + "core_dataquality_test/dataquality_test_model_legend.txt"), + code, TestDataQualityLambdaGenerator.class); + } -} + private static Function> extensions() + { + return (p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); + } +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityValidationArtifactGenerationExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityValidationArtifactGenerationExtension.java index 740a5604c59..3d3ad38c285 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityValidationArtifactGenerationExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityValidationArtifactGenerationExtension.java @@ -14,20 +14,17 @@ package org.finos.legend.engine.generation.dataquality; +import org.eclipse.collections.impl.list.mutable.FastList; import org.finos.legend.engine.language.pure.compiler.Compiler; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; -import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtensionLoader; import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.language.pure.grammar.test.GrammarParseTestUtils; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; -import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.finos.legend.engine.shared.core.deployment.DeploymentMode; import org.finos.legend.engine.shared.core.identity.Identity; import org.junit.Test; -import java.io.IOException; -import java.net.URL; import java.util.List; -import java.util.Objects; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -37,24 +34,39 @@ public class TestDataQualityValidationArtifactGenerationExtension { @Test - public void testDataQualityValidationArtifact() throws IOException + public void testDataQualityValidationArtifact() { - URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithDataQualityValidation.json")); - PureModelContextData pureModelContextData = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().readValue(url, PureModelContextData.class); - CompilerExtensionLoader.logExtensionList(); + String tree = "###DataQualityValidation\n" + + "DataQualityValidation meta::dataquality::Validation\n" + + "{\n" + + " context: fromMappingAndRuntime(meta::external::dataquality::tests::domain::dataqualitymappings, meta::external::dataquality::tests::domain::DataQualityRuntime);\n" + + " validationTree: $[\n" + + " meta::external::dataquality::tests::domain::Person{\n" + + " name\n" + + " }\n" + + " ]$;\n" + + " filter: p: meta::external::dataquality::tests::domain::Person[1]|$p.name == 'John';\n" + + "}"; + + PureModelContextData pureModelContextData = GrammarParseTestUtils.loadPureModelContextFromResources( + FastList.newListWith( + "core_dataquality_test/dataquality_test_model.pure", + "core_dataquality_test/dataquality_test_model_legend.txt"), + tree, TestDataQualityValidationArtifactGenerationExtension.class); PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); DataQualityValidationArtifactGenerationExtension extension = new DataQualityValidationArtifactGenerationExtension(); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation"); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::Validation"); assertTrue(extension.canGenerate(packageableElement)); List outputs = extension.generate(packageableElement, model, pureModelContextData, "vX_X_X"); assertEquals(2, outputs.size()); + Optional artifactOptional = outputs.stream().filter(artifact -> "dataQualityRulesMetadata.json".equalsIgnoreCase(artifact.path)).findAny(); assertTrue(artifactOptional.isPresent()); Artifact dataQualityMetaData = artifactOptional.get(); - assertEquals("{\"dqRules\":[{\"constraintGrammar\":\"($this.age >= 18)\",\"constraintName\":\"mustBeOfLegalAge\",\"constraintType\":\"Alloy_Constraint_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"Class\",\"constraintName\":\"Person\",\"constraintType\":\"Alloy_Class_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"name\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::name\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"age\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::age\"}]}", dataQualityMetaData.content); + assertEquals("{\"dqRules\":[{\"constraintGrammar\":\"($x.age >= 18)\",\"constraintName\":\"mustBeOfLegalAge\",\"constraintType\":\"Alloy_Constraint_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"Class\",\"constraintName\":\"Person\",\"constraintType\":\"Alloy_Class_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"name\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::name\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"age\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::age\"}]}", dataQualityMetaData.content); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement2 = model.getPackageableElement("meta::dataquality::Person"); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement2 = model.getPackageableElement("meta::external::dataquality::tests::domain::Person"); assertFalse(extension.canGenerate(packageableElement2)); } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/resources/inputs/modelWithDataQualityValidation.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/resources/inputs/modelWithDataQualityValidation.json deleted file mode 100644 index ca5cafab1d9..00000000000 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/resources/inputs/modelWithDataQualityValidation.json +++ /dev/null @@ -1,1533 +0,0 @@ -{ - "_type": "data", - "elements": [ - { - "_type": "connection", - "connectionValue": { - "_type": "RelationalDatabaseConnection", - "authenticationStrategy": { - "_type": "h2Default", - "sourceInformation": { - "endColumn": 18, - "endLine": 10, - "sourceId": "", - "startColumn": 3, - "startLine": 10 - } - }, - "databaseType": "H2", - "datasourceSpecification": { - "_type": "h2Local", - "sourceInformation": { - "endColumn": 4, - "endLine": 9, - "sourceId": "", - "startColumn": 3, - "startLine": 6 - }, - "testDataSetupSqls": [] - }, - "element": "meta::dataquality::db", - "elementSourceInformation": { - "endColumn": 30, - "endLine": 4, - "sourceId": "", - "startColumn": 10, - "startLine": 4 - }, - "postProcessorWithParameter": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 11, - "sourceId": "", - "startColumn": 1, - "startLine": 2 - }, - "type": "H2" - }, - "name": "H2", - "package": "meta::dataquality", - "sourceInformation": { - "endColumn": 1, - "endLine": 11, - "sourceId": "", - "startColumn": 1, - "startLine": 2 - } - }, - { - "_type": "relational", - "filters": [], - "includedStores": [], - "joins": [ - { - "name": "Address_Person", - "operation": { - "_type": "dynaFunc", - "funcName": "equal", - "parameters": [ - { - "_type": "column", - "column": "ID", - "sourceInformation": { - "endColumn": 62, - "endLine": 20, - "sourceId": "", - "startColumn": 24, - "startLine": 20 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 35, - "endLine": 20, - "sourceId": "", - "startColumn": 24, - "startLine": 20 - }, - "table": "addressTable" - }, - "tableAlias": "addressTable" - }, - { - "_type": "column", - "column": "ADDRESSID", - "sourceInformation": { - "endColumn": 62, - "endLine": 20, - "sourceId": "", - "startColumn": 42, - "startLine": 20 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 52, - "endLine": 20, - "sourceId": "", - "startColumn": 42, - "startLine": 20 - }, - "table": "personTable" - }, - "tableAlias": "personTable" - } - ], - "sourceInformation": { - "endColumn": 62, - "endLine": 20, - "sourceId": "", - "startColumn": 40, - "startLine": 20 - } - }, - "sourceInformation": { - "endColumn": 63, - "endLine": 20, - "sourceId": "", - "startColumn": 4, - "startLine": 20 - } - }, - { - "name": "Address_Location", - "operation": { - "_type": "dynaFunc", - "funcName": "equal", - "parameters": [ - { - "_type": "column", - "column": "LOCATIONID", - "sourceInformation": { - "endColumn": 67, - "endLine": 21, - "sourceId": "", - "startColumn": 26, - "startLine": 21 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 37, - "endLine": 21, - "sourceId": "", - "startColumn": 26, - "startLine": 21 - }, - "table": "addressTable" - }, - "tableAlias": "addressTable" - }, - { - "_type": "column", - "column": "ID", - "sourceInformation": { - "endColumn": 67, - "endLine": 21, - "sourceId": "", - "startColumn": 52, - "startLine": 21 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 64, - "endLine": 21, - "sourceId": "", - "startColumn": 52, - "startLine": 21 - }, - "table": "locationTable" - }, - "tableAlias": "locationTable" - } - ], - "sourceInformation": { - "endColumn": 67, - "endLine": 21, - "sourceId": "", - "startColumn": 50, - "startLine": 21 - } - }, - "sourceInformation": { - "endColumn": 68, - "endLine": 21, - "sourceId": "", - "startColumn": 4, - "startLine": 21 - } - } - ], - "name": "db", - "package": "meta::dataquality", - "schemas": [ - { - "name": "default", - "sourceInformation": { - "endColumn": 1, - "endLine": 22, - "sourceId": "", - "startColumn": 1, - "startLine": 14 - }, - "tables": [ - { - "columns": [ - { - "name": "ID", - "nullable": false, - "sourceInformation": { - "endColumn": 40, - "endLine": 16, - "sourceId": "", - "startColumn": 23, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "FIRSTNAME", - "nullable": true, - "sourceInformation": { - "endColumn": 64, - "endLine": 16, - "sourceId": "", - "startColumn": 43, - "startLine": 16 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - }, - { - "name": "LASTNAME", - "nullable": true, - "sourceInformation": { - "endColumn": 87, - "endLine": 16, - "sourceId": "", - "startColumn": 67, - "startLine": 16 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - }, - { - "name": "AGE", - "nullable": true, - "sourceInformation": { - "endColumn": 96, - "endLine": 16, - "sourceId": "", - "startColumn": 90, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "ADDRESSID", - "nullable": true, - "sourceInformation": { - "endColumn": 111, - "endLine": 16, - "sourceId": "", - "startColumn": 99, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "FIRMID", - "nullable": true, - "sourceInformation": { - "endColumn": 123, - "endLine": 16, - "sourceId": "", - "startColumn": 114, - "startLine": 16 - }, - "type": { - "_type": "Integer" - } - } - ], - "milestoning": [], - "name": "personTable", - "primaryKey": [ - "ID" - ], - "sourceInformation": { - "endColumn": 124, - "endLine": 16, - "sourceId": "", - "startColumn": 4, - "startLine": 16 - } - }, - { - "columns": [ - { - "name": "ID", - "nullable": false, - "sourceInformation": { - "endColumn": 41, - "endLine": 17, - "sourceId": "", - "startColumn": 24, - "startLine": 17 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "LOCATIONID", - "nullable": true, - "sourceInformation": { - "endColumn": 57, - "endLine": 17, - "sourceId": "", - "startColumn": 44, - "startLine": 17 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "TYPE", - "nullable": true, - "sourceInformation": { - "endColumn": 67, - "endLine": 17, - "sourceId": "", - "startColumn": 60, - "startLine": 17 - }, - "type": { - "_type": "Integer" - } - } - ], - "milestoning": [], - "name": "addressTable", - "primaryKey": [ - "ID" - ], - "sourceInformation": { - "endColumn": 68, - "endLine": 17, - "sourceId": "", - "startColumn": 4, - "startLine": 17 - } - }, - { - "columns": [ - { - "name": "ID", - "nullable": false, - "sourceInformation": { - "endColumn": 42, - "endLine": 18, - "sourceId": "", - "startColumn": 25, - "startLine": 18 - }, - "type": { - "_type": "Integer" - } - }, - { - "name": "STREET", - "nullable": true, - "sourceInformation": { - "endColumn": 63, - "endLine": 18, - "sourceId": "", - "startColumn": 45, - "startLine": 18 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - }, - { - "name": "LOCALITY", - "nullable": true, - "sourceInformation": { - "endColumn": 86, - "endLine": 18, - "sourceId": "", - "startColumn": 66, - "startLine": 18 - }, - "type": { - "_type": "Varchar", - "size": 200 - } - } - ], - "milestoning": [], - "name": "locationTable", - "primaryKey": [ - "ID" - ], - "sourceInformation": { - "endColumn": 87, - "endLine": 18, - "sourceId": "", - "startColumn": 4, - "startLine": 18 - } - } - ], - "views": [] - } - ], - "sourceInformation": { - "endColumn": 1, - "endLine": 22, - "sourceId": "", - "startColumn": 1, - "startLine": 14 - }, - "stereotypes": [] - }, - { - "_type": "mapping", - "associationMappings": [], - "classMappings": [ - { - "_type": "relational", - "class": "meta::dataquality::Person", - "classSourceInformation": { - "endColumn": 28, - "endLine": 29, - "sourceId": "", - "startColumn": 4, - "startLine": 29 - }, - "distinct": false, - "groupBy": [], - "primaryKey": [], - "propertyMappings": [ - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Person", - "property": "name", - "sourceInformation": { - "endColumn": 10, - "endLine": 31, - "sourceId": "", - "startColumn": 7, - "startLine": 31 - } - }, - "relationalOperation": { - "_type": "column", - "column": "FIRSTNAME", - "sourceInformation": { - "endColumn": 57, - "endLine": 31, - "sourceId": "", - "startColumn": 14, - "startLine": 31 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 47, - "endLine": 31, - "sourceId": "", - "startColumn": 37, - "startLine": 31 - }, - "table": "personTable" - }, - "tableAlias": "personTable" - }, - "sourceInformation": { - "endColumn": 57, - "endLine": 31, - "sourceId": "", - "startColumn": 12, - "startLine": 31 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Person", - "property": "age", - "sourceInformation": { - "endColumn": 9, - "endLine": 32, - "sourceId": "", - "startColumn": 7, - "startLine": 32 - } - }, - "relationalOperation": { - "_type": "column", - "column": "AGE", - "sourceInformation": { - "endColumn": 50, - "endLine": 32, - "sourceId": "", - "startColumn": 13, - "startLine": 32 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 46, - "endLine": 32, - "sourceId": "", - "startColumn": 36, - "startLine": 32 - }, - "table": "personTable" - }, - "tableAlias": "personTable" - }, - "sourceInformation": { - "endColumn": 50, - "endLine": 32, - "sourceId": "", - "startColumn": 11, - "startLine": 32 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Person", - "property": "addresses", - "sourceInformation": { - "endColumn": 15, - "endLine": 33, - "sourceId": "", - "startColumn": 7, - "startLine": 33 - } - }, - "relationalOperation": { - "_type": "elemtWithJoins", - "joins": [ - { - "db": "meta::dataquality::db", - "name": "Address_Person", - "sourceInformation": { - "endColumn": 56, - "endLine": 33, - "sourceId": "", - "startColumn": 42, - "startLine": 33 - } - } - ], - "sourceInformation": { - "endColumn": 56, - "endLine": 33, - "sourceId": "", - "startColumn": 19, - "startLine": 33 - } - }, - "sourceInformation": { - "endColumn": 56, - "endLine": 33, - "sourceId": "", - "startColumn": 17, - "startLine": 33 - } - } - ], - "root": false, - "sourceInformation": { - "endColumn": 4, - "endLine": 34, - "sourceId": "", - "startColumn": 4, - "startLine": 29 - } - }, - { - "_type": "relational", - "class": "meta::dataquality::Address", - "classSourceInformation": { - "endColumn": 29, - "endLine": 36, - "sourceId": "", - "startColumn": 4, - "startLine": 36 - }, - "distinct": false, - "groupBy": [], - "primaryKey": [], - "propertyMappings": [ - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Address", - "property": "addressId", - "sourceInformation": { - "endColumn": 15, - "endLine": 38, - "sourceId": "", - "startColumn": 7, - "startLine": 38 - } - }, - "relationalOperation": { - "_type": "column", - "column": "ID", - "sourceInformation": { - "endColumn": 56, - "endLine": 38, - "sourceId": "", - "startColumn": 19, - "startLine": 38 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 53, - "endLine": 38, - "sourceId": "", - "startColumn": 42, - "startLine": 38 - }, - "table": "addressTable" - }, - "tableAlias": "addressTable" - }, - "sourceInformation": { - "endColumn": 56, - "endLine": 38, - "sourceId": "", - "startColumn": 17, - "startLine": 38 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Address", - "property": "location", - "sourceInformation": { - "endColumn": 14, - "endLine": 39, - "sourceId": "", - "startColumn": 7, - "startLine": 39 - } - }, - "relationalOperation": { - "_type": "elemtWithJoins", - "joins": [ - { - "db": "meta::dataquality::db", - "name": "Address_Location", - "sourceInformation": { - "endColumn": 57, - "endLine": 39, - "sourceId": "", - "startColumn": 41, - "startLine": 39 - } - } - ], - "sourceInformation": { - "endColumn": 57, - "endLine": 39, - "sourceId": "", - "startColumn": 18, - "startLine": 39 - } - }, - "sourceInformation": { - "endColumn": 57, - "endLine": 39, - "sourceId": "", - "startColumn": 16, - "startLine": 39 - } - } - ], - "root": false, - "sourceInformation": { - "endColumn": 4, - "endLine": 40, - "sourceId": "", - "startColumn": 4, - "startLine": 36 - } - }, - { - "_type": "relational", - "class": "meta::dataquality::Location", - "classSourceInformation": { - "endColumn": 30, - "endLine": 42, - "sourceId": "", - "startColumn": 4, - "startLine": 42 - }, - "distinct": false, - "groupBy": [], - "primaryKey": [], - "propertyMappings": [ - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Location", - "property": "street", - "sourceInformation": { - "endColumn": 12, - "endLine": 44, - "sourceId": "", - "startColumn": 7, - "startLine": 44 - } - }, - "relationalOperation": { - "_type": "column", - "column": "STREET", - "sourceInformation": { - "endColumn": 58, - "endLine": 44, - "sourceId": "", - "startColumn": 16, - "startLine": 44 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 51, - "endLine": 44, - "sourceId": "", - "startColumn": 39, - "startLine": 44 - }, - "table": "locationTable" - }, - "tableAlias": "locationTable" - }, - "sourceInformation": { - "endColumn": 58, - "endLine": 44, - "sourceId": "", - "startColumn": 14, - "startLine": 44 - } - }, - { - "_type": "relationalPropertyMapping", - "property": { - "class": "meta::dataquality::Location", - "property": "locality", - "sourceInformation": { - "endColumn": 14, - "endLine": 45, - "sourceId": "", - "startColumn": 7, - "startLine": 45 - } - }, - "relationalOperation": { - "_type": "column", - "column": "LOCALITY", - "sourceInformation": { - "endColumn": 62, - "endLine": 45, - "sourceId": "", - "startColumn": 18, - "startLine": 45 - }, - "table": { - "_type": "Table", - "database": "meta::dataquality::db", - "mainTableDb": "meta::dataquality::db", - "schema": "default", - "sourceInformation": { - "endColumn": 53, - "endLine": 45, - "sourceId": "", - "startColumn": 41, - "startLine": 45 - }, - "table": "locationTable" - }, - "tableAlias": "locationTable" - }, - "sourceInformation": { - "endColumn": 62, - "endLine": 45, - "sourceId": "", - "startColumn": 16, - "startLine": 45 - } - } - ], - "root": false, - "sourceInformation": { - "endColumn": 4, - "endLine": 46, - "sourceId": "", - "startColumn": 4, - "startLine": 42 - } - } - ], - "enumerationMappings": [], - "includedMappings": [], - "name": "dataqualitymappings", - "package": "meta::dataquality", - "sourceInformation": { - "endColumn": 1, - "endLine": 47, - "sourceId": "", - "startColumn": 1, - "startLine": 26 - }, - "tests": [] - }, - { - "_type": "runtime", - "name": "DataQualityRuntime", - "package": "meta::dataquality", - "runtimeValue": { - "_type": "engineRuntime", - "connectionStores": [], - "connections": [ - { - "sourceInformation": { - "endColumn": 5, - "endLine": 61, - "sourceId": "", - "startColumn": 5, - "startLine": 58 - }, - "store": { - "path": "meta::dataquality::db", - "sourceInformation": { - "endColumn": 25, - "endLine": 58, - "sourceId": "", - "startColumn": 5, - "startLine": 58 - }, - "type": "STORE" - }, - "storeConnections": [ - { - "connection": { - "_type": "connectionPointer", - "connection": "meta::dataquality::H2", - "sourceInformation": { - "endColumn": 41, - "endLine": 60, - "sourceId": "", - "startColumn": 21, - "startLine": 60 - } - }, - "id": "connection_1", - "sourceInformation": { - "endColumn": 41, - "endLine": 60, - "sourceId": "", - "startColumn": 7, - "startLine": 60 - } - } - ] - } - ], - "mappings": [ - { - "path": "meta::dataquality::dataqualitymappings", - "sourceInformation": { - "endColumn": 42, - "endLine": 54, - "sourceId": "", - "startColumn": 5, - "startLine": 54 - }, - "type": "MAPPING" - } - ], - "sourceInformation": { - "endColumn": 1, - "endLine": 63, - "sourceId": "", - "startColumn": 1, - "startLine": 50 - } - }, - "sourceInformation": { - "endColumn": 1, - "endLine": 63, - "sourceId": "", - "startColumn": 1, - "startLine": 50 - } - }, - { - "_type": "class", - "constraints": [ - { - "functionDefinition": { - "_type": "lambda", - "body": [ - { - "_type": "func", - "function": "greaterThanEqual", - "parameters": [ - { - "_type": "property", - "parameters": [ - { - "_type": "var", - "name": "this", - "sourceInformation": { - "endColumn": 25, - "endLine": 70, - "sourceId": "", - "startColumn": 21, - "startLine": 70 - } - } - ], - "property": "age", - "sourceInformation": { - "endColumn": 29, - "endLine": 70, - "sourceId": "", - "startColumn": 27, - "startLine": 70 - } - }, - { - "_type": "integer", - "sourceInformation": { - "endColumn": 35, - "endLine": 70, - "sourceId": "", - "startColumn": 34, - "startLine": 70 - }, - "value": 18 - } - ], - "sourceInformation": { - "endColumn": 35, - "endLine": 70, - "sourceId": "", - "startColumn": 31, - "startLine": 70 - } - } - ], - "parameters": [] - }, - "name": "mustBeOfLegalAge", - "sourceInformation": { - "endColumn": 35, - "endLine": 70, - "sourceId": "", - "startColumn": 3, - "startLine": 70 - } - } - ], - "name": "Person", - "originalMilestonedProperties": [], - "package": "meta::dataquality", - "properties": [ - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "name", - "propertyTypeSourceInformation": { - "endColumn": 16, - "endLine": 73, - "sourceId": "", - "startColumn": 11, - "startLine": 73 - }, - "sourceInformation": { - "endColumn": 20, - "endLine": 73, - "sourceId": "", - "startColumn": 4, - "startLine": 73 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "age", - "propertyTypeSourceInformation": { - "endColumn": 16, - "endLine": 74, - "sourceId": "", - "startColumn": 10, - "startLine": 74 - }, - "sourceInformation": { - "endColumn": 20, - "endLine": 74, - "sourceId": "", - "startColumn": 4, - "startLine": 74 - }, - "stereotypes": [], - "taggedValues": [], - "type": "Integer" - }, - { - "multiplicity": { - "lowerBound": 0 - }, - "name": "addresses", - "propertyTypeSourceInformation": { - "endColumn": 41, - "endLine": 75, - "sourceId": "", - "startColumn": 16, - "startLine": 75 - }, - "sourceInformation": { - "endColumn": 45, - "endLine": 75, - "sourceId": "", - "startColumn": 4, - "startLine": 75 - }, - "stereotypes": [], - "taggedValues": [], - "type": "meta::dataquality::Address" - } - ], - "qualifiedProperties": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 76, - "sourceId": "", - "startColumn": 1, - "startLine": 68 - }, - "stereotypes": [], - "superTypes": [], - "taggedValues": [] - }, - { - "_type": "class", - "constraints": [], - "name": "Address", - "originalMilestonedProperties": [], - "package": "meta::dataquality", - "properties": [ - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "location", - "propertyTypeSourceInformation": { - "endColumn": 40, - "endLine": 80, - "sourceId": "", - "startColumn": 14, - "startLine": 80 - }, - "sourceInformation": { - "endColumn": 44, - "endLine": 80, - "sourceId": "", - "startColumn": 4, - "startLine": 80 - }, - "stereotypes": [], - "taggedValues": [], - "type": "meta::dataquality::Location" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "locationStreet", - "propertyTypeSourceInformation": { - "endColumn": 25, - "endLine": 81, - "sourceId": "", - "startColumn": 20, - "startLine": 81 - }, - "sourceInformation": { - "endColumn": 29, - "endLine": 81, - "sourceId": "", - "startColumn": 4, - "startLine": 81 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "addressId", - "propertyTypeSourceInformation": { - "endColumn": 20, - "endLine": 82, - "sourceId": "", - "startColumn": 15, - "startLine": 82 - }, - "sourceInformation": { - "endColumn": 24, - "endLine": 82, - "sourceId": "", - "startColumn": 4, - "startLine": 82 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - } - ], - "qualifiedProperties": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 83, - "sourceId": "", - "startColumn": 1, - "startLine": 78 - }, - "stereotypes": [], - "superTypes": [], - "taggedValues": [] - }, - { - "_type": "class", - "constraints": [], - "name": "Location", - "originalMilestonedProperties": [], - "package": "meta::dataquality", - "properties": [ - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "street", - "propertyTypeSourceInformation": { - "endColumn": 17, - "endLine": 87, - "sourceId": "", - "startColumn": 12, - "startLine": 87 - }, - "sourceInformation": { - "endColumn": 21, - "endLine": 87, - "sourceId": "", - "startColumn": 4, - "startLine": 87 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - }, - { - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "locality", - "propertyTypeSourceInformation": { - "endColumn": 19, - "endLine": 88, - "sourceId": "", - "startColumn": 14, - "startLine": 88 - }, - "sourceInformation": { - "endColumn": 23, - "endLine": 88, - "sourceId": "", - "startColumn": 4, - "startLine": 88 - }, - "stereotypes": [], - "taggedValues": [], - "type": "String" - } - ], - "qualifiedProperties": [], - "sourceInformation": { - "endColumn": 1, - "endLine": 89, - "sourceId": "", - "startColumn": 1, - "startLine": 85 - }, - "stereotypes": [], - "superTypes": [], - "taggedValues": [] - }, - { - "_type": "dataQualityValidation", - "context": { - "_type": "mappingAndRuntimeDataQualityExecutionContext", - "_type": "mappingAndRuntimeDataQualityExecutionContext", - "mapping": { - "path": "meta::dataquality::dataqualitymappings", - "sourceInformation": { - "endColumn": 73, - "endLine": 94, - "sourceId": "", - "startColumn": 36, - "startLine": 94 - }, - "type": "MAPPING" - }, - "runtime": { - "path": "meta::dataquality::DataQualityRuntime", - "sourceInformation": { - "endColumn": 112, - "endLine": 94, - "sourceId": "", - "startColumn": 76, - "startLine": 94 - }, - "type": "RUNTIME" - } - }, - "dataQualityRootGraphFetchTree": { - "_type": "dataQualityRootGraphFetchTree", - "_type": "dataQualityRootGraphFetchTree", - "class": "meta::dataquality::Person", - "constraints": [ - "mustBeOfLegalAge" - ], - "sourceInformation": { - "endColumn": 31, - "endLine": 97, - "sourceId": "", - "startColumn": 7, - "startLine": 97 - }, - "subTrees": [ - { - "_type": "dataQualityPropertyGraphFetchTree", - "_type": "dataQualityPropertyGraphFetchTree", - "constraints": [], - "parameters": [], - "property": "name", - "sourceInformation": { - "endColumn": 12, - "endLine": 98, - "sourceId": "", - "startColumn": 9, - "startLine": 98 - }, - "subTrees": [], - "subTypeTrees": [] - } - ], - "subTypeTrees": [] - }, - "filter": { - "_type": "lambda", - "body": [ - { - "_type": "func", - "function": "equal", - "parameters": [ - { - "_type": "property", - "parameters": [ - { - "_type": "var", - "name": "p", - "sourceInformation": { - "endColumn": 47, - "endLine": 95, - "sourceId": "", - "startColumn": 46, - "startLine": 95 - } - } - ], - "property": "name", - "sourceInformation": { - "endColumn": 52, - "endLine": 95, - "sourceId": "", - "startColumn": 49, - "startLine": 95 - } - }, - { - "_type": "string", - "sourceInformation": { - "endColumn": 60, - "endLine": 95, - "sourceId": "", - "startColumn": 55, - "startLine": 95 - }, - "value": "John" - } - ], - "sourceInformation": { - "endColumn": 54, - "endLine": 95, - "sourceId": "", - "startColumn": 53, - "startLine": 95 - } - } - ], - "parameters": [ - { - "_type": "var", - "class": "meta::dataquality::Person", - "multiplicity": { - "lowerBound": 1, - "upperBound": 1 - }, - "name": "p", - "sourceInformation": { - "endColumn": 39, - "endLine": 95, - "sourceId": "", - "startColumn": 15, - "startLine": 95 - } - } - ], - "sourceInformation": { - "endColumn": 60, - "endLine": 95, - "sourceId": "", - "startColumn": 44, - "startLine": 95 - } - }, - "name": "PersonDataQualityValidation", - "package": "meta::dataquality", - "sourceInformation": { - "endColumn": 1, - "endLine": 101, - "sourceId": "", - "startColumn": 1, - "startLine": 92 - }, - "stereotypes": [], - "taggedValues": [] - }, - { - "_type": "sectionIndex", - "name": "SectionIndex", - "package": "__internal__", - "sections": [ - { - "_type": "importAware", - "elements": [], - "imports": [], - "parserName": "Pure", - "sourceInformation": { - "endColumn": 8, - "endLine": 1, - "sourceId": "", - "startColumn": 1, - "startLine": 1 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::H2" - ], - "imports": [], - "parserName": "Connection", - "sourceInformation": { - "endColumn": 2, - "endLine": 13, - "sourceId": "", - "startColumn": 8, - "startLine": 2 - } - }, - { - "_type": "default", - "elements": [ - "meta::dataquality::db" - ], - "parserName": "Relational", - "sourceInformation": { - "endColumn": 1, - "endLine": 25, - "sourceId": "", - "startColumn": 1, - "startLine": 14 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::dataqualitymappings" - ], - "imports": [], - "parserName": "Mapping", - "sourceInformation": { - "endColumn": 2, - "endLine": 49, - "sourceId": "", - "startColumn": 1, - "startLine": 26 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::DataQualityRuntime" - ], - "imports": [], - "parserName": "Runtime", - "sourceInformation": { - "endColumn": 1, - "endLine": 67, - "sourceId": "", - "startColumn": 1, - "startLine": 50 - } - }, - { - "_type": "importAware", - "elements": [ - "meta::dataquality::Person", - "meta::dataquality::Address", - "meta::dataquality::Location" - ], - "imports": [], - "parserName": "Pure", - "sourceInformation": { - "endColumn": 2, - "endLine": 91, - "sourceId": "", - "startColumn": 1, - "startLine": 68 - } - }, - { - "_type": "default", - "elements": [ - "meta::dataquality::PersonDataQualityValidation" - ], - "parserName": "DataQualityValidation", - "sourceInformation": { - "endColumn": 1, - "endLine": 103, - "sourceId": "", - "startColumn": 1, - "startLine": 92 - } - } - ] - } - ] -} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/pom.xml index a34830ee180..118b32cffc4 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/pom.xml +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/pom.xml @@ -85,6 +85,11 @@ legend-engine-pure-platform-dsl-graph-java ${project.version} + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-legendCompiler + ${project.version} + org.finos.legend.pure legend-pure-m2-store-relational-pure @@ -177,6 +182,11 @@ org.finos.legend.pure legend-pure-m2-store-relational-pure + runtime + + + org.finos.legend.pure + legend-pure-m2-dsl-graph-pure org.finos.legend.engine @@ -193,6 +203,7 @@ org.finos.legend.pure legend-pure-m2-dsl-mapping-pure + runtime org.finos.legend.pure @@ -224,6 +235,16 @@ legend-engine-xt-dataquality-pure ${project.version} + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-legendCompiler + ${project.version} + runtime + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-unclassified + junit @@ -244,6 +265,16 @@ jackson-core test + + org.finos.legend.engine + legend-engine-xt-dataquality-compiler + runtime + + + org.finos.legend.engine + legend-engine-xt-relationalStore-grammar + runtime + \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test.definition.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test.definition.json index 3ada292dae1..3e605360a54 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test.definition.json +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test.definition.json @@ -5,11 +5,13 @@ "core", "core_relational", "core_data_space_metamodel", + "core_functions_unclassified", "platform", "platform_dsl_store", "platform_dsl_graph", "platform_dsl_mapping", "platform_store_relational", - "core_dataquality" + "core_dataquality", + "core_external_compiler" ] } \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure index 3f00dcdf09a..f02b9552eda 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure @@ -1,118 +1,173 @@ -import meta::pure::router::platform::execution::*; -import meta::pure::graphFetch::*; -import meta::external::dataquality::*; +import meta::pure::graphFetch::execution::*; import meta::external::dataquality::tests::*; -import meta::pure::metamodel::dataSpace::*; -import meta::pure::mapping::*; -import meta::core::runtime::*; -import meta::external::store::relational::tests::*; -import meta::relational::metamodel::*; -import meta::pure::executionPlan::*; -import meta::relational::tests::milestoning::*; -import meta::pure::profiles::*; -import meta::pure::metamodel::serialization::grammar::*; +import meta::external::dataquality::tests::domain::*; - -function <> meta::external::dataquality::tests::testLambdaGeneration():Boolean[1] -{ let dqRootConstraints = meta::external::dataquality::tests::Person->getAllTypeGeneralisations()->filter(x| $x->instanceOf(ElementWithConstraints))->cast(@ElementWithConstraints).constraints->filter(x| $x.name == 'addressIDGreaterThan2'); - - let validationTree = ^DataQualityRootGraphFetchTree( - class=meta::external::dataquality::tests::Person, - constraints=$dqRootConstraints->at(0), - subTrees=meta::external::dataquality::tests::Person->hierarchicalProperties()->filter(p | $p->isPrimitiveValueProperty())->filter(p| $p.name=='age')->map(p | ^DataQualityPropertyGraphFetchTree(property=$p)) - ); - let filter = p:Person[1]| $p.name=='John'; - let dataquality = ^DataQuality(validationTree=$validationTree, - context=^meta::external::dataquality::MappingAndRuntimeDataQualityExecutionContext(runtime=^meta::core::runtime::EngineRuntime(connectionStores = testDatabaseConnection(meta::relational::tests::db, [])), mapping=meta::external::dataquality::tests::dqValidationPersonMapping), - filter=$filter); - - // fetch lambda from DQ func; - let dqLambda = meta::external::dataquality::executeDataQualityValidation($dataquality, []); - assert($dqLambda->isNotEmpty()); +function <> meta::external::dataquality::tests::testLambdaGenerationMultipleConstraints():Boolean[1] +{ + doTest('$[' + + ' meta::external::dataquality::tests::domain::Person{' + + ' name,' + + ' addresses{' + + ' addressId' + + ' }' + + ' }' + + ' ]$', {|Person.all() + ->filter(x|(!(($x.age >= 0)) || (!(($x.age >= 18)) || !(($x.name->length() < 1000))))) + ->graphFetchChecked(#{ + Person { + addresses { + addressId + }, + age, + name + } + }#)->serialize(#{ + Person { + addresses { + addressId + }, + age, + name + } + }#) + }, true, false) } -function <> meta::external::dataquality::tests::testLambdaGeneration_OnlyStructuralConstraints():Boolean[1] -{ let dqRootConstraints = meta::external::dataquality::tests::Person->getAllTypeGeneralisations()->filter(x| $x->instanceOf(ElementWithConstraints))->cast(@ElementWithConstraints).constraints->filter(x| $x.name == 'addressIDGreaterThan2'); - - let validationTree = ^DataQualityRootGraphFetchTree( - class=meta::external::dataquality::tests::Person, - constraints=[], - subTrees=meta::external::dataquality::tests::Person->hierarchicalProperties()->filter(p | $p->isPrimitiveValueProperty())->filter(p| $p.name=='age')->map(p | ^DataQualityPropertyGraphFetchTree(property=$p)) - ); - let filter = p:Person[1]| $p.name=='John'; - let dataquality = ^DataQuality(validationTree=$validationTree, - context=^meta::external::dataquality::MappingAndRuntimeDataQualityExecutionContext(runtime=^meta::core::runtime::EngineRuntime(connectionStores = testDatabaseConnection(meta::relational::tests::db, [])), mapping=meta::external::dataquality::tests::dqValidationPersonMapping), - filter=$filter); - - let dqLambda = meta::external::dataquality::executeDataQualityValidation($dataquality, []); - assert($dqLambda->isNotEmpty()); +function <> meta::external::dataquality::tests::testLambdaGenerationNonTemporalToBusinessTemporal():Boolean[1] +{ + doTest('$[' + + ' meta::external::dataquality::tests::domain::NonTemporal{' + + ' name,' + + ' businessTemporal{' + + ' name' + + ' }' + + ' }' + + ' ]$', {businessDate:Date[1] | meta::external::dataquality::tests::domain::NonTemporal.all() + ->graphFetchChecked(#{ + NonTemporal { + name, + businessTemporal($businessDate) { + name + } + } + }#)->serialize(#{ + NonTemporal { + name, + businessTemporal($businessDate) { + name + } + } + }#) + }) } -function <> meta::external::dataquality::tests::testMetadataGeneration():Boolean[1] -{ let dqRootConstraints = meta::external::dataquality::tests::Person->getAllTypeGeneralisations()->filter(x| $x->instanceOf(ElementWithConstraints))->cast(@ElementWithConstraints).constraints->filter(x| $x.name == 'addressIDGreaterThan2' || $x.name == 'nameMustNotBeBlank'); - - let validationTree = ^DataQualityRootGraphFetchTree( - class=meta::external::dataquality::tests::Person, - constraints=$dqRootConstraints, - subTrees=meta::external::dataquality::tests::Person->hierarchicalProperties()->filter(p | $p->isPrimitiveValueProperty()->not())->filter(p| $p.name=='addresses')->map(p | ^DataQualityPropertyGraphFetchTree(property=$p, - subTrees=meta::external::dataquality::tests::Address->hierarchicalProperties()->filter(p | $p->isPrimitiveValueProperty())->filter(p| $p.name=='addressId')->map(p | ^DataQualityPropertyGraphFetchTree(property=$p)))) - ); - let filter = p:Person[1]| $p.name=='John'; - let dataquality = ^DataQuality(validationTree=$validationTree, - context=^meta::external::dataquality::MappingAndRuntimeDataQualityExecutionContext(runtime=^meta::core::runtime::EngineRuntime(connectionStores = testDatabaseConnection(meta::relational::tests::db, [])), mapping=meta::external::dataquality::tests::dqValidationPersonMapping), - filter=$filter); - - let dqMetadata = meta::external::dataquality::generateDQMetaDataForDQValidation($dataquality); - assert($dqMetadata->isNotEmpty()); +function <> meta::external::dataquality::tests::testLambdaGenerationNonTemporalToProcessingTemporal():Boolean[1] +{ + doTest('$[' + + ' meta::external::dataquality::tests::domain::NonTemporal{' + + ' name,' + + ' processingTemporal{' + + ' name' + + ' }' + + ' }' + + ' ]$', {processingDate:Date[1] | meta::external::dataquality::tests::domain::NonTemporal.all() + ->graphFetchChecked(#{ + NonTemporal { + name, + processingTemporal($processingDate) { + name + } + } + }#)->serialize(#{ + NonTemporal { + name, + processingTemporal($processingDate) { + name + } + } + }#) + }) } - - - -// -------------------------------------- Test Setup -------------------------------------------------------------------------// -// relational mappings -Class meta::external::dataquality::tests::Person -[ - nameMustNotBeBlank: $this.name->length() > 0, - ageMustBePositive: $this.age > 0, - nameAtLeast4Chars: $this.name->length() > 4, - ageMustBeGreaterThan10: $this.age > 100, - //addressIDGreaterThan1: $this.addresses->filter(a | $a.addressId > 1)->isNotEmpty(), - addressIDGreaterThan2: $this.addresses->filter(a | $a.addressId > 2)->isNotEmpty() -] +function <> meta::external::dataquality::tests::testLambdaGenerationNonTemporalToBusinessAndProcessingTemporal():Boolean[1] { - name : String[1]; - age : Integer[1]; - addresses : meta::external::dataquality::tests::Address[*]; + doTest('$[' + + ' meta::external::dataquality::tests::domain::NonTemporal{' + + ' name,' + + ' processingTemporal{' + + ' name' + + ' },' + + ' businessTemporal{' + + ' name' + + ' }' + + ' }' + + ' ]$', {processingDate:Date[1], businessDate:Date[1] | meta::external::dataquality::tests::domain::NonTemporal.all() + ->graphFetchChecked(#{ + NonTemporal { + name, + processingTemporal($processingDate) { + name + }, + businessTemporal($businessDate) { + name + } + } + }#)->serialize(#{ + NonTemporal { + name, + processingTemporal($processingDate) { + name + }, + businessTemporal($businessDate) { + name + } + } + }#) + }) } -Class meta::external::dataquality::tests::Address -[ - addressIDMustNotBeBlank: $this.addressId > 0 -] +function <> meta::external::dataquality::tests::testLambdaGenerationNonTemporalToBiTemporal():Boolean[1] { - // location: Location[1]; - // locationStreet: String[1]; - addressId: Integer[1]; + doTest('$[' + + ' meta::external::dataquality::tests::domain::NonTemporal{' + + ' name,' + + ' biTemporal{' + + ' name' + + ' }' + + ' }' + + ' ]$', {processingDate:Date[1], businessDate:Date[1] | meta::external::dataquality::tests::domain::NonTemporal.all() + ->graphFetchChecked(#{ + NonTemporal { + name, + biTemporal($processingDate, $businessDate) { + name + } + } + }#)->serialize(#{ + NonTemporal { + name, + biTemporal($processingDate, $businessDate) { + name + } + } + }#) + }) } -###Mapping -import meta::relational::validation::complex::*; - -Mapping meta::external::dataquality::tests::dqValidationPersonMapping -( - meta::external::dataquality::tests::Person : Relational - { - // ~filter [db] (INNER)@Address_Person | [db] ActiveAddressFilter - name : [db]personTable.FIRSTNAME, - age : [db]personTable.AGE, - addresses : [db]@Address_Person - } - meta::external::dataquality::tests::Address : Relational - { - // ~filter [db] (INNER)@Address_Location | [db] LocationStreetLengthLessThan10 - addressId : [db]addressTable.ID - // location : [db]@Address_Location - } - -) +function <> meta::external::dataquality::tests::testMetadata():Boolean[1] +{ + let dq = meta::external::dataquality::tests::loadDataQuality('$[' + + ' meta::external::dataquality::tests::domain::NonTemporal{' + + ' name,' + + ' biTemporal{' + + ' name' + + ' }' + + ' }' + + ' ]$'); + + + let metadata = meta::external::dataquality::generateDQMetaDataForDQValidation($dq); + + //TODO assert on structure. + assert($metadata->isNotEmpty()); +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_model.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_model.pure new file mode 100644 index 00000000000..03d7e955acf --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_model.pure @@ -0,0 +1,113 @@ +###Relational +Database meta::external::dataquality::tests::domain::db +( + Table personTable + ( + ID INTEGER PRIMARY KEY, + FIRSTNAME VARCHAR(200), + LASTNAME VARCHAR(200), + AGE INTEGER, + ADDRESSID INTEGER, + FIRMID INTEGER + ) + Table addressTable + ( + ID INTEGER PRIMARY KEY, + LOCATIONID INTEGER, + TYPE INTEGER + ) + Table locationTable + ( + ID INTEGER PRIMARY KEY, + STREET VARCHAR(200), + LOCALITY VARCHAR(200) + ) + + Join Address_Person(addressTable.ID = personTable.ADDRESSID) + Join Address_Location(addressTable.LOCATIONID = locationTable.ID) +) + + +###Mapping +Mapping meta::external::dataquality::tests::domain::dataqualitymappings +( + meta::external::dataquality::tests::domain::Person: Relational + { + name: [meta::external::dataquality::tests::domain::db]personTable.FIRSTNAME, + age: [meta::external::dataquality::tests::domain::db]personTable.AGE, + addresses: [meta::external::dataquality::tests::domain::db]@Address_Person + } + meta::external::dataquality::tests::domain::Address: Relational + { + addressId: [meta::external::dataquality::tests::domain::db]addressTable.ID, + location: [meta::external::dataquality::tests::domain::db]@Address_Location + } + meta::external::dataquality::tests::domain::Location: Relational + { + street: [meta::external::dataquality::tests::domain::db]locationTable.STREET, + locality: [meta::external::dataquality::tests::domain::db]locationTable.LOCALITY + } +) + +###Pure +Class meta::external::dataquality::tests::domain::Person +[ + mustBeOfLegalAge: $this.age >= 18, + validNameLength: $this.name->length() < 1000, + ageMustBePositive: $this.age >= 0 +] +{ + name: String[1]; + age: Integer[1]; + addresses: meta::external::dataquality::tests::domain::Address[*]; +} + +Class meta::external::dataquality::tests::domain::Address +[ + validAddressId: $this.addressId->isNotEmpty() +] +{ + location: meta::external::dataquality::tests::domain::Location[1]; + locationStreet: String[1]; + addressId: String[1]; +} + +Class meta::external::dataquality::tests::domain::Location +{ + street: String[1]; + locality: String[1]; +} + + +Class meta::external::dataquality::tests::domain::NonTemporal +[ + nameLength: $this.name->length() > 1 +] +{ + name: String[1]; + businessTemporal: meta::external::dataquality::tests::domain::BusinessTemporal[0..1]; + processingTemporal: meta::external::dataquality::tests::domain::ProcessingTemporal[0..1]; + biTemporal: meta::external::dataquality::tests::domain::BiTemporalEntity[0..1]; +} + +Class <> meta::external::dataquality::tests::domain::BusinessTemporal +{ + name: String[1]; + parent: meta::external::dataquality::tests::domain::BusinessTemporal[0..1]; + nonTemporal: meta::external::dataquality::tests::domain::NonTemporal[0..1]; + processingTemporal: meta::external::dataquality::tests::domain::ProcessingTemporal[0..1]; +} + +Class <> meta::external::dataquality::tests::domain::ProcessingTemporal +{ + name: String[1]; + parent: meta::external::dataquality::tests::domain::ProcessingTemporal[0..1]; + nonTemporal: meta::external::dataquality::tests::domain::NonTemporal[0..1]; + businessTemporal: meta::external::dataquality::tests::domain::BusinessTemporal[0..1]; +} + +Class <> meta::external::dataquality::tests::domain::BiTemporalEntity +{ + name: String[1]; + nonTemporal: meta::external::dataquality::tests::domain::NonTemporal[0..1]; +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_model_legend.txt b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_model_legend.txt new file mode 100644 index 00000000000..9dd40e90259 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_model_legend.txt @@ -0,0 +1,26 @@ +###Runtime +Runtime meta::external::dataquality::tests::domain::DataQualityRuntime +{ + mappings: + [ + meta::external::dataquality::tests::domain::dataqualitymappings + ]; + connections: + [ + meta::external::dataquality::tests::domain::db: + [ + connection_1: meta::external::dataquality::tests::domain::H2 + ] + ]; +} + +###Connection +RelationalDatabaseConnection meta::external::dataquality::tests::domain::H2 +{ + store: meta::external::dataquality::tests::domain::db; + type: H2; + specification: LocalH2 + { + }; + auth: DefaultH2; +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure new file mode 100644 index 00000000000..68b58560b6c --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure @@ -0,0 +1,78 @@ +import meta::pure::graphFetch::*; +import meta::pure::extension::*; +import meta::pure::metamodel::serialization::grammar::*; +import meta::external::dataquality::tests::*; +import meta::external::dataquality::*; + +function meta::external::dataquality::tests::loadDataQuality(tree:String[1]):DataQuality[1] +{ + let model = readFile('/core_dataquality_test/dataquality_test_model.pure')->toOne(); + let extra = readFile('/core_dataquality_test/dataquality_test_model_legend.txt')->toOne(); + + let dq = + '###DataQualityValidation\n' + + 'DataQualityValidation meta::external::dataquality::tests::domain::Validation' + + '{' + + ' context: fromMappingAndRuntime(meta::external::dataquality::tests::domain::dataqualitymappings, meta::external::dataquality::tests::domain::DataQualityRuntime);' + + ' validationTree: ' + $tree + ';' + + '}'; + + let code = $model + '\n' + $extra + '\n' + $dq; + + let compiled = meta::legend::compileLegendGrammar($code); + + $compiled->filter(c | $c->elementToPath() == 'meta::external::dataquality::tests::domain::Validation')->toOne()->cast(@DataQuality); +} + +function meta::external::dataquality::tests::doTest(tree:String[1], expected:FunctionDefinition[1]):Boolean[1] +{ + let dataQuality = loadDataQuality($tree); + + let actual = meta::external::dataquality::generateDataQualityQuery($dataQuality, [], false); + + assertLambdaAndJSONEquals($expected, $actual); +} + +function meta::external::dataquality::tests::doTest(tree:String[1], expected:FunctionDefinition[1], assertLambda:Boolean[1], assertJson:Boolean[1]):Boolean[1] +{ + let dataQuality = loadDataQuality($tree); + + let actual = meta::external::dataquality::generateDataQualityQuery($dataQuality, [], false); + + if ($assertLambda, | assertLambdaEquals($expected, $actual), | true); + if ($assertJson, | assertLambdaJSONEquals($expected, $actual), | true); +} + + + +function meta::external::dataquality::tests::assertLambdaAndJSONEquals(expected:FunctionDefinition[1], actual:LambdaFunction[1]): Boolean[1] +{ + assertLambdaEquals($expected, $actual); + assertLambdaJSONEquals($expected, $actual); +} + +function meta::external::dataquality::tests::assertLambdaEquals(expected:FunctionDefinition[1], actual:FunctionDefinition[1]): Boolean[1] +{ + let config = ^meta::pure::metamodel::serialization::grammar::Configuration + ( + fullPath = true, + extensions = ^meta::pure::metamodel::serialization::grammar::GrammarExtension( + extraInstanceValueHandlers = + [ + g:GraphFetchTree[1]| treeToString($g) + ] + ) + ); + + assertEquals($expected->printFunctionDefinition($config, ''), $actual->printFunctionDefinition($config, '')); +} + +function meta::external::dataquality::tests::assertLambdaJSONEquals(expected:FunctionDefinition[1], actual:FunctionDefinition[1]): Boolean[1] +{ + assertEquals($expected->functionJSON(), $actual->functionJSON()); +} + +function meta::external::dataquality::tests::functionJSON(func:FunctionDefinition[1]): String[1] +{ + $func->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::transformLambda(defaultExtensions())->meta::json::toJSON(100); +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json index 0195b4c6683..76b939b78f1 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json @@ -5,6 +5,7 @@ "core", "core_relational", "core_data_space_metamodel", + "core_functions_unclassified", "platform", "platform_dsl_store", "platform_dsl_graph", diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure index 919c7742e61..0683e1e5014 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure @@ -14,15 +14,79 @@ import meta::pure::metamodel::constraint::*; import meta::pure::lineage::scanProperties::propertyTree::*; +Class meta::external::dataquality::MilestoningContext +{ + rootBusinessTemporal: Boolean[1]; + rootProcessingTemporal: Boolean[1]; + nonRootBusinessTemporal: Boolean[1]; + nonRootProcessingTemporal: Boolean[1]; + + businessTemporal(){ + $this.rootBusinessTemporal || $this.nonRootBusinessTemporal + }:Boolean[1]; + + processingTemporal(){ + $this.rootProcessingTemporal || $this.nonRootProcessingTemporal + }:Boolean[1]; +} + +function meta::external::dataquality::generateDataQualityQuery(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*]): LambdaFunction[1] +{ + generateDataQualityQuery($dataquality, $limit, true); +} + + +function meta::external::dataquality::getMilestoningContext(tree:RootGraphFetchTree[1]): MilestoningContext[1] +{ + let properties = getAllProperties($tree); + let propertyTypes = $properties.genericType.rawType; + + ^MilestoningContext( + rootBusinessTemporal = $tree.class->meta::pure::milestoning::isBusinessTemporal(), + rootProcessingTemporal = $tree.class->meta::pure::milestoning::isProcessingTemporal(), + nonRootBusinessTemporal = $propertyTypes->exists(t | $t->meta::pure::milestoning::isBusinessTemporal()), + nonRootProcessingTemporal = $propertyTypes->exists(t | $t->meta::pure::milestoning::isProcessingTemporal()) + ); +} + +function meta::external::dataquality::addMilestoningParameters(tree:GraphFetchTree[1]):GraphFetchTree[1] +{ + $tree->match([ + p:PropertyGraphFetchTree[1] | + let parameters = if ($p.parameters->isEmpty(), + | let isBusinessTemporal = $p.property.genericType.rawType->toOne()->meta::pure::milestoning::isBusinessTemporal(); + let isProcessingTemporal = $p.property.genericType.rawType->toOne()->meta::pure::milestoning::isProcessingTemporal(); + getTemporalParameters($isProcessingTemporal, $isBusinessTemporal);, + | $p.parameters + ); + + ^$p( + subTrees = $p.subTrees->map(t | addMilestoningParameters($t)), + parameters = $parameters, + subTypeTrees = $p.subTypeTrees->map(t | addMilestoningParameters($t))->cast(@SubTypeGraphFetchTree) + );, + g:GraphFetchTree[1] | + ^$g( + subTrees = $g.subTrees->map(t | addMilestoningParameters($t)), + subTypeTrees = $g.subTypeTrees->map(t | addMilestoningParameters($t))->cast(@SubTypeGraphFetchTree) + ) + ]) +} -function meta::external::dataquality::executeDataQualityValidation(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*]): LambdaFunction[1] +function meta::external::dataquality::generateDataQualityQuery(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*], useFrom:Boolean[1]): LambdaFunction[1] { $dataquality.validationTree->validateTreeForNestedConstraints(true); + + let milestonedTree = $dataquality.validationTree->addMilestoningParameters(); + // 1. enrich tree with selected constraint properties - let enrichedTree = $dataquality.validationTree->ensureFunctionRequirementsForDataQuality($dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); + let enrichedTree = $milestonedTree->ensureFunctionRequirementsForDataQuality($dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); + + let milestoningContext = getMilestoningContext($enrichedTree); // 2. build query - let getAll = $dataquality.validationTree.class->createGetAllParameterized(); + let getAll = $dataquality.validationTree.class->createGetAll($milestoningContext.rootProcessingTemporal, $milestoningContext.rootBusinessTemporal); + let getExpr = if ($dataquality.filter->isNotEmpty(), | $dataquality.validationTree.class->generateFilterQuery($getAll, $dataquality.filter->toOne());, | $getAll); @@ -43,8 +107,8 @@ function meta::external::dataquality::executeDataQualityValidation(dataquality:m multiplicity = ZeroMany, importGroup=system::imports::coreImport); // 2.1 - let serialized = ^SimpleFunctionExpression(func=serialize_T_MANY__RootGraphFetchTree_1__String_1_, - functionName=serialize_T_MANY__RootGraphFetchTree_1__String_1_.name, + let serialized = ^SimpleFunctionExpression(func=serialize_Checked_MANY__RootGraphFetchTree_1__String_1_, + functionName=serialize_Checked_MANY__RootGraphFetchTree_1__String_1_.name, importGroup=system::imports::coreImport, genericType=^GenericType(rawType=String), multiplicity=ZeroMany, @@ -55,19 +119,15 @@ function meta::external::dataquality::executeDataQualityValidation(dataquality:m let deactivatedMapping = ^InstanceValue(values=$mappingAndRuntime.first->evaluateAndDeactivate(), genericType=^GenericType(rawType=Mapping), multiplicity=PureOne) ; let deactivatedRuntime = ^InstanceValue(values=$mappingAndRuntime.second->evaluateAndDeactivate(), genericType=^GenericType(rawType=NonExecutableValueSpecification), multiplicity=PureOne); - let from = ^SimpleFunctionExpression(func=from_T_m__Mapping_1__Runtime_1__T_m_, + let from = if ($useFrom, | ^SimpleFunctionExpression(func=from_T_m__Mapping_1__Runtime_1__T_m_, functionName=from_T_m__Mapping_1__Runtime_1__T_m_.name, importGroup=system::imports::coreImport, genericType=^GenericType(rawType=String), multiplicity=ZeroMany, - parametersValues=[$serialized, $deactivatedMapping, $deactivatedRuntime])->evaluateAndDeactivate(); + parametersValues=[$serialized, $deactivatedMapping, $deactivatedRuntime])->evaluateAndDeactivate(), | $serialized); // 3. build lambda - if ($dataquality.validationTree.class->meta::pure::milestoning::isTemporal(), - | $dataquality.validationTree.class->createLambdaParameterized($from), - | let lambda = {| ''}; - ^$lambda(expressionSequence = [$from]); - ); + createLambda($from, $milestoningContext.processingTemporal, $milestoningContext.businessTemporal); } function meta::external::dataquality::validateTreeForNestedConstraints(node:GraphFetchTree[1], isRoot:Boolean[1]): Boolean[1] @@ -127,118 +187,58 @@ function <> meta::external::dataquality::createGetAll(c: Clas ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); } -function <> meta::external::dataquality::createGetAllParameterized(c: Class[1]):FunctionExpression[1] -{ - if ($c->meta::pure::milestoning::isBiTemporal(), - | $c->createGetAllBiTemporal(), - | $c->createGetAllUniTemporal()); -} - -function <> meta::external::dataquality::createGetAllUniTemporal(c: Class[1]):FunctionExpression[1] +function <> meta::external::dataquality::getAllProperties(tree:GraphFetchTree[1]):AbstractProperty[*] { - if ($c->meta::pure::milestoning::isProcessingTemporal(), - | $c->createGetAllProcessingTemporal(), - | $c->createGetAllBusinessTemporal()); + let properties = $tree->match([ + p:PropertyGraphFetchTree[1] | $p.property, + g:GraphFetchTree[1] | [] + ]); + + $tree.subTrees->concatenate($tree.subTypeTrees) + ->map(t | $t->getAllProperties()) + ->concatenate($properties); } -function <> meta::external::dataquality::createLambdaParameterized(c: Class[1], from: SimpleFunctionExpression[1]):LambdaFunction[1] +function meta::external::dataquality::createLambda(body:ValueSpecification[1], processingTemporal:Boolean[1], businessTemporal:Boolean[1]):LambdaFunction[1] { - if ($c->meta::pure::milestoning::isBiTemporal(), - | $from->createLambdaBiTemporal(), - | $c->createLambdaUniTemporal($from)); -} + let parameters = getTemporalParameters($processingTemporal, $businessTemporal); + let functionType = ^FunctionType(returnMultiplicity = $body.multiplicity, returnType = $body.genericType, parameters = $parameters); -function <> meta::external::dataquality::createLambdaUniTemporal(c: Class[1], from: SimpleFunctionExpression[1]):LambdaFunction[1] -{ - if ($c->meta::pure::milestoning::isProcessingTemporal(), - | $from->createLambdaProcessingTemporal(), - | $from->createLambdaBusinessTemporal()); + let lambda = newLambdaFunction($functionType); + ^$lambda(expressionSequence = $body); } -function <> meta::external::dataquality::createLambdaProcessingTemporal(from: SimpleFunctionExpression[1]):LambdaFunction[1] +function <> meta::external::dataquality::createGetAll(c: Class[1], processingTemporal:Boolean[1], businessTemporal:Boolean[1]):FunctionExpression[1] { - let lambda = {processingDate: Date[1] | ''}; - ^$lambda(expressionSequence = [$from]); -} + let func = [ + pair($processingTemporal && $businessTemporal, getAll_Class_1__Date_1__Date_1__T_MANY_), + pair($processingTemporal || $businessTemporal, getAll_Class_1__Date_1__T_MANY_), + pair(true, getAll_Class_1__T_MANY_) + ]->filter(t | $t.first == true)->at(0).second; -function <> meta::external::dataquality::createLambdaBusinessTemporal(from: SimpleFunctionExpression[1]):LambdaFunction[1] -{ - let lambda = {businessDate: Date[1] | ''}; - ^$lambda(expressionSequence = [$from]); -} + let parameters = getTemporalParameters($processingTemporal, $businessTemporal); -function <> meta::external::dataquality::createLambdaBiTemporal(from: SimpleFunctionExpression[1]):LambdaFunction[1] -{ - let lambda = {businessDate: Date[1], processingDate: Date[1] | ''}; - ^$lambda(expressionSequence = [$from]); -} - - -function <> meta::external::dataquality::createGetAllBiTemporal(c: Class[1]):FunctionExpression[1] -{ - let getAllExpression = ^SimpleFunctionExpression + ^SimpleFunctionExpression ( - func = getAll_Class_1__Date_1__Date_1__T_MANY_, - functionName = getAll_Class_1__Date_1__Date_1__T_MANY_.name, + func = $func, + functionName = $func.name, importGroup = system::imports::coreImport, genericType = ^GenericType(rawType = $c), multiplicity = ZeroMany, - parametersValues = [ - ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + parametersValues = ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), multiplicity = PureOne, values = $c - ), - ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate(), - ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate() - ] + )->concatenate($parameters) )->evaluateAndDeactivate(); - let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); - let lambda = {|[]}; - ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); } -function <> meta::external::dataquality::createGetAllBusinessTemporal(c: Class[1]):FunctionExpression[1] +function <> meta::external::dataquality::getTemporalParameters(processingTemporal:Boolean[1], businessTemporal:Boolean[1]):VariableExpression[*] { - let getAllExpression = ^SimpleFunctionExpression - ( - func = getAll_Class_1__Date_1__T_MANY_, - functionName = getAll_Class_1__Date_1__T_MANY_.name, - importGroup = system::imports::coreImport, - genericType = ^GenericType(rawType = $c), - multiplicity = ZeroMany, - parametersValues = [ - ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), - multiplicity = PureOne, - values = $c - ), - ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne) - ] - )->evaluateAndDeactivate(); - let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); - let lambda = {|[]}; - ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); -} -function <> meta::external::dataquality::createGetAllProcessingTemporal(c: Class[1]):FunctionExpression[1] -{ - let getAllExpression = ^SimpleFunctionExpression - ( - func = getAll_Class_1__Date_1__T_MANY_, - functionName = getAll_Class_1__Date_1__T_MANY_.name, - importGroup = system::imports::coreImport, - genericType = ^GenericType(rawType = $c), - multiplicity = ZeroMany, - parametersValues = [ - ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), - multiplicity = PureOne, - values = $c - ), - ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne) - ] - )->evaluateAndDeactivate(); - let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); - let lambda = {|[]}; - ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); + [ + pair($processingTemporal, ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate()), + pair($businessTemporal, ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate()) + ]->filter(f | $f.first).second->evaluateAndDeactivate(); } function meta::external::dataquality::generateFilterQuery(c:Class[1], f: FunctionExpression[1], filter:LambdaFunction<{T[1]->Boolean[1]}>[1]):FunctionExpression[1] { @@ -252,19 +252,36 @@ function meta::external::dataquality::generateFilterQuery(c:Class[1], f: F parametersValues = [ $f, ^InstanceValue ( - genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='that', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), + genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='x', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), multiplicity = PureOne, - values = ^$dummyLambda - ( - classifierGenericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='that', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), - expressionSequence = $filter.expressionSequence->evaluateAndDeactivate() - ) + values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $filter.expressionSequence->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) ) ] )->evaluateAndDeactivate(); } +function meta::external::dataquality::lambda(functionType:FunctionType[1], expressionSequence:ValueSpecification[*]):LambdaFunction[1] +{ + let lambda = meta::pure::functions::meta::newLambdaFunction($functionType); + ^$lambda(expressionSequence = $expressionSequence->toOneMany()); +} + +function meta::external::dataquality::functionType(name:String[1], type:Type[1], multiplicity:Multiplicity[1], returnType:Type[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] +{ + functionType($name, ^GenericType(rawType = $type), $multiplicity, ^GenericType(rawType = $returnType), $returnMultiplicity); +} + +function meta::external::dataquality::functionType(name:String[1], type:GenericType[1], multiplicity:Multiplicity[1], returnType:GenericType[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] +{ + ^FunctionType(parameters = ^VariableExpression(genericType = $type, name = $name, multiplicity = $multiplicity), returnMultiplicity = $returnMultiplicity, returnType = $returnType); +} + +function meta::external::dataquality::functionType(parameters:VariableExpression[*], returnType:GenericType[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] +{ + ^FunctionType(parameters = $parameters, returnMultiplicity = $returnMultiplicity, returnType = $returnType); +} + function meta::external::dataquality::generateConstraintsNegatedORQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] { if ($constraints.values->isEmpty(), @@ -288,8 +305,7 @@ function meta::external::dataquality::generateORNegatedQuery(c:Class[1], f let final_expr = $constraints.values->drop(2)->fold({c3, curr_or_exp | let c3_negatedExprSequence = $c3.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); $c3_negatedExprSequence->orFunctionExpression($curr_or_exp);}, $c1_c2_or_expr); - - let dummyLambda = {|'ok'}; + ^SimpleFunctionExpression ( func = filter_T_MANY__Function_1__T_MANY_, @@ -301,23 +317,47 @@ function meta::external::dataquality::generateORNegatedQuery(c:Class[1], f ^InstanceValue ( genericType = $c1.functionDefinition.classifierGenericType->toOne(), multiplicity = PureOne, - values = ^$dummyLambda - ( - classifierGenericType = $c1.functionDefinition.classifierGenericType, - expressionSequence = $final_expr->evaluateAndDeactivate() - ) + values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $final_expr->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) ) ] )->evaluateAndDeactivate(); } +function meta::external::dataquality::replaceVariableWithVariable(v:ValueSpecification[1..*], from:String[1], to:String[1]):ValueSpecification[1..*] +{ + $v->map(a | $a->replaceVariableWithVariable($from, $to)) +} + +function meta::external::dataquality::replaceVariableWithVariable(v:ValueSpecification[1], from:String[1], to:String[1]):ValueSpecification[1] +{ + $v->match([ + v:VariableExpression[1] | if ($v.name == $from, | ^$v(name = $to), | $v), + s:SimpleFunctionExpression[1] | ^$s(parametersValues = $s.parametersValues->map(p | replaceVariableWithVariable($p, $from, $to))), + i:InstanceValue[1] | ^$i(values = $i.values->map(v | $v->replaceVariableWithVariable($from, $to))) + ]) +} + +function meta::external::dataquality::replaceVariableWithVariable(l:FunctionDefinition[1], from:String[1], to:String[1]):FunctionDefinition[1] +{ + ^$l(expressionSequence = $l.expressionSequence->replaceVariableWithVariable($from, $to)) +} + +function meta::external::dataquality::replaceVariableWithVariable(a:Any[1], from:String[1], to:String[1]):Any[1] +{ + $a->match([ + v:ValueSpecification[1] | $v->replaceVariableWithVariable($from, $to), + l:LambdaFunction[1] | $l->replaceVariableWithVariable($from, $to), + a:Any[1] | $a + ]) +} + function meta::external::dataquality::generateConstraintNegatedQuery(c:Class[1], f: FunctionExpression[1], constraint: Constraint[1]):FunctionExpression[1] { let constraintExprSequence = $constraint.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne(); let negatedExprSequence = $constraintExprSequence->negatedFunctionExpression(); - let dummyLambda = {|'ok'}; + ^SimpleFunctionExpression ( func = filter_T_MANY__Function_1__T_MANY_, @@ -329,11 +369,7 @@ function meta::external::dataquality::generateConstraintNegatedQuery(c:Class< ^InstanceValue ( genericType = $constraint.functionDefinition.classifierGenericType->toOne(), multiplicity = PureOne, - values = ^$dummyLambda - ( - classifierGenericType = $constraint.functionDefinition.classifierGenericType, - expressionSequence = $negatedExprSequence - ) + values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $negatedExprSequence->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) ) ] )->evaluateAndDeactivate(); @@ -432,8 +468,7 @@ function meta::external::dataquality::getEnrichedTreeForStructuralValidations(va function meta::external::dataquality::generateDQMetaDataForDQValidation(dataquality:meta::external::dataquality::DataQuality[1]): DataQualityRule[*] { let enrichedTree = ensureFunctionRequirementsForDataQuality($dataquality.validationTree, $dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); - let dqRules = $dataquality.validationTree->nodeToDqRule(true, '')->concatenate($enrichedTree->nodeToDqRule(false, '')); - $dqRules; + $dataquality.validationTree->nodeToDqRule(true, '')->concatenate($enrichedTree->nodeToDqRule(false, '')); } function meta::external::dataquality::nodeToDqRule(node:GraphFetchTree[1], processOnlyConstraints:Boolean[1], path:String[*]):DataQualityRule[*] @@ -466,5 +501,5 @@ function <> meta::external::dataquality::propertyNodeToDqRule(no function <> meta::external::dataquality::constraintToDqRule(constraint:Constraint[1], path:String[1]):DataQualityRule[1] { - ^DataQualityRule(constraintName=$constraint.name->toOne(), constraintGrammar=$constraint.functionDefinition->meta::pure::metamodel::serialization::grammar::printFunctionDefinitionExpressions('')->toOne(), constraintType='Alloy_Constraint_Validation', propertyPath=$path); + ^DataQualityRule(constraintName=$constraint.name->toOne(), constraintGrammar=$constraint.functionDefinition->replaceVariableWithVariable('this', 'x')->meta::pure::metamodel::serialization::grammar::printFunctionDefinitionExpressions('')->toOne(), constraintType='Alloy_Constraint_Validation', propertyPath=$path); } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2371b3a5204..aa69d55bcd7 100644 --- a/pom.xml +++ b/pom.xml @@ -2524,6 +2524,22 @@ ${project.version} + + org.finos.legend.engine + legend-engine-xt-dataquality-grammar + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-compiler + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-pure-test + ${project.version} + + org.finos.legend.engine