add tests for relation validation lambda generation for dq

finos · Nov 25, 2024 · 8f74853 · 8f74853
1 parent a9292a2
commit 8f74853
Show file tree

Hide file tree

Showing 6 changed files with 94 additions and 2 deletions.
diff --git a/...ine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure b/...ine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure
@@ -111,6 +111,12 @@ function meta::pure::metamodel::serialization::grammar::printInstanceValue(i:Ins
                               $configuration.extensions.extraInstanceValueHandlers->concatenate([
                                     f:FunctionDefinition<Any>[1] | $f->printFunctionDefinition($configuration, $space+'    '),
                                     f:FunctionExpression[1]      | $f->printPropertyOrFunctionExpression($configuration, $space),
+                                    cs:ColSpec<Any>[1] | '~' + $cs.name->printColName(),
+                                    fsc:FuncColSpec<Any, Any>[1] | '~' + $fsc.name->printColName() + ':' + $fsc.function->cast(@FunctionDefinition<Any>)->printFunctionDefinition(''),
+                                    asc:AggColSpec<Any, Any, Any>[1] | '~' + $asc.name->printColName() + ':' + $asc.map->cast(@FunctionDefinition<Any>)->printFunctionDefinition('') + ':' + $asc.reduce->cast(@FunctionDefinition<Any>)->printFunctionDefinition(''),
+                                    csa:ColSpecArray<Any>[1] | '~['+$csa.names->map(c|$c->printColName())->joinStrings(',')+']',
+                                    fcsa:FuncColSpecArray<Any, Any>[1] | '~[' + $fcsa.funcSpecs->map(x| $x.name->printColName() + ':' + $x.function->cast(@FunctionDefinition<Any>)->printFunctionDefinition(''))->joinStrings(',') + ']',
+                                    acsa:AggColSpecArray<Any, Any, Any>[1] | '~[' + $acsa.aggSpecs->map(x| $x.name->printColName() + ':' + $x.map->cast(@FunctionDefinition<Any>)->printFunctionDefinition('') + ':' + $x.reduce->cast(@FunctionDefinition<Any>)->printFunctionDefinition(''))->joinStrings(',') + ']',
                                     x:InstanceValue[1] | $x->printInstanceValue($configuration, $space),
                                     z:KeyExpression[*] | $z->map(f|$f.key.values->toOne()->toString())->joinStrings(','),
                                     v:VariableExpression[1] | $v->printValueSpecification($configuration, $space),

diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/pom.xml
@@ -95,6 +95,11 @@
                         <artifactId>legend-pure-m2-store-relational-pure</artifactId>
                         <version>${legend.pure.version}</version>
                     </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-pure-functions-relation-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
                 </dependencies>
             </plugin>
             <plugin>
@@ -173,6 +178,11 @@
                         <artifactId>legend-pure-m2-store-relational-pure</artifactId>
                         <version>${legend.pure.version}</version>
                     </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-pure-functions-relation-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
                 </dependencies>
             </plugin>
         </plugins>
@@ -245,6 +255,15 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-pure-runtime-java-extension-compiled-functions-unclassified</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-runtime-java-extension-compiled-functions-relation</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.pure</groupId>
+            <artifactId>legend-pure-m2-dsl-store-pure</artifactId>
+        </dependency>
         <!-- TEST -->
         <dependency>
             <groupId>junit</groupId>

diff --git a/...-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test.definition.json b/...-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test.definition.json
@@ -4,6 +4,7 @@
   "dependencies": [
     "core",
     "core_relational",
+    "core_functions_relation",
     "core_data_space_metamodel",
     "core_functions_unclassified",
     "platform",

diff --git a/...e-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure b/...e-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure
@@ -170,4 +170,38 @@ function <<test.Test>> meta::external::dataquality::tests::testMetadata():Boolea
 
     //TODO assert on structure.
     assert($metadata->isNotEmpty());
+}
+
+function <<test.Test>> meta::external::dataquality::tests::testLambdaGeneration_relationValidation():Boolean[1]
+{
+  doRelationTest('###DataQualityValidation' +
+    '      DataQualityRelationValidation meta::external::dataquality::tests::domain::RelationValidation' +
+    '      {' +
+    '        query: #>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->extend(~fullname:x|[$x.FIRSTNAME->toOne(), $x.LASTNAME->toOne()]->joinStrings());' +
+    '        validations: [' +
+    '          {' +
+    '             name: \'validFirstName\';' +
+    '             description: \'first name should not be empty\';' +
+    '             assertion: row2|$row2.FIRSTNAME->isNotEmpty();' +
+    '          }' +
+    '         ];' +
+    '      }','validFirstName', {|#>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->extend(~fullname:x|[$x.FIRSTNAME->toOne(), $x.LASTNAME->toOne()]->joinStrings())->filter(row2|$row2.FIRSTNAME->isNotEmpty()->not())
+    })
+}
+
+function <<test.Test>> meta::external::dataquality::tests::testLambdaGeneration_parameterized_relationValidation():Boolean[1]
+{
+  doRelationTest('###DataQualityValidation' +
+    '      DataQualityRelationValidation meta::external::dataquality::tests::domain::RelationValidation' +
+    '      {' +
+    '        query: name:String[1] | #>{meta::external::dataquality::tests::domain::db.personTable}#->select(~FIRSTNAME)->filter(n|$n == $name)->groupBy(~[FIRSTNAME], ~[count: x | $x.FIRSTNAME : y | $y->count()]);' +
+    '        validations: [' +
+    '          {' +
+    '             name: \'validFirstName\';' +
+    '             description: \'first name should not exceed 100 chars\';' +
+    '             assertion: row2|$row2.FIRSTNAME->size() <= 100;' +
+    '          }' +
+    '         ];' +
+    '      }','validFirstName', {name:String[1]|#>{meta::external::dataquality::tests::domain::db.personTable}#->select(~FIRSTNAME)->filter({n|($n == $name)})->groupBy(~[FIRSTNAME], ~[count:{x|$x.FIRSTNAME}:{y|$y->count()}])->filter(row2|not(($row2.FIRSTNAME->size() <= 100)))
+    })
 }
diff --git a/...ataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure b/...ataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure
@@ -1,3 +1,4 @@
+import meta::pure::metamodel::relation::*;
 import meta::pure::graphFetch::*;
 import meta::pure::extension::*;
 import meta::pure::metamodel::serialization::grammar::*;
@@ -24,6 +25,18 @@ function meta::external::dataquality::tests::loadDataQuality(tree:String[1]):Dat
   $compiled->filter(c | $c->elementToPath() == 'meta::external::dataquality::tests::domain::Validation')->toOne()->cast(@DataQuality<Any>);
 }
 
+function meta::external::dataquality::tests::loadDataQualityRelationValidation(dq:String[1]):DataQualityRelationValidation[1]
+{
+  let model = readFile('/core_dataquality_test/dataquality_test_model.pure')->toOne();
+  let extra = readFile('/core_dataquality_test/dataquality_test_model_legend.txt')->toOne();
+
+  let code = $model + '\n' + $extra + '\n' + $dq;
+
+  let compiled = meta::legend::compileLegendGrammar($code);
+
+  $compiled->filter(c | $c->elementToPath() == 'meta::external::dataquality::tests::domain::RelationValidation')->toOne()->cast(@DataQualityRelationValidation);
+}
+
 function meta::external::dataquality::tests::doTest(tree:String[1], expected:FunctionDefinition<Any>[1]):Boolean[1]
 {
   let dataQuality = loadDataQuality($tree);
@@ -43,6 +56,15 @@ function meta::external::dataquality::tests::doTest(tree:String[1], expected:Fun
   if ($assertJson, | assertLambdaJSONEquals($expected, $actual), | true);
 }
 
+function meta::external::dataquality::tests::doRelationTest(dq:String[1], validationName:String[1], expected:FunctionDefinition<Any>[1]):Boolean[1]
+{
+  let dataqualityRelationValidation = $dq->loadDataQualityRelationValidation();
+
+  let actual = $dataqualityRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationName);
+
+  assertRelationLambdaEquals($expected, $actual);
+}
+
 
 
 function meta::external::dataquality::tests::assertLambdaAndJSONEquals(expected:FunctionDefinition<Any>[1], actual:LambdaFunction<Any>[1]): Boolean[1]
@@ -63,7 +85,15 @@ function meta::external::dataquality::tests::assertLambdaEquals(expected:Functio
                                   ]
                             )
             );
+  assertEquals($expected->printFunctionDefinition($config, ''), $actual->printFunctionDefinition($config, ''));
+}
 
+function meta::external::dataquality::tests::assertRelationLambdaEquals(expected:FunctionDefinition<Any>[1], actual:FunctionDefinition<Any>[1]): Boolean[1]
+{
+  let config = ^meta::pure::metamodel::serialization::grammar::Configuration
+            (
+              fullPath = false
+            );
   assertEquals($expected->printFunctionDefinition($config, ''), $actual->printFunctionDefinition($config, ''));
 }
 

diff --git a/...ngine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure b/...ngine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure
@@ -514,6 +514,7 @@ function meta::external::dataquality::generateDataqualityRelationValidationLambd
 function meta::external::dataquality::buildRelationFilterExpression(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationName: String[1]):FunctionExpression[1]
 {
   let relationType = $dqRelationValidation.query->evaluateAndDeactivate()->functionReturnType().typeArguments.rawType->toOne();
+  let assertion = $dqRelationValidation.validations->filter(val| $val.name == $validationName)->toOne().assertion; 
 
    ^SimpleFunctionExpression
    (
@@ -526,9 +527,10 @@ function meta::external::dataquality::buildRelationFilterExpression(dqRelationVa
          ^InstanceValue (
             genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='validationFilter', genericType=^GenericType(rawType=$relationType), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))),
             multiplicity = PureOne,
-            values = meta::external::dataquality::lambda(^FunctionType(returnMultiplicity = PureOne, returnType = ^GenericType(rawType = Boolean), parameters = [^VariableExpression(multiplicity=PureOne,genericType=^GenericType(rawType=$relationType),name='assertion')]),
-                                                          $dqRelationValidation.validations->filter(val| $val.name == $validationName)->toOne().assertion.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression())
+            values = meta::external::dataquality::lambda(^FunctionType(returnMultiplicity = PureOne, returnType = ^GenericType(rawType = Boolean), parameters = [^VariableExpression(multiplicity=PureOne,genericType=^GenericType(rawType=$relationType),name=$assertion->meta::pure::metamodel::serialization::grammar::printLambdaParameters())]),
+                                                          $assertion.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression())
          )
      ]
    )->evaluateAndDeactivate();
 }
+