DQ multiple fixes and test framework

legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/HelperModelBuilder.java

@@ -265,7 +265,7 @@ public static RichIterable<org.finos.legend.pure.m3.coreinstance.meta.pure.metam
                    In the absence of a message the system will fall back on the default failure message.
                  */
             }
-            org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.constraint.Constraint pureConstraint = new Root_meta_pure_metamodel_constraint_Constraint_Impl(constraintSourceId);
+            org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.constraint.Constraint pureConstraint = new Root_meta_pure_metamodel_constraint_Constraint_Impl(constraintSourceId, SourceInformationHelper.toM3SourceInformation(c.sourceInformation), context.pureModel.getClass("meta::pure::metamodel::constraint::Constraint"));
             pureConstraint.setSourceInformation(SourceInformationHelper.toM3SourceInformation(c.sourceInformation));
             pureConstraint._functionDefinition(lf);
             pureConstraint._name(c.name);

legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/pom.xml

@@ -185,6 +185,11 @@
             <artifactId>hamcrest-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>test</scope>
+        </dependency>        
         <!-- TEST -->
     </dependencies>
-</project>
+</project>

legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-grammar/src/test/java/org/finos/legend/engine/language/pure/grammar/test/GrammarParseTestUtils.java

@@ -0,0 +1,64 @@
+//  Copyright 2024 Goldman Sachs
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package org.finos.legend.engine.language.pure.grammar.test;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.collections.impl.list.mutable.FastList;
+import org.eclipse.collections.impl.utility.internal.IterableIterate;
+import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser;
+import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class GrammarParseTestUtils
+{
+    public static PureModelContextData loadPureModelContextFromResource(String resource, Class<?> clazz)
+    {
+        return loadPureModelContextFromResource(resource, null, clazz);
+    }
+
+    public static PureModelContextData loadPureModelContextFromResource(String resource, String extraCode, Class<?> clazz)
+    {
+        return loadPureModelContextFromResources(FastList.newListWith(resource), extraCode, clazz);
+    }
+
+    public static PureModelContextData loadPureModelContextFromResources(Class<?> clazz, String... resources)
+    {
+        return loadPureModelContextFromResources(FastList.newListWith(resources), null, clazz);
+    }
+
+    public static PureModelContextData loadPureModelContextFromResources(Iterable<String> resources, String extraCode, Class<?> clazz)
+    {
+        String resourceCode = StringUtils.join(IterableIterate.collect(resources, r -> getResource(r, clazz)), "\n");
+        String code = extraCode != null ? resourceCode + "\n" + extraCode : resourceCode;
+
+        return PureModelContextData.newBuilder().withPureModelContextData(PureGrammarParser.newInstance().parseModel(code)).build();
+    }
+
+
+    public static String getResource(String resource, Class<?> clazz)
+    {
+        try
+        {
+            return IOUtils.toString(Objects.requireNonNull(clazz.getClassLoader().getResourceAsStream(resource)));
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+}

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProperties.pure

@@ -719,7 +719,8 @@ function meta::pure::lineage::scanProperties::printPropertyNodeLists(properyList
 function <<access.private>> meta::pure::lineage::scanProperties::propertyTree::recurseBuildTree(list:List<PropertyPathNode>[*]):PropertyPathTree[*]
 {
    let m = $list->filter(l|!$l.values->isEmpty())->groupBy(l|$l.values->at(0));
-   $m->keys()->map(
+
+   $m->keys()->sortBy(v | $v.property.name->toOne())->map(
                   k|^PropertyPathTree(display = $k.property->match([p:Property<Nil,Any|*>[1]|'',q:QualifiedProperty<Any>[1]|'(Q)'])+$k.property.name->toOne(),
                                       value = $k,
                                       children = $m->get($k).values->map(l|^$l(values=$l.values->slice(1, $l.values->size())))->recurseBuildTree(),

legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure

@@ -113,6 +113,7 @@ function meta::pure::metamodel::serialization::grammar::printInstanceValue(i:Ins
                                     f:FunctionExpression[1]      | $f->printPropertyOrFunctionExpression($configuration, $space),
                                     x:InstanceValue[1] | $x->printInstanceValue($configuration, $space),
                                     z:KeyExpression[*] | $z->map(f|$f.key.values->toOne()->toString())->joinStrings(','),
+                                    v:VariableExpression[1] | $v->printValueSpecification($configuration, $space),
                                     r:Runtime[1]|'^meta::core::runtime::Runtime('+
                                                   'connectionStores='+$r.connectionStores->map(s|'^meta::core::runtime::ConnectionStore(connection='+$s.connection->printConnection($configuration)+', element='+$s.element->match([x:String[1]|$x,x:Store[1]|$x->elementToPath()])+')')->makeString(',')+
                                                   ')',

legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml

@@ -271,6 +271,12 @@
             <artifactId>h2</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-language-pure-grammar</artifactId>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>        
 
     </dependencies>
 </project>

legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java

@@ -118,7 +118,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu
             // 1. load pure model from PureModelContext
             PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
             // 2. call DQ PURE func to generate lambda
-            LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath);
+            LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, this.extensions);
             // 3. Generate Plan from the lambda generated in the previous step
             SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context
             LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString());
@@ -146,7 +146,7 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteT
             // 1. load pure model from PureModelContext
             PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
             // 2. call DQ PURE func to generate lambda
-            LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit);
+            LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions);
             // 3. Generate Plan from the lambda generated in the previous step
             SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context
             // 4. execute plan
@@ -176,7 +176,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
         // 1. load pure model from PureModelContext
         PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
         // 2. call DQ PURE func to generate lambda
-        LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit);
+        LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions);
         Lambda lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions);
         return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper);
     }

legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java

@@ -18,6 +18,7 @@
 import org.finos.legend.engine.language.pure.compiler.Compiler;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtensionLoader;
+import org.finos.legend.engine.language.pure.grammar.test.GrammarParseTestUtils;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree;
 import org.finos.legend.engine.shared.core.ObjectMapperFactory;
@@ -42,8 +43,7 @@ public class TestDataQualityApi
     @Test
     public void testDataQualityGetPropertyPathTree() throws IOException
     {
-        URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithDataQualityValidation.json"));
-        PureModelContextData pureModelContextData = objectMapper.readValue(url, PureModelContextData.class);
+        PureModelContextData pureModelContextData = GrammarParseTestUtils.loadPureModelContextFromResource("inputs/modelWithDataQualityValidation.pure", TestDataQualityApi.class);
         CompilerExtensionLoader.logExtensionList();
         PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName());
         org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation");
@@ -59,8 +59,7 @@ public void testDataQualityGetPropertyPathTree() throws IOException
     @Test
     public void testDataQualityGetPropertyPathTree_WithNullDataQualityValidation() throws IOException
     {
-        URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithNullDataQualityValidation.json"));
-        PureModelContextData pureModelContextData = objectMapper.readValue(url, PureModelContextData.class);
+        PureModelContextData pureModelContextData = GrammarParseTestUtils.loadPureModelContextFromResource("inputs/modelWithNullDataQualityValidation.pure", TestDataQualityApi.class);
         CompilerExtensionLoader.logExtensionList();
         PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName());
         org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation");