Skip to content

Commit

Permalink
Dataquality - support for relation validations (finos#3257)
Browse files Browse the repository at this point in the history 
* [maven-release-plugin] prepare release legend-engine-4.66.0

* [maven-release-plugin] prepare for next development iteration

* Dataquality - support for relation validations

---------

Co-authored-by: FINOS Administrator <[email protected]>
Co-authored-by: Mauricio Uyaguari <[email protected]>
  • Loading branch information
@keenkeystrokes @finos-admin @MauricioUyaguari
3 people authored Nov 20, 2024
1 parent 028987c commit a9292a2
Show file tree
Hide file tree
Showing 21 changed files with 1,085 additions and 557 deletions.
6 changes: 3 additions & 3 deletions ...pi/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, this.extensions);
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.validationName);
// 3. Generate Plan from the lambda generated in the previous step
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString());
Expand Down Expand Up @@ -146,7 +146,7 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteT
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions);
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName);
// 3. Generate Plan from the lambda generated in the previous step
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context
// 4. execute plan
Expand Down Expand Up @@ -176,7 +176,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions);
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName);
Lambda lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions);
return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper);
}
Expand Down
1 change: 1 addition & 0 deletions ...n/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,5 @@ public class DataQualityExecuteTrialInput
public String packagePath;
public Integer queryLimit;
public List<ParameterValue> lambdaParameterValues;
public String validationName;
}
140 changes: 115 additions & 25 deletions .../finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,55 +14,69 @@

package org.finos.legend.engine.language.pure.compiler.toPureGraph;

import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ListIterable;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.impl.list.mutable.FastList;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.language.pure.compiler.Compiler;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor;
import org.finos.legend.engine.protocol.dataquality.metamodel.DataQuality;
import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree;
import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree;
import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext;
import org.finos.legend.engine.protocol.dataquality.metamodel.DataqualityRelationValidation;
import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext;
import org.finos.legend.engine.protocol.dataquality.metamodel.RelationValidation;
import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.runtime.PackageableRuntime;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.Variable;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree;
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree;
import org.finos.legend.engine.shared.core.operational.Assert;
import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityExecutionContext;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree_Impl;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRelationValidation_Impl;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree_Impl;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality_Impl;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext_Impl;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext_Impl;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_RelationValidation;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_RelationValidation_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_metamodel_constraint_Constraint_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_Class_Impl;
import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_generics_GenericType_Impl;
import org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.GraphFetchTree;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.constraint.Constraint;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.property.AbstractProperty;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType;
import org.finos.legend.pure.m3.navigation._class._Class;
import org.finos.legend.pure.m4.coreinstance.CoreInstance;
import org.finos.legend.pure.m4.coreinstance.SourceInformation;

import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

public class DataQualityCompilerExtension implements CompilerExtension
{
Expand Down Expand Up @@ -93,34 +107,110 @@ private GenericType genericType(CompileContext context, String _class)
public Iterable<? extends Processor<?>> getExtraProcessors()
{
return Lists.fixedSize.of(
Processor.newProcessor(
DataQuality.class,
org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class),
(dataquality, compileContext) ->
{
Root_meta_external_dataquality_DataQuality<Object> metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>(
dataquality.name,
SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation),
compileContext.pureModel.getClass("meta::external::dataquality::DataQuality")
)._classifierGenericType(genericType(compileContext, dataquality.dataQualityRootGraphFetchTree._class));
return metamodel;
},
(dataquality, compileContext) ->
{

},
(dataquality, compileContext) ->
{
Root_meta_external_dataquality_DataQuality<Object> metamodel = (Root_meta_external_dataquality_DataQuality<Object>) compileContext.pureModel.getPackageableElement(compileContext.pureModel.buildPackageString(dataquality._package, dataquality.name));
metamodel._context(buildDataQualityExecutionContext(dataquality, compileContext))
._filter(getFilterLambda(dataquality, compileContext))
._validationTree(buildRootGraphFetchTree(dataquality.dataQualityRootGraphFetchTree, compileContext, compileContext.pureModel.getClass(dataquality.dataQualityRootGraphFetchTree._class), null, new ProcessingContext("DataQuality")));
metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), compileContext.getExecutionSupport());
}
)
getDataQualityModelContraintProcessor(),
getDataQualityRelationValidationProcessor()
);
}

private Processor<DataQuality> getDataQualityModelContraintProcessor()
{
return Processor.newProcessor(
DataQuality.class,
org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class),
(dataquality, compileContext) ->
{
Root_meta_external_dataquality_DataQuality<Object> metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>(
dataquality.name,
SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation),
compileContext.pureModel.getClass("meta::external::dataquality::DataQuality")
)._classifierGenericType(genericType(compileContext, dataquality.dataQualityRootGraphFetchTree._class));
return metamodel;
},
(dataquality, compileContext) ->
{

},
(dataquality, compileContext) ->
{
Root_meta_external_dataquality_DataQuality<Object> metamodel = (Root_meta_external_dataquality_DataQuality<Object>) compileContext.pureModel.getPackageableElement(compileContext.pureModel.buildPackageString(dataquality._package, dataquality.name));
metamodel._context(buildDataQualityExecutionContext(dataquality, compileContext))
._filter(getFilterLambda(dataquality, compileContext))
._validationTree(buildRootGraphFetchTree(dataquality.dataQualityRootGraphFetchTree, compileContext, compileContext.pureModel.getClass(dataquality.dataQualityRootGraphFetchTree._class), null, new ProcessingContext("DataQuality")));
metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), compileContext.getExecutionSupport());
}
);
}

private Processor<DataqualityRelationValidation> getDataQualityRelationValidationProcessor()
{
return Processor.newProcessor(
DataqualityRelationValidation.class,
org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class),
(dataqualityRelationValidation, compileContext) ->
{
Root_meta_external_dataquality_DataQualityRelationValidation_Impl metamodel = new Root_meta_external_dataquality_DataQualityRelationValidation_Impl(
dataqualityRelationValidation.name,
SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation),
compileContext.pureModel.getClass("meta::external::dataquality::DataQualityRelationValidation")
);
return metamodel;
},
(dataqualityRelationValidation, compileContext) ->
{

},
(dataqualityRelationValidation, compileContext) ->
{
Root_meta_external_dataquality_DataQualityRelationValidation_Impl metamodel = (Root_meta_external_dataquality_DataQualityRelationValidation_Impl) compileContext.pureModel.getPackageableElement(compileContext.pureModel.buildPackageString(dataqualityRelationValidation._package, dataqualityRelationValidation.name));
LambdaFunction<?> relationValidationQuery = buildDataqualityRelationValidationQuery(dataqualityRelationValidation, compileContext);
metamodel._query(relationValidationQuery);
metamodel._validations(buildDataQualityRelationValidations(dataqualityRelationValidation.validations, relationValidationQuery, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext));
metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext.getExecutionSupport());
}
);
}

private LambdaFunction<?> buildDataqualityRelationValidationQuery(DataqualityRelationValidation dataqualityRelationValidation, CompileContext compileContext)
{
LambdaFunction<?> relationQuery = HelperValueSpecificationBuilder.buildLambda(dataqualityRelationValidation.query, compileContext);
String relationQueryReturnType = Compiler.getLambdaReturnType(dataqualityRelationValidation.query, compileContext.pureModel);
Assert.assertTrue("meta::pure::metamodel::relation::Relation".equals(relationQueryReturnType), () -> "Relation expected from lambda");
return relationQuery;
}

private RichIterable<Root_meta_external_dataquality_RelationValidation> buildDataQualityRelationValidations(List<RelationValidation> relationValidations, LambdaFunction<?> relationQuery, SourceInformation sourceInformation, CompileContext compileContext)
{
Set<String> allValidationNames = new HashSet<>();
return Lists.mutable.withAll(relationValidations.stream()
.map(relationValidation -> buildRelationValidation(relationValidation, relationQuery, allValidationNames, sourceInformation, compileContext))
.collect(Collectors.toList()));
}

private Root_meta_external_dataquality_RelationValidation_Impl buildRelationValidation(RelationValidation relationalValidation, LambdaFunction<?> relationQuery, Set<String> allValidationNames, SourceInformation sourceInformation, CompileContext compileContext)
{
Assert.assertTrue(relationalValidation.assertion.parameters.size() == 1, () -> "Assertion should have one input param!");
Root_meta_external_dataquality_RelationValidation_Impl relationValidation = new Root_meta_external_dataquality_RelationValidation_Impl("", sourceInformation, compileContext.pureModel.getClass("meta::external::dataquality::RelationValidation"));

if (!allValidationNames.add(relationalValidation.name))
{
throw new EngineException("Duplicated validation '" + relationalValidation.name + "'", EngineErrorType.COMPILATION);
}
relationValidation._name(relationalValidation.name);
relationValidation._description(relationalValidation.description);
Variable assertionInputParam = relationalValidation.assertion.parameters.get(0);
assertionInputParam.multiplicity = Multiplicity.PURE_ONE;
assertionInputParam.genericType = new org.finos.legend.engine.protocol.pure.v1.model.type.GenericType(RelationTypeHelper.convert(fetchQueryRelationType(relationQuery)));

LambdaFunction<?> assertion = HelperValueSpecificationBuilder.buildLambda(relationalValidation.assertion, compileContext);
relationValidation._assertion(assertion);
return relationValidation;
}

private RelationType<?> fetchQueryRelationType(LambdaFunction<?> relationQuery)
{
FunctionType functionType = (FunctionType) relationQuery._classifierGenericType()._typeArguments().getLast()._rawType();
return (RelationType<?>) functionType._returnType()._typeArguments().getLast()._rawType();
}

private LambdaFunction<?> getFilterLambda(DataQuality app, CompileContext compileContext)
{
Expand Down
Loading

0 comments on commit a9292a2

Please sign in to comment.