Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dataquality - Relation validation enhancements for optional from clause, query limit, unit tests #3295

Merged
merged 5 commits into from
Dec 11, 2024
Merged

Dataquality - Relation validation enhancements for optional from clause, query limit, unit tests #3295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a23da15
run query/support for query limit/make from optional
keenkeystrokes Dec 6, 2024
dec2056
remove println
keenkeystrokes Dec 6, 2024
bbbaa6f
change limit multiplicity to PureZeroOne
keenkeystrokes Dec 10, 2024
00074de
dummy commit
keenkeystrokes Dec 10, 2024
9381354
fix import
keenkeystrokes Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions ...pi/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,11 @@

import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
Expand Down Expand Up @@ -118,7 +120,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.validationName);
LambdaFunction<Object> dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.validationName, dataQualityExecuteInput.runQuery, dataQualityExecuteInput.queryLimit);
// 3. Generate Plan from the lambda generated in the previous step
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context
LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString());
Expand All @@ -135,7 +137,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu
@Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB})
@Produces(MediaType.APPLICATION_JSON)
//@Prometheus(name = "generate plan")
public Response execute(@Context HttpServletRequest request, DataQualityExecuteTrialInput dataQualityExecuteInput, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager<CommonProfile> pm, @Context UriInfo uriInfo)
public Response execute(@Context HttpServletRequest request, DataQualityExecuteTrialInput dataQualityExecuteInput, @DefaultValue(SerializationFormat.defaultFormatString) @QueryParam("serializationFormat") SerializationFormat format, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager<CommonProfile> pm, @Context UriInfo uriInfo)
{
MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(pm);
Identity identity = Identity.makeIdentity(profiles);
Expand All @@ -146,12 +148,12 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteT
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName);
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName, dataQualityExecuteInput.runQuery);
// 3. Generate Plan from the lambda generated in the previous step
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context
// 4. execute plan
Map<String, Object> lambdaParameterMap = dataQualityExecuteInput.lambdaParameterValues != null ? dataQualityExecuteInput.lambdaParameterValues.stream().collect(Collectors.<ParameterValue, String, Object>toMap(p -> p.name, p -> p.value.accept(new PrimitiveValueSpecificationToObjectVisitor()))) : Maps.mutable.empty();
Response response = executePlan(request, identity, start, singleExecutionPlan, lambdaParameterMap);
Response response = executePlan(request, identity, format, start, singleExecutionPlan, lambdaParameterMap);
if (response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL))
{
MetricsHandler.observeRequest(uriInfo != null ? uriInfo.getPath() : null, start, System.currentTimeMillis());
Expand All @@ -176,7 +178,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName);
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName, dataQualityExecuteInput.runQuery);
Lambda lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions);
return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper);
}
Expand All @@ -187,7 +189,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
@Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB})
@Produces(MediaType.APPLICATION_JSON)
//@Prometheus(name = "generate plan")
public Response execute(@Context HttpServletRequest request, DataQualityExecuteInput dataQualityParameterValue, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager<CommonProfile> pm, @Context UriInfo uriInfo)
public Response execute(@Context HttpServletRequest request, DataQualityExecuteInput dataQualityParameterValue, @DefaultValue(SerializationFormat.defaultFormatString) @QueryParam("serializationFormat") SerializationFormat format, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager<CommonProfile> pm, @Context UriInfo uriInfo)
{
MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(pm);
Identity identity = Identity.makeIdentity(profiles);
Expand All @@ -198,9 +200,9 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteI
// 1. fetch plan from artifacts
SingleExecutionPlan singleExecutionPlan = this.dataQualityPlanLoader.fetchPlanFromSDLC(identity, dataQualityParameterValue);

// 2. execute plan
// 2. execute plan
Map<String, Object> lambdaParameterMap = dataQualityParameterValue.lambdaParameterValues != null ? dataQualityParameterValue.lambdaParameterValues.stream().collect(Collectors.<ParameterValue, String, Object>toMap(p -> p.name, p -> p.value.accept(new PrimitiveValueSpecificationToObjectVisitor()))) : Maps.mutable.empty();
Response response = executePlan(request, identity, start, singleExecutionPlan, lambdaParameterMap);
Response response = executePlan(request, identity, format, start, singleExecutionPlan, lambdaParameterMap);
if (response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL))
{
MetricsHandler.observeRequest(uriInfo != null ? uriInfo.getPath() : null, start, System.currentTimeMillis());
Expand All @@ -214,12 +216,12 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteI
}
}

private Response executePlan(HttpServletRequest request, Identity identity, long start, SingleExecutionPlan singleExecutionPlan, Map<String, Object> lambdaParameterMap)
private Response executePlan(HttpServletRequest request, Identity identity, SerializationFormat format, long start, SingleExecutionPlan singleExecutionPlan, Map<String, Object> lambdaParameterMap)
{
MutableMap<String, Result> parametersToConstantResult = Maps.mutable.empty();
ExecuteNodeParameterTransformationHelper.buildParameterToConstantResult(singleExecutionPlan, lambdaParameterMap, parametersToConstantResult);
Result result = planExecutor.execute(singleExecutionPlan, parametersToConstantResult, request.getRemoteUser(), identity, null, RequestContextHelper.RequestContext(request));
return this.wrapInResponse(identity, SerializationFormat.DEFAULT, start, result);
return this.wrapInResponse(identity, format, start, result);
}


Expand Down
1 change: 1 addition & 0 deletions ...n/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,5 @@ public class DataQualityExecuteTrialInput
public Integer queryLimit;
public List<ParameterValue> lambdaParameterValues;
public String validationName;
public Boolean runQuery;
}
5 changes: 5 additions & 0 deletions legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@
<artifactId>legend-engine-language-pure-compiler</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-platform-dsl-store-java</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-javaGeneration-pure</artifactId>
Expand Down
12 changes: 12 additions & 0 deletions .../finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext;
import org.finos.legend.engine.protocol.dataquality.metamodel.RelationValidation;
import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping;
Expand All @@ -41,6 +42,7 @@
import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree;
import org.finos.legend.engine.shared.core.operational.Assert;
import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
import org.finos.legend.pure.generated.Root_meta_core_runtime_Runtime;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityExecutionContext;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree;
Expand Down Expand Up @@ -165,6 +167,7 @@ private Processor<DataqualityRelationValidation> getDataQualityRelationValidatio
LambdaFunction<?> relationValidationQuery = buildDataqualityRelationValidationQuery(dataqualityRelationValidation, compileContext);
metamodel._query(relationValidationQuery);
metamodel._validations(buildDataQualityRelationValidations(dataqualityRelationValidation.validations, relationValidationQuery, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext));
metamodel._runtime(buildRelationValidationRuntime(dataqualityRelationValidation.runtime, relationValidationQuery, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext));
metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext.getExecutionSupport());
}
);
Expand Down Expand Up @@ -343,4 +346,13 @@ private static String getPropertyIdentifier(PropertyGraphFetchTree propertyGraph
return propertyGraphFetchTree.alias != null ? propertyGraphFetchTree.alias : propertyGraphFetchTree.property;
}


private RichIterable<? extends Root_meta_core_runtime_Runtime> buildRelationValidationRuntime(PackageableElementPointer runtime, LambdaFunction<?> relationValidationQuery, SourceInformation toM3SourceInformation, CompileContext compileContext)
{
if (Objects.isNull(runtime))
{
return Lists.immutable.empty();
}
return Lists.immutable.of(compileContext.resolveRuntime(runtime.path, runtime.sourceInformation));
}
}
47 changes: 42 additions & 5 deletions ...gend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,21 +129,58 @@ public void testRootClassEmptyTree()
@Test
public void testRelationValidation()
{
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(RELATION_COMPILATION_PREREQUISITE_CODE +
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE +
"###DataQualityValidation\n" +
"DataQualityRelationValidation meta::external::dataquality::testvalidation\n" +
"{\n" +
" query: #>{my::Store.myTable}#->select(~name);\n" +
" query: #>{meta::dataquality::db.personTable}#->select(~FIRSTNAME)->from(meta::dataquality::DataQualityRuntime);\n" +
" validations: [\n" +
" {\n" +
" name: 'validFirstName';\n" +
" description: 'First name cannot be empty';\n" +
" assertion: row|$row.FIRSTNAME->isNotEmpty();\n" +
" }\n" +
" ];\n" +
"}");
}

@Test
public void testRelationValidation_separateRuntime()
{
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE +
"###DataQualityValidation\n" +
"DataQualityRelationValidation meta::external::dataquality::Validation\n" +
"{\n" +
" query: #>{meta::dataquality::db.personTable}#->select(~FIRSTNAME);\n" +
" runtime: meta::dataquality::DataQualityRuntime;\n" +
" validations: [\n" +
" {\n" +
" name: 'testValidation';\n" +
" description: 'test validation';\n" +
" assertion: row|$row.name != 'error';\n" +
" name: 'validFirstName';\n" +
" description: 'First name cannot be empty';\n" +
" assertion: row|$row.FIRSTNAME->isNotEmpty();\n" +
" }\n" +
" ];\n" +
"}");
}

@Test
public void testRelationValidation_noRuntime()
{
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE +
"###DataQualityValidation\n" +
"DataQualityRelationValidation meta::external::dataquality::Validation\n" +
"{\n" +
" query: #>{meta::dataquality::db.personTable}#->select(~FIRSTNAME);\n" +
" validations: [\n" +
" {\n" +
" name: 'validFirstName';\n" +
" description: 'First name cannot be empty';\n" +
" assertion: row|$row.FIRSTNAME->isNotEmpty();\n" +
" }\n" +
" ];\n" +
"}", " at [104:1-114:1]: Error in 'meta::external::dataquality::Validation': Execution error at (resource: lines:104c1-114c1), \"Constraint :[mustHaveOneRuntime] violated in the Class DataQualityRelationValidation\"");
}


private static final String COMPILATION_PREREQUISITE_CODE = "###Connection\n" +
"RelationalDatabaseConnection meta::dataquality::H2\n" +
Expand Down
Loading
Loading