diff --git a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/pom.xml b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/pom.xml index bafb2d278bc..0075e84873d 100644 --- a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/pom.xml +++ b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/pom.xml @@ -393,6 +393,31 @@ org.finos.legend.engine legend-engine-xt-snowflakeApp-protocol + + + org.finos.legend.engine + legend-engine-xt-dataquality-compiler + ${project.version} + runtime + + + org.finos.legend.engine + legend-engine-xt-dataquality-grammar + ${project.version} + runtime + + + org.finos.legend.engine + legend-engine-xt-dataquality-api + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-protocol + ${project.version} + runtime + + org.finos.legend.engine legend-engine-xt-bigqueryFunction-compiler diff --git a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java index c2cc75aca29..b20e345daaf 100644 --- a/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -63,6 +63,7 @@ import org.finos.legend.engine.external.shared.format.model.api.ExternalFormats; import org.finos.legend.engine.functionActivator.api.FunctionActivatorAPI; import org.finos.legend.engine.generation.artifact.api.ArtifactGenerationExtensionApi; +import org.finos.legend.engine.language.dataquality.api.DataQualityExecute; import org.finos.legend.engine.language.hostedService.api.HostedServiceService; import org.finos.legend.engine.language.memsql.api.MemSqlFunctionService; import org.finos.legend.engine.language.pure.compiler.api.Compile; @@ -416,6 +417,10 @@ public void run(T serverConfiguration, Environment environment) environment.jersey().register(new LineageAnalytics(modelManager)); environment.jersey().register(new StoreEntitlementAnalytics(modelManager, entitlementServiceExtensions)); + // DataQuality + environment.jersey().register(new DataQualityExecute(modelManager, planExecutor, routerExtensions, generatorExtensions.flatCollect(PlanGeneratorExtension::getExtraPlanTransformers), serverConfiguration.metadataserver, null)); + + // Testable environment.jersey().register(new TestableApi(modelManager)); diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java index 2691c6cfb1c..3b32a056e9b 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java @@ -107,6 +107,7 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-arrow/legend-engine-xt-arrow-pure", "external-format-arrow")) .with(this.buildCore("legend-engine-xts-relationalai/legend-engine-xt-relationalai-pure", "external-query-relationalai")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure", "external_test_connection")) + .with(this.buildCore("legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure", "dataquality")) ; } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml new file mode 100644 index 00000000000..cbf9c4e7617 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml @@ -0,0 +1,275 @@ + + + + legend-engine-xts-dataquality + org.finos.legend.engine + 4.48.2-SNAPSHOT + + 4.0.0 + + legend-engine-xt-dataquality-api + jar + Legend Engine - XT - DataQuality - Api + + + 11 + 11 + + + + + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + + + org.finos.legend.pure + legend-pure-m3-core + + + + + + org.finos.legend.engine + legend-engine-shared-core + + + org.finos.legend.engine + legend-engine-language-pure-modelManager + + + org.finos.legend.engine + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-language-pure-compiler + + + org.finos.legend.engine + legend-engine-language-pure-compiler + test-jar + test + + + + org.finos.legend.engine + legend-engine-xt-relationalStore-javaPlatformBinding-pure + runtime + + + org.finos.legend.engine + legend-engine-pure-code-core-extension + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + runtime + + + org.finos.legend.engine + legend-engine-xt-relationalStore-grammar + runtime + + + org.finos.legend.engine + legend-engine-executionPlan-execution + + + org.finos.legend.engine + legend-engine-executionPlan-execution-http-api + ${project.version} + + + org.finos.legend.engine + legend-engine-language-pure-modelManager-sdlc + + + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + + + org.finos.legend.engine + legend-engine-xt-data-space-compiler + runtime + + + org.finos.legend.engine + legend-engine-xt-relationalStore-trino-grammar + ${project.version} + runtime + + + + javax.servlet + javax.servlet-api + + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + + org.finos.legend.engine + legend-engine-xt-dataquality-protocol + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-pure + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-grammar + runtime + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-compiler + runtime + ${project.version} + + + + + org.finos.legend.pure + legend-pure-m2-dsl-graph-pure + + + + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + + + + + + org.pac4j.jax-rs + core + + + org.pac4j + pac4j-core + + + + + org.slf4j + slf4j-api + + + + + io.opentracing + opentracing-util + + + io.opentracing + opentracing-api + + + + + + javax.ws.rs + javax.ws.rs-api + + + io.swagger + swagger-annotations + + + + + + log4j + log4j + test + + + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-shared-core + test-jar + test + + + org.glassfish.jersey.core + jersey-common + test + + + org.finos.legend.engine + legend-engine-executionPlan-generation + + + org.finos.legend.engine + legend-engine-xt-identity-pac4j + + + org.finos.legend.engine + legend-engine-identity-core + ${project.version} + + + org.finos.legend.engine + legend-engine-language-pure-dsl-generation + + + com.github.tomakehurst + wiremock-jre8 + test + + + com.h2database + h2 + test + + + + \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java new file mode 100644 index 00000000000..dc1de429242 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java @@ -0,0 +1,248 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.opentracing.Scope; +import io.opentracing.util.GlobalTracer; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.apache.http.impl.client.CloseableHttpClient; +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.map.MutableMap; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.modelManager.ModelManager; +import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.MetaDataServerConfiguration; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.api.request.RequestContextHelper; +import org.finos.legend.engine.plan.execution.api.result.ResultManager; +import org.finos.legend.engine.plan.execution.nodes.helpers.ExecuteNodeParameterTransformationHelper; +import org.finos.legend.engine.plan.execution.planHelper.PrimitiveValueSpecificationToObjectVisitor; +import org.finos.legend.engine.plan.execution.result.Result; +import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.generation.transformers.PlanTransformer; +import org.finos.legend.engine.plan.platform.PlanPlatform; +import org.finos.legend.engine.protocol.dataquality.model.DataQualityExecuteInput; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.ParameterValue; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.api.result.ManageConstantResult; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.engine.shared.core.kerberos.ProfileManagerHelper; +import org.finos.legend.engine.shared.core.operational.errorManagement.ExceptionTool; +import org.finos.legend.engine.shared.core.operational.http.InflateInterceptor; +import org.finos.legend.engine.shared.core.operational.logs.LogInfo; +import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType; +import org.finos.legend.engine.shared.core.operational.prometheus.MetricsHandler; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.pac4j.core.profile.CommonProfile; +import org.pac4j.core.profile.ProfileManager; +import org.pac4j.jax.rs.annotations.Pac4JProfileManager; +import org.slf4j.Logger; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.finos.legend.engine.shared.core.operational.http.InflateInterceptor.APPLICATION_ZLIB; + +@Api(tags = "DataQuality - Execution") +@Path("pure/v1/dataquality") +@Produces(MediaType.APPLICATION_JSON) +public class DataQualityExecute +{ + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DataQualityExecute.class); + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + private final ModelManager modelManager; + private final Function> extensions; + private final Iterable transformers; + private final PlanExecutor planExecutor; + private final DataQualityPlanLoader dataQualityPlanLoader; + + + public DataQualityExecute(ModelManager modelManager, PlanExecutor planExecutor, Function> extensions, Iterable transformers, MetaDataServerConfiguration metaDataServerConfiguration, CloseableHttpClient httpClientProvider) + { + this.modelManager = modelManager; + this.extensions = extensions; + this.transformers = transformers; + this.planExecutor = planExecutor; + this.dataQualityPlanLoader = new DataQualityPlanLoader(metaDataServerConfiguration.sdlc, httpClientProvider); + MetricsHandler.createMetrics(this.getClass()); + + } + + @POST + @Path("generatePlan") + @Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB}) + @Produces(MediaType.APPLICATION_JSON) + //@Prometheus(name = "generate plan") + public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInput, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager pm) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(pm); + Identity identity = Identity.makeIdentity(profiles); + long start = System.currentTimeMillis(); + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_START).toString()); + try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: planGeneration").startActive(true)) + { + // 1. load pure model from PureModelContext + PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); + // 2. call DQ PURE func to generate lambda + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath); + // 3. Generate Plan from the lambda generated in the previous step + SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString()); + return ManageConstantResult.manageResult(identity.getName(), singleExecutionPlan, objectMapper); + } + catch (Exception ex) + { + return ExceptionTool.exceptionManager(ex, LoggingEventType.GENERATE_PLAN_ERROR, identity.getName()); + } + } + + @POST + @Path("execute") + @Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB}) + @Produces(MediaType.APPLICATION_JSON) + //@Prometheus(name = "generate plan") + public Response execute(@Context HttpServletRequest request, DataQualityExecuteTrialInput dataQualityExecuteInput, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager pm, @Context UriInfo uriInfo) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(pm); + Identity identity = Identity.makeIdentity(profiles); + long start = System.currentTimeMillis(); + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PLAN_EXECUTION_START).toString()); + try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: executeTrial").startActive(true)) + { + // 1. load pure model from PureModelContext + PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); + // 2. call DQ PURE func to generate lambda + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit); + // 3. Generate Plan from the lambda generated in the previous step + SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context + // 4. execute plan + Map lambdaParameterMap = dataQualityExecuteInput.lambdaParameterValues != null ? dataQualityExecuteInput.lambdaParameterValues.stream().collect(Collectors.toMap(p -> p.name, p -> p.value.accept(new PrimitiveValueSpecificationToObjectVisitor()))) : Maps.mutable.empty(); + Response response = executePlan(request, identity, start, singleExecutionPlan, lambdaParameterMap); + if (response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) + { + MetricsHandler.observeRequest(uriInfo != null ? uriInfo.getPath() : null, start, System.currentTimeMillis()); + } + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PLAN_EXECUTION_END, System.currentTimeMillis() - start).toString()); + return response; + } + catch (Exception ex) + { + return ExceptionTool.exceptionManager(ex, LoggingEventType.EXECUTION_PLAN_EXEC_ERROR, identity.getName()); + } + } + + + @POST + @Path("executeArtifacts") + @Consumes({MediaType.APPLICATION_JSON, APPLICATION_ZLIB}) + @Produces(MediaType.APPLICATION_JSON) + //@Prometheus(name = "generate plan") + public Response execute(@Context HttpServletRequest request, DataQualityExecuteInput dataQualityParameterValue, @ApiParam(hidden = true) @Pac4JProfileManager() ProfileManager pm, @Context UriInfo uriInfo) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(pm); + Identity identity = Identity.makeIdentity(profiles); + long start = System.currentTimeMillis(); + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_ARTIFACT_PLAN_EXECUTION_START).toString()); + try (Scope scope = GlobalTracer.get().buildSpan("DataQuality: execute").startActive(true)) + { + // 1. fetch plan from artifacts + SingleExecutionPlan singleExecutionPlan = this.dataQualityPlanLoader.fetchPlanFromSDLC(identity, dataQualityParameterValue); + + // 2. execute plan + Map lambdaParameterMap = dataQualityParameterValue.lambdaParameterValues != null ? dataQualityParameterValue.lambdaParameterValues.stream().collect(Collectors.toMap(p -> p.name, p -> p.value.accept(new PrimitiveValueSpecificationToObjectVisitor()))) : Maps.mutable.empty(); + Response response = executePlan(request, identity, start, singleExecutionPlan, lambdaParameterMap); + if (response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) + { + MetricsHandler.observeRequest(uriInfo != null ? uriInfo.getPath() : null, start, System.currentTimeMillis()); + } + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_ARTIFACT_PLAN_EXECUTION_END, System.currentTimeMillis() - start).toString()); + return response; + } + catch (Exception ex) + { + return ExceptionTool.exceptionManager(ex, LoggingEventType.EXECUTION_PLAN_EXEC_ERROR, identity.getName()); + } + } + + private Response executePlan(HttpServletRequest request, Identity identity, long start, SingleExecutionPlan singleExecutionPlan, Map lambdaParameterMap) + { + MutableMap parametersToConstantResult = Maps.mutable.empty(); + ExecuteNodeParameterTransformationHelper.buildParameterToConstantResult(singleExecutionPlan, lambdaParameterMap, parametersToConstantResult); + Result result = planExecutor.execute(singleExecutionPlan, parametersToConstantResult, request.getRemoteUser(), identity, null, RequestContextHelper.RequestContext(request)); + return this.wrapInResponse(identity, SerializationFormat.DEFAULT, start, result); + } + + + private Response wrapInResponse(Identity identity, SerializationFormat format, long start, Result result) + { + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_EXECUTE_INTERACTIVE_STOP, (double) System.currentTimeMillis() - start).toString()); + MetricsHandler.observe("execute", start, System.currentTimeMillis()); + try (Scope scope = GlobalTracer.get().buildSpan("Manage Results").startActive(true)) + { + return ResultManager.manageResult(identity.getName(), result, format, LoggingEventType.EXECUTE_INTERACTIVE_ERROR); + } + } + + @POST + @Path("propertyPathTree") + @ApiOperation(value = "Analyze the DataQuality tree to generate property path tree for given constraints") + @Consumes({MediaType.APPLICATION_JSON, InflateInterceptor.APPLICATION_ZLIB}) + @Produces(MediaType.APPLICATION_JSON) + public Response generatePropertyPathTree(DataQualityExecuteTrialInput input, + @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm) + { + MutableList profiles = ProfileManagerHelper.extractProfiles(pm); + Identity identity = Identity.makeIdentity(profiles); + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PROPERTY_PATH_TREE_START).toString()); + try + { + PureModel pureModel = this.modelManager.loadModel(input.model, input.clientVersion, identity, null); + PackageableElement packageableElement = pureModel.getPackageableElement(input.packagePath); + if (!DataQualityPropertyPathTreeGenerator.isDataQualityInstance(packageableElement)) + { + return ExceptionTool.exceptionManager("Invalid Element", LoggingEventType.MODEL_RESOLVE_ERROR, identity.getName()); + } + RootGraphFetchTree rootGraphFetchTree = DataQualityPropertyPathTreeGenerator.getPropertyPathTree(((Root_meta_external_dataquality_DataQuality)packageableElement)._validationTree(), input.clientVersion, pureModel, this.extensions.apply(pureModel)); + LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_PROPERTY_PATH_TREE_END).toString()); + return ManageConstantResult.manageResult(identity.getName(), rootGraphFetchTree); + } + catch (Exception ex) + { + return ExceptionTool.exceptionManager(ex, LoggingEventType.MODEL_RESOLVE_ERROR, identity.getName()); + } + + } + +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java new file mode 100644 index 00000000000..88d98aa31e2 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java @@ -0,0 +1,32 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.ParameterValue; + +import java.util.List; + +public class DataQualityExecuteTrialInput +{ + public String clientVersion; + @JsonProperty(required = true) + public PureModelContext model; + @JsonProperty + public String packagePath; + public Integer queryLimit; + public List lambdaParameterValues; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecutionLoggingEventType.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecutionLoggingEventType.java new file mode 100644 index 00000000000..4e53dd6e728 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecutionLoggingEventType.java @@ -0,0 +1,33 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import org.finos.legend.engine.shared.core.operational.logs.ILoggingEventType; + +public enum DataQualityExecutionLoggingEventType implements ILoggingEventType +{ + DATAQUALITY_PLAN_EXECUTION_START, + DATAQUALITY_PLAN_EXECUTION_END, + DATAQUALITY_PLAN_EXECUTION_TRIAL_ERROR, + DATAQUALITY_ARTIFACT_PLAN_EXECUTION_START, + DATAQUALITY_ARTIFACT_PLAN_EXECUTION_END, + DATAQUALITY_PLAN_EXECUTION_ERROR, + DATAQUALITY_GENERATE_PLAN_START, + DATAQUALITY_GENERATE_PLAN_END, + DATAQUALITY_EXECUTE_INTERACTIVE_STOP, + DATAQUALITY_EXECUTE_INTERACTIVE_ERROR, + DATAQUALITY_PROPERTY_PATH_TREE_START, + DATAQUALITY_PROPERTY_PATH_TREE_END +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityLambdaGenerator.java new file mode 100644 index 00000000000..3af91656fcc --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityLambdaGenerator.java @@ -0,0 +1,51 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.finos.legend.pure.generated.core_dataquality_generation_dataquality; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; + +import java.util.Objects; + +public class DataQualityLambdaGenerator +{ + public static final int DEFAULT_QUERY_LIMIT = 100; + + public static LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath) + { + PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); + return generateLambda(pureModel, packageableElement); + } + + public static LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement) + { + return core_dataquality_generation_dataquality.Root_meta_external_dataquality_executeDataQualityValidation_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.empty(), pureModel.getExecutionSupport()); + } + + public static LambdaFunction generateLambdaForTrial(PureModel pureModel, String qualifiedPath, Integer queryLimit) + { + PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); + int trialQueryLimit = DEFAULT_QUERY_LIMIT; + if (Objects.nonNull(queryLimit)) + { + trialQueryLimit = queryLimit; + } + return core_dataquality_generation_dataquality.Root_meta_external_dataquality_executeDataQualityValidation_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.of((long)trialQueryLimit), pureModel.getExecutionSupport()); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityPlanLoader.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityPlanLoader.java new file mode 100644 index 00000000000..c8b5852198f --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityPlanLoader.java @@ -0,0 +1,97 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.opentracing.Scope; +import io.opentracing.util.GlobalTracer; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.client.CloseableHttpClient; +import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader; +import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.ServerConnectionConfiguration; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.protocol.dataquality.model.DataQualityExecuteInput; +import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.engine.shared.core.kerberos.HttpClientBuilder; +import org.finos.legend.engine.shared.core.operational.Assert; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; + +import java.io.InputStream; +import java.util.List; + +public class DataQualityPlanLoader +{ + private static final TypeReference> ARTIFACT_TYPE = new TypeReference>() + { + }; + + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + public static final String EXECUTION_PLAN_FILE_NAME = "dataQualityValidationExecutionPlan.json"; + + private final ServerConnectionConfiguration sdlcServerConnectionConfig; + private final CloseableHttpClient httpClientProvider; + + public DataQualityPlanLoader(ServerConnectionConfiguration sdlcServerConnectionConfig, CloseableHttpClient httpClientProvider) + { + this.sdlcServerConnectionConfig = sdlcServerConnectionConfig; + this.httpClientProvider = httpClientProvider; + } + + public SingleExecutionPlan fetchPlanFromSDLC(Identity identity, DataQualityExecuteInput dataQualityParameterValue) + { + List metaDataArtifactList = loadPlanDataQualityFromHTTPURL(getMetaDataApiUrl(dataQualityParameterValue.elementPath, dataQualityParameterValue.alloySDLC)); + return getPlanFromArtifactResponse(metaDataArtifactList); + } + + private String getMetaDataApiUrl(String elementPath, AlloySDLC alloySDLC) + { + Assert.assertTrue(alloySDLC != null && alloySDLC.groupId != null && alloySDLC.artifactId != null && alloySDLC.version != null, () -> "AlloySDLC info must contain and group and artifact IDs to access metadata services"); + return String.format("%s/generations/%s/%s/versions/%s/%s", sdlcServerConnectionConfig.getBaseUrl(), alloySDLC.groupId, alloySDLC.artifactId, alloySDLC.version, elementPath); + } + + private List loadPlanDataQualityFromHTTPURL(String url) + { + + try ( + CloseableHttpClient client = httpClientProvider == null ? (CloseableHttpClient) HttpClientBuilder.getHttpClient(new BasicCookieStore()) : httpClientProvider; + Scope scope = GlobalTracer.get().buildSpan("Load project upstream dependencies").startActive(true) + ) + { + HttpGet httpRequest = new HttpGet(url); + HttpEntity entity = SDLCLoader.execHttpRequest(scope.span(), client, httpRequest); + try (InputStream content = entity.getContent()) + { + return objectMapper.readValue(content, ARTIFACT_TYPE); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + private SingleExecutionPlan getPlanFromArtifactResponse(List metaDataArtifactList) + { + Artifact planArtifact = metaDataArtifactList.stream().filter(artifact -> artifact.path.contains(EXECUTION_PLAN_FILE_NAME)).findAny().orElseThrow(() -> new EngineException("Error fetching DataQuality Execution plan")); + return PlanGenerator.stringToPlan(planArtifact.content); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityPropertyPathTreeGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityPropertyPathTreeGenerator.java new file mode 100644 index 00000000000..00511892e44 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityPropertyPathTreeGenerator.java @@ -0,0 +1,84 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.protocol.pure.PureClientVersions; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.generated.core_dataquality_generation_dataquality; +import org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.GraphFetchTree; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; +import org.finos.legend.pure.m3.execution.ExecutionSupport; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.support.map.PureMap; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import static org.finos.legend.pure.generated.core_pure_protocol_protocol.Root_meta_alloy_metadataServer_alloyToJSON_Any_1__String_1_; + +public class DataQualityPropertyPathTreeGenerator +{ + public static RootGraphFetchTree getPropertyPathTree(Root_meta_external_dataquality_DataQualityRootGraphFetchTree dataQualityRootGraphFetchTree, String clientVersion, PureModel pureModel, RichIterable extensions) throws JsonProcessingException + { + org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.RootGraphFetchTree rootGraphFetchTree = core_dataquality_generation_dataquality.Root_meta_external_dataquality_getEnrichedTreeForStructuralValidations_DataQualityRootGraphFetchTree_1__RootGraphFetchTree_1_(dataQualityRootGraphFetchTree, pureModel.getExecutionSupport()); + return serializeToJSON(rootGraphFetchTree, clientVersion, pureModel, extensions); + } + + private static RootGraphFetchTree serializeToJSON(org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.RootGraphFetchTree purePlan, String clientVersion, PureModel pureModel, RichIterable extensions) throws JsonProcessingException + { + String cl = (clientVersion == null || !supports(clientVersion)) ? PureClientVersions.production : clientVersion; + Object transformed = transformToVersionedModel(purePlan, cl, extensions, pureModel.getExecutionSupport()); + return serializeToJSON(transformed, pureModel); + } + + private static Object transformToVersionedModel(org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.RootGraphFetchTree pureGraph, String version, RichIterable extensions, ExecutionSupport executionSupport) + { + try + { + Class cl = Class.forName("org.finos.legend.pure.generated.core_pure_protocol_" + version + "_transfers_valueSpecification"); + Method graphFetchProtocolMethod = cl.getMethod("Root_meta_protocols_pure_" + version + "_transformation_fromPureGraph_valueSpecification_transformGraphFetchTree_GraphFetchTree_1__String_MANY__Map_1__Extension_MANY__GraphFetchTree_1_", GraphFetchTree.class, RichIterable.class, PureMap.class, RichIterable.class, org.finos.legend.pure.m3.execution.ExecutionSupport.class); + return graphFetchProtocolMethod.invoke(null, pureGraph, Lists.mutable.empty(), new PureMap(Maps.mutable.empty()), extensions, executionSupport); + } + catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) + { + throw new RuntimeException(e); + } + } + + private static RootGraphFetchTree serializeToJSON(Object protocolPlan, PureModel pureModel) throws JsonProcessingException + { + String asJSON = Root_meta_alloy_metadataServer_alloyToJSON_Any_1__String_1_(protocolPlan, pureModel.getExecutionSupport()); + return ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().readValue(asJSON, RootGraphFetchTree.class); + } + + public static boolean isDataQualityInstance(PackageableElement element) + { + return element instanceof Root_meta_external_dataquality_DataQuality + && ((Root_meta_external_dataquality_DataQuality) element)._validationTree() != null; + } + + private static boolean supports(String version) + { + return "vX_X_X".equals(version) || PureClientVersions.versionAGreaterThanOrEqualsVersionB(version, "v1_20_0"); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/deployment/DataQualityValidationArtifactGenerationExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/deployment/DataQualityValidationArtifactGenerationExtension.java new file mode 100644 index 00000000000..585ee6f1d61 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/deployment/DataQualityValidationArtifactGenerationExtension.java @@ -0,0 +1,97 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.deployment; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.factory.Lists; +import org.finos.legend.engine.language.dataquality.api.DataQualityLambdaGenerator; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.generation.transformers.LegendPlanTransformers; +import org.finos.legend.engine.plan.platform.PlanPlatform; +import org.finos.legend.engine.protocol.dataquality.model.DataQualityMetadata; +import org.finos.legend.engine.protocol.dataquality.model.DataQualityRule; +import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRule; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.generated.core_dataquality_generation_dataquality; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.slf4j.Logger; + +import java.util.List; + +public class DataQualityValidationArtifactGenerationExtension implements ArtifactGenerationExtension +{ + private final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DataQualityValidationArtifactGenerationExtension.class); + private final ObjectMapper mapper = ObjectMapperFactory.withStandardConfigurations(PureProtocolObjectMapperFactory.withPureProtocolExtensions(new ObjectMapper())); + + public static final String ROOT_PATH = "dataQualityValidation"; + public static final String META_DATA_FILE_NAME = "dataQualityRulesMetadata.json"; + public static final String EXECUTION_PLAN_FILE_NAME = "dataQualityValidationExecutionPlan.json"; + + + @Override + public String getKey() + { + return ROOT_PATH; + } + + @Override + public boolean canGenerate(PackageableElement element) + { + return element instanceof Root_meta_external_dataquality_DataQuality + && ((Root_meta_external_dataquality_DataQuality) element)._validationTree() != null; + } + + @Override + public List generate(PackageableElement packageableElement, PureModel pureModel, PureModelContextData pureModelContextData, String clientVersion) + { + List artifacts = Lists.mutable.empty(); + + try + { + RichIterable dataQualityRules = + core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDQMetaDataForDQValidation_DataQuality_1__DataQualityRule_MANY_((Root_meta_external_dataquality_DataQuality)packageableElement, pureModel.getExecutionSupport()); + List dataQualityRuleList = Lists.mutable.withAll(dataQualityRules).collect(d -> new DataQualityRule(d._constraintName(), d._constraintType(), d._constraintGrammar(), d._propertyPath())); + DataQualityMetadata dataQualityMetadata = new DataQualityMetadata(); + dataQualityMetadata.dqRules = dataQualityRuleList; + artifacts.add(new Artifact(mapper.writeValueAsString(dataQualityMetadata), META_DATA_FILE_NAME, "json")); + + Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); + // 1. call DQ PURE func to generate lambda + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement); + // 2. Generate Plan from the lambda generated in the previous step + SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null, + routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context + artifacts.add(new Artifact(mapper.writeValueAsString(singleExecutionPlan), EXECUTION_PLAN_FILE_NAME, "json")); + } + catch (Exception e) + { + LOGGER.error("Unable to compute dataQuality validation artifact for dqValidation: " + packageableElement.getName() + ". Exception:" + e.getMessage()); + } + return artifacts; + } + +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension new file mode 100644 index 00000000000..892b5e69787 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.dataquality.deployment.DataQualityValidationArtifactGenerationExtension diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java new file mode 100644 index 00000000000..48c3106bb85 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityApi.java @@ -0,0 +1,69 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.finos.legend.engine.language.pure.compiler.Compiler; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtensionLoader; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.deployment.DeploymentMode; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestDataQualityApi +{ + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + + @Test + public void testDataQualityGetPropertyPathTree() throws IOException + { + URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithDataQualityValidation.json")); + PureModelContextData pureModelContextData = objectMapper.readValue(url, PureModelContextData.class); + CompilerExtensionLoader.logExtensionList(); + PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName()); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation"); + assertTrue(DataQualityPropertyPathTreeGenerator.isDataQualityInstance(packageableElement)); + RootGraphFetchTree rootGraphFetchTree = DataQualityPropertyPathTreeGenerator.getPropertyPathTree(((Root_meta_external_dataquality_DataQuality)packageableElement)._validationTree(), "vX_X_X", model, null); + assertEquals("{\"_type\":\"rootGraphFetchTree\",\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::dataquality::Person\",\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"name\",\"subTrees\":[],\"subTypeTrees\":[]},{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"age\",\"subTrees\":[],\"subTypeTrees\":[]}],\"subTypeTrees\":[]}", objectMapper.writeValueAsString(rootGraphFetchTree)); + + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement2 = model.getPackageableElement("meta::dataquality::Person"); + assertFalse(DataQualityPropertyPathTreeGenerator.isDataQualityInstance(packageableElement2)); + } + + @Ignore + @Test + public void testDataQualityGetPropertyPathTree_WithNullDataQualityValidation() throws IOException + { + URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithNullDataQualityValidation.json")); + PureModelContextData pureModelContextData = objectMapper.readValue(url, PureModelContextData.class); + CompilerExtensionLoader.logExtensionList(); + PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST, Identity.getAnonymousIdentity().getName()); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation"); + assertFalse(DataQualityPropertyPathTreeGenerator.isDataQualityInstance(packageableElement)); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityPlanLoader.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityPlanLoader.java new file mode 100644 index 00000000000..20e5ab6112e --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/api/TestDataQualityPlanLoader.java @@ -0,0 +1,105 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.api; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit.WireMockClassRule; +import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.ServerConnectionConfiguration; +import org.finos.legend.engine.protocol.dataquality.model.DataQualityExecuteInput; +import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Objects; + +import static org.junit.Assert.assertNotNull; + +public class TestDataQualityPlanLoader +{ + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + + private static final TypeReference> ARTIFACT_TYPE = new TypeReference>() + { + }; + + @ClassRule + public static WireMockClassRule wireMockServer = new WireMockClassRule(); + + @Rule + public WireMockClassRule rule = wireMockServer; + + @Test + public void testFetchPlanFromSDLC() throws IOException + { + configureWireMockForNoRetries(); + DataQualityPlanLoader dataQualityPlanLoader = new DataQualityPlanLoader(createServerConnectionConfiguration(), null); + SingleExecutionPlan singleExecutionPlan = dataQualityPlanLoader.fetchPlanFromSDLC(Identity.getAnonymousIdentity(), createDataQualityParameterValue()); + assertNotNull(singleExecutionPlan); + } + + @Test + public void testFetchPlanFromSDLC_EmptyProjectCoordinates_Exception() throws IOException + { + configureWireMockForNoRetries(); + DataQualityPlanLoader dataQualityPlanLoader = new DataQualityPlanLoader(createServerConnectionConfiguration(), null); + DataQualityExecuteInput dataQualityParameterValue = new DataQualityExecuteInput(); + dataQualityParameterValue.alloySDLC = null; + dataQualityParameterValue.elementPath = "meta::dataquality::PersonDataQualityValidation"; + Assert.assertEquals("AlloySDLC info must contain and group and artifact IDs to access metadata services", Assert.assertThrows(EngineException.class, () -> dataQualityPlanLoader.fetchPlanFromSDLC(Identity.getAnonymousIdentity(), dataQualityParameterValue)).getMessage()); + } + + private DataQualityExecuteInput createDataQualityParameterValue() + { + AlloySDLC alloySDLC = new AlloySDLC(); + alloySDLC.groupId = "com.dq.test"; + alloySDLC.artifactId = "test-sandbox"; + alloySDLC.version = "master-SNAPSHOT"; + DataQualityExecuteInput dataQualityParameterValue = new DataQualityExecuteInput(); + dataQualityParameterValue.alloySDLC = alloySDLC; + dataQualityParameterValue.elementPath = "meta::dataquality::PersonDataQualityValidation"; + return dataQualityParameterValue; + } + + private ServerConnectionConfiguration createServerConnectionConfiguration() + { + ServerConnectionConfiguration serverConfiguration = new ServerConnectionConfiguration(); + + serverConfiguration.host = "localhost"; + serverConfiguration.port = rule.port(); + serverConfiguration.prefix = "/api"; + + return serverConfiguration; + } + + private static void configureWireMockForNoRetries() throws IOException + { + URL url = Objects.requireNonNull(TestDataQualityPlanLoader.class.getClassLoader().getResource("inputs/dataQualityArtifacts.json")); + List response = objectMapper.readValue(url, ARTIFACT_TYPE); + WireMock.stubFor(WireMock.get("/api/generations/com.dq.test/test-sandbox/versions/master-SNAPSHOT/meta::dataquality::PersonDataQualityValidation") + .willReturn(WireMock.aResponse().withStatus(200).withBody(objectMapper.writeValueAsString(response)))); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/deployment/TestDataQualityValidationArtifactGenerationExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/deployment/TestDataQualityValidationArtifactGenerationExtension.java new file mode 100644 index 00000000000..6020161953c --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/java/org/finos/legend/engine/language/dataquality/deployment/TestDataQualityValidationArtifactGenerationExtension.java @@ -0,0 +1,61 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.deployment; + +import org.finos.legend.engine.language.pure.compiler.Compiler; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtensionLoader; +import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.deployment.DeploymentMode; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.junit.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestDataQualityValidationArtifactGenerationExtension +{ + @Test + public void testDataQualityValidationArtifact() throws IOException + { + URL url = Objects.requireNonNull(getClass().getClassLoader().getResource("inputs/modelWithDataQualityValidation.json")); + PureModelContextData pureModelContextData = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().readValue(url, PureModelContextData.class); + CompilerExtensionLoader.logExtensionList(); + + PureModel model = Compiler.compile(pureModelContextData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); + DataQualityValidationArtifactGenerationExtension extension = new DataQualityValidationArtifactGenerationExtension(); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = model.getPackageableElement("meta::dataquality::PersonDataQualityValidation"); + assertTrue(extension.canGenerate(packageableElement)); + List outputs = extension.generate(packageableElement, model, pureModelContextData, "vX_X_X"); + assertEquals(2, outputs.size()); + Optional artifactOptional = outputs.stream().filter(artifact -> "dataQualityRulesMetadata.json".equalsIgnoreCase(artifact.path)).findAny(); + assertTrue(artifactOptional.isPresent()); + Artifact dataQualityMetaData = artifactOptional.get(); + assertEquals("{\"dqRules\":[{\"constraintGrammar\":\"($this.age >= 18)\",\"constraintName\":\"mustBeOfLegalAge\",\"constraintType\":\"Alloy_Constraint_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"Class\",\"constraintName\":\"Person\",\"constraintType\":\"Alloy_Class_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"name\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::name\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"age\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::age\"}]}", dataQualityMetaData.content); + + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement2 = model.getPackageableElement("meta::dataquality::Person"); + assertFalse(extension.canGenerate(packageableElement2)); + + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/dataQualityArtifacts.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/dataQualityArtifacts.json new file mode 100644 index 00000000000..9d457909686 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/dataQualityArtifacts.json @@ -0,0 +1,12 @@ +[ + { + "content": "{\"dqRules\":[{\"constraintGrammar\":\"($this.age >= 18)\",\"constraintName\":\"mustBeOfLegalAge\",\"constraintType\":\"Alloy_Constraint_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"Class\",\"constraintName\":\"Person\",\"constraintType\":\"Alloy_Class_Validation\",\"propertyPath\":\"Person\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"name\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::name\"},{\"constraintGrammar\":\"1\",\"constraintName\":\"age\",\"constraintType\":\"Alloy_Structural_Validation\",\"propertyPath\":\"Person::age\"}]}", + "format": "json", + "path": "dataQualityRulesMetadata.json" + }, + { + "content": "{\"_type\":\"simple\",\"authDependent\":false,\"globalImplementationSupport\":{\"_type\":\"java\",\"classes\":[{\"name\":\"Person\",\"package\":\"_pure.app.meta.dataquality\",\"source\":\"package _pure.app.meta.dataquality;\\n\\nimport java.math.*;\\nimport java.util.*;\\nimport org.finos.legend.engine.plan.dependencies.domain.date.PureDate;\\n\\npublic interface Person extends org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject\\n{\\n default String typeName$()\\n {\\n return \\\"Person\\\";\\n }\\n\\n default String typePath$()\\n {\\n return \\\"meta::dataquality::Person\\\";\\n }\\n\\n String getName();\\n long getAge();\\n String getAlloyStoreObjectReference$();\\n}\"},{\"name\":\"Serialize\",\"package\":\"_pure.plan.root\",\"source\":\"package _pure.plan.root;\\n\\nimport org.finos.legend.engine.plan.dependencies.store.platform.IGraphSerializer;\\nimport org.finos.legend.engine.plan.dependencies.store.platform.IPlatformPureExpressionExecutionNodeSerializeSpecifics;\\nimport org.finos.legend.engine.plan.dependencies.store.platform.ISerializationWriter;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IExecutionNodeContext;\\n\\npublic class Serialize implements IPlatformPureExpressionExecutionNodeSerializeSpecifics\\n{\\n public IGraphSerializer serializer(ISerializationWriter writer,\\n IExecutionNodeContext context)\\n {\\n return new Serializer(writer, context);\\n }\\n}\"},{\"name\":\"Serializer\",\"package\":\"_pure.plan.root\",\"source\":\"package _pure.plan.root;\\n\\nimport _pure.app.meta.dataquality.Person;\\nimport java.util.List;\\nimport org.finos.legend.engine.plan.dependencies.domain.dataQuality.EnforcementLevel;\\nimport org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked;\\nimport org.finos.legend.engine.plan.dependencies.domain.dataQuality.IDefect;\\nimport org.finos.legend.engine.plan.dependencies.domain.dataQuality.RelativePathNode;\\nimport org.finos.legend.engine.plan.dependencies.domain.dataQuality.RuleType;\\nimport org.finos.legend.engine.plan.dependencies.store.platform.IGraphSerializer;\\nimport org.finos.legend.engine.plan.dependencies.store.platform.ISerializationWriter;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IExecutionNodeContext;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject;\\n\\npublic class Serializer implements IGraphSerializer\\n{\\n private ISerializationWriter writer;\\n private IExecutionNodeContext context;\\n\\n Serializer(ISerializationWriter writer, IExecutionNodeContext context)\\n {\\n this.writer = writer;\\n this.context = context;\\n }\\n\\n public void serialize(IChecked value)\\n {\\n this.writer.startObject(\\\"meta::pure::dataQuality::Checked\\\");\\n this.writer\\n .writeComplexProperty(\\\"defects\\\",\\n value.getDefects(),\\n this::writeIDefect_defects);\\n this.writer\\n .writeComplexProperty(\\\"value\\\",\\n (Person) value.getValue(),\\n this::writePerson_value);\\n this.writer.endObject();\\n }\\n\\n public void writeIDefect_defects(IDefect value)\\n {\\n this.writer.startObject(\\\"meta::pure::dataQuality::Defect\\\");\\n this.writer.writeStringProperty(\\\"id\\\", value.getId());\\n this.writer.writeStringProperty(\\\"externalId\\\", value.getExternalId());\\n this.writer.writeStringProperty(\\\"message\\\", value.getMessage());\\n this.writer\\n .writeEnumProperty(\\\"enforcementLevel\\\",\\n \\\"meta::pure::dataQuality::EnforcementLevel\\\",\\n value.getEnforcementLevel() == null\\n ? null\\n : value.getEnforcementLevel().getName());\\n this.writer\\n .writeEnumProperty(\\\"ruleType\\\",\\n \\\"meta::pure::dataQuality::RuleType\\\",\\n value.getRuleType() == null\\n ? null\\n : value.getRuleType().getName());\\n this.writer\\n .writeStringProperty(\\\"ruleDefinerPath\\\",\\n value.getRuleDefinerPath());\\n this.writer\\n .writeComplexProperty(\\\"path\\\",\\n value.getPath(),\\n this::writeRelativePathNode_defects_path);\\n this.writer.endObject();\\n }\\n\\n public void writeRelativePathNode_defects_path(RelativePathNode value)\\n {\\n this.writer.startObject(\\\"meta::pure::dataQuality::RelativePathNode\\\");\\n this.writer\\n .writeStringProperty(\\\"propertyName\\\",\\n value.getPropertyName());\\n this.writer.writeIntegerProperty(\\\"index\\\", value.getIndex());\\n this.writer.endObject();\\n }\\n\\n public void writePerson_value(Person value)\\n {\\n if (value instanceof IReferencedObject)\\n {\\n this.writer\\n .startObject(value.typePath$(),\\n ((IReferencedObject) value).getAlloyStoreObjectReference$());\\n }\\n else\\n {\\n this.writer.startObject(value.typePath$());\\n }\\n this.writer.writeStringProperty(\\\"name\\\", value.getName());\\n this.writer.writeIntegerProperty(\\\"age\\\", value.getAge());\\n this.writer.endObject();\\n }\\n}\"},{\"name\":\"Execute\",\"package\":\"_pure.plan.root.n1.localGraph\",\"source\":\"package _pure.plan.root.n1.localGraph;\\n\\nimport java.lang.reflect.Method;\\nimport java.sql.ResultSet;\\nimport java.util.*;\\nimport java.util.function.*;\\nimport java.util.stream.*;\\nimport org.eclipse.collections.api.tuple.Pair;\\nimport org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance;\\nimport org.finos.legend.engine.plan.dependencies.store.relational.graphFetch.IRelationalRootQueryTempTableGraphFetchExecutionNodeSpecifics;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IExecutionNodeContext;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IReferencedObject;\\n\\npublic class Execute implements IRelationalRootQueryTempTableGraphFetchExecutionNodeSpecifics\\n{\\n private Specifics specifics;\\n\\n public Execute()\\n {\\n this.specifics = new Specifics();\\n }\\n\\n public void prepare(ResultSet resultSet, String databaseTimeZone, String databaseConnection)\\n {\\n this.specifics.prepare(resultSet, databaseTimeZone, databaseConnection);\\n }\\n\\n public IGraphInstance nextGraphInstance()\\n {\\n return this.specifics.nextGraphInstance();\\n }\\n\\n public List primaryKeyGetters()\\n {\\n return this.specifics.primaryKeyGetters();\\n }\\n\\n public List> allInstanceSetImplementations()\\n {\\n return this.specifics.allInstanceSetImplementations();\\n }\\n\\n public List primaryKeyColumns(int setIndex)\\n {\\n return this.specifics.primaryKeyColumns(setIndex);\\n }\\n\\n public boolean supportsCaching()\\n {\\n return true;\\n }\\n}\"},{\"name\":\"GraphFetch_Node0_Person_Impl\",\"package\":\"_pure.plan.root.n1.localGraph\",\"source\":\"package _pure.plan.root.n1.localGraph;\\n\\nimport java.math.*;\\nimport java.util.*;\\nimport java.util.function.*;\\nimport java.util.stream.*;\\nimport org.finos.legend.engine.plan.dependencies.domain.date.DayOfWeek;\\nimport org.finos.legend.engine.plan.dependencies.domain.date.DurationUnit;\\nimport org.finos.legend.engine.plan.dependencies.domain.date.PureDate;\\nimport org.finos.legend.engine.plan.dependencies.util.Library;\\nimport com.fasterxml.jackson.annotation.JsonInclude;\\nimport com.fasterxml.jackson.core.JsonGenerator;\\nimport com.fasterxml.jackson.databind.JsonSerializer;\\nimport com.fasterxml.jackson.databind.ObjectMapper;\\nimport com.fasterxml.jackson.databind.SerializerProvider;\\nimport com.fasterxml.jackson.databind.module.SimpleModule;\\nimport java.io.IOException;\\n\\npublic class GraphFetch_Node0_Person_Impl implements _pure.app.meta.dataquality.Person, org.finos.legend.engine.plan.dependencies.domain.dataQuality.Constrained<_pure.app.meta.dataquality.Person>, java.io.Serializable\\n{\\n private String name;\\n private long age;\\n private Object pk$_0;\\n private static final ObjectMapper objectMapper$ = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL).registerModule(new SimpleModule().addSerializer(PureDate.class, new JsonSerializer() { @Override public void serialize(PureDate value, JsonGenerator gen, SerializerProvider serializers) throws IOException { gen.writeRawValue(\\\"\\\\\\\"\\\" + value.toString() + \\\"\\\\\\\"\\\"); } }));\\n private String setId$;\\n public static String databaseConnection$;\\n private String alloyStoreObjectReference$;\\n private static final long serialVersionUID = 1461153389L;\\n\\n public String getName()\\n {\\n return this.name;\\n }\\n\\n public void setName(String name)\\n {\\n this.name = name;\\n }\\n\\n public void addName(String object)\\n {\\n if ((Object) this.name != null)\\n {\\n throw new IllegalStateException(\\\"Found multiple objects for property 'name' of multiplicity with bound 1\\\");\\n }\\n this.name = object;\\n }\\n\\n public long getAge()\\n {\\n return this.age;\\n }\\n\\n public void setAge(long age)\\n {\\n this.age = age;\\n }\\n\\n public void addAge(long object)\\n {\\n if ((Object) new Long(this.age) != null)\\n {\\n throw new IllegalStateException(\\\"Found multiple objects for property 'age' of multiplicity with bound 1\\\");\\n }\\n this.age = object;\\n }\\n\\n public List allConstraints()\\n {\\n return this.allConstraints(new org.finos.legend.engine.plan.dependencies.domain.dataQuality.GraphContext());\\n }\\n\\n public _pure.app.meta.dataquality.Person withConstraintsApplied()\\n {\\n java.util.List defects = allConstraints();\\n if (!defects.isEmpty())\\n {\\n throw new IllegalStateException(defects.stream().map(org.finos.legend.engine.plan.dependencies.domain.dataQuality.IDefect::getMessage).collect(java.util.stream.Collectors.joining(\\\"\\\\n\\\")));\\n }\\n return this;\\n }\\n\\n public org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked<_pure.app.meta.dataquality.Person> toChecked()\\n {\\n return this.toChecked(null, true);\\n }\\n\\n public org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked<_pure.app.meta.dataquality.Person> toChecked(boolean applyConstraints)\\n {\\n return this.toChecked(null, applyConstraints);\\n }\\n\\n public org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked<_pure.app.meta.dataquality.Person> toChecked(Object source)\\n {\\n return this.toChecked(source, true);\\n }\\n\\n public org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked<_pure.app.meta.dataquality.Person> toChecked(Object source,\\n boolean applyConstraints)\\n {\\n java.util.List defects = applyConstraints ? allConstraints() : java.util.Collections.emptyList();\\n return new org.finos.legend.engine.plan.dependencies.domain.dataQuality.IChecked<_pure.app.meta.dataquality.Person>() {\\n public java.util.List getDefects() { return defects; }\\n public Object getSource() { return source; }\\n public _pure.app.meta.dataquality.Person getValue() { return GraphFetch_Node0_Person_Impl.this; }\\n };\\n }\\n\\n public List allConstraints(org.finos.legend.engine.plan.dependencies.domain.dataQuality.GraphContext context)\\n {\\n List result = new ArrayList();\\n if (!context.visited.contains(this))\\n {\\n context.visited.add(this);\\n this.constraint_mustBeOfLegalAge().ifPresent(result::add);\\n }\\n return result;\\n }\\n\\n public Optional constraint_mustBeOfLegalAge()\\n {\\n try\\n {\\n if (!(this.getAge() >= 18L))\\n {\\n String message = \\\"Constraint :[mustBeOfLegalAge] violated in the Class Person\\\";\\n return Optional.of(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicDefect.newConstraintDefect(\\\"mustBeOfLegalAge\\\",\\n null,\\n message,\\n org.finos.legend.engine.plan.dependencies.domain.dataQuality.EnforcementLevel.Error,\\n \\\"meta::dataquality::Person\\\"));\\n }\\n return Optional.empty();\\n }\\n catch (Exception e)\\n {\\n String message = \\\"Unable to evaluate constraint [mustBeOfLegalAge]: \\\" + (e.getMessage() == null\\n ? \\\"data not available - check your mappings\\\"\\n : e.getMessage());\\n return Optional.of(org.finos.legend.engine.plan.dependencies.domain.dataQuality.BasicDefect.newConstraintDefect(\\\"mustBeOfLegalAge\\\",\\n null,\\n message,\\n org.finos.legend.engine.plan.dependencies.domain.dataQuality.EnforcementLevel.Error,\\n \\\"meta::dataquality::Person\\\"));\\n }\\n }\\n\\n public Object getPk$_0()\\n {\\n return this.pk$_0;\\n }\\n\\n public void setPk$_0(Object pk$_0)\\n {\\n this.pk$_0 = pk$_0;\\n }\\n\\n public String getSetId$()\\n {\\n return this.setId$;\\n }\\n\\n public void setSetId$(String setId)\\n {\\n this.setId$ = setId;\\n }\\n\\n public String getAlloyStoreObjectReference$()\\n {\\n if (this.alloyStoreObjectReference$ == null)\\n {\\n try\\n {\\n StringBuilder referenceBuilder = new StringBuilder();\\n referenceBuilder.append(\\\"001:\\\");\\n referenceBuilder.append(\\\"010:\\\");\\n\\n referenceBuilder.append(\\\"0000000010:\\\");\\n referenceBuilder.append(\\\"Relational:\\\");\\n\\n referenceBuilder.append(\\\"0000000038:\\\");\\n referenceBuilder.append(\\\"meta::dataquality::dataqualitymappings:\\\");\\n\\n referenceBuilder.append(\\\"0000000023:\\\");\\n referenceBuilder.append(\\\"meta_dataquality_Person:\\\");\\n\\n String setId = this.getSetId$();\\n referenceBuilder.append(String.format(\\\"%010d\\\", setId.length()));\\n referenceBuilder.append(\\\":\\\");\\n referenceBuilder.append(setId);\\n referenceBuilder.append(\\\":\\\");\\n\\n String databaseConnectionString = _pure.plan.root.n1.localGraph.GraphFetch_Node0_Person_Impl.databaseConnection$;\\n referenceBuilder.append(String.format(\\\"%010d\\\", databaseConnectionString.length()));\\n referenceBuilder.append(\\\":\\\");\\n referenceBuilder.append(databaseConnectionString);\\n referenceBuilder.append(\\\":\\\");\\n\\n Map pkMap = new HashMap<>();\\n\\n pkMap.put(\\\"pk$_0\\\", this.getPk$_0());\\n String pkMapString = objectMapper$.writeValueAsString(pkMap);\\n referenceBuilder.append(String.format(\\\"%010d\\\", pkMapString.length()));\\n referenceBuilder.append(\\\":\\\");\\n referenceBuilder.append(pkMapString);\\n\\n this.alloyStoreObjectReference$ = \\\"ASOR:\\\" + org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString(referenceBuilder.toString().getBytes());\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n }\\n\\n return this.alloyStoreObjectReference$;\\n }\\n\\n public void setAlloyStoreObjectReference$(String reference)\\n {\\n this.alloyStoreObjectReference$ = reference;\\n }\\n\\n private static long getClassSize$()\\n {\\n return 116L;\\n }\\n\\n public long getInstanceSize$()\\n {\\n long size = GraphFetch_Node0_Person_Impl.getClassSize$();\\n if (this.name != null)\\n {\\n size = size + this.name.length();\\n }\\n if (this.setId$ != null)\\n {\\n size = size + this.setId$.length();\\n }\\n if (this.alloyStoreObjectReference$ != null)\\n {\\n size = size + this.alloyStoreObjectReference$.length();\\n }\\n return size;\\n }\\n}\"},{\"name\":\"Specifics\",\"package\":\"_pure.plan.root.n1.localGraph\",\"source\":\"package _pure.plan.root.n1.localGraph;\\n\\nimport java.lang.reflect.Method;\\nimport java.sql.JDBCType;\\nimport java.sql.ResultSet;\\nimport java.sql.ResultSetMetaData;\\nimport java.sql.Types;\\nimport java.util.*;\\nimport java.util.function.*;\\nimport java.util.stream.*;\\nimport org.eclipse.collections.api.tuple.Pair;\\nimport org.eclipse.collections.impl.tuple.Tuples;\\nimport org.finos.legend.engine.plan.dependencies.domain.date.PureDate;\\nimport org.finos.legend.engine.plan.dependencies.domain.graphFetch.IGraphInstance;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IConstantResult;\\nimport org.finos.legend.engine.plan.dependencies.store.shared.IExecutionNodeContext;\\n\\nclass Specifics\\n{\\n private static final List STRING_TYPES = Arrays.asList(Types.CHAR, Types.VARCHAR, Types.LONGVARCHAR, Types.NCHAR, Types.NVARCHAR, Types.LONGNVARCHAR, Types.OTHER, Types.NULL);\\n private static final List INT_TYPES = Arrays.asList(Types.TINYINT, Types.SMALLINT, Types.INTEGER, Types.BIGINT, Types.NULL);\\n private static final List FLOAT_TYPES = Arrays.asList(Types.REAL, Types.FLOAT, Types.DOUBLE, Types.DECIMAL, Types.NUMERIC, Types.NULL);\\n private static final List DECIMAL_TYPES = Arrays.asList(Types.DECIMAL, Types.NUMERIC, Types.NULL);\\n private static final List BOOL_TYPES = Arrays.asList(Types.BIT, Types.BOOLEAN, Types.NULL);\\n private static final List STRICT_DATE_TYPES = Arrays.asList(Types.DATE, Types.NULL);\\n private static final List DATE_TIME_TYPES = Arrays.asList(Types.TIMESTAMP, Types.NULL);\\n private ResultSet resultSet;\\n private String databaseTimeZone;\\n private String databaseConnection;\\n private List columnTypes;\\n private List> propertyIndices;\\n private List>> propertyGetters;\\n private Calendar calendar;\\n private Method parentPropertyAdder;\\n private Method parentEdgePointPropertyAdder;\\n\\n private Object getAlloyNativeValueFromResultSet(ResultSet resultSet,\\n int columnIndex,\\n int columnType)\\n {\\n try\\n {\\n Object result = null;\\n switch (columnType)\\n {\\n case Types.DATE:\\n {\\n java.sql.Date date = resultSet.getDate(columnIndex);\\n if (date != null)\\n {\\n result = PureDate.fromSQLDate(date);\\n }\\n break;\\n }\\n case Types.TIMESTAMP:\\n {\\n java.sql.Timestamp timestamp = resultSet.getTimestamp(columnIndex, this.calendar);\\n if (timestamp != null)\\n {\\n result = PureDate.fromSQLTimestamp(timestamp);\\n }\\n break;\\n }\\n case Types.TINYINT:\\n case Types.SMALLINT:\\n case Types.INTEGER:\\n case Types.BIGINT:\\n {\\n long num = resultSet.getLong(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n result = Long.valueOf(num);\\n }\\n break;\\n }\\n case Types.REAL:\\n case Types.FLOAT:\\n case Types.DOUBLE:\\n {\\n double num = resultSet.getDouble(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n result = Double.valueOf(num);\\n }\\n break;\\n }\\n case Types.DECIMAL:\\n case Types.NUMERIC:\\n {\\n result = resultSet.getBigDecimal(columnIndex);\\n break;\\n }\\n case Types.CHAR:\\n case Types.VARCHAR:\\n case Types.LONGVARCHAR:\\n case Types.NCHAR:\\n case Types.NVARCHAR:\\n case Types.LONGNVARCHAR:\\n case Types.OTHER:\\n {\\n result = resultSet.getString(columnIndex);\\n break;\\n }\\n case Types.BIT:\\n case Types.BOOLEAN:\\n {\\n boolean bool = resultSet.getBoolean(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n result = Boolean.valueOf(bool);\\n }\\n break;\\n }\\n case Types.BINARY:\\n case Types.VARBINARY:\\n case Types.LONGVARBINARY:\\n {\\n byte[] bytes = resultSet.getBytes(columnIndex);\\n if (bytes != null)\\n {\\n result = this.encodeHex(bytes);\\n }\\n break;\\n }\\n case Types.NULL:\\n {\\n // do nothing: value is already assigned to null\\n break;\\n }\\n default:\\n {\\n result = resultSet.getObject(columnIndex);\\n }\\n }\\n return result;}\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n }\\n\\n private String encodeHex(byte[] data)\\n {\\n final char[] DIGITS_LOWER = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};\\n final int l = data.length;\\n final char[] out = new char[l << 1];\\n for (int i = 0, j = 0; i < l; i++)\\n {\\n out[j++] = DIGITS_LOWER[(0xF0 & data[i]) >>> 4];\\n out[j++] = DIGITS_LOWER[0x0F & data[i]];\\n }\\n return new String(out);\\n }\\n\\n private Supplier getResultSetPropertyGetterForStringProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (STRING_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n return resultSet.getString(columnIndex);\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n else\\n {\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type String from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n }\\n\\n private Supplier getResultSetPropertyGetterForIntegerProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (INT_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n Long res = null;\\n long r = resultSet.getLong(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = Long.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type Integer from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n private Supplier getResultSetPropertyGetterForFloatProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (FLOAT_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n Double res = null;\\n double r = resultSet.getDouble(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = Double.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (INT_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n Double res = null;\\n long r = resultSet.getLong(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = Double.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type Float from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n private Supplier getResultSetPropertyGetterForDecimalProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (DECIMAL_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n return resultSet.getBigDecimal(columnIndex);\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (FLOAT_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n java.math.BigDecimal res = null;\\n double r = resultSet.getDouble(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = java.math.BigDecimal.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (INT_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n java.math.BigDecimal res = null;\\n long r = resultSet.getLong(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = java.math.BigDecimal.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type Decimal from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n private Supplier getResultSetPropertyGetterForBooleanProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (BOOL_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n Boolean res = null;\\n boolean r = resultSet.getBoolean(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = Boolean.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (STRING_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n Boolean res = null;\\n String r = resultSet.getString(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = Boolean.valueOf(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (INT_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n Boolean res = null;\\n long r = resultSet.getLong(columnIndex);\\n if (!resultSet.wasNull())\\n {\\n res = Boolean.valueOf(r == 1);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type Boolean from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n private Supplier getResultSetPropertyGetterForStrictDateProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (STRICT_DATE_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n java.sql.Date r = resultSet.getDate(columnIndex);\\n if (r != null)\\n {\\n res = PureDate.fromSQLDate(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (STRING_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n String r = resultSet.getString(columnIndex);\\n if (r != null)\\n {\\n try\\n {\\n res = PureDate.parsePureDate(r);\\n }\\n catch (java.lang.IllegalArgumentException dateTimeParseException)\\n {\\n res = PureDate.fromSQLDate(java.sql.Date.valueOf(r));\\n }\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type StrictDate from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n private Supplier getResultSetPropertyGetterForDateTimeProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (DATE_TIME_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n java.sql.Timestamp r = resultSet.getTimestamp(columnIndex, this.calendar);\\n if (r != null)\\n {\\n res = PureDate.fromSQLTimestamp(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (STRING_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n String r = resultSet.getString(columnIndex);\\n if (r != null)\\n {\\n try\\n {\\n res = PureDate.parsePureDate(r);\\n }\\n catch (java.lang.IllegalArgumentException dateTimeParseException)\\n {\\n res = PureDate.fromSQLTimestamp(java.sql.Timestamp.valueOf(r));\\n }\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type DateTime from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n private Supplier getResultSetPropertyGetterForDateProperty(ResultSet resultSet,\\n int columnIndex,\\n int columnType,\\n String propertyName)\\n {\\n if (STRICT_DATE_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n java.sql.Date r = resultSet.getDate(columnIndex);\\n if (r != null)\\n {\\n res = PureDate.fromSQLDate(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (DATE_TIME_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n java.sql.Timestamp r = resultSet.getTimestamp(columnIndex, this.calendar);\\n if (r != null)\\n {\\n res = PureDate.fromSQLTimestamp(r);\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n if (STRING_TYPES.contains(columnType))\\n {\\n return () -> {\\n try\\n {\\n PureDate res = null;\\n String r = resultSet.getString(columnIndex);\\n if (r != null)\\n {\\n try\\n {\\n res = PureDate.parsePureDate(r);\\n }\\n catch (java.lang.IllegalArgumentException dateTimeParseException1)\\n {\\n try\\n {\\n res = PureDate.fromSQLTimestamp(java.sql.Timestamp.valueOf(r));\\n }\\n catch (java.time.format.DateTimeParseException dateTimeParseException2)\\n {\\n res = PureDate.fromSQLDate(java.sql.Date.valueOf(r));\\n }\\n }\\n }\\n return res;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n };\\n }\\n throw new RuntimeException(\\\"Error reading in property '\\\" + propertyName + \\\"' of type Date from SQL column of type '\\\" + JDBCType.valueOf(columnType) + \\\"'.\\\");\\n }\\n\\n void prepare(ResultSet resultSet, String databaseTimeZone, String databaseConnection)\\n {\\n try\\n {\\n this.resultSet = resultSet;\\n this.databaseTimeZone = databaseTimeZone;\\n this.databaseConnection = databaseConnection;\\n this.calendar = new GregorianCalendar(TimeZone.getTimeZone(this.databaseTimeZone));\\n ResultSetMetaData resultSetMetaData = this.resultSet.getMetaData();\\n int columnCount = resultSetMetaData.getColumnCount();\\n this.columnTypes = new ArrayList();\\n List columnNames = new ArrayList();\\n for (int i = 1; i <= columnCount; i++)\\n {\\n String columnLabel = resultSetMetaData.getColumnLabel(i);\\n columnNames.add(columnLabel.startsWith(\\\"\\\\\\\"\\\") && columnLabel\\n .endsWith(\\\"\\\\\\\"\\\")\\n ? columnLabel.substring(1, columnLabel.length() - 1)\\n .toUpperCase()\\n : columnLabel.toUpperCase());\\n this.columnTypes.add(resultSetMetaData.getColumnType(i));\\n }\\n this.propertyIndices = new ArrayList>();\\n List index_0 = new ArrayList();\\n index_0.add(columnNames.indexOf(\\\"NAME\\\") + 1);\\n index_0.add(columnNames.indexOf(\\\"AGE\\\") + 1);\\n index_0.add(columnNames.indexOf(\\\"PK_0\\\") + 1);\\n this.propertyIndices.add(index_0);\\n GraphFetch_Node0_Person_Impl.databaseConnection$ = databaseConnection;\\n this.propertyGetters = new ArrayList>>();\\n int propertyIndex;\\n Supplier propertyGetter = null;\\n List> propertyGetter_0 = new ArrayList>();\\n propertyIndex = this.propertyIndices.get(0).get(0);\\n propertyGetter = this.getResultSetPropertyGetterForStringProperty(this.resultSet,\\n propertyIndex,\\n resultSetMetaData.getColumnType(propertyIndex),\\n \\\"name\\\");\\n propertyGetter_0.add(propertyGetter);\\n propertyIndex = this.propertyIndices.get(0).get(1);\\n propertyGetter = this.getResultSetPropertyGetterForIntegerProperty(this.resultSet,\\n propertyIndex,\\n resultSetMetaData.getColumnType(propertyIndex),\\n \\\"age\\\");\\n propertyGetter_0.add(propertyGetter);\\n this.propertyGetters.add(propertyGetter_0);\\n }\\n catch (RuntimeException e)\\n {\\n throw e;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n }\\n\\n IGraphInstance nextGraphInstance()\\n {\\n try\\n {\\n final GraphFetch_Node0_Person_Impl object = new GraphFetch_Node0_Person_Impl();\\n object.setSetId$(\\\"meta_dataquality_Person\\\");\\n int pkColIndex;\\n int propertyIndex;\\n pkColIndex = this.propertyIndices.get(0).get(2);\\n Object pk$_0 = this.getAlloyNativeValueFromResultSet(resultSet,\\n pkColIndex,\\n this.columnTypes.get(pkColIndex - 1));\\n object.setPk$_0(pk$_0);\\n propertyIndex = this.propertyIndices.get(0).get(0);\\n {\\n String res = (String) this.propertyGetters.get(0).get(0).get();\\n if (res == null)\\n {\\n throw new RuntimeException(\\\"Error reading in property 'name'. Property of multiplicity [1] can not be null\\\");\\n }\\n object.setName(res);\\n }\\n propertyIndex = this.propertyIndices.get(0).get(1);\\n {\\n Long res = (Long) this.propertyGetters.get(0).get(1).get();\\n if (res == null)\\n {\\n throw new RuntimeException(\\\"Error reading in property 'age'. Property of multiplicity [1] can not be null\\\");\\n }\\n object.setAge(res);\\n }\\n return new IGraphInstance()\\n {\\n public GraphFetch_Node0_Person_Impl getValue()\\n {\\n return object;\\n }\\n public long instanceSize()\\n {\\n return object.getInstanceSize$();\\n }\\n };\\n }\\n catch (RuntimeException e)\\n {\\n throw e;\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n }\\n\\n List primaryKeyGetters()\\n {\\n try\\n {\\n return Arrays.asList(GraphFetch_Node0_Person_Impl.class.getMethod(\\\"getPk$_0\\\"));\\n }\\n catch (Exception e)\\n {\\n throw new RuntimeException(e);\\n }\\n }\\n\\n List> allInstanceSetImplementations()\\n {\\n return Arrays.asList(Tuples.pair(\\\"meta::dataquality::dataqualitymappings\\\",\\n \\\"meta_dataquality_Person\\\"));\\n }\\n\\n List primaryKeyColumns(int setIndex)\\n {\\n if (setIndex == 0)\\n {\\n return Arrays.asList(\\\"pk_0\\\");\\n }\\n return null;\\n }\\n}\"}]},\"rootExecutionNode\":{\"_type\":\"platform\",\"authDependent\":false,\"executionNodes\":[{\"_type\":\"storeMappingGlobalGraphFetchExecutionNode\",\"authDependent\":false,\"checked\":true,\"enableConstraints\":true,\"executionNodes\":[],\"graphFetchTree\":{\"_type\":\"rootGraphFetchTree\",\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::dataquality::Person\",\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"name\",\"subTrees\":[],\"subTypeTrees\":[]},{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"age\",\"subTrees\":[],\"subTypeTrees\":[]}],\"subTypeTrees\":[]},\"localGraphFetchExecutionNode\":{\"_type\":\"relationalRootQueryTempTableGraphFetch\",\"authDependent\":false,\"checked\":true,\"columns\":[{\"dataType\":\"INTEGER\",\"label\":\"pk_0\"}],\"executionNodes\":[{\"_type\":\"sql\",\"authDependent\":false,\"connection\":{\"_type\":\"RelationalDatabaseConnection\",\"authenticationStrategy\":{\"_type\":\"h2Default\"},\"datasourceSpecification\":{\"_type\":\"h2Local\"},\"element\":\"\",\"postProcessorWithParameter\":[],\"postProcessors\":[],\"type\":\"H2\"},\"executionNodes\":[],\"resultColumns\":[{\"dataType\":\"INTEGER\",\"label\":\"\\\"pk_0\\\"\"},{\"dataType\":\"VARCHAR(200)\",\"label\":\"\\\"name\\\"\"},{\"dataType\":\"INTEGER\",\"label\":\"\\\"age\\\"\"}],\"resultType\":{\"_type\":\"dataType\",\"dataType\":\"meta::pure::metamodel::type::Any\"},\"sqlComment\":\"-- \\\"executionTraceID\\\" : \\\"${execID}\\\"\",\"sqlQuery\":\"select \\\"root\\\".ID as \\\"pk_0\\\", \\\"root\\\".FIRSTNAME as \\\"name\\\", \\\"root\\\".AGE as \\\"age\\\" from personTable as \\\"root\\\" where \\\"root\\\".FIRSTNAME = 'John' and not \\\"root\\\".AGE >= 18\"}],\"graphFetchTree\":{\"_type\":\"rootGraphFetchTree\",\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::dataquality::Person\",\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"name\",\"subTrees\":[],\"subTypeTrees\":[]},{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"age\",\"subTrees\":[],\"subTypeTrees\":[]}],\"subTypeTrees\":[]},\"implementation\":{\"_type\":\"java\",\"executionClassFullName\":\"_pure.plan.root.n1.localGraph.Execute\"},\"nodeIndex\":0,\"processedTempTableName\":\"temp_table_node_0\",\"resultType\":{\"_type\":\"partialClass\",\"class\":\"meta::dataquality::Person\",\"propertiesWithParameters\":[{\"property\":\"name\"},{\"property\":\"age\"}],\"setImplementations\":[{\"class\":\"meta::dataquality::Person\",\"id\":\"meta_dataquality_Person\",\"mapping\":\"meta::dataquality::dataqualitymappings\",\"propertyMappings\":[{\"enumMapping\":{},\"property\":\"name\",\"type\":\"String\"},{\"enumMapping\":{},\"property\":\"age\",\"type\":\"Integer\"}]}]},\"tempTableName\":\"temp_table_node_0\"},\"localTreeIndices\":[0,1,2],\"resultSizeRange\":{\"lowerBound\":0},\"resultType\":{\"_type\":\"partialClass\",\"class\":\"meta::dataquality::Person\",\"propertiesWithParameters\":[{\"property\":\"name\"},{\"property\":\"age\"}],\"setImplementations\":[{\"class\":\"meta::dataquality::Person\",\"id\":\"meta_dataquality_Person\",\"mapping\":\"meta::dataquality::dataqualitymappings\",\"propertyMappings\":[{\"enumMapping\":{},\"property\":\"name\",\"type\":\"String\"},{\"enumMapping\":{},\"property\":\"age\",\"type\":\"Integer\"}]}]},\"store\":\"meta::dataquality::db\"}],\"implementation\":{\"_type\":\"java\",\"executionClassFullName\":\"_pure.plan.root.Serialize\"},\"pure\":{\"_type\":\"func\",\"fControl\":\"serialize_T_MANY__RootGraphFetchTree_1__String_1_\",\"function\":\"serialize\",\"parameters\":[{\"_type\":\"collection\",\"multiplicity\":{\"lowerBound\":0},\"values\":[]},{\"_type\":\"classInstance\",\"type\":\"rootGraphFetchTree\",\"value\":{\"_type\":\"rootGraphFetchTree\",\"class\":\"meta::dataquality::Person\",\"subTrees\":[{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"name\",\"subTrees\":[],\"subTypeTrees\":[]},{\"_type\":\"propertyGraphFetchTree\",\"_type\":\"propertyGraphFetchTree\",\"parameters\":[],\"property\":\"age\",\"subTrees\":[],\"subTypeTrees\":[]}],\"subTypeTrees\":[]}}]},\"resultType\":{\"_type\":\"dataType\",\"dataType\":\"String\"}},\"serializer\":{\"name\":\"pure\",\"version\":\"vX_X_X\"},\"templateFunctions\":[\"<#function renderCollection collection separator prefix suffix replacementMap defaultValue><#if collection?size == 0><#return defaultValue><#assign newCollection = collection><#list replacementMap as oldValue, newValue> <#assign newCollection = collection?map(ele -> ele?replace(oldValue, newValue))><#return prefix + newCollection?join(suffix + separator + prefix) + suffix>\",\"<#function collectionSize collection> <#return collection?size?c> \",\"<#function optionalVarPlaceHolderOperationSelector optionalParameter trueClause falseClause><#if optionalParameter?has_content || optionalParameter?is_string><#return trueClause><#else><#return falseClause>\",\"<#function varPlaceHolderToString optionalParameter prefix suffix replacementMap defaultValue><#if optionalParameter?is_enumerable && !optionalParameter?has_content><#return defaultValue><#else><#assign newParam = optionalParameter><#list replacementMap as oldValue, newValue> <#assign newParam = newParam?replace(oldValue, newValue)><#return prefix + newParam + suffix>\",\"<#function equalEnumOperationSelector enumVal inDyna equalDyna><#assign enumList = enumVal?split(\\\",\\\")><#if enumList?size = 1><#return equalDyna><#else><#return inDyna>\"]}", + "format": "json", + "path": "dataQualityValidationExecutionPlan.json" + } +] \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.json new file mode 100644 index 00000000000..ca5cafab1d9 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithDataQualityValidation.json @@ -0,0 +1,1533 @@ +{ + "_type": "data", + "elements": [ + { + "_type": "connection", + "connectionValue": { + "_type": "RelationalDatabaseConnection", + "authenticationStrategy": { + "_type": "h2Default", + "sourceInformation": { + "endColumn": 18, + "endLine": 10, + "sourceId": "", + "startColumn": 3, + "startLine": 10 + } + }, + "databaseType": "H2", + "datasourceSpecification": { + "_type": "h2Local", + "sourceInformation": { + "endColumn": 4, + "endLine": 9, + "sourceId": "", + "startColumn": 3, + "startLine": 6 + }, + "testDataSetupSqls": [] + }, + "element": "meta::dataquality::db", + "elementSourceInformation": { + "endColumn": 30, + "endLine": 4, + "sourceId": "", + "startColumn": 10, + "startLine": 4 + }, + "postProcessorWithParameter": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 11, + "sourceId": "", + "startColumn": 1, + "startLine": 2 + }, + "type": "H2" + }, + "name": "H2", + "package": "meta::dataquality", + "sourceInformation": { + "endColumn": 1, + "endLine": 11, + "sourceId": "", + "startColumn": 1, + "startLine": 2 + } + }, + { + "_type": "relational", + "filters": [], + "includedStores": [], + "joins": [ + { + "name": "Address_Person", + "operation": { + "_type": "dynaFunc", + "funcName": "equal", + "parameters": [ + { + "_type": "column", + "column": "ID", + "sourceInformation": { + "endColumn": 62, + "endLine": 20, + "sourceId": "", + "startColumn": 24, + "startLine": 20 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 35, + "endLine": 20, + "sourceId": "", + "startColumn": 24, + "startLine": 20 + }, + "table": "addressTable" + }, + "tableAlias": "addressTable" + }, + { + "_type": "column", + "column": "ADDRESSID", + "sourceInformation": { + "endColumn": 62, + "endLine": 20, + "sourceId": "", + "startColumn": 42, + "startLine": 20 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 52, + "endLine": 20, + "sourceId": "", + "startColumn": 42, + "startLine": 20 + }, + "table": "personTable" + }, + "tableAlias": "personTable" + } + ], + "sourceInformation": { + "endColumn": 62, + "endLine": 20, + "sourceId": "", + "startColumn": 40, + "startLine": 20 + } + }, + "sourceInformation": { + "endColumn": 63, + "endLine": 20, + "sourceId": "", + "startColumn": 4, + "startLine": 20 + } + }, + { + "name": "Address_Location", + "operation": { + "_type": "dynaFunc", + "funcName": "equal", + "parameters": [ + { + "_type": "column", + "column": "LOCATIONID", + "sourceInformation": { + "endColumn": 67, + "endLine": 21, + "sourceId": "", + "startColumn": 26, + "startLine": 21 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 37, + "endLine": 21, + "sourceId": "", + "startColumn": 26, + "startLine": 21 + }, + "table": "addressTable" + }, + "tableAlias": "addressTable" + }, + { + "_type": "column", + "column": "ID", + "sourceInformation": { + "endColumn": 67, + "endLine": 21, + "sourceId": "", + "startColumn": 52, + "startLine": 21 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 64, + "endLine": 21, + "sourceId": "", + "startColumn": 52, + "startLine": 21 + }, + "table": "locationTable" + }, + "tableAlias": "locationTable" + } + ], + "sourceInformation": { + "endColumn": 67, + "endLine": 21, + "sourceId": "", + "startColumn": 50, + "startLine": 21 + } + }, + "sourceInformation": { + "endColumn": 68, + "endLine": 21, + "sourceId": "", + "startColumn": 4, + "startLine": 21 + } + } + ], + "name": "db", + "package": "meta::dataquality", + "schemas": [ + { + "name": "default", + "sourceInformation": { + "endColumn": 1, + "endLine": 22, + "sourceId": "", + "startColumn": 1, + "startLine": 14 + }, + "tables": [ + { + "columns": [ + { + "name": "ID", + "nullable": false, + "sourceInformation": { + "endColumn": 40, + "endLine": 16, + "sourceId": "", + "startColumn": 23, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "FIRSTNAME", + "nullable": true, + "sourceInformation": { + "endColumn": 64, + "endLine": 16, + "sourceId": "", + "startColumn": 43, + "startLine": 16 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + }, + { + "name": "LASTNAME", + "nullable": true, + "sourceInformation": { + "endColumn": 87, + "endLine": 16, + "sourceId": "", + "startColumn": 67, + "startLine": 16 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + }, + { + "name": "AGE", + "nullable": true, + "sourceInformation": { + "endColumn": 96, + "endLine": 16, + "sourceId": "", + "startColumn": 90, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "ADDRESSID", + "nullable": true, + "sourceInformation": { + "endColumn": 111, + "endLine": 16, + "sourceId": "", + "startColumn": 99, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "FIRMID", + "nullable": true, + "sourceInformation": { + "endColumn": 123, + "endLine": 16, + "sourceId": "", + "startColumn": 114, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + } + ], + "milestoning": [], + "name": "personTable", + "primaryKey": [ + "ID" + ], + "sourceInformation": { + "endColumn": 124, + "endLine": 16, + "sourceId": "", + "startColumn": 4, + "startLine": 16 + } + }, + { + "columns": [ + { + "name": "ID", + "nullable": false, + "sourceInformation": { + "endColumn": 41, + "endLine": 17, + "sourceId": "", + "startColumn": 24, + "startLine": 17 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "LOCATIONID", + "nullable": true, + "sourceInformation": { + "endColumn": 57, + "endLine": 17, + "sourceId": "", + "startColumn": 44, + "startLine": 17 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "TYPE", + "nullable": true, + "sourceInformation": { + "endColumn": 67, + "endLine": 17, + "sourceId": "", + "startColumn": 60, + "startLine": 17 + }, + "type": { + "_type": "Integer" + } + } + ], + "milestoning": [], + "name": "addressTable", + "primaryKey": [ + "ID" + ], + "sourceInformation": { + "endColumn": 68, + "endLine": 17, + "sourceId": "", + "startColumn": 4, + "startLine": 17 + } + }, + { + "columns": [ + { + "name": "ID", + "nullable": false, + "sourceInformation": { + "endColumn": 42, + "endLine": 18, + "sourceId": "", + "startColumn": 25, + "startLine": 18 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "STREET", + "nullable": true, + "sourceInformation": { + "endColumn": 63, + "endLine": 18, + "sourceId": "", + "startColumn": 45, + "startLine": 18 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + }, + { + "name": "LOCALITY", + "nullable": true, + "sourceInformation": { + "endColumn": 86, + "endLine": 18, + "sourceId": "", + "startColumn": 66, + "startLine": 18 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + } + ], + "milestoning": [], + "name": "locationTable", + "primaryKey": [ + "ID" + ], + "sourceInformation": { + "endColumn": 87, + "endLine": 18, + "sourceId": "", + "startColumn": 4, + "startLine": 18 + } + } + ], + "views": [] + } + ], + "sourceInformation": { + "endColumn": 1, + "endLine": 22, + "sourceId": "", + "startColumn": 1, + "startLine": 14 + }, + "stereotypes": [] + }, + { + "_type": "mapping", + "associationMappings": [], + "classMappings": [ + { + "_type": "relational", + "class": "meta::dataquality::Person", + "classSourceInformation": { + "endColumn": 28, + "endLine": 29, + "sourceId": "", + "startColumn": 4, + "startLine": 29 + }, + "distinct": false, + "groupBy": [], + "primaryKey": [], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Person", + "property": "name", + "sourceInformation": { + "endColumn": 10, + "endLine": 31, + "sourceId": "", + "startColumn": 7, + "startLine": 31 + } + }, + "relationalOperation": { + "_type": "column", + "column": "FIRSTNAME", + "sourceInformation": { + "endColumn": 57, + "endLine": 31, + "sourceId": "", + "startColumn": 14, + "startLine": 31 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 47, + "endLine": 31, + "sourceId": "", + "startColumn": 37, + "startLine": 31 + }, + "table": "personTable" + }, + "tableAlias": "personTable" + }, + "sourceInformation": { + "endColumn": 57, + "endLine": 31, + "sourceId": "", + "startColumn": 12, + "startLine": 31 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Person", + "property": "age", + "sourceInformation": { + "endColumn": 9, + "endLine": 32, + "sourceId": "", + "startColumn": 7, + "startLine": 32 + } + }, + "relationalOperation": { + "_type": "column", + "column": "AGE", + "sourceInformation": { + "endColumn": 50, + "endLine": 32, + "sourceId": "", + "startColumn": 13, + "startLine": 32 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 46, + "endLine": 32, + "sourceId": "", + "startColumn": 36, + "startLine": 32 + }, + "table": "personTable" + }, + "tableAlias": "personTable" + }, + "sourceInformation": { + "endColumn": 50, + "endLine": 32, + "sourceId": "", + "startColumn": 11, + "startLine": 32 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Person", + "property": "addresses", + "sourceInformation": { + "endColumn": 15, + "endLine": 33, + "sourceId": "", + "startColumn": 7, + "startLine": 33 + } + }, + "relationalOperation": { + "_type": "elemtWithJoins", + "joins": [ + { + "db": "meta::dataquality::db", + "name": "Address_Person", + "sourceInformation": { + "endColumn": 56, + "endLine": 33, + "sourceId": "", + "startColumn": 42, + "startLine": 33 + } + } + ], + "sourceInformation": { + "endColumn": 56, + "endLine": 33, + "sourceId": "", + "startColumn": 19, + "startLine": 33 + } + }, + "sourceInformation": { + "endColumn": 56, + "endLine": 33, + "sourceId": "", + "startColumn": 17, + "startLine": 33 + } + } + ], + "root": false, + "sourceInformation": { + "endColumn": 4, + "endLine": 34, + "sourceId": "", + "startColumn": 4, + "startLine": 29 + } + }, + { + "_type": "relational", + "class": "meta::dataquality::Address", + "classSourceInformation": { + "endColumn": 29, + "endLine": 36, + "sourceId": "", + "startColumn": 4, + "startLine": 36 + }, + "distinct": false, + "groupBy": [], + "primaryKey": [], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Address", + "property": "addressId", + "sourceInformation": { + "endColumn": 15, + "endLine": 38, + "sourceId": "", + "startColumn": 7, + "startLine": 38 + } + }, + "relationalOperation": { + "_type": "column", + "column": "ID", + "sourceInformation": { + "endColumn": 56, + "endLine": 38, + "sourceId": "", + "startColumn": 19, + "startLine": 38 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 53, + "endLine": 38, + "sourceId": "", + "startColumn": 42, + "startLine": 38 + }, + "table": "addressTable" + }, + "tableAlias": "addressTable" + }, + "sourceInformation": { + "endColumn": 56, + "endLine": 38, + "sourceId": "", + "startColumn": 17, + "startLine": 38 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Address", + "property": "location", + "sourceInformation": { + "endColumn": 14, + "endLine": 39, + "sourceId": "", + "startColumn": 7, + "startLine": 39 + } + }, + "relationalOperation": { + "_type": "elemtWithJoins", + "joins": [ + { + "db": "meta::dataquality::db", + "name": "Address_Location", + "sourceInformation": { + "endColumn": 57, + "endLine": 39, + "sourceId": "", + "startColumn": 41, + "startLine": 39 + } + } + ], + "sourceInformation": { + "endColumn": 57, + "endLine": 39, + "sourceId": "", + "startColumn": 18, + "startLine": 39 + } + }, + "sourceInformation": { + "endColumn": 57, + "endLine": 39, + "sourceId": "", + "startColumn": 16, + "startLine": 39 + } + } + ], + "root": false, + "sourceInformation": { + "endColumn": 4, + "endLine": 40, + "sourceId": "", + "startColumn": 4, + "startLine": 36 + } + }, + { + "_type": "relational", + "class": "meta::dataquality::Location", + "classSourceInformation": { + "endColumn": 30, + "endLine": 42, + "sourceId": "", + "startColumn": 4, + "startLine": 42 + }, + "distinct": false, + "groupBy": [], + "primaryKey": [], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Location", + "property": "street", + "sourceInformation": { + "endColumn": 12, + "endLine": 44, + "sourceId": "", + "startColumn": 7, + "startLine": 44 + } + }, + "relationalOperation": { + "_type": "column", + "column": "STREET", + "sourceInformation": { + "endColumn": 58, + "endLine": 44, + "sourceId": "", + "startColumn": 16, + "startLine": 44 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 51, + "endLine": 44, + "sourceId": "", + "startColumn": 39, + "startLine": 44 + }, + "table": "locationTable" + }, + "tableAlias": "locationTable" + }, + "sourceInformation": { + "endColumn": 58, + "endLine": 44, + "sourceId": "", + "startColumn": 14, + "startLine": 44 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Location", + "property": "locality", + "sourceInformation": { + "endColumn": 14, + "endLine": 45, + "sourceId": "", + "startColumn": 7, + "startLine": 45 + } + }, + "relationalOperation": { + "_type": "column", + "column": "LOCALITY", + "sourceInformation": { + "endColumn": 62, + "endLine": 45, + "sourceId": "", + "startColumn": 18, + "startLine": 45 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 53, + "endLine": 45, + "sourceId": "", + "startColumn": 41, + "startLine": 45 + }, + "table": "locationTable" + }, + "tableAlias": "locationTable" + }, + "sourceInformation": { + "endColumn": 62, + "endLine": 45, + "sourceId": "", + "startColumn": 16, + "startLine": 45 + } + } + ], + "root": false, + "sourceInformation": { + "endColumn": 4, + "endLine": 46, + "sourceId": "", + "startColumn": 4, + "startLine": 42 + } + } + ], + "enumerationMappings": [], + "includedMappings": [], + "name": "dataqualitymappings", + "package": "meta::dataquality", + "sourceInformation": { + "endColumn": 1, + "endLine": 47, + "sourceId": "", + "startColumn": 1, + "startLine": 26 + }, + "tests": [] + }, + { + "_type": "runtime", + "name": "DataQualityRuntime", + "package": "meta::dataquality", + "runtimeValue": { + "_type": "engineRuntime", + "connectionStores": [], + "connections": [ + { + "sourceInformation": { + "endColumn": 5, + "endLine": 61, + "sourceId": "", + "startColumn": 5, + "startLine": 58 + }, + "store": { + "path": "meta::dataquality::db", + "sourceInformation": { + "endColumn": 25, + "endLine": 58, + "sourceId": "", + "startColumn": 5, + "startLine": 58 + }, + "type": "STORE" + }, + "storeConnections": [ + { + "connection": { + "_type": "connectionPointer", + "connection": "meta::dataquality::H2", + "sourceInformation": { + "endColumn": 41, + "endLine": 60, + "sourceId": "", + "startColumn": 21, + "startLine": 60 + } + }, + "id": "connection_1", + "sourceInformation": { + "endColumn": 41, + "endLine": 60, + "sourceId": "", + "startColumn": 7, + "startLine": 60 + } + } + ] + } + ], + "mappings": [ + { + "path": "meta::dataquality::dataqualitymappings", + "sourceInformation": { + "endColumn": 42, + "endLine": 54, + "sourceId": "", + "startColumn": 5, + "startLine": 54 + }, + "type": "MAPPING" + } + ], + "sourceInformation": { + "endColumn": 1, + "endLine": 63, + "sourceId": "", + "startColumn": 1, + "startLine": 50 + } + }, + "sourceInformation": { + "endColumn": 1, + "endLine": 63, + "sourceId": "", + "startColumn": 1, + "startLine": 50 + } + }, + { + "_type": "class", + "constraints": [ + { + "functionDefinition": { + "_type": "lambda", + "body": [ + { + "_type": "func", + "function": "greaterThanEqual", + "parameters": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "this", + "sourceInformation": { + "endColumn": 25, + "endLine": 70, + "sourceId": "", + "startColumn": 21, + "startLine": 70 + } + } + ], + "property": "age", + "sourceInformation": { + "endColumn": 29, + "endLine": 70, + "sourceId": "", + "startColumn": 27, + "startLine": 70 + } + }, + { + "_type": "integer", + "sourceInformation": { + "endColumn": 35, + "endLine": 70, + "sourceId": "", + "startColumn": 34, + "startLine": 70 + }, + "value": 18 + } + ], + "sourceInformation": { + "endColumn": 35, + "endLine": 70, + "sourceId": "", + "startColumn": 31, + "startLine": 70 + } + } + ], + "parameters": [] + }, + "name": "mustBeOfLegalAge", + "sourceInformation": { + "endColumn": 35, + "endLine": 70, + "sourceId": "", + "startColumn": 3, + "startLine": 70 + } + } + ], + "name": "Person", + "originalMilestonedProperties": [], + "package": "meta::dataquality", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "name", + "propertyTypeSourceInformation": { + "endColumn": 16, + "endLine": 73, + "sourceId": "", + "startColumn": 11, + "startLine": 73 + }, + "sourceInformation": { + "endColumn": 20, + "endLine": 73, + "sourceId": "", + "startColumn": 4, + "startLine": 73 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "age", + "propertyTypeSourceInformation": { + "endColumn": 16, + "endLine": 74, + "sourceId": "", + "startColumn": 10, + "startLine": 74 + }, + "sourceInformation": { + "endColumn": 20, + "endLine": 74, + "sourceId": "", + "startColumn": 4, + "startLine": 74 + }, + "stereotypes": [], + "taggedValues": [], + "type": "Integer" + }, + { + "multiplicity": { + "lowerBound": 0 + }, + "name": "addresses", + "propertyTypeSourceInformation": { + "endColumn": 41, + "endLine": 75, + "sourceId": "", + "startColumn": 16, + "startLine": 75 + }, + "sourceInformation": { + "endColumn": 45, + "endLine": 75, + "sourceId": "", + "startColumn": 4, + "startLine": 75 + }, + "stereotypes": [], + "taggedValues": [], + "type": "meta::dataquality::Address" + } + ], + "qualifiedProperties": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 76, + "sourceId": "", + "startColumn": 1, + "startLine": 68 + }, + "stereotypes": [], + "superTypes": [], + "taggedValues": [] + }, + { + "_type": "class", + "constraints": [], + "name": "Address", + "originalMilestonedProperties": [], + "package": "meta::dataquality", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "location", + "propertyTypeSourceInformation": { + "endColumn": 40, + "endLine": 80, + "sourceId": "", + "startColumn": 14, + "startLine": 80 + }, + "sourceInformation": { + "endColumn": 44, + "endLine": 80, + "sourceId": "", + "startColumn": 4, + "startLine": 80 + }, + "stereotypes": [], + "taggedValues": [], + "type": "meta::dataquality::Location" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "locationStreet", + "propertyTypeSourceInformation": { + "endColumn": 25, + "endLine": 81, + "sourceId": "", + "startColumn": 20, + "startLine": 81 + }, + "sourceInformation": { + "endColumn": 29, + "endLine": 81, + "sourceId": "", + "startColumn": 4, + "startLine": 81 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "addressId", + "propertyTypeSourceInformation": { + "endColumn": 20, + "endLine": 82, + "sourceId": "", + "startColumn": 15, + "startLine": 82 + }, + "sourceInformation": { + "endColumn": 24, + "endLine": 82, + "sourceId": "", + "startColumn": 4, + "startLine": 82 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + } + ], + "qualifiedProperties": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 83, + "sourceId": "", + "startColumn": 1, + "startLine": 78 + }, + "stereotypes": [], + "superTypes": [], + "taggedValues": [] + }, + { + "_type": "class", + "constraints": [], + "name": "Location", + "originalMilestonedProperties": [], + "package": "meta::dataquality", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "street", + "propertyTypeSourceInformation": { + "endColumn": 17, + "endLine": 87, + "sourceId": "", + "startColumn": 12, + "startLine": 87 + }, + "sourceInformation": { + "endColumn": 21, + "endLine": 87, + "sourceId": "", + "startColumn": 4, + "startLine": 87 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "locality", + "propertyTypeSourceInformation": { + "endColumn": 19, + "endLine": 88, + "sourceId": "", + "startColumn": 14, + "startLine": 88 + }, + "sourceInformation": { + "endColumn": 23, + "endLine": 88, + "sourceId": "", + "startColumn": 4, + "startLine": 88 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + } + ], + "qualifiedProperties": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 89, + "sourceId": "", + "startColumn": 1, + "startLine": 85 + }, + "stereotypes": [], + "superTypes": [], + "taggedValues": [] + }, + { + "_type": "dataQualityValidation", + "context": { + "_type": "mappingAndRuntimeDataQualityExecutionContext", + "_type": "mappingAndRuntimeDataQualityExecutionContext", + "mapping": { + "path": "meta::dataquality::dataqualitymappings", + "sourceInformation": { + "endColumn": 73, + "endLine": 94, + "sourceId": "", + "startColumn": 36, + "startLine": 94 + }, + "type": "MAPPING" + }, + "runtime": { + "path": "meta::dataquality::DataQualityRuntime", + "sourceInformation": { + "endColumn": 112, + "endLine": 94, + "sourceId": "", + "startColumn": 76, + "startLine": 94 + }, + "type": "RUNTIME" + } + }, + "dataQualityRootGraphFetchTree": { + "_type": "dataQualityRootGraphFetchTree", + "_type": "dataQualityRootGraphFetchTree", + "class": "meta::dataquality::Person", + "constraints": [ + "mustBeOfLegalAge" + ], + "sourceInformation": { + "endColumn": 31, + "endLine": 97, + "sourceId": "", + "startColumn": 7, + "startLine": 97 + }, + "subTrees": [ + { + "_type": "dataQualityPropertyGraphFetchTree", + "_type": "dataQualityPropertyGraphFetchTree", + "constraints": [], + "parameters": [], + "property": "name", + "sourceInformation": { + "endColumn": 12, + "endLine": 98, + "sourceId": "", + "startColumn": 9, + "startLine": 98 + }, + "subTrees": [], + "subTypeTrees": [] + } + ], + "subTypeTrees": [] + }, + "filter": { + "_type": "lambda", + "body": [ + { + "_type": "func", + "function": "equal", + "parameters": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "p", + "sourceInformation": { + "endColumn": 47, + "endLine": 95, + "sourceId": "", + "startColumn": 46, + "startLine": 95 + } + } + ], + "property": "name", + "sourceInformation": { + "endColumn": 52, + "endLine": 95, + "sourceId": "", + "startColumn": 49, + "startLine": 95 + } + }, + { + "_type": "string", + "sourceInformation": { + "endColumn": 60, + "endLine": 95, + "sourceId": "", + "startColumn": 55, + "startLine": 95 + }, + "value": "John" + } + ], + "sourceInformation": { + "endColumn": 54, + "endLine": 95, + "sourceId": "", + "startColumn": 53, + "startLine": 95 + } + } + ], + "parameters": [ + { + "_type": "var", + "class": "meta::dataquality::Person", + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "p", + "sourceInformation": { + "endColumn": 39, + "endLine": 95, + "sourceId": "", + "startColumn": 15, + "startLine": 95 + } + } + ], + "sourceInformation": { + "endColumn": 60, + "endLine": 95, + "sourceId": "", + "startColumn": 44, + "startLine": 95 + } + }, + "name": "PersonDataQualityValidation", + "package": "meta::dataquality", + "sourceInformation": { + "endColumn": 1, + "endLine": 101, + "sourceId": "", + "startColumn": 1, + "startLine": 92 + }, + "stereotypes": [], + "taggedValues": [] + }, + { + "_type": "sectionIndex", + "name": "SectionIndex", + "package": "__internal__", + "sections": [ + { + "_type": "importAware", + "elements": [], + "imports": [], + "parserName": "Pure", + "sourceInformation": { + "endColumn": 8, + "endLine": 1, + "sourceId": "", + "startColumn": 1, + "startLine": 1 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::H2" + ], + "imports": [], + "parserName": "Connection", + "sourceInformation": { + "endColumn": 2, + "endLine": 13, + "sourceId": "", + "startColumn": 8, + "startLine": 2 + } + }, + { + "_type": "default", + "elements": [ + "meta::dataquality::db" + ], + "parserName": "Relational", + "sourceInformation": { + "endColumn": 1, + "endLine": 25, + "sourceId": "", + "startColumn": 1, + "startLine": 14 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::dataqualitymappings" + ], + "imports": [], + "parserName": "Mapping", + "sourceInformation": { + "endColumn": 2, + "endLine": 49, + "sourceId": "", + "startColumn": 1, + "startLine": 26 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::DataQualityRuntime" + ], + "imports": [], + "parserName": "Runtime", + "sourceInformation": { + "endColumn": 1, + "endLine": 67, + "sourceId": "", + "startColumn": 1, + "startLine": 50 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::Person", + "meta::dataquality::Address", + "meta::dataquality::Location" + ], + "imports": [], + "parserName": "Pure", + "sourceInformation": { + "endColumn": 2, + "endLine": 91, + "sourceId": "", + "startColumn": 1, + "startLine": 68 + } + }, + { + "_type": "default", + "elements": [ + "meta::dataquality::PersonDataQualityValidation" + ], + "parserName": "DataQualityValidation", + "sourceInformation": { + "endColumn": 1, + "endLine": 103, + "sourceId": "", + "startColumn": 1, + "startLine": 92 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithNullDataQualityValidation.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithNullDataQualityValidation.json new file mode 100644 index 00000000000..0306d5c1330 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/test/resources/inputs/modelWithNullDataQualityValidation.json @@ -0,0 +1,1500 @@ +{ + "_type": "data", + "elements": [ + { + "_type": "connection", + "connectionValue": { + "_type": "RelationalDatabaseConnection", + "authenticationStrategy": { + "_type": "h2Default", + "sourceInformation": { + "endColumn": 18, + "endLine": 10, + "sourceId": "", + "startColumn": 3, + "startLine": 10 + } + }, + "databaseType": "H2", + "datasourceSpecification": { + "_type": "h2Local", + "sourceInformation": { + "endColumn": 4, + "endLine": 9, + "sourceId": "", + "startColumn": 3, + "startLine": 6 + }, + "testDataSetupSqls": [] + }, + "element": "meta::dataquality::db", + "elementSourceInformation": { + "endColumn": 30, + "endLine": 4, + "sourceId": "", + "startColumn": 10, + "startLine": 4 + }, + "postProcessorWithParameter": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 11, + "sourceId": "", + "startColumn": 1, + "startLine": 2 + }, + "type": "H2" + }, + "name": "H2", + "package": "meta::dataquality", + "sourceInformation": { + "endColumn": 1, + "endLine": 11, + "sourceId": "", + "startColumn": 1, + "startLine": 2 + } + }, + { + "_type": "relational", + "filters": [], + "includedStores": [], + "joins": [ + { + "name": "Address_Person", + "operation": { + "_type": "dynaFunc", + "funcName": "equal", + "parameters": [ + { + "_type": "column", + "column": "ID", + "sourceInformation": { + "endColumn": 62, + "endLine": 20, + "sourceId": "", + "startColumn": 24, + "startLine": 20 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 35, + "endLine": 20, + "sourceId": "", + "startColumn": 24, + "startLine": 20 + }, + "table": "addressTable" + }, + "tableAlias": "addressTable" + }, + { + "_type": "column", + "column": "ADDRESSID", + "sourceInformation": { + "endColumn": 62, + "endLine": 20, + "sourceId": "", + "startColumn": 42, + "startLine": 20 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 52, + "endLine": 20, + "sourceId": "", + "startColumn": 42, + "startLine": 20 + }, + "table": "personTable" + }, + "tableAlias": "personTable" + } + ], + "sourceInformation": { + "endColumn": 62, + "endLine": 20, + "sourceId": "", + "startColumn": 40, + "startLine": 20 + } + }, + "sourceInformation": { + "endColumn": 63, + "endLine": 20, + "sourceId": "", + "startColumn": 4, + "startLine": 20 + } + }, + { + "name": "Address_Location", + "operation": { + "_type": "dynaFunc", + "funcName": "equal", + "parameters": [ + { + "_type": "column", + "column": "LOCATIONID", + "sourceInformation": { + "endColumn": 67, + "endLine": 21, + "sourceId": "", + "startColumn": 26, + "startLine": 21 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 37, + "endLine": 21, + "sourceId": "", + "startColumn": 26, + "startLine": 21 + }, + "table": "addressTable" + }, + "tableAlias": "addressTable" + }, + { + "_type": "column", + "column": "ID", + "sourceInformation": { + "endColumn": 67, + "endLine": 21, + "sourceId": "", + "startColumn": 52, + "startLine": 21 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 64, + "endLine": 21, + "sourceId": "", + "startColumn": 52, + "startLine": 21 + }, + "table": "locationTable" + }, + "tableAlias": "locationTable" + } + ], + "sourceInformation": { + "endColumn": 67, + "endLine": 21, + "sourceId": "", + "startColumn": 50, + "startLine": 21 + } + }, + "sourceInformation": { + "endColumn": 68, + "endLine": 21, + "sourceId": "", + "startColumn": 4, + "startLine": 21 + } + } + ], + "name": "db", + "package": "meta::dataquality", + "schemas": [ + { + "name": "default", + "sourceInformation": { + "endColumn": 1, + "endLine": 22, + "sourceId": "", + "startColumn": 1, + "startLine": 14 + }, + "tables": [ + { + "columns": [ + { + "name": "ID", + "nullable": false, + "sourceInformation": { + "endColumn": 40, + "endLine": 16, + "sourceId": "", + "startColumn": 23, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "FIRSTNAME", + "nullable": true, + "sourceInformation": { + "endColumn": 64, + "endLine": 16, + "sourceId": "", + "startColumn": 43, + "startLine": 16 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + }, + { + "name": "LASTNAME", + "nullable": true, + "sourceInformation": { + "endColumn": 87, + "endLine": 16, + "sourceId": "", + "startColumn": 67, + "startLine": 16 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + }, + { + "name": "AGE", + "nullable": true, + "sourceInformation": { + "endColumn": 96, + "endLine": 16, + "sourceId": "", + "startColumn": 90, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "ADDRESSID", + "nullable": true, + "sourceInformation": { + "endColumn": 111, + "endLine": 16, + "sourceId": "", + "startColumn": 99, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "FIRMID", + "nullable": true, + "sourceInformation": { + "endColumn": 123, + "endLine": 16, + "sourceId": "", + "startColumn": 114, + "startLine": 16 + }, + "type": { + "_type": "Integer" + } + } + ], + "milestoning": [], + "name": "personTable", + "primaryKey": [ + "ID" + ], + "sourceInformation": { + "endColumn": 124, + "endLine": 16, + "sourceId": "", + "startColumn": 4, + "startLine": 16 + } + }, + { + "columns": [ + { + "name": "ID", + "nullable": false, + "sourceInformation": { + "endColumn": 41, + "endLine": 17, + "sourceId": "", + "startColumn": 24, + "startLine": 17 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "LOCATIONID", + "nullable": true, + "sourceInformation": { + "endColumn": 57, + "endLine": 17, + "sourceId": "", + "startColumn": 44, + "startLine": 17 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "TYPE", + "nullable": true, + "sourceInformation": { + "endColumn": 67, + "endLine": 17, + "sourceId": "", + "startColumn": 60, + "startLine": 17 + }, + "type": { + "_type": "Integer" + } + } + ], + "milestoning": [], + "name": "addressTable", + "primaryKey": [ + "ID" + ], + "sourceInformation": { + "endColumn": 68, + "endLine": 17, + "sourceId": "", + "startColumn": 4, + "startLine": 17 + } + }, + { + "columns": [ + { + "name": "ID", + "nullable": false, + "sourceInformation": { + "endColumn": 42, + "endLine": 18, + "sourceId": "", + "startColumn": 25, + "startLine": 18 + }, + "type": { + "_type": "Integer" + } + }, + { + "name": "STREET", + "nullable": true, + "sourceInformation": { + "endColumn": 63, + "endLine": 18, + "sourceId": "", + "startColumn": 45, + "startLine": 18 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + }, + { + "name": "LOCALITY", + "nullable": true, + "sourceInformation": { + "endColumn": 86, + "endLine": 18, + "sourceId": "", + "startColumn": 66, + "startLine": 18 + }, + "type": { + "_type": "Varchar", + "size": 200 + } + } + ], + "milestoning": [], + "name": "locationTable", + "primaryKey": [ + "ID" + ], + "sourceInformation": { + "endColumn": 87, + "endLine": 18, + "sourceId": "", + "startColumn": 4, + "startLine": 18 + } + } + ], + "views": [] + } + ], + "sourceInformation": { + "endColumn": 1, + "endLine": 22, + "sourceId": "", + "startColumn": 1, + "startLine": 14 + }, + "stereotypes": [] + }, + { + "_type": "mapping", + "associationMappings": [], + "classMappings": [ + { + "_type": "relational", + "class": "meta::dataquality::Person", + "classSourceInformation": { + "endColumn": 28, + "endLine": 29, + "sourceId": "", + "startColumn": 4, + "startLine": 29 + }, + "distinct": false, + "groupBy": [], + "primaryKey": [], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Person", + "property": "name", + "sourceInformation": { + "endColumn": 10, + "endLine": 31, + "sourceId": "", + "startColumn": 7, + "startLine": 31 + } + }, + "relationalOperation": { + "_type": "column", + "column": "FIRSTNAME", + "sourceInformation": { + "endColumn": 57, + "endLine": 31, + "sourceId": "", + "startColumn": 14, + "startLine": 31 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 47, + "endLine": 31, + "sourceId": "", + "startColumn": 37, + "startLine": 31 + }, + "table": "personTable" + }, + "tableAlias": "personTable" + }, + "sourceInformation": { + "endColumn": 57, + "endLine": 31, + "sourceId": "", + "startColumn": 12, + "startLine": 31 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Person", + "property": "age", + "sourceInformation": { + "endColumn": 9, + "endLine": 32, + "sourceId": "", + "startColumn": 7, + "startLine": 32 + } + }, + "relationalOperation": { + "_type": "column", + "column": "AGE", + "sourceInformation": { + "endColumn": 50, + "endLine": 32, + "sourceId": "", + "startColumn": 13, + "startLine": 32 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 46, + "endLine": 32, + "sourceId": "", + "startColumn": 36, + "startLine": 32 + }, + "table": "personTable" + }, + "tableAlias": "personTable" + }, + "sourceInformation": { + "endColumn": 50, + "endLine": 32, + "sourceId": "", + "startColumn": 11, + "startLine": 32 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Person", + "property": "addresses", + "sourceInformation": { + "endColumn": 15, + "endLine": 33, + "sourceId": "", + "startColumn": 7, + "startLine": 33 + } + }, + "relationalOperation": { + "_type": "elemtWithJoins", + "joins": [ + { + "db": "meta::dataquality::db", + "name": "Address_Person", + "sourceInformation": { + "endColumn": 56, + "endLine": 33, + "sourceId": "", + "startColumn": 42, + "startLine": 33 + } + } + ], + "sourceInformation": { + "endColumn": 56, + "endLine": 33, + "sourceId": "", + "startColumn": 19, + "startLine": 33 + } + }, + "sourceInformation": { + "endColumn": 56, + "endLine": 33, + "sourceId": "", + "startColumn": 17, + "startLine": 33 + } + } + ], + "root": false, + "sourceInformation": { + "endColumn": 4, + "endLine": 34, + "sourceId": "", + "startColumn": 4, + "startLine": 29 + } + }, + { + "_type": "relational", + "class": "meta::dataquality::Address", + "classSourceInformation": { + "endColumn": 29, + "endLine": 36, + "sourceId": "", + "startColumn": 4, + "startLine": 36 + }, + "distinct": false, + "groupBy": [], + "primaryKey": [], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Address", + "property": "addressId", + "sourceInformation": { + "endColumn": 15, + "endLine": 38, + "sourceId": "", + "startColumn": 7, + "startLine": 38 + } + }, + "relationalOperation": { + "_type": "column", + "column": "ID", + "sourceInformation": { + "endColumn": 56, + "endLine": 38, + "sourceId": "", + "startColumn": 19, + "startLine": 38 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 53, + "endLine": 38, + "sourceId": "", + "startColumn": 42, + "startLine": 38 + }, + "table": "addressTable" + }, + "tableAlias": "addressTable" + }, + "sourceInformation": { + "endColumn": 56, + "endLine": 38, + "sourceId": "", + "startColumn": 17, + "startLine": 38 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Address", + "property": "location", + "sourceInformation": { + "endColumn": 14, + "endLine": 39, + "sourceId": "", + "startColumn": 7, + "startLine": 39 + } + }, + "relationalOperation": { + "_type": "elemtWithJoins", + "joins": [ + { + "db": "meta::dataquality::db", + "name": "Address_Location", + "sourceInformation": { + "endColumn": 57, + "endLine": 39, + "sourceId": "", + "startColumn": 41, + "startLine": 39 + } + } + ], + "sourceInformation": { + "endColumn": 57, + "endLine": 39, + "sourceId": "", + "startColumn": 18, + "startLine": 39 + } + }, + "sourceInformation": { + "endColumn": 57, + "endLine": 39, + "sourceId": "", + "startColumn": 16, + "startLine": 39 + } + } + ], + "root": false, + "sourceInformation": { + "endColumn": 4, + "endLine": 40, + "sourceId": "", + "startColumn": 4, + "startLine": 36 + } + }, + { + "_type": "relational", + "class": "meta::dataquality::Location", + "classSourceInformation": { + "endColumn": 30, + "endLine": 42, + "sourceId": "", + "startColumn": 4, + "startLine": 42 + }, + "distinct": false, + "groupBy": [], + "primaryKey": [], + "propertyMappings": [ + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Location", + "property": "street", + "sourceInformation": { + "endColumn": 12, + "endLine": 44, + "sourceId": "", + "startColumn": 7, + "startLine": 44 + } + }, + "relationalOperation": { + "_type": "column", + "column": "STREET", + "sourceInformation": { + "endColumn": 58, + "endLine": 44, + "sourceId": "", + "startColumn": 16, + "startLine": 44 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 51, + "endLine": 44, + "sourceId": "", + "startColumn": 39, + "startLine": 44 + }, + "table": "locationTable" + }, + "tableAlias": "locationTable" + }, + "sourceInformation": { + "endColumn": 58, + "endLine": 44, + "sourceId": "", + "startColumn": 14, + "startLine": 44 + } + }, + { + "_type": "relationalPropertyMapping", + "property": { + "class": "meta::dataquality::Location", + "property": "locality", + "sourceInformation": { + "endColumn": 14, + "endLine": 45, + "sourceId": "", + "startColumn": 7, + "startLine": 45 + } + }, + "relationalOperation": { + "_type": "column", + "column": "LOCALITY", + "sourceInformation": { + "endColumn": 62, + "endLine": 45, + "sourceId": "", + "startColumn": 18, + "startLine": 45 + }, + "table": { + "_type": "Table", + "database": "meta::dataquality::db", + "mainTableDb": "meta::dataquality::db", + "schema": "default", + "sourceInformation": { + "endColumn": 53, + "endLine": 45, + "sourceId": "", + "startColumn": 41, + "startLine": 45 + }, + "table": "locationTable" + }, + "tableAlias": "locationTable" + }, + "sourceInformation": { + "endColumn": 62, + "endLine": 45, + "sourceId": "", + "startColumn": 16, + "startLine": 45 + } + } + ], + "root": false, + "sourceInformation": { + "endColumn": 4, + "endLine": 46, + "sourceId": "", + "startColumn": 4, + "startLine": 42 + } + } + ], + "enumerationMappings": [], + "includedMappings": [], + "name": "dataqualitymappings", + "package": "meta::dataquality", + "sourceInformation": { + "endColumn": 1, + "endLine": 47, + "sourceId": "", + "startColumn": 1, + "startLine": 26 + }, + "tests": [] + }, + { + "_type": "runtime", + "name": "DataQualityRuntime", + "package": "meta::dataquality", + "runtimeValue": { + "_type": "engineRuntime", + "connectionStores": [], + "connections": [ + { + "sourceInformation": { + "endColumn": 5, + "endLine": 61, + "sourceId": "", + "startColumn": 5, + "startLine": 58 + }, + "store": { + "path": "meta::dataquality::db", + "sourceInformation": { + "endColumn": 25, + "endLine": 58, + "sourceId": "", + "startColumn": 5, + "startLine": 58 + }, + "type": "STORE" + }, + "storeConnections": [ + { + "connection": { + "_type": "connectionPointer", + "connection": "meta::dataquality::H2", + "sourceInformation": { + "endColumn": 41, + "endLine": 60, + "sourceId": "", + "startColumn": 21, + "startLine": 60 + } + }, + "id": "connection_1", + "sourceInformation": { + "endColumn": 41, + "endLine": 60, + "sourceId": "", + "startColumn": 7, + "startLine": 60 + } + } + ] + } + ], + "mappings": [ + { + "path": "meta::dataquality::dataqualitymappings", + "sourceInformation": { + "endColumn": 42, + "endLine": 54, + "sourceId": "", + "startColumn": 5, + "startLine": 54 + }, + "type": "MAPPING" + } + ], + "sourceInformation": { + "endColumn": 1, + "endLine": 63, + "sourceId": "", + "startColumn": 1, + "startLine": 50 + } + }, + "sourceInformation": { + "endColumn": 1, + "endLine": 63, + "sourceId": "", + "startColumn": 1, + "startLine": 50 + } + }, + { + "_type": "class", + "constraints": [ + { + "functionDefinition": { + "_type": "lambda", + "body": [ + { + "_type": "func", + "function": "greaterThanEqual", + "parameters": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "this", + "sourceInformation": { + "endColumn": 25, + "endLine": 70, + "sourceId": "", + "startColumn": 21, + "startLine": 70 + } + } + ], + "property": "age", + "sourceInformation": { + "endColumn": 29, + "endLine": 70, + "sourceId": "", + "startColumn": 27, + "startLine": 70 + } + }, + { + "_type": "integer", + "sourceInformation": { + "endColumn": 35, + "endLine": 70, + "sourceId": "", + "startColumn": 34, + "startLine": 70 + }, + "value": 18 + } + ], + "sourceInformation": { + "endColumn": 35, + "endLine": 70, + "sourceId": "", + "startColumn": 31, + "startLine": 70 + } + } + ], + "parameters": [] + }, + "name": "mustBeOfLegalAge", + "sourceInformation": { + "endColumn": 35, + "endLine": 70, + "sourceId": "", + "startColumn": 3, + "startLine": 70 + } + } + ], + "name": "Person", + "originalMilestonedProperties": [], + "package": "meta::dataquality", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "name", + "propertyTypeSourceInformation": { + "endColumn": 16, + "endLine": 73, + "sourceId": "", + "startColumn": 11, + "startLine": 73 + }, + "sourceInformation": { + "endColumn": 20, + "endLine": 73, + "sourceId": "", + "startColumn": 4, + "startLine": 73 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "age", + "propertyTypeSourceInformation": { + "endColumn": 16, + "endLine": 74, + "sourceId": "", + "startColumn": 10, + "startLine": 74 + }, + "sourceInformation": { + "endColumn": 20, + "endLine": 74, + "sourceId": "", + "startColumn": 4, + "startLine": 74 + }, + "stereotypes": [], + "taggedValues": [], + "type": "Integer" + }, + { + "multiplicity": { + "lowerBound": 0 + }, + "name": "addresses", + "propertyTypeSourceInformation": { + "endColumn": 41, + "endLine": 75, + "sourceId": "", + "startColumn": 16, + "startLine": 75 + }, + "sourceInformation": { + "endColumn": 45, + "endLine": 75, + "sourceId": "", + "startColumn": 4, + "startLine": 75 + }, + "stereotypes": [], + "taggedValues": [], + "type": "meta::dataquality::Address" + } + ], + "qualifiedProperties": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 76, + "sourceId": "", + "startColumn": 1, + "startLine": 68 + }, + "stereotypes": [], + "superTypes": [], + "taggedValues": [] + }, + { + "_type": "class", + "constraints": [], + "name": "Address", + "originalMilestonedProperties": [], + "package": "meta::dataquality", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "location", + "propertyTypeSourceInformation": { + "endColumn": 40, + "endLine": 80, + "sourceId": "", + "startColumn": 14, + "startLine": 80 + }, + "sourceInformation": { + "endColumn": 44, + "endLine": 80, + "sourceId": "", + "startColumn": 4, + "startLine": 80 + }, + "stereotypes": [], + "taggedValues": [], + "type": "meta::dataquality::Location" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "locationStreet", + "propertyTypeSourceInformation": { + "endColumn": 25, + "endLine": 81, + "sourceId": "", + "startColumn": 20, + "startLine": 81 + }, + "sourceInformation": { + "endColumn": 29, + "endLine": 81, + "sourceId": "", + "startColumn": 4, + "startLine": 81 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "addressId", + "propertyTypeSourceInformation": { + "endColumn": 20, + "endLine": 82, + "sourceId": "", + "startColumn": 15, + "startLine": 82 + }, + "sourceInformation": { + "endColumn": 24, + "endLine": 82, + "sourceId": "", + "startColumn": 4, + "startLine": 82 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + } + ], + "qualifiedProperties": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 83, + "sourceId": "", + "startColumn": 1, + "startLine": 78 + }, + "stereotypes": [], + "superTypes": [], + "taggedValues": [] + }, + { + "_type": "class", + "constraints": [], + "name": "Location", + "originalMilestonedProperties": [], + "package": "meta::dataquality", + "properties": [ + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "street", + "propertyTypeSourceInformation": { + "endColumn": 17, + "endLine": 87, + "sourceId": "", + "startColumn": 12, + "startLine": 87 + }, + "sourceInformation": { + "endColumn": 21, + "endLine": 87, + "sourceId": "", + "startColumn": 4, + "startLine": 87 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + }, + { + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "locality", + "propertyTypeSourceInformation": { + "endColumn": 19, + "endLine": 88, + "sourceId": "", + "startColumn": 14, + "startLine": 88 + }, + "sourceInformation": { + "endColumn": 23, + "endLine": 88, + "sourceId": "", + "startColumn": 4, + "startLine": 88 + }, + "stereotypes": [], + "taggedValues": [], + "type": "String" + } + ], + "qualifiedProperties": [], + "sourceInformation": { + "endColumn": 1, + "endLine": 89, + "sourceId": "", + "startColumn": 1, + "startLine": 85 + }, + "stereotypes": [], + "superTypes": [], + "taggedValues": [] + }, + { + "_type": "dataQualityValidation", + "context": { + "_type": "mappingAndRuntimeDataQualityExecutionContext", + "_type": "mappingAndRuntimeDataQualityExecutionContext", + "mapping": { + "path": "meta::dataquality::dataqualitymappings", + "sourceInformation": { + "endColumn": 73, + "endLine": 94, + "sourceId": "", + "startColumn": 36, + "startLine": 94 + }, + "type": "MAPPING" + }, + "runtime": { + "path": "meta::dataquality::DataQualityRuntime", + "sourceInformation": { + "endColumn": 112, + "endLine": 94, + "sourceId": "", + "startColumn": 76, + "startLine": 94 + }, + "type": "RUNTIME" + } + }, + "dataQualityRootGraphFetchTree": null, + "filter": { + "_type": "lambda", + "body": [ + { + "_type": "func", + "function": "equal", + "parameters": [ + { + "_type": "property", + "parameters": [ + { + "_type": "var", + "name": "p", + "sourceInformation": { + "endColumn": 47, + "endLine": 95, + "sourceId": "", + "startColumn": 46, + "startLine": 95 + } + } + ], + "property": "name", + "sourceInformation": { + "endColumn": 52, + "endLine": 95, + "sourceId": "", + "startColumn": 49, + "startLine": 95 + } + }, + { + "_type": "string", + "sourceInformation": { + "endColumn": 60, + "endLine": 95, + "sourceId": "", + "startColumn": 55, + "startLine": 95 + }, + "value": "John" + } + ], + "sourceInformation": { + "endColumn": 54, + "endLine": 95, + "sourceId": "", + "startColumn": 53, + "startLine": 95 + } + } + ], + "parameters": [ + { + "_type": "var", + "class": "meta::dataquality::Person", + "multiplicity": { + "lowerBound": 1, + "upperBound": 1 + }, + "name": "p", + "sourceInformation": { + "endColumn": 39, + "endLine": 95, + "sourceId": "", + "startColumn": 15, + "startLine": 95 + } + } + ], + "sourceInformation": { + "endColumn": 60, + "endLine": 95, + "sourceId": "", + "startColumn": 44, + "startLine": 95 + } + }, + "name": "PersonDataQualityValidation", + "package": "meta::dataquality", + "sourceInformation": { + "endColumn": 1, + "endLine": 101, + "sourceId": "", + "startColumn": 1, + "startLine": 92 + }, + "stereotypes": [], + "taggedValues": [] + }, + { + "_type": "sectionIndex", + "name": "SectionIndex", + "package": "__internal__", + "sections": [ + { + "_type": "importAware", + "elements": [], + "imports": [], + "parserName": "Pure", + "sourceInformation": { + "endColumn": 8, + "endLine": 1, + "sourceId": "", + "startColumn": 1, + "startLine": 1 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::H2" + ], + "imports": [], + "parserName": "Connection", + "sourceInformation": { + "endColumn": 2, + "endLine": 13, + "sourceId": "", + "startColumn": 8, + "startLine": 2 + } + }, + { + "_type": "default", + "elements": [ + "meta::dataquality::db" + ], + "parserName": "Relational", + "sourceInformation": { + "endColumn": 1, + "endLine": 25, + "sourceId": "", + "startColumn": 1, + "startLine": 14 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::dataqualitymappings" + ], + "imports": [], + "parserName": "Mapping", + "sourceInformation": { + "endColumn": 2, + "endLine": 49, + "sourceId": "", + "startColumn": 1, + "startLine": 26 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::DataQualityRuntime" + ], + "imports": [], + "parserName": "Runtime", + "sourceInformation": { + "endColumn": 1, + "endLine": 67, + "sourceId": "", + "startColumn": 1, + "startLine": 50 + } + }, + { + "_type": "importAware", + "elements": [ + "meta::dataquality::Person", + "meta::dataquality::Address", + "meta::dataquality::Location" + ], + "imports": [], + "parserName": "Pure", + "sourceInformation": { + "endColumn": 2, + "endLine": 91, + "sourceId": "", + "startColumn": 1, + "startLine": 68 + } + }, + { + "_type": "default", + "elements": [ + "meta::dataquality::PersonDataQualityValidation" + ], + "parserName": "DataQualityValidation", + "sourceInformation": { + "endColumn": 1, + "endLine": 103, + "sourceId": "", + "startColumn": 1, + "startLine": 92 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/pom.xml new file mode 100644 index 00000000000..4dd4a28105e --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/pom.xml @@ -0,0 +1,162 @@ + + + + legend-engine-xts-dataquality + org.finos.legend.engine + 4.48.2-SNAPSHOT + + 4.0.0 + + legend-engine-xt-dataquality-compiler + jar + Legend Engine - XT - DataQuality - Compiler + + + 11 + 11 + + + + + + org.finos.legend.pure + legend-pure-m3-core + + + + + + + org.finos.legend.engine + legend-engine-pure-platform-java + + + org.finos.legend.engine + legend-engine-shared-core + + + org.finos.legend.pure + legend-pure-m2-dsl-graph-pure + + + org.finos.legend.pure + legend-pure-m4 + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-planExecution + runtime + + + + + org.finos.legend.engine + legend-engine-xt-dataquality-pure + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-protocol + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-dataquality-grammar + ${project.version} + runtime + + + + org.finos.legend.engine + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-xt-relationalStore-grammar + + + org.finos.legend.engine + legend-engine-xt-relationalStore-trino-grammar + ${project.version} + + + org.finos.legend.engine + legend-engine-language-pure-compiler + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-javaGeneration-pure + test + + + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + + + org.finos.legend.engine + legend-engine-xt-data-space-compiler + + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test-jar + test + + + org.finos.legend.engine + legend-engine-language-pure-compiler + test-jar + test + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + dependency-analyze + + + + org.finos.legend.engine:legend-engine-xt-relationalStore-trino-grammar + org.finos.legend.engine:legend-engine-xt-relationalStore-grammar:jar + + + + + + + + + \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java new file mode 100644 index 00000000000..e7f298ddfeb --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java @@ -0,0 +1,249 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.pure.compiler.toPureGraph; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.list.mutable.FastList; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQuality; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.runtime.PackageableRuntime; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.Variable; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityExecutionContext; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree_Impl; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree_Impl; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality_Impl; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext_Impl; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext_Impl; +import org.finos.legend.pure.generated.Root_meta_pure_metamodel_constraint_Constraint_Impl; +import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_generics_GenericType_Impl; +import org.finos.legend.pure.m3.coreinstance.meta.pure.graphFetch.GraphFetchTree; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.constraint.Constraint; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.property.AbstractProperty; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; +import org.finos.legend.pure.m3.navigation._class._Class; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +public class DataQualityCompilerExtension implements CompilerExtension +{ + + @Override + public MutableList group() + { + return org.eclipse.collections.impl.factory.Lists.mutable.with("PackageableElement", "DataQualityValidation"); + } + + static final ConcurrentHashMap dataQualityIndex = new ConcurrentHashMap<>(); + + @Override + public CompilerExtension build() + { + return new DataQualityCompilerExtension(); + } + + @Override + public Iterable> getExtraProcessors() + { + return Lists.fixedSize.of( + Processor.newProcessor( + DataQuality.class, + org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class), + (dataquality, compileContext) -> + { + Root_meta_external_dataquality_DataQuality_Impl metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>( + dataquality.name, + SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), + compileContext.pureModel.getClass("meta::external::dataquality::DataQuality") + ); + dataQualityIndex.put(compileContext.pureModel.buildPackageString(dataquality._package, dataquality.name), metamodel); + return metamodel; + }, + (dataquality, compileContext) -> + { + + }, + (dataquality, compileContext) -> + { + + }, + (dataquality, compileContext) -> + { + + }, + (dataquality, compileContext) -> + { + Root_meta_external_dataquality_DataQuality metamodel = dataQualityIndex.get(compileContext.pureModel.buildPackageString(dataquality._package, dataquality.name)); + metamodel._context(buildDataQualityExecutionContext(dataquality, compileContext)) + ._filter(getFilterLambda(dataquality, compileContext)) + ._validationTree(buildRootGraphFetchTree(dataquality.dataQualityRootGraphFetchTree, compileContext, compileContext.pureModel.getClass(dataquality.dataQualityRootGraphFetchTree._class), null, new ProcessingContext("DataQuality"))); + } + ) + ); + } + + + private LambdaFunction getFilterLambda(DataQuality app, CompileContext compileContext) + { + if (Objects.isNull(app.filter)) + { + return null; + } + return HelperValueSpecificationBuilder.buildLambda(app.filter, compileContext); + } + + private Root_meta_external_dataquality_DataQualityExecutionContext buildDataQualityExecutionContext(DataQuality app, CompileContext context) + { + if (app.context instanceof MappingAndRuntimeDataQualityExecutionContext) + { + return buildMappingAndRuntimeExecutionContext(((MappingAndRuntimeDataQualityExecutionContext) app.context), context); + } + else if (app.context instanceof DataSpaceDataQualityExecutionContext) + { + return buildDataSpaceExecutionContext(((DataSpaceDataQualityExecutionContext) app.context), context); + } + throw new EngineException("Unsupported DataQuality ExecutionContext"); + } + + private Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext buildMappingAndRuntimeExecutionContext(MappingAndRuntimeDataQualityExecutionContext mappingAndRuntimeDataQualityExecutionContext, CompileContext compileContext) + { + return new Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext_Impl( + null, + SourceInformationHelper.toM3SourceInformation(mappingAndRuntimeDataQualityExecutionContext.sourceInformation), + compileContext.pureModel.getClass("meta::external::dataquality::MappingAndRuntimeDataQualityExecutionContext") + ) + ._mapping(compileContext.resolveMapping(mappingAndRuntimeDataQualityExecutionContext.mapping.path, mappingAndRuntimeDataQualityExecutionContext.mapping.sourceInformation)) + ._runtime(compileContext.resolveRuntime(mappingAndRuntimeDataQualityExecutionContext.runtime.path, mappingAndRuntimeDataQualityExecutionContext.runtime.sourceInformation)); + + } + + private Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext buildDataSpaceExecutionContext(DataSpaceDataQualityExecutionContext dataSpaceDataQualityExecutionContext, CompileContext compileContext) + { + return new Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext_Impl( + null, + SourceInformationHelper.toM3SourceInformation(dataSpaceDataQualityExecutionContext.sourceInformation), + compileContext.pureModel.getClass("meta::external::dataquality::DataSpaceDataQualityExecutionContext") + ) + ._dataSpace(HelperDataSpaceBuilder.resolveDataSpace(dataSpaceDataQualityExecutionContext.dataSpace.path, dataSpaceDataQualityExecutionContext.dataSpace.sourceInformation, compileContext)) + ._contextName(dataSpaceDataQualityExecutionContext.context); + + } + + private static GraphFetchTree buildPropertyGraphFetchTree(DataQualityPropertyGraphFetchTree propertyGraphFetchTree, CompileContext context, Class parentClass, MutableList openVariables, ProcessingContext processingContext) + { + AbstractProperty property; + MutableList pureParameters = org.eclipse.collections.impl.factory.Lists.mutable.empty(); + + Variable thisVariable = new Variable("this", HelperModelBuilder.getElementFullPath(parentClass, context.pureModel.getExecutionSupport()), new Multiplicity(1, 1)); + MutableList originalParams = org.eclipse.collections.impl.factory.Lists.mutable.with(thisVariable).withAll(propertyGraphFetchTree.parameters); + property = HelperModelBuilder.getAppliedProperty(context, parentClass, Optional.of(originalParams), propertyGraphFetchTree.property, propertyGraphFetchTree.sourceInformation); + processingContext.push("PropertyTree"); + processingContext.addInferredVariables("this", HelperModelBuilder.createThisVariableForClass(context, HelperModelBuilder.getElementFullPath(parentClass, context.pureModel.getExecutionSupport()))); + Class subType = propertyGraphFetchTree.subType == null ? null : context.resolveClass(propertyGraphFetchTree.subType, propertyGraphFetchTree.sourceInformation); + Type returnType = subType == null ? property._genericType()._rawType() : subType; + + ListIterable children = ListIterate.collect(propertyGraphFetchTree.subTrees, subTree -> buildPropertyGraphFetchTree((DataQualityPropertyGraphFetchTree) subTree, context, (Class) returnType, openVariables, processingContext)); + return new Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree_Impl("", SourceInformationHelper.toM3SourceInformation(propertyGraphFetchTree.sourceInformation), context.pureModel.getClass("meta::external::dataquality::DataQualityPropertyGraphFetchTree")) + ._property(property) + ._parameters(pureParameters) + ._alias(propertyGraphFetchTree.alias) + ._subType(subType) + ._subTrees(children) + ._constraints(resolveNodeConstraints(context, parentClass, propertyGraphFetchTree.constraints)); // returnType - current property type - for constraints + } + + private static Root_meta_external_dataquality_DataQualityRootGraphFetchTree buildRootGraphFetchTree(DataQualityRootGraphFetchTree rootGraphFetchTree, CompileContext context, Class parentClass, MutableList openVariables, ProcessingContext processingContext) + { + HashSet subTypeClasses = new HashSet(); + HashSet propertieIdentifiersAtRootLevel = new HashSet(); + for (org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.GraphFetchTree propertyGraphFetchTree : rootGraphFetchTree.subTrees) + { + propertieIdentifiersAtRootLevel.add(getPropertyIdentifier((PropertyGraphFetchTree) propertyGraphFetchTree)); + } + for (SubTypeGraphFetchTree subTypeGraphFetchTree : rootGraphFetchTree.subTypeTrees) + { + if (!subTypeClasses.add(subTypeGraphFetchTree.subTypeClass)) + { + throw new EngineException("There are multiple subTypeTrees having subType " + subTypeGraphFetchTree.subTypeClass + ", Only one is allowed", subTypeGraphFetchTree.sourceInformation, EngineErrorType.COMPILATION); + } + for (org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.GraphFetchTree propertyGraphFetchTree : subTypeGraphFetchTree.subTrees) + { + String propertyIdentifier = getPropertyIdentifier((PropertyGraphFetchTree) propertyGraphFetchTree); + if (propertieIdentifiersAtRootLevel.contains(propertyIdentifier)) + { + throw new EngineException("Property \"" + propertyIdentifier + "\" is present at root level hence should not be specified at subType level", subTypeGraphFetchTree.sourceInformation, EngineErrorType.COMPILATION); + } + } + } + ListIterable children = ListIterate.collect(rootGraphFetchTree.subTrees, subTree -> buildPropertyGraphFetchTree((DataQualityPropertyGraphFetchTree) subTree, context, parentClass, openVariables, processingContext)); + Class classifier = context.pureModel.getClass("meta::external::dataquality::DataQualityRootGraphFetchTree"); + GenericType genericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) + ._rawType(classifier) + ._typeArguments(org.eclipse.collections.impl.factory.Lists.fixedSize.of(context.pureModel.getGenericType(parentClass))); + return new Root_meta_external_dataquality_DataQualityRootGraphFetchTree_Impl<>("", SourceInformationHelper.toM3SourceInformation(rootGraphFetchTree.sourceInformation), classifier) + ._class(parentClass) + ._classifierGenericType(genericType) + ._subTrees(children) + ._constraints(resolveNodeConstraints(context, parentClass, rootGraphFetchTree.constraints)); + } + + /** + * Recursively go through hierarchical/generalization chain and find the property. + */ + public static FastList resolveNodeConstraints(CompileContext context, org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class _class, List treeConstraintNames) + { + FastList constraints = FastList.newList(); + for (CoreInstance c_type : _Class.computeConstraintsInHierarchy(_class, context.pureModel.getExecutionSupport().getProcessorSupport())) + { + Root_meta_pure_metamodel_constraint_Constraint_Impl constraint = (Root_meta_pure_metamodel_constraint_Constraint_Impl) c_type; + if (treeConstraintNames.contains(constraint._name)) + { + constraints.add(constraint); + } + } + return constraints; + } + + private static String getPropertyIdentifier(PropertyGraphFetchTree propertyGraphFetchTree) + { + return propertyGraphFetchTree.alias != null ? propertyGraphFetchTree.alias : propertyGraphFetchTree.property; + } + +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension new file mode 100644 index 00000000000..b0638094e6c --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.pure.compiler.toPureGraph.DataQualityCompilerExtension \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java new file mode 100644 index 00000000000..dd6ddfca470 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java @@ -0,0 +1,174 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.pure.compiler.toPureGraph; + +import org.finos.legend.engine.language.pure.compiler.test.TestCompilationFromGrammar; +import org.junit.Test; + +public class TestDataQualityCompilationFromGrammar extends TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite +{ + @Override + public String getDuplicatedElementTestCode() + { + return COMPILATION_PREREQUISITE_CODE + + "###DataQualityValidation\n" + + "DataQualityValidation meta::dataquality::Person\n" + // duplicated the validation name, same name as the class + "{\n" + + " context: fromMappingAndRuntime(meta::dataquality::dataqualitymappings, meta::dataquality::DataQualityRuntime);\n" + + " filter: p:meta::dataquality::Person[1] | $p.name=='John';\n" + + " validationTree: $[\n" + + " meta::dataquality::Person{\n" + + " name\n" + + " }\n" + + " ]$;\n" + + "}"; + } + + @Override + public String getDuplicatedElementTestExpectedErrorMessage() + { + return "COMPILATION error at [92:1-101:1]: Duplicated element 'meta::dataquality::Person'"; + } + + @Test + public void testHappyPath() + { + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE + + "###DataQualityValidation\n" + + "DataQualityValidation meta::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromMappingAndRuntime(meta::dataquality::dataqualitymappings, meta::dataquality::DataQualityRuntime);\n" + + " filter: p:meta::dataquality::Person[1] | $p.name=='John';\n" + + " validationTree: $[\n" + + " meta::dataquality::Person{\n" + + " name\n" + + " }\n" + + " ]$;\n" + + "}"); + } + + @Test + public void testRootClassNoModelConstraints() + { + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE + + "###DataQualityValidation\n" + + "DataQualityValidation meta::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromMappingAndRuntime(meta::dataquality::dataqualitymappings, meta::dataquality::DataQualityRuntime);\n" + + " filter: p:meta::dataquality::Person[1] | $p.name=='John';\n" + + " validationTree: $[\n" + + " meta::dataquality::Person{\n" + + " name\n" + + " }\n" + + " ]$;\n" + + "}"); + } + + + + + + private static final String COMPILATION_PREREQUISITE_CODE = "###Connection\n" + + "RelationalDatabaseConnection meta::dataquality::H2\n" + + "{\n" + + " store: meta::dataquality::db;\n" + + " type: H2;\n" + + " specification: LocalH2\n" + + " { \n" + + " testDataSetupSqls: [];\n" + + " };\n" + + " auth: DefaultH2;\n" + + "}\n" + + "\n" + + "###Relational\n" + + "Database meta::dataquality::db\n" + + "(\n" + + " Table personTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT)\n" + + " Table addressTable (ID INT PRIMARY KEY, LOCATIONID INT, TYPE INT)\n" + + " Table locationTable (ID INT PRIMARY KEY, STREET VARCHAR(200), LOCALITY VARCHAR(200))\n" + + "\n" + + " Join Address_Person(addressTable.ID = personTable.ADDRESSID)\n" + + " Join Address_Location(addressTable.LOCATIONID = locationTable.ID)\n" + + ")\n" + + "\n" + + "\n" + + "###Mapping\n" + + "Mapping meta::dataquality::dataqualitymappings\n" + + "(\n" + + "\n" + + " meta::dataquality::Person : Relational\n" + + " {\n" + + " name : [meta::dataquality::db]personTable.FIRSTNAME,\n" + + " age : [meta::dataquality::db]personTable.AGE,\n" + + " addresses : [meta::dataquality::db]@Address_Person\n" + + " }\n" + + "\n" + + " meta::dataquality::Address : Relational\n" + + " {\n" + + " addressId : [meta::dataquality::db]addressTable.ID,\n" + + " location : [meta::dataquality::db]@Address_Location\n" + + " }\n" + + "\n" + + " meta::dataquality::Location : Relational\n" + + " {\n" + + " street : [meta::dataquality::db]locationTable.STREET,\n" + + " locality : [meta::dataquality::db]locationTable.LOCALITY\n" + + " }\n" + + ")\n" + + "\n" + + "###Runtime\n" + + "Runtime meta::dataquality::DataQualityRuntime\n" + + "{\n" + + " mappings:\n" + + " [\n" + + " meta::dataquality::dataqualitymappings\n" + + " ];\n" + + " connections:\n" + + " [\n" + + " meta::dataquality::db:\n" + + " [\n" + + " connection_1: meta::dataquality::H2\n" + + " ]\n" + + " ];\n" + + "}\n" + + "\n" + + "\n" + + "\n" + + "###Pure\n" + + "Class meta::dataquality::Person\n" + + "[\n" + + " mustBeOfLegalAge: $this.age >= 18\n" + + "]\n" + + "{\n" + + " name : String[1];\n" + + " age : Integer[1];\n" + + " addresses : meta::dataquality::Address[*];\n" + + "}\n" + + "\n" + + "Class meta::dataquality::Address\n" + + "{\n" + + " location: meta::dataquality::Location[1];\n" + + " locationStreet: String[1];\n" + + " addressId: String[1];\n" + + "}\n" + + "\n" + + "Class meta::dataquality::Location\n" + + "{\n" + + " street: String[1];\n" + + " locality: String[1];\n" + + "}\n" + + "\n"; + +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/pom.xml new file mode 100644 index 00000000000..2303979bdf8 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/pom.xml @@ -0,0 +1,145 @@ + + + + legend-engine-xts-dataquality + org.finos.legend.engine + 4.48.2-SNAPSHOT + + 4.0.0 + + legend-engine-xt-dataquality-grammar + Legend Engine - XT - DataQuality - Grammar + + + 11 + 11 + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-antlr-core-grammar + initialize + + unpack + + + + + org.finos.legend.engine + legend-engine-language-pure-grammar + jar + false + ${project.build.directory} + antlr/*.g4 + + + + + + + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + true + true + true + target/antlr + ${project.build.directory}/generated-sources + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + + + org.finos.legend.engine + legend-engine-language-pure-grammar + + + org.finos.legend.engine + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-protocol + + + org.finos.legend.engine + legend-engine-shared-core + + + + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + + + + org.antlr + antlr4-runtime + compile + + + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-shared-core + test-jar + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test-jar + test + + + org.finos.legend.engine + legend-engine-xt-dataquality-protocol + ${project.version} + compile + + + + + \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 new file mode 100644 index 00000000000..b557cbbc04f --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 @@ -0,0 +1,18 @@ +lexer grammar DataQualityLexerGrammar; + +import M3LexerGrammar; + +DATAQUALITYVALIDATION: 'DataQualityValidation'; +DQVALIDATIONCONSTRAINTS: 'validationTree'; +DQCONTEXT: 'context'; +FROM_DATASPACE: 'fromDataSpace'; +FROM_MAPPING_AND_RUNTIME: 'fromMappingAndRuntime'; +FILTER: 'filter'; +GRAPH_START: '$['; +GRAPH_END: ']$'; + + + +// ---------------------------------- BUILDING BLOCK -------------------------------------- + +SUBTYPE_START: '->subType(@'; diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 new file mode 100644 index 00000000000..1d365b951f7 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 @@ -0,0 +1,100 @@ +parser grammar DataQualityParserGrammar; + +import M3ParserGrammar; + +options +{ + tokenVocab = DataQualityLexerGrammar; +} + +// -------------------------------------- IDENTIFIER -------------------------------------- + +identifier: VALID_STRING | STRING + | ALL | LET | ALL_VERSIONS | ALL_VERSIONS_IN_RANGE | TO_BYTES_FUNCTION + | DATAQUALITYVALIDATION + | DQVALIDATIONCONSTRAINTS + | FROM_DATASPACE + | FROM_MAPPING_AND_RUNTIME + | DQCONTEXT + | FILTER +; + + +// -------------------------------------- DEFINITION -------------------------------------- + +definition: (validationDefinition)* + EOF +; +validationDefinition: DATAQUALITYVALIDATION stereotypes? taggedValues? qualifiedName + BRACE_OPEN + ( + dqContext + | filter + | validationTree + )* + BRACE_CLOSE +; +stereotypes: LESS_THAN LESS_THAN stereotype (COMMA stereotype)* GREATER_THAN GREATER_THAN; +stereotype: qualifiedName DOT identifier; +taggedValues: BRACE_OPEN taggedValue (COMMA taggedValue)* BRACE_CLOSE; +taggedValue: qualifiedName DOT identifier EQUAL STRING; + + +dqContext: DQCONTEXT COLON (fromDataSpace| fromMappingAndRuntime) + SEMI_COLON +; +fromDataSpace: FROM_DATASPACE PAREN_OPEN dataspace COMMA contextName PAREN_CLOSE; +fromMappingAndRuntime: FROM_MAPPING_AND_RUNTIME PAREN_OPEN mapping COMMA runtime PAREN_CLOSE; +mapping: qualifiedName; +runtime: qualifiedName; +dataspace: qualifiedName; +contextName: STRING; +validationTree: DQVALIDATIONCONSTRAINTS COLON dqGraphDefinition SEMI_COLON +; +filter: FILTER COLON combinedExpression SEMI_COLON; + + +// -------------------------------------- TREE DEFINITION -------------------------------------- + +dqGraphDefinition: GRAPH_START qualifiedName(constraintList)? graphDefinition GRAPH_END +; +graphDefinition: BRACE_OPEN + graphPaths + BRACE_CLOSE +; +graphPaths: (graphPath | subTypeGraphPath) (COMMA (graphPath | subTypeGraphPath))* +; +graphPath: alias? identifier(constraintList)? propertyParameters? subtype? graphDefinition? +; +subTypeGraphPath: subtype graphDefinition +; +alias: STRING COLON +; +propertyParameters: PAREN_OPEN + (parameter (COMMA parameter)*)? + PAREN_CLOSE +; +subtype: SUBTYPE_START qualifiedName PAREN_CLOSE +; +parameter: scalarParameter | collectionParameter +; +scalarParameter: LATEST_DATE | instanceLiteral | variable | enumReference +; +collectionParameter: BRACKET_OPEN + (scalarParameter (COMMA scalarParameter)*)? + BRACKET_CLOSE +; +instanceLiteral: instanceLiteralToken | (MINUS INTEGER) | (MINUS FLOAT) | (MINUS DECIMAL) | (PLUS INTEGER) | (PLUS FLOAT) | (PLUS DECIMAL) +; +instanceLiteralToken: STRING | INTEGER | FLOAT | DECIMAL | DATE | BOOLEAN +; +variable: DOLLAR identifier +; +enumReference: qualifiedName DOT identifier +; +constraintList: LESS_THAN + dqConstraintName (COMMA dqConstraintName)* + GREATER_THAN +; +dqConstraintName: STRING | identifier +; \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java new file mode 100644 index 00000000000..97daa25377c --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java @@ -0,0 +1,83 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.grammar.from; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.grammar.from.ParserErrorListener; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserContext; +import org.finos.legend.engine.language.pure.grammar.from.SectionSourceCode; +import org.finos.legend.engine.language.pure.grammar.from.SourceCodeParserInfo; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataQualityLexerGrammar; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataQualityParserGrammar; +import org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension; +import org.finos.legend.engine.language.pure.grammar.from.extension.SectionParser; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.Section; + +import java.util.function.Consumer; + +public class DataQualityGrammarParserExtension implements PureGrammarParserExtension +{ + public static final String NAME = "DataQualityValidation"; + + @Override + public MutableList group() + { + return org.eclipse.collections.impl.factory.Lists.mutable.with("PackageableElement", "DataQualityValidation"); + } + + @Override + public Iterable getExtraSectionParsers() + { + return Lists.fixedSize.of(SectionParser.newParser(NAME, DataQualityGrammarParserExtension::parseSection)); + } + + private static Section parseSection(SectionSourceCode sectionSourceCode, Consumer elementConsumer, PureGrammarParserContext context) + { + SourceCodeParserInfo parserInfo = getDataQualityInfo(sectionSourceCode); + DefaultCodeSection section = new DefaultCodeSection(); + section.parserName = sectionSourceCode.sectionType; + section.sourceInformation = parserInfo.sourceInformation; + + DataQualityTreeWalker walker = new DataQualityTreeWalker(parserInfo.input, parserInfo.walkerSourceInformation, elementConsumer, section); + walker.visit((DataQualityParserGrammar.DefinitionContext) parserInfo.rootContext); + + return section; + } + + private static SourceCodeParserInfo getDataQualityInfo(SectionSourceCode sectionSourceCode) + { + CharStream input = CharStreams.fromString(sectionSourceCode.code); + ParserErrorListener errorListener = new ParserErrorListener(sectionSourceCode.walkerSourceInformation, DataQualityLexerGrammar.VOCABULARY); + DataQualityLexerGrammar lexer = new DataQualityLexerGrammar(input); + lexer.removeErrorListeners(); + lexer.addErrorListener(errorListener); + DataQualityParserGrammar parser = new DataQualityParserGrammar(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); + parser.addErrorListener(errorListener); + return new SourceCodeParserInfo(sectionSourceCode.code, + input, + sectionSourceCode.sourceInformation, + sectionSourceCode.walkerSourceInformation, + lexer, + parser, + parser.definition()); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java new file mode 100644 index 00000000000..95846467aa8 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java @@ -0,0 +1,269 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.grammar.from; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.misc.Interval; +import org.eclipse.collections.impl.factory.Lists; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.grammar.from.ParseTreeWalkerSourceInformation; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserUtility; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataQualityParserGrammar; +import org.finos.legend.engine.language.pure.grammar.from.domain.DomainParser; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQuality; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation; +import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; +import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementType; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.StereotypePtr; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TagPtr; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TaggedValue; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.GraphFetchTree; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +public class DataQualityTreeWalker +{ + private final CharStream input; + private final ParseTreeWalkerSourceInformation walkerSourceInformation; + private final Consumer elementConsumer; + private final DefaultCodeSection section; + + public DataQualityTreeWalker(CharStream input, ParseTreeWalkerSourceInformation walkerSourceInformation, Consumer elementConsumer, DefaultCodeSection section) + { + this.input = input; + this.walkerSourceInformation = walkerSourceInformation; + this.elementConsumer = elementConsumer; + this.section = section; + } + + public void visit(DataQualityParserGrammar.DefinitionContext definitionContext) + { + if (definitionContext.validationDefinition() != null && !definitionContext.validationDefinition().isEmpty()) + { + definitionContext.validationDefinition().stream().map(this::visitDataQualityValidation).peek(e -> this.section.elements.add(e.getPath())).forEach(this.elementConsumer); + } + } + + private DataQuality visitDataQualityValidation(DataQualityParserGrammar.ValidationDefinitionContext validationDefinitionContext) + { + DataQuality dataQuality = new DataQuality(); + dataQuality.name = PureGrammarParserUtility.fromIdentifier(validationDefinitionContext.qualifiedName().identifier()); + dataQuality._package = validationDefinitionContext.qualifiedName().packagePath() == null ? "" : PureGrammarParserUtility.fromPath(validationDefinitionContext.qualifiedName().packagePath().identifier()); + dataQuality.sourceInformation = walkerSourceInformation.getSourceInformation(validationDefinitionContext); + dataQuality.stereotypes = validationDefinitionContext.stereotypes() == null ? Lists.mutable.empty() : this.visitStereotypes(validationDefinitionContext.stereotypes()); + dataQuality.taggedValues = validationDefinitionContext.taggedValues() == null ? Lists.mutable.empty() : this.visitTaggedValues(validationDefinitionContext.taggedValues()); + dataQuality.sourceInformation = walkerSourceInformation.getSourceInformation(validationDefinitionContext); + + // context + DataQualityParserGrammar.DqContextContext dqContextContext = PureGrammarParserUtility.validateAndExtractRequiredField(validationDefinitionContext.dqContext(), + "context", + dataQuality.sourceInformation); + dataQuality.context = visitDqContext(dqContextContext, dataQuality.sourceInformation); + // constraints + DataQualityParserGrammar.ValidationTreeContext validationTreeContext = PureGrammarParserUtility.validateAndExtractRequiredField(validationDefinitionContext.validationTree(), + "validationTree", + dataQuality.sourceInformation); + dataQuality.dataQualityRootGraphFetchTree = this.visitRootGraphDefinition(validationTreeContext.dqGraphDefinition().graphDefinition(), + validationTreeContext.dqGraphDefinition()); + if (Objects.nonNull(validationDefinitionContext.filter()) && !validationDefinitionContext.filter().isEmpty()) + { + dataQuality.filter = visitLambda(validationDefinitionContext.filter().get(0).combinedExpression()); + } + + return dataQuality; + } + + private DataQualityRootGraphFetchTree visitRootGraphDefinition(DataQualityParserGrammar.GraphDefinitionContext graphDefinitionContext, + DataQualityParserGrammar.DqGraphDefinitionContext validationDefinitionContext) + { + List subTrees = new ArrayList<>(); + List subTypeTrees = new ArrayList<>(); + for (DataQualityParserGrammar.GraphPathContext graphPathContext : graphDefinitionContext.graphPaths().graphPath()) + { + subTrees.add(this.visitGraphPathContext(graphPathContext)); + } +// for (GraphFetchTreeParserGrammar.SubTypeGraphPathContext subTypeGraphPathContext : graphDefinitionContext.graphPaths().subTypeGraphPath()) +// { +// subTypeTrees.add(this.visitSubTypeGraphPathContext(subTypeGraphPathContext)); +// } + DataQualityRootGraphFetchTree result = new DataQualityRootGraphFetchTree(); + result._class = PureGrammarParserUtility.fromQualifiedName(validationDefinitionContext.qualifiedName().packagePath() == null ? Collections.emptyList() : validationDefinitionContext.qualifiedName().packagePath().identifier(), validationDefinitionContext.qualifiedName().identifier()); + result.sourceInformation = walkerSourceInformation.getSourceInformation(validationDefinitionContext.qualifiedName()); + result.subTrees = subTrees; + result.subTypeTrees = subTypeTrees; + + if (Objects.nonNull(validationDefinitionContext.constraintList())) + { + result.constraints = new ArrayList<>(); + for (DataQualityParserGrammar.DqConstraintNameContext dqConstraintNameContext : validationDefinitionContext.constraintList().dqConstraintName()) + { + result.constraints.add(dqConstraintNameContext.getText()); + } + } + return result; + } + + private PropertyGraphFetchTree visitGraphPathContext(DataQualityParserGrammar.GraphPathContext graphPathContext) + { + List subTrees = new ArrayList<>(); + if (graphPathContext.graphDefinition() != null) + { + // validationForSubTypeTrees(graphPathContext.graphDefinition()); + for (DataQualityParserGrammar.GraphPathContext subGraphPathContext : graphPathContext.graphDefinition().graphPaths().graphPath()) + { + subTrees.add(this.visitGraphPathContext(subGraphPathContext)); + } + } + + List parameters = new ArrayList<>(); +// if (graphPathContext.propertyParameters() != null) +// { +// for (GraphFetchTreeParserGrammar.ParameterContext parameterContext : graphPathContext.propertyParameters().parameter()) +// { +// parameters.add(this.visitParameterContext(parameterContext)); +// } +// } + + DataQualityPropertyGraphFetchTree result = new DataQualityPropertyGraphFetchTree(); + result.property = PureGrammarParserUtility.fromIdentifier(graphPathContext.identifier()); + if (Objects.nonNull(graphPathContext.constraintList())) + { + result.constraints = new ArrayList<>(); + for (DataQualityParserGrammar.DqConstraintNameContext dqConstraintNameContext : graphPathContext.constraintList().dqConstraintName()) + { + result.constraints.add(dqConstraintNameContext.getText()); + } + } + result.sourceInformation = walkerSourceInformation.getSourceInformation(graphPathContext.identifier()); + result.parameters = parameters; + result.subTrees = subTrees; + if (graphPathContext.alias() != null) + { + String withQuote = graphPathContext.alias().STRING().getText(); + result.alias = withQuote.substring(1, withQuote.length() - 1); + } + if (graphPathContext.subtype() != null) + { + result.subType = graphPathContext.subtype().qualifiedName().getText(); + } + return result; + } + + + private DataQualityExecutionContext visitDqContext(DataQualityParserGrammar.DqContextContext dqContextContext, SourceInformation sourceInformation) + { + if (Objects.nonNull(dqContextContext.fromMappingAndRuntime())) + { + PackageableElementPointer mappingPointer = new PackageableElementPointer(); + mappingPointer.type = PackageableElementType.MAPPING; + mappingPointer.path = visitQualifiedName(dqContextContext.fromMappingAndRuntime().mapping().qualifiedName()); + mappingPointer.sourceInformation = walkerSourceInformation.getSourceInformation(dqContextContext.fromMappingAndRuntime().mapping()); + + PackageableElementPointer runtimePointer = new PackageableElementPointer(); + runtimePointer.type = PackageableElementType.RUNTIME; + runtimePointer.path = visitQualifiedName(dqContextContext.fromMappingAndRuntime().runtime().qualifiedName()); + runtimePointer.sourceInformation = walkerSourceInformation.getSourceInformation(dqContextContext.fromMappingAndRuntime().runtime()); + + final MappingAndRuntimeDataQualityExecutionContext mappingAndRuntimeDataQualityExecutionContext = new MappingAndRuntimeDataQualityExecutionContext(); + mappingAndRuntimeDataQualityExecutionContext.mapping = mappingPointer; + mappingAndRuntimeDataQualityExecutionContext.runtime = runtimePointer; + return mappingAndRuntimeDataQualityExecutionContext; + } + PackageableElementPointer dataSpaceElementPointer = new PackageableElementPointer(); + dataSpaceElementPointer.type = PackageableElementType.DATASPACE; + dataSpaceElementPointer.path = visitQualifiedName(dqContextContext.fromDataSpace().dataspace().qualifiedName()); + dataSpaceElementPointer.sourceInformation = walkerSourceInformation.getSourceInformation(dqContextContext.fromDataSpace().dataspace()); + + final DataSpaceDataQualityExecutionContext dataSpaceDataQualityExecutionContext = new DataSpaceDataQualityExecutionContext(); + dataSpaceDataQualityExecutionContext.dataSpace = dataSpaceElementPointer; + dataSpaceDataQualityExecutionContext.context = PureGrammarParserUtility.fromGrammarString(dqContextContext.fromDataSpace().contextName().STRING().getText(), true); + return dataSpaceDataQualityExecutionContext; + } + + private String visitQualifiedName(DataQualityParserGrammar.QualifiedNameContext qualifiedNameContext) + { + return PureGrammarParserUtility.fromQualifiedName(qualifiedNameContext.packagePath() == null ? Collections.emptyList() : + qualifiedNameContext.packagePath().identifier(), qualifiedNameContext.identifier()); + } + + private List visitTaggedValues(DataQualityParserGrammar.TaggedValuesContext ctx) + { + return ListIterate.collect(ctx.taggedValue(), taggedValueContext -> + { + TaggedValue taggedValue = new TaggedValue(); + TagPtr tagPtr = new TagPtr(); + taggedValue.tag = tagPtr; + tagPtr.profile = PureGrammarParserUtility.fromQualifiedName(taggedValueContext.qualifiedName().packagePath() == null ? Collections.emptyList() : taggedValueContext.qualifiedName().packagePath().identifier(), taggedValueContext.qualifiedName().identifier()); + tagPtr.value = PureGrammarParserUtility.fromIdentifier(taggedValueContext.identifier()); + taggedValue.value = PureGrammarParserUtility.fromGrammarString(taggedValueContext.STRING().getText(), true); + taggedValue.tag.profileSourceInformation = this.walkerSourceInformation.getSourceInformation(taggedValueContext.qualifiedName()); + taggedValue.tag.sourceInformation = this.walkerSourceInformation.getSourceInformation(taggedValueContext.identifier()); + taggedValue.sourceInformation = this.walkerSourceInformation.getSourceInformation(taggedValueContext); + return taggedValue; + }); + } + + private List visitStereotypes(DataQualityParserGrammar.StereotypesContext ctx) + { + return ListIterate.collect(ctx.stereotype(), stereotypeContext -> + { + StereotypePtr stereotypePtr = new StereotypePtr(); + stereotypePtr.profile = PureGrammarParserUtility.fromQualifiedName(stereotypeContext.qualifiedName().packagePath() == null ? Collections.emptyList() : stereotypeContext.qualifiedName().packagePath().identifier(), stereotypeContext.qualifiedName().identifier()); + stereotypePtr.value = PureGrammarParserUtility.fromIdentifier(stereotypeContext.identifier()); + stereotypePtr.profileSourceInformation = this.walkerSourceInformation.getSourceInformation(stereotypeContext.qualifiedName()); + stereotypePtr.sourceInformation = this.walkerSourceInformation.getSourceInformation(stereotypeContext); + return stereotypePtr; + }); + } + + private Lambda visitLambda(DataQualityParserGrammar.CombinedExpressionContext combinedExpressionContext) + { + DomainParser parser = new DomainParser(); + // prepare island grammar walker source information + int startLine = combinedExpressionContext.getStart().getLine(); + int lineOffset = walkerSourceInformation.getLineOffset() + startLine - 1; + // only add current walker source information column offset if this is the first line + int columnOffset = (startLine == 1 ? walkerSourceInformation.getColumnOffset() : 0) + combinedExpressionContext.getStart().getCharPositionInLine(); + ParseTreeWalkerSourceInformation combineExpressionSourceInformation = new ParseTreeWalkerSourceInformation.Builder(walkerSourceInformation.getSourceId(), lineOffset, columnOffset).withReturnSourceInfo(this.walkerSourceInformation.getReturnSourceInfo()).build(); + String lambdaString = this.input.getText(new Interval(combinedExpressionContext.start.getStartIndex(), combinedExpressionContext.stop.getStopIndex())); + ValueSpecification valueSpecification = parser.parseCombinedExpression(lambdaString, combineExpressionSourceInformation, null); + if (valueSpecification instanceof Lambda) + { + return (Lambda) valueSpecification; + } + // NOTE: If the user just provides the body of the lambda, we will wrap a lambda around it + // we might want to reconsider this behavior and throw error if this convenience causes any trouble + Lambda lambda = new Lambda(); + lambda.body = new ArrayList<>(); + lambda.body.add(valueSpecification); + lambda.parameters = new ArrayList<>(); + return lambda; + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java new file mode 100644 index 00000000000..f746fdd3644 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java @@ -0,0 +1,235 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.grammar.to; + +import org.eclipse.collections.api.block.function.Function3; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.utility.Iterate; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.dataquality.grammar.from.DataQualityGrammarParserExtension; +import org.finos.legend.engine.language.pure.grammar.to.DEPRECATED_PureGrammarComposerCore; +import org.finos.legend.engine.language.pure.grammar.to.HelperValueSpecificationGrammarComposer; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext; +import org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQuality; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree; +import org.finos.legend.engine.shared.core.api.grammar.RenderStyle; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.finos.legend.engine.language.pure.grammar.to.HelperDomainGrammarComposer.renderAnnotations; +import static org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerUtility.convertString; +import static org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerUtility.getTabSize; + +public class DataQualityGrammarComposerExtension implements PureGrammarComposerExtension +{ + private static final int initialTabSize = 3; + private static final RenderStyle renderStyle = RenderStyle.PRETTY; // get this from context + + @Override + public MutableList group() + { + return org.eclipse.collections.impl.factory.Lists.mutable.with("PackageableElement", "DataQualityValidation"); + } + + private static String renderElement(PackageableElement element, PureGrammarComposerContext context) + { + if (element instanceof DataQuality) + { + return renderDataQuality((DataQuality) element, context); + } + return "/* Can't transform element '" + element.getPath() + "' in this section */"; + } + + private static String renderDataQuality(DataQuality dataQuality, PureGrammarComposerContext context) + { + String packageName = dataQuality._package == null || dataQuality._package.isEmpty() ? dataQuality.name : dataQuality._package + "::" + dataQuality.name; + DEPRECATED_PureGrammarComposerCore grammarTransformer = DEPRECATED_PureGrammarComposerCore.Builder.newInstance(context).withIndentation(1).withRenderStyle(RenderStyle.PRETTY).build(); + return "DataQualityValidation " + renderAnnotations(dataQuality.stereotypes, dataQuality.taggedValues) + packageName + "\n" + + "{\n" + + " context: " + getContextFunc(dataQuality) + ";\n" + + " validationTree: " + processGraphFetchTree(dataQuality.dataQualityRootGraphFetchTree, grammarTransformer) + ";\n" + + renderLambda(dataQuality, context) + + "}"; + } + + private static String renderLambda(DataQuality dataQuality, PureGrammarComposerContext context) + { + if (Objects.isNull(dataQuality.filter)) + { + return ""; + } + return " filter: " + dataQuality.filter.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance(context).build()) + ";\n"; + } + + private static String getContextFunc(DataQuality dataQuality) + { + if (dataQuality.context instanceof MappingAndRuntimeDataQualityExecutionContext) + { + MappingAndRuntimeDataQualityExecutionContext context = (MappingAndRuntimeDataQualityExecutionContext) dataQuality.context; + return "fromMappingAndRuntime(" + context.mapping.path + ", " + context.runtime.path + ")"; + } + else if (dataQuality.context instanceof DataSpaceDataQualityExecutionContext) + { + DataSpaceDataQualityExecutionContext context = (DataSpaceDataQualityExecutionContext) dataQuality.context; + return "fromDataSpace(" + context.dataSpace.path + ", '" + context.context + "')"; + } + throw new EngineException("Unsupported DataQuality ExecutionContext"); + } + + private static String processGraphFetchTree(DataQualityRootGraphFetchTree rootGraphFetchTree, DEPRECATED_PureGrammarComposerCore grammarTransformer) + { + String subTreeString = ""; + String subTypeTreeString = ""; + if (rootGraphFetchTree.subTrees != null && !rootGraphFetchTree.subTrees.isEmpty()) + { + subTreeString = rootGraphFetchTree.subTrees.stream().map(x -> processGraphFetchTree((DataQualityPropertyGraphFetchTree) x, initialTabSize + 1, grammarTransformer)).collect(Collectors.joining("," + (isRenderingPretty() ? returnChar() : ""))); + } + if (rootGraphFetchTree.subTypeTrees != null && !rootGraphFetchTree.subTypeTrees.isEmpty()) + { + subTypeTreeString = subTypeTreeString + rootGraphFetchTree.subTypeTrees.stream().map(x -> processGraphFetchTree(x, initialTabSize, grammarTransformer)).collect(Collectors.joining("," + (isRenderingPretty() ? returnChar() : ""))); + } + return "$[" + (isRenderingPretty() ? returnChar() : "") + + DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(initialTabSize)) + printFullPath(rootGraphFetchTree._class) + printConstraints(rootGraphFetchTree.constraints) + "{" + (isRenderingPretty() ? returnChar() : "") + + subTreeString + (!subTreeString.isEmpty() && !subTypeTreeString.isEmpty() ? (isRenderingPretty() ? "," + returnChar() : ",") : "") + subTypeTreeString + (isRenderingPretty() ? returnChar() : "") + + DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(initialTabSize)) + "}" + (isRenderingPretty() ? returnChar() : "") + + (isRenderingPretty() ? DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(initialTabSize - 1)) : DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(1))) + "]$"; + } + + private static String printConstraints(List constraints) + { + if (Objects.isNull(constraints) || constraints.isEmpty()) + { + return ""; + } + return "<" + String.join(", ", constraints) + ">"; + } + + public static String printFullPath(String fullPath) + { + if (isRenderingHTML()) + { + int index = fullPath.lastIndexOf("::"); + if (index == -1) + { + return "" + fullPath + ""; + } + return "" + fullPath.substring(0, index + 2) + "" + fullPath.substring(index + 2) + ""; + } + return fullPath; + } + + public static String processGraphFetchTree(DataQualityPropertyGraphFetchTree propertyGraphFetchTree, int tabSize, DEPRECATED_PureGrammarComposerCore grammarTransformer) + { + String aliasString = ""; + if (propertyGraphFetchTree.alias != null) + { + aliasString = convertString(propertyGraphFetchTree.alias, false) + ":"; // we do not need to escape here because the alias need to be a valid string + } + + String subTreeString = ""; + if (propertyGraphFetchTree.subTrees != null && !propertyGraphFetchTree.subTrees.isEmpty()) + { + subTreeString = "{" + (isRenderingPretty() ? returnChar() : "") + + propertyGraphFetchTree.subTrees.stream().map(x -> processGraphFetchTree((DataQualityPropertyGraphFetchTree) x, tabSize + 1, grammarTransformer)).collect(Collectors.joining("," + (isRenderingPretty() ? returnChar() : ""))) + (isRenderingPretty() ? returnChar() : "") + + DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(tabSize)) + "}"; + } + + String parametersString = ""; + if (propertyGraphFetchTree.parameters != null && !propertyGraphFetchTree.parameters.isEmpty()) + { + parametersString = propertyGraphFetchTree.parameters.stream().map(x -> x.accept(grammarTransformer)).collect(Collectors.joining(", ", "(", ")")); + } + + String subTypeString = ""; + if (propertyGraphFetchTree.subType != null) + { + subTypeString = "->subType(@" + printFullPath(propertyGraphFetchTree.subType) + ")"; + } + + return DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(tabSize)) + aliasString + propertyGraphFetchTree.property + printConstraints(propertyGraphFetchTree.constraints) + parametersString + subTypeString + subTreeString; + } + + public static String processGraphFetchTree(SubTypeGraphFetchTree subTypeGraphFetchTree, int tabSize, DEPRECATED_PureGrammarComposerCore grammarTransformer) + { + String subTreeString = ""; + if (subTypeGraphFetchTree.subTrees != null && !subTypeGraphFetchTree.subTrees.isEmpty()) + { + subTreeString = "{" + (isRenderingPretty() ? returnChar() : "") + + subTypeGraphFetchTree.subTrees.stream().map(x -> DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withIndentation(getTabSize(1)).build().processGraphFetchTree(x, getTabSize(1))).collect(Collectors.joining("," + (isRenderingPretty() ? returnChar() : ""))) + (isRenderingPretty() ? returnChar() : "") + + DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(tabSize)) + "}"; + } + String subTypeString = "->subType(@" + HelperValueSpecificationGrammarComposer.printFullPath(subTypeGraphFetchTree.subTypeClass, grammarTransformer) + ")"; + + return DEPRECATED_PureGrammarComposerCore.computeIndentationString(grammarTransformer, getTabSize(1)) + subTypeString + subTreeString; + } + + public static boolean isRenderingPretty() + { + return RenderStyle.PRETTY.equals(renderStyle) || RenderStyle.PRETTY_HTML.equals(renderStyle); + } + + public static boolean isRenderingHTML() + { + return RenderStyle.PRETTY_HTML.equals(renderStyle); + } + + public static String returnChar() + { + return isRenderingHTML() ? "
\n" : "\n"; + } + + @Override + public List, PureGrammarComposerContext, String, String>> getExtraSectionComposers() + { + return Lists.fixedSize.with((elements, context, sectionName) -> + { + if (!DataQualityGrammarParserExtension.NAME.equals(sectionName)) + { + return null; + } + return ListIterate.collect(elements, element -> + { + if (element instanceof DataQuality) + { + return renderDataQuality((DataQuality) element, context); + } + return "/* Can't transform element '" + element.getPath() + "' in this section */"; + }).makeString("\n\n"); + }); + } + + @Override + public List, PureGrammarComposerContext, List, PureFreeSectionGrammarComposerResult>> getExtraFreeSectionComposers() + { + return Collections.singletonList((elements, context, composedSections) -> // TODO: use context for render style etc - dont hardcode + { + MutableList composableElements = Iterate.select(elements, e -> (e instanceof DataQuality), Lists.mutable.empty()); + return composableElements.isEmpty() + ? null + : new PureFreeSectionGrammarComposerResult(composableElements.asLazy().collect(element -> renderElement(element, context)).makeString("###" + DataQualityGrammarParserExtension.NAME + "\n", "\n\n", ""), composableElements); + }); + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension new file mode 100644 index 00000000000..e92f11cdab7 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.dataquality.grammar.from.DataQualityGrammarParserExtension \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension new file mode 100644 index 00000000000..2aecb62cbe5 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.dataquality.grammar.to.DataQualityGrammarComposerExtension \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java new file mode 100644 index 00000000000..fc52bce2ee3 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java @@ -0,0 +1,125 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.grammar.from; + +import org.antlr.v4.runtime.Vocabulary; +import org.eclipse.collections.impl.list.mutable.ListAdapter; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataQualityParserGrammar; +import org.finos.legend.engine.language.pure.grammar.test.TestGrammarParser; +import org.junit.Test; + +import java.util.List; + +public class TestDataQualityParsing extends TestGrammarParser.TestGrammarParserTestSuite +{ + @Override + public Vocabulary getParserGrammarVocabulary() + { + return DataQualityParserGrammar.VOCABULARY; + } + + @Override + public String getParserGrammarIdentifierInclusionTestCode(List keywords) + { + return "###DataQualityValidation\n" + + "DataQualityValidation " + ListAdapter.adapt(keywords).makeString("::") + "\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " age,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + " filter: p: Person[1]|$p.age >= 18;\n" + + "}"; + } + + @Test + public void testParserErrorForMandatoryFields() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " age,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + " filter: p: Person[1]|$p.age >= 18;\n" + + "}", "PARSER error at [2:1-15:1]: Field 'context' is required"); + + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " filter: p: Person[1]|$p.age >= 18;\n" + + "}", "PARSER error at [2:1-6:1]: Field 'validationTree' is required"); + } + + @Test + public void testParserForValidGrammar_withNoConstraints() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " age,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + " filter: p: Person[1]|$p.age >= 18;\n" + + "}"); + + } + + @Test + public void testParserForValidGrammar() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + " filter: p:Person[1] | p.age >= 18;\n" + + "}"); + } + + + +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java new file mode 100644 index 00000000000..aef7835a3eb --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java @@ -0,0 +1,104 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.dataquality.grammar.to; + +import org.finos.legend.engine.language.pure.grammar.test.TestGrammarRoundtrip; +import org.junit.Test; + +public class TestDataQualityRoundtrip extends TestGrammarRoundtrip.TestGrammarRoundtripTestSuite +{ + + @Test + public void testDataQuality_withModelAndStructuralConstraints() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + "}\n"); + } + + @Test + public void testDataQuality_withFilter() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " age,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + " filter: p: Person[1]|$p.age >= 18;\n" + + "}\n"); + } + + @Test + public void testDataQuality_withMappingAndRuntime() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " age,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + "}\n"); + } + + @Test + public void testDataQuality_withOnlyStructuralConstraints() + { + test("###DataQualityValidation\n" + + "DataQualityValidation meta::external::dataquality::PersonDataQualityValidation\n" + + "{\n" + + " context: fromDataSpace(meta::external::dataquality::PersonDataSpace, 'Local_Context');\n" + + " validationTree: $[\n" + + " Person{\n" + + " lastName,\n" + + " name,\n" + + " age,\n" + + " addresses{\n" + + " addressId\n" + + " }\n" + + " }\n" + + " ]$;\n" + + "}\n"); + } + + +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/pom.xml new file mode 100644 index 00000000000..bff5cd48212 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/pom.xml @@ -0,0 +1,53 @@ + + + + legend-engine-xts-dataquality + org.finos.legend.engine + 4.48.2-SNAPSHOT + + 4.0.0 + + legend-engine-xt-dataquality-protocol + jar + Legend Engine - XT - DataQuality - Protocol + + + 11 + 11 + + + + + org.finos.legend.engine + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-protocol + + + com.fasterxml.jackson.core + jackson-annotations + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + + + junit + junit + test + + + + + + \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQuality.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQuality.java new file mode 100644 index 00000000000..dd157d59e1e --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQuality.java @@ -0,0 +1,39 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.StereotypePtr; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TaggedValue; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; + +import java.util.Collections; +import java.util.List; + + +//------------------------------------------------------------ +// Should be generated out of the Pure protocol specification +//------------------------------------------------------------ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataQuality extends PackageableElement +{ + public List stereotypes = Collections.emptyList(); + public List taggedValues = Collections.emptyList(); + + public DataQualityExecutionContext context; + public DataQualityRootGraphFetchTree dataQualityRootGraphFetchTree; + public Lambda filter; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityExecutionContext.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityExecutionContext.java new file mode 100644 index 00000000000..2857666a542 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityExecutionContext.java @@ -0,0 +1,29 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = MappingAndRuntimeDataQualityExecutionContext.class, name = "mappingAndRuntimeDataQualityExecutionContext"), + @JsonSubTypes.Type(value = DataSpaceDataQualityExecutionContext.class, name = "dataSpaceDataQualityExecutionContext") +}) +public abstract class DataQualityExecutionContext +{ //add only stuff what users write + public SourceInformation sourceInformation; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityPropertyGraphFetchTree.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityPropertyGraphFetchTree.java new file mode 100644 index 00000000000..ff476995c73 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityPropertyGraphFetchTree.java @@ -0,0 +1,34 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.GraphFetchTreeVisitor; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree; + +import java.util.Collections; +import java.util.List; + +public class DataQualityPropertyGraphFetchTree extends PropertyGraphFetchTree +{ + public String _type = "dataQualityPropertyGraphFetchTree"; + + public List constraints = Collections.emptyList(); + + @Override + public T accept(GraphFetchTreeVisitor visitor) + { + throw new UnsupportedOperationException(); // TODO: revisit this + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java new file mode 100644 index 00000000000..da899dabb21 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java @@ -0,0 +1,61 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import org.eclipse.collections.api.block.function.Function0; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.protocol.pure.v1.extension.ProtocolSubTypeInfo; +import org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree; + +import java.util.List; +import java.util.Map; + +public class DataQualityProtocolExtension implements PureProtocolExtension +{ + public static String packageJSONType = "dataQualityValidation"; + + @Override + public MutableList group() + { + return org.eclipse.collections.impl.factory.Lists.mutable.with("PackageableElement", "DataQualityValidation"); + } + + @Override + public List>>> getExtraProtocolSubTypeInfoCollectors() + { + return Lists.fixedSize.with(() -> Lists.mutable.with( + ProtocolSubTypeInfo.newBuilder(PackageableElement.class) + .withSubtype(DataQuality.class, packageJSONType) + .build(), + ProtocolSubTypeInfo.newBuilder(RootGraphFetchTree.class) + .withSubtype(DataQualityRootGraphFetchTree.class, "dataQualityRootGraphFetchTree") + .build(), + ProtocolSubTypeInfo.newBuilder(PropertyGraphFetchTree.class) + .withSubtype(DataQualityPropertyGraphFetchTree.class, "dataQualityPropertyGraphFetchTree") + .build() + )); + } + + @Override + public Map, String> getExtraProtocolToClassifierPathMap() + { + return Maps.mutable.with(DataQuality.class, "meta::external::dataquality::DataQuality"); // TODO: check classifier path + } +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityRootGraphFetchTree.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityRootGraphFetchTree.java new file mode 100644 index 00000000000..0b75a8188f2 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityRootGraphFetchTree.java @@ -0,0 +1,34 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.GraphFetchTreeVisitor; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.RootGraphFetchTree; + +import java.util.Collections; +import java.util.List; + +public class DataQualityRootGraphFetchTree extends RootGraphFetchTree +{ + public String _type = "dataQualityRootGraphFetchTree"; + public List constraints = Collections.emptyList(); + + @Override + public T accept(GraphFetchTreeVisitor visitor) + { + throw new UnsupportedOperationException(); // TODO: revisit this + } + +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataSpaceDataQualityExecutionContext.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataSpaceDataQualityExecutionContext.java new file mode 100644 index 00000000000..f2c1852cb4e --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataSpaceDataQualityExecutionContext.java @@ -0,0 +1,25 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; + +public class DataSpaceDataQualityExecutionContext extends DataQualityExecutionContext +{ + public String _type = "dataSpaceDataQualityExecutionContext"; + public PackageableElementPointer dataSpace; + public String context; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/MappingAndRuntimeDataQualityExecutionContext.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/MappingAndRuntimeDataQualityExecutionContext.java new file mode 100644 index 00000000000..b45ab423fa0 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/MappingAndRuntimeDataQualityExecutionContext.java @@ -0,0 +1,24 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; + +public class MappingAndRuntimeDataQualityExecutionContext extends DataQualityExecutionContext +{ + public String _type = "mappingAndRuntimeDataQualityExecutionContext"; + public PackageableElementPointer mapping; + public PackageableElementPointer runtime; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityExecuteInput.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityExecuteInput.java new file mode 100644 index 00000000000..a69410f74be --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityExecuteInput.java @@ -0,0 +1,30 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.ParameterValue; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataQualityExecuteInput +{ + public AlloySDLC alloySDLC; + public String rootClass; + public String elementPath; + public List lambdaParameterValues; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityMetadata.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityMetadata.java new file mode 100644 index 00000000000..b3ce3c31c4b --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityMetadata.java @@ -0,0 +1,22 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.model; + +import java.util.List; + +public class DataQualityMetadata +{ + public List dqRules; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityRule.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityRule.java new file mode 100644 index 00000000000..c2aa16c81ee --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/model/DataQualityRule.java @@ -0,0 +1,31 @@ +// Copyright 2020 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.model; + +public class DataQualityRule +{ + public String constraintName; + public String constraintType; + public String constraintGrammar; + public String propertyPath; + + public DataQualityRule(String constraintName, String constraintType, String constraintGrammar, String propertyPath) + { + this.constraintName = constraintName; + this.constraintType = constraintType; + this.constraintGrammar = constraintGrammar; + this.propertyPath = propertyPath; + } +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension new file mode 100644 index 00000000000..d11a818a5cb --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension @@ -0,0 +1 @@ +org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityProtocolExtension \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml new file mode 100644 index 00000000000..69ec1e6e34e --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml @@ -0,0 +1,298 @@ + + + + legend-engine-xts-dataquality + org.finos.legend.engine + 4.48.2-SNAPSHOT + + 4.0.0 + + legend-engine-xt-dataquality-pure + jar + Legend Engine - XT - DataQuality - PAR/JAVA + + + 11 + 11 + + + + + + org.finos.legend.pure + legend-pure-maven-generation-par + + src/main/resources + ${legend.pure.version} + + + + + + + + + + ${project.basedir}/src/main/resources/core_dataquality.definition.json + + + + + generate-sources + + build-pure-jar + + + + + + org.finos.legend.engine + legend-engine-pure-code-compiled-functions + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + ${project.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + + + + + + + + + + + + + + org.finos.legend.pure + legend-pure-m2-dsl-store-pure + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-planExecution + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-javaGeneration-pure + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + org.finos.legend.pure + legend-pure-maven-generation-java + + + compile + + build-pure-compiled-jar + + + true + true + modular + true + + core_dataquality + + + + + + + org.finos.legend.pure + legend-pure-m2-dsl-store-pure + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-pure-code-compiled-functions + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + ${project.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + + + + + + + + + + + + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-planExecution + ${project.version} + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-legendCompiler + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-javaGeneration-pure + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + + + + + + + + + + + + + org.finos.legend.engine + legend-engine-pure-runtime-java-extension-compiled-functions-planExecution + + + org.finos.legend.pure + legend-pure-m4 + + + org.finos.legend.pure + legend-pure-m3-core + + + org.finos.legend.pure + legend-pure-m2-dsl-mapping-pure + + + org.finos.legend.engine + legend-engine-pure-platform-dsl-graph-java + + + org.finos.legend.pure + legend-pure-m2-dsl-graph-pure + + + + + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + + + + + + + org.finos.legend.engine + legend-engine-xt-data-space-pure-metamodel + + + + + + + org.eclipse.collections + eclipse-collections + + + org.eclipse.collections + eclipse-collections-api + + + org.finos.legend.engine + legend-engine-pure-platform-dsl-store-java + + + org.finos.legend.engine + legend-engine-pure-platform-java + + + org.finos.legend.engine + legend-engine-pure-platform-functions-java + + + + + + + + + org.finos.legend.pure + legend-pure-m2-functions-json-pure + test + + + com.fasterxml.jackson.core + jackson-annotations + test + + + com.fasterxml.jackson.core + jackson-databind + test + + + com.fasterxml.jackson.core + jackson-core + test + + + + \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/java/org/finos/legend/pure/code/core/CoreDataQualityCodeRepositoryProvider.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/java/org/finos/legend/pure/code/core/CoreDataQualityCodeRepositoryProvider.java new file mode 100644 index 00000000000..7659eb54396 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/java/org/finos/legend/pure/code/core/CoreDataQualityCodeRepositoryProvider.java @@ -0,0 +1,29 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core; + +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository; +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider; +import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository; + +public class CoreDataQualityCodeRepositoryProvider implements CodeRepositoryProvider +{ + @Override + public CodeRepository repository() + { + return GenericCodeRepository.build("core_dataquality.definition.json"); + } +} + diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider new file mode 100644 index 00000000000..109ba94c6d4 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider @@ -0,0 +1 @@ +org.finos.legend.pure.code.core.CoreDataQualityCodeRepositoryProvider \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json new file mode 100644 index 00000000000..3e2ff32569b --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json @@ -0,0 +1,17 @@ +{ + "name": "core_dataquality", + "pattern": "(meta::external::dataquality|meta::protocols)(::.*)?", + "dependencies": [ + "core", + "core_relational", + "core_functions", + "core_relational", + "core_data_space_metamodel", + "platform", + "platform_dsl_store", + "platform_dsl_graph", + "platform_dsl_mapping", + "platform_store_relational", + "platform_functions" + ] +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure new file mode 100644 index 00000000000..7af27d2240c --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure @@ -0,0 +1,458 @@ +import meta::pure::dataQuality::*; +import meta::pure::metamodel::serialization::grammar::*; +import meta::pure::lineage::scanProperties::*; +import meta::external::dataquality::*; +import meta::pure::graphFetch::*; +import meta::pure::functions::collection::*; +import meta::pure::functions::meta::*; +import meta::pure::graphFetch::execution::*; +import meta::core::runtime::*; +import meta::pure::executionPlan::*; +import meta::pure::mapping::*; +import meta::pure::functions::boolean::*; +import meta::pure::metamodel::constraint::*; +import meta::pure::lineage::scanProperties::propertyTree::*; + + + +function meta::external::dataquality::executeDataQualityValidation(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*]): LambdaFunction[1] +{ + // 1. enrich tree with selected constraint properties + let enrichedTree = $dataquality.validationTree->ensureFunctionRequirementsForDataQuality($dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); + + // 2. build query + let getAll = $dataquality.validationTree.class->createGetAll(); + let getExpr = if ($dataquality.filter->isNotEmpty(), + | $dataquality.validationTree.class->generateFilterQuery($getAll, $dataquality.filter->toOne());, + | $getAll); + let dqRootConstraints = $dataquality.validationTree.constraints; + let constraintQueryExpr = $dataquality.validationTree.class->meta::external::dataquality::generateConstraintsNegatedORQuery($getExpr, ^List(values=$dqRootConstraints)); + let limitQueryExpr = if ($limit->isNotEmpty(), + | ^SimpleFunctionExpression(func=take_T_MANY__Integer_1__T_MANY_, + parametersValues=[$constraintQueryExpr, ^InstanceValue(values=$limit->toOne(), genericType=^GenericType(rawType=Integer), multiplicity=PureOne)], + functionName=take_T_MANY__Integer_1__T_MANY_.name, // todo: function ref+name + genericType=^GenericType(rawType=$dataquality.validationTree.class), + multiplicity = ZeroMany, + importGroup=system::imports::coreImport), + | $constraintQueryExpr); + let graphFetchChecked = ^SimpleFunctionExpression(func=graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_ , + parametersValues=[$limitQueryExpr, ^InstanceValue(values=$enrichedTree, genericType=^GenericType(rawType=RootGraphFetchTree), multiplicity=PureOne)], + functionName=graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_.name, + genericType=^GenericType(rawType=Checked, typeArguments=^GenericType(rawType=$dataquality.validationTree.class)), + multiplicity = ZeroMany, + importGroup=system::imports::coreImport); + // 2.1 + let serialized = ^SimpleFunctionExpression(func=serialize_T_MANY__RootGraphFetchTree_1__String_1_, + functionName=serialize_T_MANY__RootGraphFetchTree_1__String_1_.name, + importGroup=system::imports::coreImport, + genericType=^GenericType(rawType=String), + multiplicity=ZeroMany, + parametersValues=[$graphFetchChecked, ^InstanceValue(values=$enrichedTree, genericType=^GenericType(rawType=RootGraphFetchTree), multiplicity=PureOne)])->evaluateAndDeactivate(); + + // 2.2 extract mapping and runtime here + let mappingAndRuntime = $dataquality.context->getMappingAndRuntime(); + let deactivatedMapping = ^InstanceValue(values=$mappingAndRuntime.first->evaluateAndDeactivate(), genericType=^GenericType(rawType=Mapping), multiplicity=PureOne) ; + let deactivatedRuntime = ^InstanceValue(values=$mappingAndRuntime.second->evaluateAndDeactivate(), genericType=^GenericType(rawType=NonExecutableValueSpecification), multiplicity=PureOne); + + let from = ^SimpleFunctionExpression(func=from_T_m__Mapping_1__Runtime_1__T_m_, + functionName=from_T_m__Mapping_1__Runtime_1__T_m_.name, + importGroup=system::imports::coreImport, + genericType=^GenericType(rawType=String), + multiplicity=ZeroMany, + parametersValues=[$serialized, $deactivatedMapping, $deactivatedRuntime])->evaluateAndDeactivate(); + + // 3. build lambda + if ($dataquality.validationTree.class->meta::pure::milestoning::isTemporal(), + | $dataquality.validationTree.class->createLambdaParameterized($from), + | let lambda = {| ''}; + ^$lambda(expressionSequence = [$from]); + ); +} + + +function meta::external::dataquality::ensureFunctionRequirementsForDataQuality(node:GraphFetchTree[1], constraints:Constraint[*], class:Class[1], processed:Class[*], ensureConstraintsForSubTrees:Boolean[1]): GraphFetchTree[1] +{ + let constraintResult = pathsForConstraintFunctions($class, $constraints.functionDefinition->concatenate($constraints.messageFunction)); + let qualifiedPropertyPaths = $constraintResult->filter(path| $path.values->exists(x| $x.property->instanceOf(QualifiedProperty))); //QualifiedProperty/derived property - methods within a class - first try with inline properties, milestoning properties - used in loans usecase + let inlinedPropertyTree = $constraintResult->meta::pure::lineage::scanProperties::propertyTree::buildPropertyTree()->meta::pure::lineage::scanProperties::inlineQualifiedPropertyNodes(); + let inlinedGraphTree = $inlinedPropertyTree->propertyTreeToGraphFetchTree($class); + let inlinedPropertyGraphTrees = $inlinedGraphTree.subTrees->cast(@PropertyGraphFetchTree); + let withFoundProperties = $node->addSubTrees($inlinedPropertyGraphTrees); + let updatedForClass = $qualifiedPropertyPaths->fold({path, gt| $gt->meta::external::dataquality::recordQualifiedProperties($path)}, $withFoundProperties); + let updatedProcessed = $processed->add($class); + + if($ensureConstraintsForSubTrees, + {| + let newSubTrees = $updatedForClass.subTrees->map({st| + let returns = if($st->cast(@PropertyGraphFetchTree).subType->isEmpty(), + | $st->cast(@PropertyGraphFetchTree).property->functionReturnType().rawType->toOne(), + | $st->cast(@PropertyGraphFetchTree).subType->toOne() + ); + if($returns->instanceOf(Class) && !$updatedProcessed->contains($returns), + | $st->ensureFunctionRequirementsForDataQuality($constraints, $returns->cast(@Class), $updatedProcessed, $ensureConstraintsForSubTrees), + | $st + ); + }); + + ^$updatedForClass(subTrees=$newSubTrees); + }, + | $updatedForClass + ); +} + +function <> meta::external::dataquality::createGetAll(c: Class[1]):FunctionExpression[1] +{ + let getAllExpression = ^SimpleFunctionExpression + ( + func = getAll_Class_1__T_MANY_, + functionName = getAll_Class_1__T_MANY_.name, + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $c), + multiplicity = ZeroMany, + parametersValues = ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + multiplicity = PureOne, + values = $c + ))->evaluateAndDeactivate(); + let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); + let lambda = {|[]}; + ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); +} + +function <> meta::external::dataquality::createGetAllParameterized(c: Class[1]):FunctionExpression[1] +{ + if ($c->meta::pure::milestoning::isBiTemporal(), + | $c->createGetAllBiTemporal(), + | $c->createGetAllUniTemporal()); +} + +function <> meta::external::dataquality::createGetAllUniTemporal(c: Class[1]):FunctionExpression[1] +{ + if ($c->meta::pure::milestoning::isProcessingTemporal(), + | $c->createGetAllProcessingTemporal(), + | $c->createGetAllBusinessTemporal()); +} + +function <> meta::external::dataquality::createLambdaParameterized(c: Class[1], from: SimpleFunctionExpression[1]):LambdaFunction[1] +{ + if ($c->meta::pure::milestoning::isBiTemporal(), + | $from->createLambdaBiTemporal(), + | $c->createLambdaUniTemporal($from)); +} + +function <> meta::external::dataquality::createLambdaUniTemporal(c: Class[1], from: SimpleFunctionExpression[1]):LambdaFunction[1] +{ + if ($c->meta::pure::milestoning::isProcessingTemporal(), + | $from->createLambdaProcessingTemporal(), + | $from->createLambdaBusinessTemporal()); +} + +function <> meta::external::dataquality::createLambdaProcessingTemporal(from: SimpleFunctionExpression[1]):LambdaFunction[1] +{ + let lambda = {processingDate: Date[1] | ''}; + ^$lambda(expressionSequence = [$from]); +} + +function <> meta::external::dataquality::createLambdaBusinessTemporal(from: SimpleFunctionExpression[1]):LambdaFunction[1] +{ + let lambda = {businessDate: Date[1] | ''}; + ^$lambda(expressionSequence = [$from]); +} + +function <> meta::external::dataquality::createLambdaBiTemporal(from: SimpleFunctionExpression[1]):LambdaFunction[1] +{ + let lambda = {businessDate: Date[1], processingDate: Date[1] | ''}; + ^$lambda(expressionSequence = [$from]); +} + + +function <> meta::external::dataquality::createGetAllBiTemporal(c: Class[1]):FunctionExpression[1] +{ + let getAllExpression = ^SimpleFunctionExpression + ( + func = getAll_Class_1__Date_1__Date_1__T_MANY_, + functionName = getAll_Class_1__Date_1__Date_1__T_MANY_.name, + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $c), + multiplicity = ZeroMany, + parametersValues = [ + ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + multiplicity = PureOne, + values = $c + ), + ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate(), + ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate() + ] + )->evaluateAndDeactivate(); + let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); + let lambda = {|[]}; + ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); +} + +function <> meta::external::dataquality::createGetAllBusinessTemporal(c: Class[1]):FunctionExpression[1] +{ + let getAllExpression = ^SimpleFunctionExpression + ( + func = getAll_Class_1__Date_1__T_MANY_, + functionName = getAll_Class_1__Date_1__T_MANY_.name, + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $c), + multiplicity = ZeroMany, + parametersValues = [ + ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + multiplicity = PureOne, + values = $c + ), + ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne) + ] + )->evaluateAndDeactivate(); + let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); + let lambda = {|[]}; + ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); +} + +function <> meta::external::dataquality::createGetAllProcessingTemporal(c: Class[1]):FunctionExpression[1] +{ + let getAllExpression = ^SimpleFunctionExpression + ( + func = getAll_Class_1__Date_1__T_MANY_, + functionName = getAll_Class_1__Date_1__T_MANY_.name, + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $c), + multiplicity = ZeroMany, + parametersValues = [ + ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + multiplicity = PureOne, + values = $c + ), + ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne) + ] + )->evaluateAndDeactivate(); + let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); + let lambda = {|[]}; + ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); +} + +function meta::external::dataquality::generateFilterQuery(c:Class[1], f: FunctionExpression[1], filter:LambdaFunction<{T[1]->Boolean[1]}>[1]):FunctionExpression[1] { + let dummyLambda = {|'ok'}; + ^SimpleFunctionExpression + ( + func = filter_T_MANY__Function_1__T_MANY_, + multiplicity = ZeroMany, + genericType = ^GenericType(rawType = $c), + importGroup = system::imports::coreImport, + parametersValues = + [ $f, + ^InstanceValue ( + genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='that', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), + multiplicity = PureOne, + values = ^$dummyLambda + ( + classifierGenericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='that', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), + expressionSequence = $filter.expressionSequence->evaluateAndDeactivate() + ) + ) + ] + )->evaluateAndDeactivate(); +} + + + +function meta::external::dataquality::generateConstraintsNegatedORQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] { + if ($constraints.values->size() == 1, + | $c->generateConstraintNegatedQuery($f, $constraints.values->at(0)) , + | $c->generateORNegatedQuery($f, $constraints) + ); +} + + +function meta::external::dataquality::generateORNegatedQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] +{ + let c1 = $constraints.values->at(0); + let c2 = $constraints.values->at(1); + let c1_negatedExprSequence = $c1.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); + let c2_negatedExprSequence = $c2.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); + let c1_c2_or_expr = $c1_negatedExprSequence->orFunctionExpression($c2_negatedExprSequence); + + let final_expr = $constraints->slice(2, $constraints->size()).values->fold({c3, curr_or_exp | + let c3_negatedExprSequence = $c3.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); + $c3_negatedExprSequence->orFunctionExpression($curr_or_exp);}, $c1_c2_or_expr); + + let dummyLambda = {|'ok'}; + ^SimpleFunctionExpression + ( + func = filter_T_MANY__Function_1__T_MANY_, + multiplicity = ZeroMany, + genericType = ^GenericType(rawType = $c), + importGroup = system::imports::coreImport, + parametersValues = + [ $f, + ^InstanceValue ( + genericType = $c1.functionDefinition.classifierGenericType->toOne(), + multiplicity = PureOne, + values = ^$dummyLambda + ( + classifierGenericType = $c1.functionDefinition.classifierGenericType, + expressionSequence = $final_expr->evaluateAndDeactivate() + ) + ) + ] + )->evaluateAndDeactivate(); + +} + + +function meta::external::dataquality::generateConstraintNegatedQuery(c:Class[1], f: FunctionExpression[1], constraint: Constraint[1]):FunctionExpression[1] +{ + let constraintExprSequence = $constraint.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne(); + let negatedExprSequence = $constraintExprSequence->negatedFunctionExpression(); + let dummyLambda = {|'ok'}; + ^SimpleFunctionExpression + ( + func = filter_T_MANY__Function_1__T_MANY_, + multiplicity = ZeroMany, + genericType = ^GenericType(rawType = $c), + importGroup = system::imports::coreImport, + parametersValues = + [ $f, + ^InstanceValue ( + genericType = $constraint.functionDefinition.classifierGenericType->toOne(), + multiplicity = PureOne, + values = ^$dummyLambda + ( + classifierGenericType = $constraint.functionDefinition.classifierGenericType, + expressionSequence = $negatedExprSequence + ) + ) + ] + )->evaluateAndDeactivate(); +} + +function <> {doc.doc = 'Genrates a "!$f" query'} +meta::external::dataquality::negatedFunctionExpression(f: ValueSpecification[1]):ValueSpecification[1] +{ + if($f->instanceOf(SimpleFunctionExpression) && $f->cast(@SimpleFunctionExpression).func->evaluateAndDeactivate() == not_Boolean_1__Boolean_1_, + | $f->cast(@SimpleFunctionExpression).parametersValues->evaluateAndDeactivate()->at(0), + | ^SimpleFunctionExpression + ( + func = not_Boolean_1__Boolean_1_, //native function meta::pure::functions::boolean::not(bool:Boolean[1]):Boolean[1]; + multiplicity = PureOne, + genericType = ^GenericType(rawType = Boolean), + importGroup = system::imports::coreImport, + parametersValues = $f + )->evaluateAndDeactivate() + ) +} + +function {doc.doc = 'Generates a OR query'} +meta::external::dataquality::orFunctionExpression(f1: ValueSpecification[1], f2: ValueSpecification[1]):ValueSpecification[1] +{ +^SimpleFunctionExpression + ( + func = meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_ , // meta::pure::functions::boolean::or **use OR** + multiplicity = PureOne, + genericType = ^GenericType(rawType = Boolean), // check types + importGroup = system::imports::coreImport, + parametersValues = [$f1, $f2] + )->evaluateAndDeactivate(); +} + +function {doc.doc = 'Generates a AND query'} +meta::external::dataquality::andFunctionExpression(f1: ValueSpecification[1], f2: ValueSpecification[1]):ValueSpecification[1] +{ +^SimpleFunctionExpression + ( + func = meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_ , + multiplicity = PureOne, + genericType = ^GenericType(rawType = Boolean), + importGroup = system::imports::coreImport, + parametersValues = [$f1, $f2] + )->evaluateAndDeactivate(); +} + +function meta::external::dataquality::getMappingAndRuntime(context: DataQualityExecutionContext[1]):Pair[1] { + if($context->instanceOf(MappingAndRuntimeDataQualityExecutionContext), + | let mappingAndRuntimeContext = $context->cast(@MappingAndRuntimeDataQualityExecutionContext); + ^Pair(first=$mappingAndRuntimeContext.mapping, second=$mappingAndRuntimeContext.runtime);, + | let dataSpaceDataQualityExecutionContext = $context->cast(@DataSpaceDataQualityExecutionContext); + let dataSpaceExecutionContext = $dataSpaceDataQualityExecutionContext.dataSpace.executionContexts->filter(execContext| $execContext.name == $dataSpaceDataQualityExecutionContext.contextName)->toOne(); + ^Pair(first=$dataSpaceExecutionContext.mapping, second=$dataSpaceExecutionContext.defaultRuntime.runtimeValue); + ); +} + +function meta::external::dataquality::recordQualifiedProperties(tree:GraphFetchTree[1], path:List[1]): GraphFetchTree[1] +{ + if($path.values->isEmpty(), + | $tree, + {| + let head = $path.values->at(0); + let tail = if($head.property->instanceOf(QualifiedProperty), + | list($head.nestedQualifierReturn.values->tail()->concatenate($path.values->tail())), + | list($path.values->tail()) + ); + let nextProperty = if($head.property->instanceOf(QualifiedProperty), + | $head.nestedQualifierReturn.values->first().property, + | $head.property + ); + let withQp = if($head.property->instanceOf(QualifiedProperty), + {| + let qp = $head.property->cast(@QualifiedProperty); + $tree->match([ + ergft: ExtendedRootGraphFetchTree[1] | ^$ergft(requiredQualifiedProperties=$ergft.requiredQualifiedProperties->concatenate($qp)->removeDuplicates()), + epgft: ExtendedPropertyGraphFetchTree[1] | ^$epgft(requiredQualifiedProperties=$epgft.requiredQualifiedProperties->concatenate($qp)->removeDuplicates()), + rgft : RootGraphFetchTree[1] | ^ExtendedRootGraphFetchTree(requiredQualifiedProperties=$qp, class=$rgft.class, subTrees=$rgft.subTrees), + pgft : PropertyGraphFetchTree[1] | ^ExtendedPropertyGraphFetchTree(requiredQualifiedProperties=$qp, property=$pgft.property, subTrees=$pgft.subTrees) + ]); + }, + | $tree; + ); + + ^$withQp(subTrees=$tree.subTrees->cast(@PropertyGraphFetchTree)->map(st|if($st.property == $nextProperty, |$st->meta::external::dataquality::recordQualifiedProperties($tail), |$st))); + } + ); +} + +function meta::external::dataquality::getEnrichedTreeForStructuralValidations(validationTree:meta::external::dataquality::DataQualityRootGraphFetchTree[1]): RootGraphFetchTree[1] +{ + // 1. enrich tree with selected constraint properties + $validationTree->ensureFunctionRequirementsForDataQuality($validationTree.constraints, $validationTree.class, [], true)->cast(@RootGraphFetchTree); +} + +function meta::external::dataquality::generateDQMetaDataForDQValidation(dataquality:meta::external::dataquality::DataQuality[1]): DataQualityRule[*] +{ + let enrichedTree = ensureFunctionRequirementsForDataQuality($dataquality.validationTree, $dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); + let dqRules = $dataquality.validationTree->nodeToDqRule(true, '')->concatenate($enrichedTree->nodeToDqRule(false, '')); + $dqRules; +} + +function meta::external::dataquality::nodeToDqRule(node:GraphFetchTree[1], processOnlyConstraints:Boolean[1], path:String[1]):DataQualityRule[*] +{ + let dqRules = $node->match([ + dr: DataQualityRootGraphFetchTree[1] | $dr->rootNodeToDqRule($processOnlyConstraints), + dp: DataQualityPropertyGraphFetchTree[1] | $dp->propertyNodeToDqRule($processOnlyConstraints, $path), + r : RootGraphFetchTree[1] | $r->rootNodeToDqRule($processOnlyConstraints), + p : PropertyGraphFetchTree[1] | $p->propertyNodeToDqRule($processOnlyConstraints, $path) + ]); + + $dqRules->concatenate($node.subTrees->map(st|$st->nodeToDqRule($processOnlyConstraints, $dqRules->at(0).propertyPath))); +} + +function <> meta::external::dataquality::rootNodeToDqRule(node:RootGraphFetchTree[1], processOnlyConstraints: Boolean[1]):DataQualityRule[*] +{ + if ( $processOnlyConstraints, + | $node->cast(@DataQualityRootGraphFetchTree).constraints->map(c|$c->constraintToDqRule($node.class.name->toOne())), + | ^DataQualityRule(constraintName=$node.class.name->toOne(), constraintGrammar='Class', constraintType='Alloy_Class_Validation', propertyPath=$node.class.name->toOne()) + ); +} + +function <> meta::external::dataquality::propertyNodeToDqRule(node:PropertyGraphFetchTree[1], processOnlyConstraints: Boolean[1], path:String[1]):DataQualityRule[*] +{ + if ( $processOnlyConstraints, + | $node->cast(@DataQualityPropertyGraphFetchTree).constraints->map(c|$c->constraintToDqRule([$path, $node.property.name->toOne()]->joinStrings('::'))), + | ^DataQualityRule(constraintName=$node.property.name->toOne(), constraintGrammar=$node.property.multiplicity->printMultiplicity(), constraintType='Alloy_Structural_Validation', propertyPath=[$path, $node.property.name->toOne()]->joinStrings('::')) + ); +} + +function <> meta::external::dataquality::constraintToDqRule(constraint:Constraint[1], path:String[1]):DataQualityRule[1] +{ + ^DataQualityRule(constraintName=$constraint.name->toOne(), constraintGrammar=$constraint.functionDefinition->meta::pure::metamodel::serialization::grammar::printFunctionDefinitionExpressions('')->toOne(), constraintType='Alloy_Constraint_Validation', propertyPath=$path); +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure new file mode 100644 index 00000000000..4ddc27f5c70 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure @@ -0,0 +1,48 @@ +import meta::pure::metamodel::dataSpace::*; +import meta::pure::graphFetch::*; +import meta::pure::metamodel::constraint::*; +import meta::core::runtime::*; + + + +Class meta::external::dataquality::DataQualityRootGraphFetchTree extends RootGraphFetchTree +{ + constraints: Constraint[*]; + +} + +Class meta::external::dataquality::DataQualityPropertyGraphFetchTree extends PropertyGraphFetchTree +{ + constraints: Constraint[*]; +} + +Class meta::external::dataquality::DataQuality extends PackageableElement +{ + context: meta::external::dataquality::DataQualityExecutionContext[1]; + validationTree: meta::external::dataquality::DataQualityRootGraphFetchTree[1]; + filter : LambdaFunction<{T[1]->Boolean[1]}>[0..1]; //typed params +} + +Class meta::external::dataquality::DataQualityExecutionContext +{ +} + +Class meta::external::dataquality::MappingAndRuntimeDataQualityExecutionContext extends meta::external::dataquality::DataQualityExecutionContext +{ + mapping: meta::pure::mapping::Mapping[1]; + runtime: meta::core::runtime::Runtime[1]; +} + +Class meta::external::dataquality::DataSpaceDataQualityExecutionContext extends meta::external::dataquality::DataQualityExecutionContext +{ + dataSpace: DataSpace[1]; + contextName: String[1]; +} + +Class meta::external::dataquality::DataQualityRule +{ + constraintName: String[1]; + constraintType : String[1]; + constraintGrammar: String[1]; + propertyPath: String[1]; +} \ No newline at end of file diff --git a/legend-engine-xts-dataquality/pom.xml b/legend-engine-xts-dataquality/pom.xml new file mode 100644 index 00000000000..d846ca27d9c --- /dev/null +++ b/legend-engine-xts-dataquality/pom.xml @@ -0,0 +1,31 @@ + + + + legend-engine + org.finos.legend.engine + 4.48.2-SNAPSHOT + + 4.0.0 + + legend-engine-xts-dataquality + pom + Legend Engine - XTS - DataQuality + + + + + legend-engine-xt-dataquality-pure + legend-engine-xt-dataquality-grammar + legend-engine-xt-dataquality-protocol + legend-engine-xt-dataquality-api + legend-engine-xt-dataquality-compiler + + + + 11 + 11 + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index a5cabd9daf0..e7d32c74bc7 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,7 @@ legend-engine-xts-data-space legend-engine-xts-changetoken legend-engine-xts-generation + legend-engine-xts-dataquality legend-engine-xts-openapi