Skip to content

Commit

Permalink
New Data Quality validation module to enable users to define data qua…
Browse files Browse the repository at this point in the history 
…lity specifications (#2933)

New Data quality module to enable users to define data quality specs in legend
  • Loading branch information
@keenkeystrokes
keenkeystrokes authored Jun 27, 2024
1 parent 6e6a971 commit 31fe17b
Show file tree
Hide file tree
Showing 52 changed files with 7,162 additions and 0 deletions.
25 changes: 25 additions & 0 deletions legend-engine-config/legend-engine-server/legend-engine-server-http-server/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,31 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-snowflakeApp-protocol</artifactId>
</dependency>
<!-- DataQuality -->
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-compiler</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-grammar</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-protocol</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<!-- DataQuality -->
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-bigqueryFunction-compiler</artifactId>
Expand Down
5 changes: 5 additions & 0 deletions ...legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.finos.legend.engine.external.shared.format.model.api.ExternalFormats;
import org.finos.legend.engine.functionActivator.api.FunctionActivatorAPI;
import org.finos.legend.engine.generation.artifact.api.ArtifactGenerationExtensionApi;
import org.finos.legend.engine.language.dataquality.api.DataQualityExecute;
import org.finos.legend.engine.language.hostedService.api.HostedServiceService;
import org.finos.legend.engine.language.memsql.api.MemSqlFunctionService;
import org.finos.legend.engine.language.pure.compiler.api.Compile;
Expand Down Expand Up @@ -416,6 +417,10 @@ public void run(T serverConfiguration, Environment environment)
environment.jersey().register(new LineageAnalytics(modelManager));
environment.jersey().register(new StoreEntitlementAnalytics(modelManager, entitlementServiceExtensions));

// DataQuality
environment.jersey().register(new DataQualityExecute(modelManager, planExecutor, routerExtensions, generatorExtensions.flatCollect(PlanGeneratorExtension::getExtraPlanTransformers), serverConfiguration.metadataserver, null));


// Testable
environment.jersey().register(new TestableApi(modelManager));

Expand Down
1 change: 1 addition & 0 deletions ...ne-pure-ide-light-http-server/src/main/java/org/finos/legend/engine/ide/PureIDELight.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ protected MutableList<RepositoryCodeStorage> buildRepositories(SourceLocationCon
.with(this.buildCore("legend-engine-xts-arrow/legend-engine-xt-arrow-pure", "external-format-arrow"))
.with(this.buildCore("legend-engine-xts-relationalai/legend-engine-xt-relationalai-pure", "external-query-relationalai"))
.with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure", "external_test_connection"))
.with(this.buildCore("legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure", "dataquality"))
;
}

Expand Down
275 changes: 275 additions & 0 deletions legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,275 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>legend-engine-xts-dataquality</artifactId>
<groupId>org.finos.legend.engine</groupId>
<version>4.48.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>legend-engine-xt-dataquality-api</artifactId>
<packaging>jar</packaging>
<name>Legend Engine - XT - DataQuality - Api</name>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>

<dependencies>

<!-- PURE -->
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-runtime-java-engine-compiled</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-code-compiled-core</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m3-core</artifactId>
</dependency>
<!-- PURE -->

<!-- ENGINE -->
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-shared-core</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-language-pure-modelManager</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-protocol-pure</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-language-pure-compiler</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-language-pure-compiler</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<!-- ENGINE -->
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-javaPlatformBinding-pure</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-code-core-extension</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-pure</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-grammar</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-execution</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-execution-http-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-language-pure-modelManager-sdlc</artifactId>
</dependency>
<!-- ENGINE -->
<!-- ECLIPSE -->
<dependency>
<groupId>org.eclipse.collections</groupId>
<artifactId>eclipse-collections-api</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.collections</groupId>
<artifactId>eclipse-collections</artifactId>
</dependency>
<!-- ECLIPSE -->

<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-data-space-compiler</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-trino-grammar</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
</dependency>
<!-- Servlet -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<!-- Servlet -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>

<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-protocol</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-pure</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-grammar</artifactId>
<scope>runtime</scope>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-dataquality-compiler</artifactId>
<scope>runtime</scope>
<version>${project.version}</version>
</dependency>


<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-dsl-graph-pure</artifactId>
</dependency>


<!-- JACKSON -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- JACKSON -->




<!-- ECLIPSE COLLECTIONS -->
<!-- ECLIPSE COLLECTIONS -->

<!-- ANNOTATIONS -->
<dependency>
<groupId>org.pac4j.jax-rs</groupId>
<artifactId>core</artifactId>
</dependency>
<dependency>
<groupId>org.pac4j</groupId>
<artifactId>pac4j-core</artifactId>
</dependency>
<!-- ANNOTATIONS -->

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<!-- OPEN TRACING -->
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-util</artifactId>
</dependency>
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-api</artifactId>
</dependency>
<!-- OPEN TRACING -->

<!-- ANNOTATIONS -->
<dependency>
<groupId>javax.ws.rs</groupId>
<artifactId>javax.ws.rs-api</artifactId>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
</dependency>
<!-- ANNOTATIONS -->

<!-- LOG -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>test</scope>
</dependency>
<!-- LOG -->

<!-- TEST -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-shared-core</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-generation</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-identity-pac4j</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-identity-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-language-pure-dsl-generation</artifactId>
</dependency>
<dependency>
<groupId>com.github.tomakehurst</groupId>
<artifactId>wiremock-jre8</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>test</scope>
</dependency>

</dependencies>
</project>
Loading

0 comments on commit 31fe17b

Please sign in to comment.