Skip to content

Commit

Permalink
Introduce legend ingest modules (#3268)
Browse files Browse the repository at this point in the history 
* [maven-release-plugin] prepare release legend-engine-4.66.0

* [maven-release-plugin] prepare for next development iteration

* Barrier DSL Initial Commmit

* Barrier DSL Initial Commmit

* Add meaningful fail

* Protocol Generation

* Update pom.xml

* Update pom.xml

* Update pom.xml

---------

Co-authored-by: FINOS Administrator <[email protected]>
Co-authored-by: Mauricio Uyaguari <[email protected]>
  • Loading branch information
@hbalasu2 @finos-admin @MauricioUyaguari
3 people authored Nov 23, 2024
1 parent 850474b commit 7bb8085
Show file tree
Hide file tree
Showing 12 changed files with 578 additions and 0 deletions.
89 changes: 89 additions & 0 deletions legend-engine-xts-ingest/legend-engine-xt-ingest-protocol/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xts-ingest</artifactId>
<version>4.67.1-SNAPSHOT</version>
</parent>

<artifactId>legend-engine-xt-ingest-protocol</artifactId>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-dependencies</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<configuration>
<addResourcesToClasspath>true</addResourcesToClasspath>
</configuration>
<executions>
<execution>
<id>generate metamodel</id>
<phase>generate-sources</phase>
<goals>
<goal>java</goal>
</goals>
<configuration>
<includePluginDependencies>true</includePluginDependencies>
<mainClass>org.finos.legend.engine.protocol.generation.GenerateMetaClasses</mainClass>
<arguments>
<argument>core_ingest_metamodel.protocol.json</argument>
<argument>${project.build.directory}/generated-sources/</argument>
</arguments>
<classpathScope>test</classpathScope>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-protocol-generation</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-ingest-pure</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/target/generated-sources/</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
7 changes: 7 additions & 0 deletions ...t/legend-engine-xt-ingest-protocol/src/main/resources/core_ingest_metamodel.protocol.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"purePackage": "meta::external::ingest::metamodel",
"javaPackage": "org.finos.legend.engine.protocol.ingest.metamodel",
"elementsToBeExcluded": [
"meta::external::ingest::metamodel::IngestConfig"
]
}
168 changes: 168 additions & 0 deletions legend-engine-xts-ingest/legend-engine-xt-ingest-pure/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xts-ingest</artifactId>
<version>4.67.1-SNAPSHOT</version>
</parent>

<artifactId>legend-engine-xt-ingest-pure</artifactId>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>

<packaging>jar</packaging>
<name>Legend Engine - XT - INGEST - PAR/JAVA</name>

<build>
<plugins>
<plugin>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-maven-generation-par</artifactId>
<configuration>
<sourceDirectory>src/main/resources</sourceDirectory>
<purePlatformVersion>${legend.pure.version}</purePlatformVersion>
<repositories>
<repository>core_ingest</repository>
</repositories>
<extraRepositories>
<extraRepository>${project.basedir}/src/main/resources/core_ingest.definition.json</extraRepository>
</extraRepositories>
</configuration>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>build-pure-jar</goal>
</goals>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-analytics-lineage-pure</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-functions-unclassified-pure</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-functionActivator-pure</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-dsl-diagram-grammar</artifactId>
<version>${legend.pure.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-core-pure</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-maven-generation-java</artifactId>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>build-pure-compiled-jar</goal>
</goals>
<configuration>
<generateSources>true</generateSources>
<preventJavaCompilation>true</preventJavaCompilation>
<generationType>modular</generationType>
<useSingleDir>true</useSingleDir>
<repositories>
<repository>core_ingest</repository>
</repositories>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-functions-unclassified-pure</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-functionActivator-pure</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-dsl-diagram-grammar</artifactId>
<version>${legend.pure.version}</version>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-core-pure</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>

<dependencies>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m3-core</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m4</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-runtime-java-engine-compiled</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-platform-java</artifactId>
</dependency>

<dependency>
<groupId>org.eclipse.collections</groupId>
<artifactId>eclipse-collections-api</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.collections</groupId>
<artifactId>eclipse-collections</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-platform-dsl-store-java</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-platform-store-relational-java</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-relationalStore-core-pure</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-store-relational-pure</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-code-compiled-core</artifactId>
</dependency>

</dependencies>
</project>
29 changes: 29 additions & 0 deletions ...-pure/src/main/java/org/finos/legend/pure/code/core/CoreIngestCodeRepositoryProvider.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright 2023 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.pure.code.core;

import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository;
import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider;
import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository;

public class CoreIngestCodeRepositoryProvider implements CodeRepositoryProvider
{
@Override
public CodeRepository repository()
{
return GenericCodeRepository.build("core_ingest.definition.json");
}
}

1 change: 1 addition & 0 deletions ...vices/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
org.finos.legend.pure.code.core.CoreIngestCodeRepositoryProvider
12 changes: 12 additions & 0 deletions ...ne-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/core_ingest.definition.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"name": "core_ingest",
"pattern": "(meta::external::ingest)(::.*)?",
"dependencies": [
"core",
"platform",
"platform_dsl_store",
"platform_store_relational",
"core_functions_json",
"core_relational"
]
}
30 changes: 30 additions & 0 deletions ...ngine-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/core_ingest/barrier.pure
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import meta::external::ingest::barrier::*;
import meta::external::ingest::metamodel::*;

function meta::external::ingest::barrier::isDatasetAvailable<T>(ingestConfig: IngestConfig<T>[1], prevPublishedWatermark: Watermark[1], currentStateWatermark: Watermark[1]):Boolean[1]
{
let prevHighBatchId = getBatchId($ingestConfig.datasetName, $prevPublishedWatermark);
let currentBatchId = getBatchId($ingestConfig.datasetName, $currentStateWatermark);
($currentBatchId > $prevHighBatchId) ;
}

function meta::external::ingest::barrier::isDatasetAvailable<T>(ingestConfig: IngestConfig<T>[1],filterFunc:Function<{T[1]->Boolean[1]}>[1], prevPublishedWatermark: Watermark[1], currentStateWatermark: Watermark[1]):Boolean[1]
{
let prevHighBatchId = getBatchId($ingestConfig.datasetName, $prevPublishedWatermark);
let currentBatchId = getBatchId($ingestConfig.datasetName, $currentStateWatermark);

//TODO : Change to T , likely need to uplift getAll to handle T properly
meta::pure::functions::collection::getAll(BatchMetadata)
->filter(x | $x.datasetName == $ingestConfig.datasetName)
->filter(y | ($y.batchId > $prevHighBatchId) && ($y.batchId <= $currentBatchId))
->cast(@T)
->filter($filterFunc)
->isNotEmpty();
}


function <<access.private>> meta::external::ingest::barrier::getBatchId(datasetName: String[1], watermark: Watermark[1]):Integer[1]
{
$watermark.datasetMarks->filter(w | $w.datasetName == $datasetName)->toOne().batchId;
}

37 changes: 37 additions & 0 deletions ...ine-xts-ingest/legend-engine-xt-ingest-pure/src/main/resources/core_ingest/metamodel.pure
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import meta::external::ingest::metamodel::*;

Class meta::external::ingest::metamodel::DatasetMark
{
datasetName: String[1];
batchId: Integer[1];
}

Class meta::external::ingest::metamodel::Watermark
{
watermarkId: String[1];
datasetMarks: DatasetMark[*];
}

Class meta::external::ingest::metamodel::IngestConfig<T>
{
datasetName: String[1];
metadataType: Class<T>[1];
}

Class meta::external::ingest::metamodel::BatchMetadata
{
datasetName: String[1];
batchId: Integer[1];
ingest_request_id: String[1];
ingest_definition_urn: String[1];
}

Class meta::external::ingest::metamodel::EodBatchMetadata extends meta::external::ingest::metamodel::BatchMetadata
{
endOfDay: Boolean[1];
}

Class meta::external::ingest::metamodel::BusinessDateBatchMetadata extends meta::external::ingest::metamodel::EodBatchMetadata
{
businessDate: StrictDate[1];
}
Loading

0 comments on commit 7bb8085

Please sign in to comment.