From 564eeeb62d5ced4c3540204c8ad533f4be77afe6 Mon Sep 17 00:00:00 2001 From: Mohammed Ibrahim Date: Tue, 2 Jan 2024 14:12:16 -0500 Subject: [PATCH] Move generation logic to Pure (#2519) * Check string length before substring * Move generation to logic to pure * fix dependencies * fix dependencies * fix dependencies --- .../core_hostedservice.definition.json | 1 - .../pom.xml | 17 +- .../core_snowflakeapp.definition.json | 8 + .../generation/generation.pure | 34 ++++ .../showcase/showcaseModel.pure | 175 ++++++++++++++++++ .../showcase/showcaseNativeApps.pure | 31 ++++ 6 files changed, 264 insertions(+), 2 deletions(-) create mode 100644 legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure create mode 100644 legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure create mode 100644 legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice.definition.json b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice.definition.json index 40429ddca81..65e20406065 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice.definition.json +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice.definition.json @@ -2,7 +2,6 @@ "name": "core_hostedservice", "pattern": "(meta::external::function::activator::hostedService|meta::protocols)(::.*)?", "dependencies": [ - "platform", "platform", "platform_dsl_graph", "platform_dsl_mapping", diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml index 80854b7fbdd..478229db19f 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml @@ -118,7 +118,22 @@ org.finos.legend.pure legend-pure-m3-core - + + org.finos.legend.pure + legend-pure-m2-dsl-mapping-pure + + + org.finos.legend.pure + legend-pure-m2-store-relational-pure + + + org.finos.legend.engine + legend-engine-pure-platform-dsl-mapping-java + + + org.finos.legend.engine + legend-engine-pure-platform-store-relational-java + org.finos.legend.pure diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp.definition.json b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp.definition.json index cc4f5c58b1d..9fb456e44dd 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp.definition.json +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp.definition.json @@ -2,8 +2,16 @@ "name": "core_snowflakeapp", "pattern": "(meta::external::function::activator::snowflakeApp|meta::protocols)(::.*)?", "dependencies": [ + "core", "platform", + "platform_dsl_graph", + "platform_dsl_mapping", + "platform_store_relational", + "platform_functions", + "platform_functions_json", + "core_relational", "core_function_activator", + "core_functions", "core_relational" ] } \ No newline at end of file diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure new file mode 100644 index 00000000000..8ea8111e0f6 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure @@ -0,0 +1,34 @@ +import meta::external::function::activator::snowflakeApp::generation::*; +import meta::relational::mapping::*; +import meta::pure::executionPlan::*; + + +function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1]):Any[*] +{ + let extensions = meta::external::format::shared::externalFormatExtension()->concatenate(meta::relational::extension::relationalExtensions()); + meta::external::function::activator::snowflakeApp::generation::generateArtifact($s, $extensions); +} + +function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1], extensions:meta::pure::extension::Extension[*]):String[1] +{ + let plan = meta::pure::executionPlan::executionPlan($s.function->cast(@ConcreteFunctionDefinition),$extensions ); + let resultStub = generateResultTypeStub($plan.rootExecutionNode.resultType->cast(@TDSResultType), $extensions); + let generatedQuery = $plan.rootExecutionNode->allNodes($extensions)->filter(n|$n->instanceOf(SQLExecutionNode))->last()->cast(@SQLExecutionNode).sqlQuery->toOne('candidate query not found'); + 'CREATE OR REPLACE SECURE FUNCTION '+ $s.applicationName->toUpper()+ ' RETURNS TABLE ('+ $resultStub+ ') LANGUAGE SQL AS \''+ $generatedQuery +'\';'; + +} + +function meta::external::function::activator::snowflakeApp::generation::generateResultTypeStub(r: TDSResultType[1], extensions:meta::pure::extension::Extension[*]):String[1] +{ + let tdsTypeToRelationalTypeMap = TdsTypeToRelationalTypeMap(); + $r.tdsColumns->map(c|'"'+ $c.name->toUpper() + '" '+ $tdsTypeToRelationalTypeMap->get($c.type->toOne('Column type missing for column: '+$c.name))->toOne('Relational type missing for type: '+ $c.type->toOne()->toString()))->joinStrings(','); +} + +function meta::external::function::activator::snowflakeApp::generation::TdsTypeToRelationalTypeMap():Map[1] +{ + [pair(String, 'VARCHAR(16777216)'), + pair(Boolean, 'boolean'), + pair(Integer, 'number'), + pair(Float, 'number') + ]->newMap() +} \ No newline at end of file diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure new file mode 100644 index 00000000000..799b1b8fdf7 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure @@ -0,0 +1,175 @@ + +import meta::external::function::activator::snowflakeApp::tests::*; +import meta::external::function::activator::snowflakeApp::tests::model::simple::*; +import meta::pure::runtime::*; +import meta::core::runtime::*; +import meta::relational::runtime::*; +import meta::external::store::relational::runtime::*; +import meta::relational::metamodel::*; + + +Class meta::external::function::activator::snowflakeApp::tests::model::simple::PersonX +{ + firstName : String[1]; + lastName : String[1]; + otherNames : String[*]; + extraInformation : String[0..1]; + manager : PersonX[0..1]; + age : Integer[0..1]; + nickName : String[0..1]; + activeEmployment: Boolean[0..1]; + name(){$this.firstName+' '+$this.lastName}:String[1]; + +} + +function meta::external::function::activator::snowflakeApp::tests::testRuntime(db:Database[1]):Runtime[1] +{ + testRuntime(testDatabaseConnection($db,[]->cast(@String))); +} + +function meta::external::function::activator::snowflakeApp::tests::testRelationalConnection():ConnectionStore[1] +{ + ^ConnectionStore( + element = dbInc ,connection= + ^RelationalDatabaseConnection( + type = DatabaseType.H2, + datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification(testDataSetupCsv = ''), + authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::TestDatabaseAuthenticationStrategy() + )) +} + +function <> meta::external::function::activator::snowflakeApp::tests::testRuntime(testConnection:ConnectionStore[1]):Runtime[1] +{ + ^Runtime(connectionStores = $testConnection) +} + +function <> meta::external::function::activator::snowflakeApp::tests::testDatabaseConnection(db:Database[1], timeZone:String[0..1]):ConnectionStore[1] +{ + ^ConnectionStore( + element = $db, + connection= + ^TestDatabaseConnection( + type = DatabaseType.H2, + timeZone = if($timeZone->isEmpty(), |'GMT', |$timeZone) + )); +} + + +###Mapping + +import meta::external::function::activator::snowflakeApp::tests::model::simple::*; +import meta::external::function::activator::snowflakeApp::tests::*; + + +Mapping meta::external::function::activator::snowflakeApp::tests::simpleRelationalMapping +( + + PersonX : Relational + { + scope([dbInc]) + ( + firstName : personTable.FIRSTNAME, + age : personTable.AGE + ), + scope([dbInc]default.personTable) + ( + lastName : LASTNAME + ), + manager : [dbInc]@Person_Manager + } + + +) + + + +Mapping meta::external::function::activator::snowflakeApp::tests::simpleRelationalMapping2 +( + + PersonX : Relational + { + scope([dbInc]) + ( + firstName : concat(personTable.FIRSTNAME,'__X'), + age : personTable.AGE + ), + scope([dbInc]default.personTable) + ( + lastName : concat(LASTNAME,'__X') + ), + manager : [dbInc]@Person_Manager + } + + +) + + +###Relational +Database meta::external::function::activator::snowflakeApp::tests::dbInc +( + Table personTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT) + Table validPersonTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT) + Table PersonTableExtension (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT, birthDate DATE) + Table differentPersonTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT) + + Table firmTable(ID INT PRIMARY KEY, LEGALNAME VARCHAR(200), ADDRESSID INT, CEOID INT) + Table firmExtensionTable(firmId INT PRIMARY KEY, legalName VARCHAR(200), establishedDate DATE) + Table otherFirmTable(ID INT PRIMARY KEY, LEGALNAME VARCHAR(200), ADDRESSID INT) + + Table addressTable(ID INT PRIMARY KEY, TYPE INT, NAME VARCHAR(200), STREET VARCHAR(100), COMMENTS VARCHAR(100)) + Table locationTable(ID INT PRIMARY KEY, PERSONID INT, PLACE VARCHAR(200),date DATE) + Table placeOfInterestTable(ID INT PRIMARY KEY,locationID INT PRIMARY KEY, NAME VARCHAR(200)) + + View PersonFirmView + ( + PERSON_ID: personTable.ID PRIMARY KEY, + lastName: personTable.LASTNAME, + firm_name : @Firm_Person | firmTable.LEGALNAME + ) + + View FirstNameAddress + ( + ~distinct + firstName: personTable.FIRSTNAME PRIMARY KEY, + address : @Address_Person | addressTable.NAME PRIMARY KEY + ) + + View personViewWithGroupBy + ( + ~groupBy(personTable.ID) + id: personTable.ID PRIMARY KEY, + maxage: max(personTable.AGE) + ) + + View PersonViewWithDistinct + ( + ~distinct + id: @PersonWithPersonView| personTable.ID PRIMARY KEY, + firstName: @PersonWithPersonView| personTable.FIRSTNAME, + lastName: @PersonWithPersonView|personTable.LASTNAME, + firmId: @PersonWithPersonView|personTable.FIRMID + ) + + Schema productSchema + ( + Table productTable(ID INT PRIMARY KEY, NAME VARCHAR(200)) + ) + + Filter FirmXFilter(firmTable.LEGALNAME = 'Firm X') + Filter FirmBFilter(firmTable.LEGALNAME = 'Firm B') + + Join personViewWithFirmTable(firmTable.ID = PersonViewWithDistinct.firmId) + Join PersonWithPersonView(personTable.ID = personViewWithGroupBy.id and personTable.AGE = personViewWithGroupBy.maxage) + Join Address_Firm(addressTable.ID = firmTable.ADDRESSID) + Join Address_Person(addressTable.ID = personTable.ADDRESSID) + Join Firm_Ceo(firmTable.CEOID = personTable.ID) + Join Firm_Person(firmTable.ID = personTable.FIRMID) + Join Firm_Person1(firmTable.ID = personTable.FIRMID and firmTable.LEGALNAME = 'Firm X') + Join Firm_Person2(firmTable.ID = personTable.FIRMID and personTable.FIRSTNAME = 'Peter') + Join FirmExtension_PersonExtension(firmExtensionTable.firmId = PersonTableExtension.FIRMID) + Join Person_Location(personTable.ID = locationTable.PERSONID) + Join Person_Manager(personTable.MANAGERID = {target}.ID) + Join location_PlaceOfInterest(locationTable.ID = placeOfInterestTable.locationID) + Join Person_OtherFirm(personTable.FIRMID = otherFirmTable.ID) + +) diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure new file mode 100644 index 00000000000..07b2df4b31d --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure @@ -0,0 +1,31 @@ +import meta::external::function::activator::snowflakeApp::generation::*; +import meta::external::store::relational::runtime::*; +import meta::pure::mapping::*; +import meta::external::function::activator::snowflakeApp::tests::model::simple::*; +import meta::external::function::activator::snowflakeApp::tests::*; +import meta::external::function::activator::snowflakeApp::*; + +function meta::external::function::activator::snowflakeApp::tests::defaultConfig():SnowflakeDeploymentConfiguration[1] +{ + ^SnowflakeDeploymentConfiguration(target = testRelationalConnection().connection->cast(@RelationalDatabaseConnection) ); +} + +function meta::external::function::activator::snowflakeApp::tests::simpleApp():Any[*] +{ + let app = ^SnowflakeApp + ( + applicationName = 'App1', + owner = 'owner1', + description = 'bla bla', + activationConfiguration = defaultConfig() , + function = meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction__TabularDataSet_1_ + ); + let generatedQuery = $app->generateArtifact(); + //isMulti +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction():TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.firstName == 'haha')->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} \ No newline at end of file