Add query generation on tabular Function

finos · Dec 20, 2024 · de0cc59 · de0cc59
1 parent 4a29395
commit de0cc59
Show file tree

Hide file tree

Showing 19 changed files with 633 additions and 3 deletions.
diff --git a/...n/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure b/...n/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure
@@ -51,6 +51,7 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::snowf
       windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_,
       semiStructuredElementProcessor = processSemiStructuredElementForSnowflake_RelationalOperationElement_1__SqlGenerationContext_1__String_1_,
       tableFunctionParamProcessor = processTableFunctionParamPlaceHolder_RelationalOperationElement_1__SqlGenerationContext_1__String_1_,
+      tabularFunctionProcessor = meta::relational::functions::sqlQueryToString::snowflake::tabularFunctionProcessor_String_1__String_1__String_1_,
       lateralJoinProcessor = processJoinTreeNodeWithLateralJoinForSnowflake_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_,
       joinProcessor = meta::relational::functions::sqlQueryToString::snowflake::processJoinForSnowflake_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_,
       joinStringsProcessor = processJoinStringsOperationForSnowflake_JoinStrings_1__SqlGenerationContext_1__String_1_,
@@ -98,6 +99,11 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::snowf
     );
 }
 
+function meta::relational::functions::sqlQueryToString::snowflake::tabularFunctionProcessor(schema:String[1], functionName: String[1]):String[1]
+{
+   'table('+ if($schema=='', | $functionName,| $schema+'.'+$functionName) + '())';
+}
+
 function meta::relational::functions::sqlQueryToString::snowflake::schemaIdentifierToString(identifier:String[1], dbConfig: DbConfig[1]):String[1]
 {
    $identifier->split('.')->map(s|$s->processIdentifierWithQuoteChar('"', $dbConfig))->joinStrings('.');

diff --git a/...re/src/main/resources/core_relational_snowflake/relational/tests/tabularFunctionModel.txt b/...re/src/main/resources/core_relational_snowflake/relational/tests/tabularFunctionModel.txt
@@ -0,0 +1,318 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+###Relational
+Database demo::udtf::DemoDb
+(
+  Schema Org
+  (
+    Table Firm
+    (
+      firmId INTEGER,
+      legalname VARCHAR(200)
+    )
+    TabularFunction Person
+    (
+      firstname VARCHAR(200),
+      lastname VARCHAR(200),
+      age INTEGER,
+      associatedFirmId INTEGER
+    )
+    TabularFunction Person2
+    (
+      firstname VARCHAR(200),
+      lastname VARCHAR(200),
+      age INTEGER,
+      associatedFirmId INTEGER
+    )
+    TabularFunction ParentAndChildren
+    (
+      firstname VARCHAR(200),
+      lastname VARCHAR(200),
+      id INTEGER,
+      age INTEGER,
+      parentId INTEGER
+    )
+  )
+
+  Join firm_person(Org.Firm.firmId = Org.Person.associatedFirmId)
+  Join firm_person2(Org.Firm.firmId = Org.Person2.associatedFirmId)
+    Join relationship(Org.ParentAndChildren.parentId = {target}.id)
+
+)
+
+
+###Pure
+Class demo::udtf::Org::Firm
+{
+  firmId: Integer[1];
+  legalname: String[1];
+}
+
+Class demo::udtf::Org::Person
+{
+  firstname: String[1];
+  lastname: String[1];
+  age: Integer[1];
+  associatedFirmId: Integer[1];
+  id: Integer[1];
+}
+
+Association demo::udtf::Person_person
+{
+  parent: demo::udtf::Org::Person[1];
+  children: demo::udtf::Org::Person[1..*];
+}
+
+Association demo::udtf::firm_person
+{
+  assoacitedFirm: demo::udtf::Org::Firm[1];
+  associatedPerson: demo::udtf::Org::Person[1..*];
+}
+
+function demo::udtf::Org::FirmToPerson(): meta::pure::tds::TabularDataSet[1]
+{
+  demo::udtf::Org::Firm.all()
+  ->project(
+    [
+      x|$x.firmId,
+      x|$x.associatedPerson.age
+    ],
+    [
+      'Firm Id',
+      'Associated Person/Age'
+    ]
+  )->from(
+    demo::udtf::DemoMapping,
+    demo::runtimes::DemoRuntime
+  )
+}
+
+function demo::udtf::Org::FirmToPersonWithFilterOnPerson(): meta::pure::tds::TabularDataSet[1]
+{
+  demo::udtf::Org::Firm.all()->filter(
+    x|$x.associatedPerson->exists(
+      x_1|$x_1.firstname == 'David'
+    )
+  )->project(
+    [
+      x|$x.firmId,
+      x|$x.associatedPerson.age
+    ],
+    [
+      'Firm Id',
+      'Associated Person/Age'
+    ]
+  )->from(
+    demo::udtf::DemoMapping,
+    demo::runtimes::DemoRuntime
+  )
+}
+
+function demo::udtf::Org::FirmToPersonWithFilterOnPersonUsingUnion(): meta::pure::tds::TabularDataSet[1]
+{
+  demo::udtf::Org::Firm.all()->filter(
+    x|$x.associatedPerson->exists(
+      x_1|$x_1.firstname == 'David'
+    )
+  )->project(
+    [
+      x|$x.firmId,
+      x|$x.associatedPerson.age
+    ],
+    [
+      'Firm Id',
+      'Associated Person/Age'
+    ]
+  )->from(
+    demo::udtf::DemoMappingUnion,
+    demo::runtimes::DemoRuntime
+  )
+}
+
+
+function demo::udtf::Org::FetchChildrenViaSelfJoin(): meta::pure::tds::TabularDataSet[1]
+{
+  demo::udtf::Org::Person.all()->project(
+    [
+      x|$x.firstname,
+      x|$x.id,
+      x|$x.children.age,
+      x|$x.children.id,
+      x|$x.children.firstname
+    ],
+    [
+      'Firstname',
+      'Id',
+      'Children/Age',
+      'Children/Id',
+      'Children/Firstname'
+    ]
+  )->from(
+    demo::udtf::DemoMappingSelfJoin,
+    demo::runtimes::DemoRuntime
+  )
+}
+
+
+###Mapping
+Mapping demo::udtf::DemoMapping
+(
+  *demo::udtf::Org::Firm[f]: Relational
+  {
+    ~primaryKey
+    (
+      [demo::udtf::DemoDb]Org.Firm.firmId,
+      [demo::udtf::DemoDb]Org.Firm.legalname
+    )
+    ~mainTable [demo::udtf::DemoDb]Org.Firm
+    firmId: [demo::udtf::DemoDb]Org.Firm.firmId,
+    legalname: [demo::udtf::DemoDb]Org.Firm.legalname
+  }
+  *demo::udtf::Org::Person[p]: Relational
+  {
+
+    ~mainTable [demo::udtf::DemoDb]Org.Person
+    firstname: [demo::udtf::DemoDb]Org.Person.firstname,
+    lastname: [demo::udtf::DemoDb]Org.Person.lastname,
+    age: [demo::udtf::DemoDb]Org.Person.age
+  }
+
+  demo::udtf::firm_person: Relational
+  {
+    AssociationMapping
+    (
+      assoacitedFirm[p,f]: [demo::udtf::DemoDb]@firm_person,
+      associatedPerson[f,p]: [demo::udtf::DemoDb]@firm_person
+    )
+  }
+)
+
+###Mapping
+Mapping demo::udtf::DemoMappingUnion
+(
+  *demo::udtf::Org::Person: Operation
+  {
+    meta::pure::router::operations::union_OperationSetImplementation_1__SetImplementation_MANY_(p1,p2)
+  }
+  *demo::udtf::Org::Firm[f]: Relational
+  {
+    ~primaryKey
+    (
+      [demo::udtf::DemoDb]Org.Firm.firmId,
+      [demo::udtf::DemoDb]Org.Firm.legalname
+    )
+    ~mainTable [demo::udtf::DemoDb]Org.Firm
+    firmId: [demo::udtf::DemoDb]Org.Firm.firmId,
+    legalname: [demo::udtf::DemoDb]Org.Firm.legalname
+  }
+  demo::udtf::Org::Person[p1]: Relational
+  {
+    ~mainTable [demo::udtf::DemoDb]Org.Person
+    firstname: [demo::udtf::DemoDb]Org.Person.firstname,
+    lastname: [demo::udtf::DemoDb]Org.Person.lastname,
+    age: [demo::udtf::DemoDb]Org.Person.age
+  }
+  demo::udtf::Org::Person[p2]: Relational
+  {
+    ~mainTable [demo::udtf::DemoDb]Org.Person2
+    firstname: [demo::udtf::DemoDb]Org.Person2.firstname,
+    lastname: [demo::udtf::DemoDb]Org.Person2.lastname,
+    age: [demo::udtf::DemoDb]Org.Person2.age
+  }
+
+  demo::udtf::firm_person: Relational
+  {
+    AssociationMapping
+    (
+      assoacitedFirm[p1,f]: [demo::udtf::DemoDb]@firm_person,
+      assoacitedFirm[p2,f]: [demo::udtf::DemoDb]@firm_person2,
+      associatedPerson[f,p1]: [demo::udtf::DemoDb]@firm_person,
+      associatedPerson[f,p2]: [demo::udtf::DemoDb]@firm_person2
+    )
+  }
+)
+
+###Mapping
+Mapping demo::udtf::DemoMappingSelfJoin
+(
+  *demo::udtf::Org::Person[p1]: Relational
+  {
+     ~primaryKey
+    (
+      [demo::udtf::DemoDb]Org.ParentAndChildren.firstname
+    )
+    ~mainTable [demo::udtf::DemoDb]Org.ParentAndChildren
+    firstname: [demo::udtf::DemoDb]Org.ParentAndChildren.firstname,
+    lastname: [demo::udtf::DemoDb]Org.ParentAndChildren.lastname,
+    id: [demo::udtf::DemoDb]Org.ParentAndChildren.id,
+    age: [demo::udtf::DemoDb]Org.ParentAndChildren.age
+  }
+
+  demo::udtf::Person_person: Relational
+  {
+    AssociationMapping
+    (
+      children[p1,p1]: [demo::udtf::DemoDb]@relationship,
+      parent[p1,p1]: [demo::udtf::DemoDb]@relationship
+    )
+  }
+)
+
+###Connection
+RelationalDatabaseConnection demo::udtf::DemoSnowflakeConnection
+{
+  store: demo::udtf::DemoDb;
+  type: Snowflake;
+  specification: Snowflake
+  {
+    name: 'SUMMIT_MDM_DATA';
+    account: 'sfcedeawseast1d01';
+    warehouse: 'DEMO_WH';
+    region: 'us-east-1';
+  };
+  auth: SnowflakePublic
+  {
+    publicUserName: 'isThis';
+    privateKeyVaultReference: 'Hi';
+    passPhraseVaultReference: 'What';
+  };
+}
+
+
+###Runtime
+Runtime demo::runtimes::DemoRuntime
+{
+  mappings:
+  [
+    demo::udtf::DemoMapping
+  ];
+  connections:
+  [
+    demo::udtf::DemoDb:
+    [
+      connection_2: demo::udtf::DemoSnowflakeConnection
+    ]
+  ];
+}
+
+###Snowflake
+SnowflakeApp demo::udtf::snowflakeApp::App1
+{
+   applicationName : 'App1_revised';
+   function : demo::udtf::Org::FirmToPersonWithFilterOnPersonUsingUnion():TabularDataSet[1];
+   ownership : Deployment { identifier: '441143'};
+   description : 'test App';
+   activationConfiguration : demo::udtf::DemoSnowflakeConnection;
+}
diff --git a/...c/main/resources/core_relational_snowflake/relational/tests/testTabularFunctionQuery.pure b/...c/main/resources/core_relational_snowflake/relational/tests/testTabularFunctionQuery.pure
@@ -0,0 +1,52 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::pure::executionPlan::toString::*;
+import meta::pure::profiles::*;
+
+function <<test.Test>> meta::relational::tests::query::snowflake::testSimpleSelect():Any[*]
+{
+  let result  = meta::legend::compileLegendGrammar(readFile('/core_relational_snowflake/relational/tests/tabularFunctionModel.txt')->toOne());
+  let func = $result->filter(f|$f->instanceOf(ConcreteFunctionDefinition))
+    ->cast(@ConcreteFunctionDefinition<Any>)->filter(c| $c.name=='FirmToPerson__TabularDataSet_1_')->toOne();
+  let query = 'select "root".firmId as "Firm Id", "person_0".age as "Associated Person/Age" from Org.Firm as "root" left outer join table(Org.Person()) as "person_0" on ("root".firmId = "person_0".associatedFirmId)';
+  assert(meta::pure::executionPlan::executionPlan($func, meta::relational::extension::relationalExtensions())->planToString(meta::relational::extension::relationalExtensions())->contains($query));
+}
+
+function <<test.Test>> meta::relational::tests::query::snowflake::testExists():Any[*]
+{
+  let result  = meta::legend::compileLegendGrammar(readFile('/core_relational_snowflake/relational/tests/tabularFunctionModel.txt')->toOne());
+  let func = $result->filter(f|$f->instanceOf(ConcreteFunctionDefinition))
+    ->cast(@ConcreteFunctionDefinition<Any>)->filter(c| $c.name=='FirmToPersonWithFilterOnPerson__TabularDataSet_1_')->toOne();
+  let query = 'select "root".firmId as "Firm Id", "person_2".age as "Associated Person/Age" from Org.Firm as "root" left outer join (select distinct "person_1".associatedFirmId from table(Org.Person()) as "person_1" where "person_1".firstname = \'David\') as "person_0" on ("root".firmId = "person_0".associatedFirmId) left outer join table(Org.Person()) as "person_2" on ("root".firmId = "person_2".associatedFirmId) where "person_0".associatedFirmId is not null';
+  assert(meta::pure::executionPlan::executionPlan($func, meta::relational::extension::relationalExtensions())->planToString(meta::relational::extension::relationalExtensions())->contains($query));
+}
+
+function <<test.Test>> meta::relational::tests::query::snowflake::testExistsAndUnion():Any[*]
+{
+  let result  = meta::legend::compileLegendGrammar(readFile('/core_relational_snowflake/relational/tests/tabularFunctionModel.txt')->toOne());
+  let func = $result->filter(f|$f->instanceOf(ConcreteFunctionDefinition))
+    ->cast(@ConcreteFunctionDefinition<Any>)->filter(c| $c.name=='FirmToPersonWithFilterOnPersonUsingUnion__TabularDataSet_1_')->toOne();
+  let query = 'select "root".firmId as "Firm Id", "unionalias_0"."Personage_Person2age" as "Associated Person/Age" from Org.Firm as "root" left outer join (select "root".associatedFirmId as associatedFirmId_0, null as associatedFirmId_1, "root".age as "Personage_Person2age" from table(Org.Person()) as "root" UNION ALL select null as associatedFirmId_0, "root".associatedFirmId as associatedFirmId_1, "root".age as "Personage_Person2age" from table(Org.Person2()) as "root") as "unionalias_0" on ("root".firmId = "unionalias_0".associatedFirmId_0 or "root".firmId = "unionalias_0".associatedFirmId_1) where exists(select 1 from (select "root".associatedFirmId as associatedFirmId_0, null as associatedFirmId_1, "root".firstname as "Personfirstname_Person2firstname" from table(Org.Person()) as "root" UNION ALL select null as associatedFirmId_0, "root".associatedFirmId as associatedFirmId_1, "root".firstname as "Personfirstname_Person2firstname" from table(Org.Person2()) as "root") as "unionalias_1" where ("root".firmId = "unionalias_1".associatedFirmId_0 or "root".firmId = "unionalias_1".associatedFirmId_1) and "unionalias_1"."Personfirstname_Person2firstname" = \'David\')';
+  assert(meta::pure::executionPlan::executionPlan($func, meta::relational::extension::relationalExtensions())->planToString(meta::relational::extension::relationalExtensions())->contains($query));
+}
+
+function <<test.Test>> meta::relational::tests::query::snowflake::testSelfJoin():Any[*]
+{
+  let result  = meta::legend::compileLegendGrammar(readFile('/core_relational_snowflake/relational/tests/tabularFunctionModel.txt')->toOne());
+  let func = $result->filter(f|$f->instanceOf(ConcreteFunctionDefinition))
+    ->cast(@ConcreteFunctionDefinition<Any>)->filter(c| $c.name=='FetchChildrenViaSelfJoin__TabularDataSet_1_')->toOne();
+  let query = 'select "root".firstname as "Firstname", "root".id as "Id", "parentandchildren_1".age as "Children/Age", "parentandchildren_1".id as "Children/Id", "parentandchildren_1".firstname as "Children/Firstname" from table(Org.ParentAndChildren()) as "root" left outer join table(Org.ParentAndChildren()) as "parentandchildren_1" on ("root".parentId = "parentandchildren_1".id)';
+  assert(meta::pure::executionPlan::executionPlan($func, meta::relational::extension::relationalExtensions())->planToString(meta::relational::extension::relationalExtensions())->contains($query));
+}