From a928b72a897dbdc32a202a832297237eaf9c8650 Mon Sep 17 00:00:00 2001 From: gs-jp1 <80327721+gs-jp1@users.noreply.github.com> Date: Wed, 29 Nov 2023 14:01:23 +0000 Subject: [PATCH 1/3] Legend SQL - Assortment of changes (#2483) - left/right support - make_date/make_timestamp/localtimestamp support - trim support --- .../toPureGraph/handlers/Handlers.java | 4 + .../core/legend/test/handlersTest.pure | 4 +- .../pure/corefunctions/stringExtension.pure | 28 +++++++ .../corefunctions/tests/string/testLeft.pure | 20 +++++ .../corefunctions/tests/string/testRight.pure | 20 +++++ .../pure/router/routing/router_routing.pure | 2 + .../sqlQueryToString/postgresExtension.pure | 4 +- .../pureToSQLQuery/pureToSQLQuery.pure | 2 + .../dbSpecific/db2/db2Extension.pure | 2 + .../testSuite/dynaFunctions/string.pure | 14 ++++ .../fromPure/tests/testToSQLString.pure | 25 ++++++ .../language/sql/grammar/from/SqlVisitor.java | 45 ++++++++++- .../sql/grammar/to/SQLGrammarComposer.java | 7 ++ .../test/roundtrip/TestSQLRoundTrip.java | 7 ++ .../metamodel.pure | 7 ++ .../binding/fromPure/fromPure.pure | 81 ++++++++++++++++++- .../binding/fromPure/tests/testTranspile.pure | 60 ++++++++++++-- .../query/sql/api/TableSourceExtractor.java | 6 ++ 18 files changed, 324 insertions(+), 14 deletions(-) create mode 100644 legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testLeft.pure create mode 100644 legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testRight.pure diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index dc7e7b2be04..d83ee915ee5 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -930,6 +930,8 @@ private void registerStrings() register("meta::pure::functions::string::trim_String_1__String_1_", true, ps -> res("String", "one")); register("meta::pure::functions::string::ltrim_String_1__String_1_", true, ps -> res("String", "one")); register("meta::pure::functions::string::rtrim_String_1__String_1_", true, ps -> res("String", "one")); + register("meta::pure::functions::string::left_String_1__Integer_1__String_1_", false, ps -> res("String", "one")); + register("meta::pure::functions::string::right_String_1__Integer_1__String_1_", false, ps -> res("String", "one")); register(m(m(h("meta::pure::functions::string::lpad_String_1__Integer_1__String_1_", false, ps -> res("String", "one"), ps -> ps.size() == 2)), m(h("meta::pure::functions::string::lpad_String_1__Integer_1__String_1__String_1_", false, ps -> res("String", "one"), ps -> true)))); @@ -2073,6 +2075,8 @@ private Map buildDispatch() map.put("meta::pure::functions::string::trim_String_1__String_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::ltrim_String_1__String_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::rtrim_String_1__String_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name()))); + map.put("meta::pure::functions::string::left_String_1__Integer_1__String_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name()))); + map.put("meta::pure::functions::string::right_String_1__Integer_1__String_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::lpad_String_1__Integer_1__String_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::lpad_String_1__Integer_1__String_1__String_1_", (List ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "String".equals(ps.get(2)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::rpad_String_1__Integer_1__String_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name()))); diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure index 145fab5829f..380dd9a2e83 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure @@ -380,7 +380,9 @@ Class meta::legend::test::handlers::model::TestString lpad(){$this.string->lpad(1)}:String[1]; lpad2(){$this.string->lpad(1, '0')}:String[1]; rpad(){$this.string->rpad(1)}:String[1]; - rpad2(){$this.string->rpad(1, '0')}:String[1]; + rpad2(){$this.string->rpad(1, '0')}:String[1]; + left(){$this.string->left(1)}:String[1]; + right(){$this.string->right(1)}:String[1]; parseBoolean(){$this.string->parseBoolean()}:Boolean[1]; parseDate(){$this.string->parseDate()}:Date[1]; diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure index 390c64a9cbd..732c2929370 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure @@ -51,6 +51,34 @@ function {doc.doc = 'Lower cases the first charater of the provided string'} met }) } +function {doc.doc = 'Return first n characters in the string'} +meta::pure::functions::string::left(string:String[1], length:Integer[1]):String[1] +{ + assert($length >= 0, 'length must be >= 0'); + + let strLength = $string->length(); + + let digits = if ($length > $strLength, + | $strLength, + | $length); + + $string->substring(0, $digits); +} + +function {doc.doc = 'Return last n characters in the string'} +meta::pure::functions::string::right(string:String[1], length:Integer[1]):String[1] +{ + assert($length >= 0, 'length must be >= 0'); + + let strLength = $string->length(); + + let digits = if ($length > $strLength, + | 0, + | $strLength - $length); + + $string->substring($digits, $strLength); +} + function meta::pure::functions::string::repeatString(str:String[0..1], times:Integer[1]):String[0..1] { if ($str->isNotEmpty(), | $str->toOne()->repeat($times)->joinStrings(), | []); diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testLeft.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testLeft.pure new file mode 100644 index 00000000000..ad16116c71f --- /dev/null +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testLeft.pure @@ -0,0 +1,20 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +function <> meta::pure::functions::string::tests::left::leftTest():Boolean[1] { + + assertEquals('ab', 'abcde'->left(2)); + assertEquals('', 'abcde'->left(0)); + assertEquals('abcde', 'abcde'->left(7)); +} diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testRight.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testRight.pure new file mode 100644 index 00000000000..da38ec4df8a --- /dev/null +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/string/testRight.pure @@ -0,0 +1,20 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +function <> meta::pure::functions::string::tests::right::rightTest():Boolean[1] { + + assertEquals('de', 'abcde'->right(2)); + assertEquals('', 'abcde'->right(0)); + assertEquals('abcde', 'abcde'->right(7)); +} diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure index 4b0c581f4aa..ff1ab740794 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure @@ -739,6 +739,8 @@ function meta::pure::router::routing::shouldStopFunctions(extensions:meta::pure: lpad_String_1__Integer_1__String_1__String_1_, rpad_String_1__Integer_1__String_1_, rpad_String_1__Integer_1__String_1__String_1_, + left_String_1__Integer_1__String_1_, + right_String_1__Integer_1__String_1_, meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_, meta::pure::functions::date::calendar::annualized_Date_1__String_1__Date_1__Number_$0_1$__Number_$0_1$_, meta::pure::functions::date::calendar::cme_Date_1__String_1__Date_1__Number_$0_1$__Number_$0_1$_, diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/src/main/resources/core_relational_postgres/relational/sqlQueryToString/postgresExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/src/main/resources/core_relational_postgres/relational/sqlQueryToString/postgresExtension.pure index db340cecdda..42fd762831f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/src/main/resources/core_relational_postgres/relational/sqlQueryToString/postgresExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/src/main/resources/core_relational_postgres/relational/sqlQueryToString/postgresExtension.pure @@ -185,6 +185,7 @@ function <> meta::relational::functions::sqlQueryToString::postg dynaFnToSql('hour', $allStates, ^ToSql(format='date_part(\'hour\', %s)')), dynaFnToSql('indexOf', $allStates, ^ToSql(format='strpos(%s, %s)')), dynaFnToSql('joinStrings', $allStates, ^ToSql(format='string_agg(%s, %s)')), + dynaFnToSql('left', $allStates, ^ToSql(format='left(%s, %s)')), dynaFnToSql('length', $allStates, ^ToSql(format='char_length(%s)')), dynaFnToSql('minute', $allStates, ^ToSql(format='date_part(\'minute\', %s)')), dynaFnToSql('mod', $allStates, ^ToSql(format='mod(%s,%s)')), @@ -195,9 +196,10 @@ function <> meta::relational::functions::sqlQueryToString::postg dynaFnToSql('parseFloat', $allStates, ^ToSql(format='cast(%s as float)')), dynaFnToSql('parseInteger', $allStates, ^ToSql(format='cast(%s as integer)')), dynaFnToSql('position', $allStates, ^ToSql(format='position(%s in %s)')), - dynaFnToSql('quarter', $allStates, ^ToSql(format='quarter(%s)')), + dynaFnToSql('quarter', $allStates, ^ToSql(format='date_part(\'quarter\', %s)')), dynaFnToSql('quarterNumber', $allStates, ^ToSql(format='date_part(\'quarter\', %s)')), dynaFnToSql('rem', $allStates, ^ToSql(format='mod(%s,%s)')), + dynaFnToSql('right', $allStates, ^ToSql(format='right(%s, %s)')), dynaFnToSql('round', $allStates, ^ToSql(format='round((%s)::numeric, %s)', transform=transformRound_String_MANY__String_MANY_)), dynaFnToSql('second', $allStates, ^ToSql(format='date_part(\'second\', %s)')), dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure index 782dfa54311..cccadba05ef 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure @@ -7739,6 +7739,8 @@ function meta::relational::functions::pureToSqlQuery::getSupportedFunctions():Ma ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::trim_String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::ltrim_String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::rtrim_String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), + ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::left_String_1__Integer_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), + ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::right_String_1__Integer_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::lpad_String_1__Integer_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::lpad_String_1__Integer_1__String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::rpad_String_1__Integer_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/db2/db2Extension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/db2/db2Extension.pure index c059e176ac7..7c3e5926461 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/db2/db2Extension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/db2/db2Extension.pure @@ -98,6 +98,7 @@ function <> meta::relational::functions::sqlQueryToString::db2:: dynaFnToSql('firstDayOfYear', $allStates, ^ToSql(format='date(1) + (year(%s)-1) YEARS')), dynaFnToSql('hour', $allStates, ^ToSql(format='hour(%s)')), dynaFnToSql('joinStrings', $allStates, ^ToSql(format='listagg(%s,%s)')), + dynaFnToSql('left', $allStates, ^ToSql(format='trim(left(%s, %s))')), dynaFnToSql('length', $allStates, ^ToSql(format='CHARACTER_LENGTH(%s,CODEUNITS32)')), dynaFnToSql('minute', $allStates, ^ToSql(format='minute(%s)')), dynaFnToSql('md5', $allStates, ^ToSql(format='hash_md5(%s)')), @@ -114,6 +115,7 @@ function <> meta::relational::functions::sqlQueryToString::db2:: dynaFnToSql('quarter', $allStates, ^ToSql(format='quarter(%s)')), dynaFnToSql('quarterNumber', $allStates, ^ToSql(format='quarter(%s)')), dynaFnToSql('rem', $allStates, ^ToSql(format='mod(%s,%s)')), + dynaFnToSql('right', $allStates, ^ToSql(format='trim(right(%s, %s))')), dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)), dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')), dynaFnToSql('sha1', $allStates, ^ToSql(format='hash_sha1(%s)')), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure index de59320059d..0f73ea1e6ed 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure @@ -251,6 +251,13 @@ function <> meta::relational::tests::dbSpecificTests::sqlQueryTe runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); } +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::left::testLonger(config:DbTestConfig[1]):Boolean[1] +{ + let dynaFunc = ^DynaFunction(name='left', parameters=[^Literal(value='Bloggs'), ^Literal(value=8)]); + let expected = ^Literal(value='Bloggs'); + runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); +} + function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::right::testNoSpace(config:DbTestConfig[1]):Boolean[1] { let dynaFunc = ^DynaFunction(name='right', parameters=[^Literal(value='Smith'), ^Literal(value=3)]); @@ -265,6 +272,13 @@ function <> meta::relational::tests::dbSpecificTests::sqlQueryTe runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); } +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::right::testLonger(config:DbTestConfig[1]):Boolean[1] +{ + let dynaFunc = ^DynaFunction(name='right', parameters=[^Literal(value='Bloggs'), ^Literal(value=8)]); + let expected = ^Literal(value='Bloggs'); + runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); +} + // contains, endsWith , startsWith function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::contains::testNoSpace(config:DbTestConfig[1]):Boolean[1] { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure index 01a668a1a26..5cc74e81a43 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure @@ -431,6 +431,31 @@ function <> meta::relational::tests::functions::sqlstring::testPad(): )->distinct() == [true]; } +function <> meta::relational::tests::functions::sqlstring::testLeftRight():Boolean[1] +{ + + let expected = [ + pair(DatabaseType.DB2, 'select trim(left("root".FIRSTNAME, 1)) as "left", trim(right("root".FIRSTNAME, 1)) as "right" from personTable as "root"'), + pair(DatabaseType.H2, 'select left("root".FIRSTNAME,1) as "left", right("root".FIRSTNAME,1) as "right" from personTable as "root"') + ]; + + $expected->map(p| + let driver = $p.first; + let expectedSql = $p.second; + + let result = toSQLString( + |Person.all()->project([ + a | $a.firstName->left(1), + a | $a.firstName->right(1) + ], + ['left', 'right']), + simpleRelationalMapping, + $driver, meta::relational::extension::relationalExtensions()); + + assertEquals($expectedSql, $result, '\nSQL not as expected for %s\n\nexpected: %s\nactual: %s', [$driver, $expectedSql, $result]); + )->distinct() == [true]; +} + function <> meta::relational::tests::functions::sqlstring::testCbrt():Boolean[1] { let common = 'select cbrt("root".quantity) as "cbrt" from tradeTable as "root"'; diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java index 55ea8368590..39933622c87 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java @@ -20,6 +20,7 @@ import org.antlr.v4.runtime.tree.TerminalNode; import org.eclipse.collections.impl.list.mutable.FastList; import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.utility.ArrayIterate; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.language.sql.grammar.from.antlr4.SqlBaseLexer; import org.finos.legend.engine.language.sql.grammar.from.antlr4.SqlBaseParser; @@ -1407,19 +1408,24 @@ public Node visitAtTimezone(SqlBaseParser.AtTimezoneContext context) @Override public Node visitLeft(SqlBaseParser.LeftContext context) { - return unsupported(); + return functionCall("left", context.strOrColName, context.len); } @Override public Node visitRight(SqlBaseParser.RightContext context) { - return unsupported(); + return functionCall("right", context.strOrColName, context.len); } @Override public Node visitTrim(SqlBaseParser.TrimContext ctx) { - return unsupported(); + Trim trim = new Trim(); + trim.value = (Expression) visit(ctx.target); + trim.characters = visitIfPresent(ctx.charsToTrim, Expression.class).orElse(null); + trim.mode = getTrimMode(ctx.trimMode); + + return trim; } @Override @@ -1906,6 +1912,25 @@ private static CurrentTimeType getDateTimeFunctionType(Token token) } } + private TrimMode getTrimMode(Token type) + { + if (type == null) + { + return TrimMode.BOTH; + } + switch (type.getType()) + { + case SqlBaseLexer.BOTH: + return TrimMode.BOTH; + case SqlBaseLexer.LEADING: + return TrimMode.LEADING; + case SqlBaseLexer.TRAILING: + return TrimMode.TRAILING; + default: + throw new UnsupportedOperationException("Unsupported trim mode: " + type.getText()); + } + } + private static SortItemNullOrdering getNullOrderingType(Token token) { if (token != null) @@ -1948,6 +1973,20 @@ private QualifiedName qualifiedName(String... parts) return qualifiedName; } + private FunctionCall functionCall(String name, ExprContext... contexts) + { + return functionCall(name, ArrayIterate.collect(contexts, c -> (Expression) visit(c))); + } + + private FunctionCall functionCall(String name, List arguments) + { + FunctionCall functionCall = new FunctionCall(); + functionCall.name = qualifiedName(name); + functionCall.arguments = arguments; + + return functionCall; + } + private static WindowFrameMode getFrameType(Token type) { switch (type.getType()) diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java index a43e9461c29..9991e4eff7e 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java @@ -533,6 +533,13 @@ public String visit(TableSubquery val) return "(" + visit(val.query) + ")"; } + @Override + public String visit(Trim val) + { + String chars = val.characters != null ? " " + visit(val.characters) : ""; + return "trim(" + val.mode.name() + chars + " FROM " + visit(val.value) + ")"; + } + @Override public String visit(Union val) { diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java index 762706172b2..76b643e9467 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java @@ -321,6 +321,13 @@ public void testFunctionCallWithOrder() check("SELECT string_agg(Col1, ', ' ORDER BY Col2 ASC, Col3 DESC) FROM myTable"); } + @Test + public void testTrim() + { + check("SELECT trim(BOTH ' ' FROM 'abc') FROM myTable"); + check("SELECT trim(BOTH FROM 'abc') FROM myTable"); + } + @Test public void testNested() { diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure index da34ebef02c..37ad513ee78 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure @@ -268,6 +268,13 @@ Class meta::external::query::sql::metamodel::Extract extends meta::external::que <> field: meta::external::query::sql::metamodel::ExtractField[1]; } +Class meta::external::query::sql::metamodel::Trim extends meta::external::query::sql::metamodel::Expression +{ + <> value: meta::external::query::sql::metamodel::Expression[1]; + <> characters: meta::external::query::sql::metamodel::Expression[0..1 ]; + <> mode: meta::external::query::sql::metamodel::TrimMode[1]; +} + Class meta::external::query::sql::metamodel::Join extends meta::external::query::sql::metamodel::Relation { <> type: meta::external::query::sql::metamodel::JoinType[1]; diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure index 0ce699dcb50..8600b749b8a 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure @@ -1382,6 +1382,7 @@ function meta::external::query::sql::transformation::queryToPure::extractNameFro let when = $s.whenClauses->map(w | $w->extractNameFromExpression($context))->joinStrings(' '); let default = if ($s.defaultValue->isEmpty(), | '', | ' ELSE ' + $s.defaultValue->toOne()->extractNameFromExpression($context)); 'CASE ' + $when + $default + ' END';, + t:Trim[1] | 'TRIM(' + $t.mode->toString() + if ($t.characters->isEmpty(), | '', | ' ' + $t.characters->toOne()->extractNameFromExpression($context)) + ' FROM ' + $t.value->extractNameFromExpression($context) + ')', w:WhenClause[1] | 'WHEN ' + $w.operand->extractNameFromExpression($context) + ' THEN ' + $w.result->extractNameFromExpression($context), e:meta::external::query::sql::metamodel::Expression[1] | '' ]) @@ -1461,6 +1462,7 @@ function <> meta::external::query::sql::transformation::queryToP q:QualifiedNameReference[1] | processQualifiedNameReference($q, $expContext, $context), s:SimpleCaseExpression[1] | processSimpleCaseExpression($s, $expContext, $context), s:SearchedCaseExpression[1] | processSearchedCaseExpression($s, $expContext, $context), + t:Trim[1] | processTrim($t, $expContext, $context), e:meta::external::query::sql::metamodel::Expression[*] | fail('Expression type not yet supported'); iv(1); ])->evaluateAndDeactivate(); } @@ -1537,7 +1539,6 @@ function <> meta::external::query::sql::transformation::queryToP let func = [ pair(CurrentTimeType.DATE, today__StrictDate_1_), - pair(CurrentTimeType.TIME, now__DateTime_1_), pair(CurrentTimeType.TIMESTAMP, now__DateTime_1_) ]->getValue($c.type); @@ -1728,6 +1729,21 @@ function <> meta::external::query::sql::transformation::queryToP ]->getValue($c.type.name->toUpper()); } +function <> meta::external::query::sql::transformation::queryToPure::processTrim(t:Trim[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):ValueSpecification[1] +{ + debug('processTrim', $context.debug); + + let name = [ + pair(TrimMode.LEADING, 'ltrim'), + pair(TrimMode.TRAILING, 'rtrim'), + pair(TrimMode.BOTH, 'btrim') + ]->getValue($t.mode); + + ^FunctionCall(name = ^QualifiedName(parts = $name), + distinct = false, + arguments = $t.value->concatenate($t.characters))->processFunctionCall($expContext, $context); +} + function <> meta::external::query::sql::transformation::queryToPure::processSearchedCaseExpression(s:SearchedCaseExpression[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):SimpleFunctionExpression[1] { debug('processSearchedCaseExpression', $context.debug); @@ -2062,15 +2078,39 @@ function meta::external::query::sql::transformation::queryToPure::functionProces processor('concat', String, {args, fc, ctx | sfe(plus_String_MANY__String_1_, iv($args)) }), + processor('decode', String, {args, fc, ctx | + assert($args->size() == 2, | 'incorrect number of args to decode'); + + let type = $args->at(1)->reactivate()->cast(@String)->toOne(); + + let func = [ + pair('base64', decodeBase64_String_1__String_1_) + ]->getValue($type->toLower()); + + nullOrSfe($func, $args->at(0)); + }), + processor('encode', String, {args, fc, ctx | + assert($args->size() == 2, | 'incorrect number of args to encode'); + + let type = $args->at(1)->reactivate()->cast(@String)->toOne(); + + let func = [ + pair('base64', encodeBase64_String_1__String_1_) + ]->getValue($type->toLower()); + + nullOrSfe($func, $args->at(0)); + }), processor('length', length_String_1__Integer_1_), processor('lower', toLower_String_1__String_1_), processor('lpad', String, {args, fc, ctx | processPad($args, true)}), processor('ltrim', String, {args, fc, ctx | processTrim(ltrim_String_1__String_1_, $args)}), + processor('left', left_String_1__Integer_1__String_1_), processor('md5', String, {args, fc, ctx | processHash($args, meta::pure::functions::hash::HashType.MD5)}), processor('regexp_like', matches_String_1__String_1__Boolean_1_), processor('repeat', repeatString_String_$0_1$__Integer_1__String_$0_1$_), processor('replace', replace_String_1__String_1__String_1__String_1_), processor('reverse', reverseString_String_1__String_1_), + processor('right', right_String_1__Integer_1__String_1_), processor('rpad', String, {args, fc, ctx | processPad($args, false)}), processor('rtrim', String, {args, fc, ctx | processTrim(rtrim_String_1__String_1_, $args)}), processor('sha256', String, {args, fc, ctx | processHash($args, meta::pure::functions::hash::HashType.SHA256)}), @@ -2137,6 +2177,37 @@ function meta::external::query::sql::transformation::queryToPure::functionProces nullOrSfe($func, $args->at(1)); }), + processor('localtimestamp', Date, {args, fc, ctx | + assertEquals(0, $args->size(), 'only zero arg localtimestamp supported'); + + sfe(now__DateTime_1_, []); + }), + processor('make_date', Date, {args, fc, ctx | + assertEquals(3, $args->size(), 'incorrect number of args for make_date'); + + let year = $args->at(0)->reactivate()->toOne()->cast(@Integer); + let month = $args->at(1)->reactivate()->toOne()->cast(@Integer); + let day = $args->at(2)->reactivate()->toOne()->cast(@Integer); + + let date = date($year, $month, $day); + + iv($date); + }), + + processor('make_timestamp', Date, {args, fc, ctx | + assertEquals(6, $args->size(), 'incorrect number of args for make_timestamp'); + + let year = $args->at(0)->reactivate()->toOne()->cast(@Integer); + let month = $args->at(1)->reactivate()->toOne()->cast(@Integer); + let day = $args->at(2)->reactivate()->toOne()->cast(@Integer); + let hours = $args->at(3)->reactivate()->toOne()->cast(@Integer); + let minutes = $args->at(4)->reactivate()->toOne()->cast(@Integer); + let seconds = $args->at(5)->reactivate()->toOne()->cast(@Number); + + let date = date($year, $month, $day, $hours, $minutes, $seconds); + + iv($date); + }), //COLLECTION processor('coalesce', [], {args, fc, ctx | @@ -2179,7 +2250,7 @@ function meta::external::query::sql::transformation::queryToPure::functionProces let type = $arg.genericType.rawType; let format = $args->at(1)->reactivate()->toOne()->cast(@String); - assert($type->isNotEmpty() && $type->toOne()->normalizeType() == Date, 'to_char currently only supported for know date inputs'); + assert($type->isNotEmpty() && $type->toOne()->normalizeType() == Date, 'to_char currently only supported for date inputs'); toChar($format, [], $arg->evaluateAndDeactivate()); }), @@ -2504,6 +2575,7 @@ function <> meta::external::query::sql::transformation::queryToP q:QualifiedNameReference[1] | $context.columnByNameParts($q.name.parts, true).type->toOne(), s:SearchedCaseExpression[1] | caseExpressionType($s.whenClauses, $s.defaultValue, $context), s:SimpleCaseExpression[1] | caseExpressionType($s.whenClauses, $s.defaultValue, $context), + t:Trim[1] | String, e:meta::external::query::sql::metamodel::Expression[1] | Any ]); } @@ -2964,6 +3036,7 @@ function <> meta::external::query::sql::transformation::queryToP n:NotExpression[1] | walk($n.value, $accumulator, $extras), s:SimpleCaseExpression[1] | $s->convertToSearchedCaseExpression()->walk($accumulator, $extras), s:SearchedCaseExpression[1] | $s.whenClauses->map(w | walk($w.operand, $w.result, $accumulator, $extras))->concatenate(walk($s.defaultValue, $accumulator, $extras)), + t:Trim[1] | walk($t.value, $t.characters, $accumulator, $extras), e:meta::external::query::sql::metamodel::Expression[0..1] | [] ])->toOneMany()); @@ -3024,6 +3097,10 @@ function <> meta::external::query::sql::transformation::queryToP defaultValue = $s.defaultValue->walk($extras), whenClauses = $s.whenClauses->map(wc | ^$wc(operand = $wc.operand->walk($extras)->toOne(), result = $wc.result->walk($extras)->toOne())) ), + t:Trim[1] | ^$t( + value = $t.value->walk($extras)->toOne(), + characters = $t.characters->walk($extras) + ), e:meta::external::query::sql::metamodel::Expression[1] | $e ])->match($extras->concatenate([ e:meta::external::query::sql::metamodel::Expression[1] | $e diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure index 12b8850ab1a..49a943b5eb9 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure @@ -1481,7 +1481,7 @@ function <> meta::external::query::sql::transformation::queryToPure:: function <> meta::external::query::sql::transformation::queryToPure::tests::testCurrentTime():Boolean[1] { test( - 'SELECT CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_DATE, CURRENT_TIME AS time, CURRENT_TIMESTAMP AS timestamp, CURRENT_DATE AS date FROM service."/service/service1"', + 'SELECT CURRENT_TIMESTAMP, CURRENT_DATE, CURRENT_TIMESTAMP AS timestamp, CURRENT_DATE AS date FROM service."/service/service1"', {| FlatInput.all() @@ -1489,16 +1489,36 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->project([ - col(row:TDSRow[1] | now(), 'CURRENT_TIME'), col(row:TDSRow[1] | now(), 'CURRENT_TIMESTAMP'), col(row:TDSRow[1] | today(), 'CURRENT_DATE'), - col(row:TDSRow[1] | now(), 'time'), col(row:TDSRow[1] | now(), 'timestamp'), col(row:TDSRow[1] | today(), 'date') ]) }) } +function <> meta::external::query::sql::transformation::queryToPure::tests::testDateFunctions():Boolean[1] +{ + test( + 'SELECT ' + + 'localtimestamp() AS "LOCALTIMESTAMP", ' + + 'make_date(2023, 01, 02) AS "MAKEDATE", ' + + 'make_timestamp(2023, 01, 02, 12, 11, 10.5) AS "MAKETIMESTAMP" ' + + 'FROM service."/service/service1"', + + {| + FlatInput.all() + ->project( + [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], + [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) + ->project([ + col(row:TDSRow[1] | now(), 'LOCALTIMESTAMP'), + col(row:TDSRow[1] | %2023-01-02, 'MAKEDATE'), + col(row:TDSRow[1] | %2023-01-02T12:11:10.5, 'MAKETIMESTAMP') + ]) + }) +} + //DATE_TRUNC function <> meta::external::query::sql::transformation::queryToPure::tests::testDateTrunc():Boolean[1] { @@ -1633,7 +1653,11 @@ function <> meta::external::query::sql::transformation::queryToPure:: 'lower(String) AS "LOWER", repeat(String, 2) AS "REPEAT", replace(String, \'A\', \'a\') AS "REPLACE", starts_with(String, \'a\') AS "STARTSWITH", strpos(String, \'abc\') AS "STRPOS",' + 'reverse(String) AS "REVERSE", rtrim(String) AS "RTRIM", rtrim(String, \' \') AS "RTRIM2", sha256(String) AS "SHA256", split_part(String, \',\', 1) AS "SPLITPART", ' + 'split_part(String, \',\', Integer) AS "SPLITPART2", substring(String, 1) AS "SUBSTRING", substr(String, 1, 2) AS "SUBSTR", btrim(String) AS "TRIM", btrim(String, \' \') AS "TRIM2",' + - 'lpad(String, 2) AS "LPAD", lpad(String, 2, \'a\') AS "LPAD2", rpad(String, 2) AS "RPAD", rpad(String, 2, \'a\') AS "RPAD2" FROM service."/service/service1"', + 'lpad(String, 2) AS "LPAD", lpad(String, 2, \'a\') AS "LPAD2", rpad(String, 2) AS "RPAD", rpad(String, 2, \'a\') AS "RPAD2", left(String, 1) AS "LEFT", right(String, 1) AS "RIGHT", ' + + 'decode(String, \'base64\') AS "DECODE", encode(String, \'base64\') AS "ENCODE", ' + + 'trim(BOTH from String) AS "TRIMB", trim(BOTH \' \' from String) AS "TRIMB2", trim(LEADING from String) AS "TRIML", trim(LEADING \' \' from String) AS "TRIML2", ' + + 'trim(TRAILING from String) AS "TRIMT", trim(TRAILING \' \' from String) AS "TRIMT2"' + + 'FROM service."/service/service1"', {| FlatInput.all() @@ -1670,7 +1694,17 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | lpad($row.getString('String'), 2), 'LPAD'), col(row:TDSRow[1] | lpad($row.getString('String'), 2, 'a'), 'LPAD2'), col(row:TDSRow[1] | rpad($row.getString('String'), 2), 'RPAD'), - col(row:TDSRow[1] | rpad($row.getString('String'), 2, 'a'), 'RPAD2') + col(row:TDSRow[1] | rpad($row.getString('String'), 2, 'a'), 'RPAD2'), + col(row:TDSRow[1] | left($row.getString('String'), 1), 'LEFT'), + col(row:TDSRow[1] | right($row.getString('String'), 1), 'RIGHT'), + col(row:TDSRow[1] | decodeBase64($row.getString('String')), 'DECODE'), + col(row:TDSRow[1] | encodeBase64($row.getString('String')), 'ENCODE'), + col(row:TDSRow[1] | trim($row.getString('String')), 'TRIMB'), + col(row:TDSRow[1] | trim($row.getString('String')), 'TRIMB2'), + col(row:TDSRow[1] | ltrim($row.getString('String')), 'TRIML'), + col(row:TDSRow[1] | ltrim($row.getString('String')), 'TRIML2'), + col(row:TDSRow[1] | rtrim($row.getString('String')), 'TRIMT'), + col(row:TDSRow[1] | rtrim($row.getString('String')), 'TRIMT2') ]) }) } @@ -1682,7 +1716,9 @@ function <> meta::external::query::sql::transformation::queryToPure:: 'ltrim(NULL, \' \') AS "LTRIM2", md5(NULL) AS "MD5", upper(NULL) AS "UPPER", lower(NULL) AS "LOWER", replace(NULL, \'A\', \'a\') AS "REPLACE", starts_with(NULL, \'a\') AS "STARTSWITH", ' + 'strpos(NULL, \'abc\') AS "STRPOS", reverse(NULL) AS "REVERSE", rtrim(NULL) AS "RTRIM", rtrim(NULL, \' \') AS "RTRIM2", sha256(NULL) AS "SHA256", substring(NULL, 1) AS "SUBSTRING", ' + 'substr(NULL, 1, 2) AS "SUBSTR", btrim(NULL) AS "TRIM", btrim(NULL, \' \') AS "TRIM2", lpad(NULL, 2) AS "LPAD", lpad(String, NULL) AS "LPAD2", lpad(String, 2, NULL) AS "LPAD3",' + - 'rpad(NULL, 2) AS "RPAD", rpad(String, NULL) AS "RPAD2", rpad(String, 2, NULL) AS "RPAD3" FROM service."/service/service1"', + 'rpad(NULL, 2) AS "RPAD", rpad(String, NULL) AS "RPAD2", rpad(String, 2, NULL) AS "RPAD3", left(NULL, 1) AS "LEFT", left(String, NULL) AS "LEFT2", right(NULL, 1) AS "RIGHT", right(String, NULL) AS "RIGHT2", ' + + 'decode(NULL, \'base64\') AS "DECODE", encode(NULL, \'base64\') AS "ENCODE", trim(BOTH from NULL) AS "TRIM3" ' + + 'FROM service."/service/service1"', {| FlatInput.all() @@ -1716,7 +1752,14 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | []->cast(@String), 'LPAD3'), col(row:TDSRow[1] | []->cast(@String), 'RPAD'), col(row:TDSRow[1] | []->cast(@String), 'RPAD2'), - col(row:TDSRow[1] | []->cast(@String), 'RPAD3') + col(row:TDSRow[1] | []->cast(@String), 'RPAD3'), + col(row:TDSRow[1] | []->cast(@String), 'LEFT'), + col(row:TDSRow[1] | []->cast(@String), 'LEFT2'), + col(row:TDSRow[1] | []->cast(@String), 'RIGHT'), + col(row:TDSRow[1] | []->cast(@String), 'RIGHT2'), + col(row:TDSRow[1] | []->cast(@String), 'DECODE'), + col(row:TDSRow[1] | []->cast(@String), 'ENCODE'), + col(row:TDSRow[1] | []->cast(@String), 'TRIM3') ]) }) } @@ -2381,6 +2424,9 @@ function <> meta::external::query::sql::transformation::queryToPure:: doNameTest(^SimpleCaseExpression(operand = ^BooleanLiteral(value = true), whenClauses = ^WhenClause(operand = ^BooleanLiteral(value = false),result = ^IntegerLiteral(value = 2)), defaultValue = ^IntegerLiteral(value = 1)), 'CASE WHEN true = false THEN 2 ELSE 1 END'); + + doNameTest(^Trim(value = ^StringLiteral(value = 'abc', quoted = true), characters = ^StringLiteral(value = ' ', quoted = true), mode =TrimMode.BOTH), 'TRIM(BOTH \' \' FROM \'abc\')'); + doNameTest(^Trim(value = ^StringLiteral(value = 'abc', quoted = true), mode = TrimMode.BOTH), 'TRIM(BOTH FROM \'abc\')'); } diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/TableSourceExtractor.java b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/TableSourceExtractor.java index 5d05297574d..06b2684782c 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/TableSourceExtractor.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/TableSourceExtractor.java @@ -423,6 +423,12 @@ public Set visit(TableSubquery val) return val.query.accept(this); } + @Override + public Set visit(Trim val) + { + return val.value.accept(this); + } + @Override public Set visit(Union val) { From 1a6b3b02ec62c587751262b6cd56266e6b25c5f2 Mon Sep 17 00:00:00 2001 From: Abhishoya Lunavat <87463332+abhishoya-gs@users.noreply.github.com> Date: Thu, 30 Nov 2023 12:50:33 +0530 Subject: [PATCH 2/3] Relational - sortBy accepts lambda function (#2460) --- .../relational/functions/tests/testSort.pure | 103 ++++++++++++++++-- .../pureToSQLQuery/pureToSQLQuery.pure | 89 +++++++++++---- 2 files changed, 162 insertions(+), 30 deletions(-) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tests/testSort.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tests/testSort.pure index 93924893267..9cabd39a0a4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tests/testSort.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tests/testSort.pure @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::pure::graphFetch::execution::*; import meta::relational::mapping::*; import meta::relational::tests::model::simple::*; import meta::relational::tests::*; import meta::external::store::relational::tests::*; import meta::pure::profiles::*; +import meta::pure::executionPlan::toString::*; function <> meta::relational::tests::query::sort::testSortSimple():Boolean[1] { @@ -26,13 +28,98 @@ function <> meta::relational::tests::query::sort::testSortSimple():Bo assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName", "root".LASTNAME as "o_lastName" from personTable as "root" order by "root".LASTNAME', $result->sqlRemoveFormatting()); } -function <> meta::relational::tests::query::sort::testSortDeep():Boolean[1] +function <> meta::relational::tests::query::sort::testSortByLambdaSimple():Boolean[1] { - let result = execute(|Person.all()->sortBy(p | $p.address->toOne().name), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); - assertSize($result.values, 9); - assertEquals('Smith', $result.values->at(0).lastName); - assertEquals('Hill', $result.values->at(1).lastName); - assertSameElements(['Harris', 'Allen', 'Johnson', 'Hill'], $result.values->drop(2)->take(4).lastName); - assertEquals('Roberts', $result.values->at(6).lastName); - assertEquals('select root.FIRSTNAME as "firstName", root.AGE as "age", root.LASTNAME as "lastName", addressTable_d1_d_m1.NAME as "o_name" from personTable as root left outer join addressTable as addressTable_d1_d_m1 on (addressTable_d1_d_m1.ID = root.ADDRESSID) order by "o_name"', $result->sqlRemoveFormatting()); + let result = execute(|Person.all()->sortBy(p | $p.lastName), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertSize($result.values, 12); + assertEquals(['Allen', 'Firm B', 'Harris', 'Hill', 'Hill', 'Johnson', 'New York', 'Roberts', 'Smith', 'Smith', 'York', 'no Firm'], $result.values.lastName); + assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" order by "root".LASTNAME', $result->sqlRemoveFormatting()); +} + +function <> meta::relational::tests::query::sort::testSortByLambdaMultiple():Boolean[1] +{ + let result = execute(|Person.all()->sortBy(p | $p.lastName)->sortBy(p | $p.firstName), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertSize($result.values, 12); + assertEquals(['Anthony', 'David', 'Don', 'Elena', 'Fabrice', 'John', 'John', 'New', 'No address', 'No firm', 'Oliver', 'Peter'], $result.values.firstName); + assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" order by "root".FIRSTNAME', $result->sqlRemoveFormatting()); +} + +function <> meta::relational::tests::query::sort::testSortByLambdaColumnAddition():Boolean[1] +{ + let result = execute(|Person.all()->sortBy(p | $p.lastName + '|' + $p.firstName), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertSize($result.values, 12); + assertEquals(['Allen|Anthony', 'Firm B|Elena', 'Harris|David', 'Hill|John', 'Hill|Oliver', 'Johnson|John', 'New York|Don', 'Roberts|Fabrice', 'Smith|No address', 'Smith|Peter', 'York|New', 'no Firm|No firm'], zip($result.values.lastName, $result.values.firstName)->map(pair | $pair.first + '|' + $pair.second)); + assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" order by concat("root".LASTNAME, \'|\', "root".FIRSTNAME)', $result->sqlRemoveFormatting()); +} + +function <> meta::relational::tests::query::sort::testSortByLambdaWithIfElseValue():Boolean[1] +{ + let result = execute(|Person.all()->filter(p|$p.lastName == 'Hill' || $p.firstName == 'John')->sortBy(p | if($p.lastName == 'Hill' && $p.firstName == 'John', | 0, |if ($p.lastName == 'Hill', | 10, | if($p.firstName == 'John',| 20, | 30)))), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertSize($result.values, 3); + assertEquals(['John Hill', 'Oliver Hill', 'John Johnson'], zip($result.values.firstName, $result.values.lastName)->map(p| $p.first + ' ' + $p.second)); + assertEquals([0, 10, 20], zip($result.values.firstName, $result.values.lastName)->map(pair | if($pair.first == 'John' && $pair.second == 'Hill', |0, |if($pair.second == 'Hill', |10, |if($pair.first == 'John', |20, |30))) )); + assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" where ("root".LASTNAME = \'Hill\' or "root".FIRSTNAME = \'John\') order by case when ("root".LASTNAME = \'Hill\' and "root".FIRSTNAME = \'John\') then 0 else case when "root".LASTNAME = \'Hill\' then 10 else case when "root".FIRSTNAME = \'John\' then 20 else 30 end end end', $result->sqlRemoveFormatting()); +} + +function <> meta::relational::tests::query::sort::testSortByLambdaWIthIfElseColumn():Boolean[1] +{ + let result = execute(|Person.all()->filter(p|$p.lastName == 'Hill' || $p.firstName == 'John')->sortBy(p | if($p.lastName == 'Hill', |$p.firstName, |$p.lastName)), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertSize($result.values, 3); + assertEquals(['John', 'Johnson', 'Oliver'], zip($result.values.firstName, $result.values.lastName)->map(pair | if($pair.second == 'Hill', | $pair.first, |$pair.second))); + assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" where ("root".LASTNAME = \'Hill\' or "root".FIRSTNAME = \'John\') order by case when "root".LASTNAME = \'Hill\' then "root".FIRSTNAME else "root".LASTNAME end', $result->sqlRemoveFormatting()); +} + +function <> meta::relational::tests::query::sort::testSortByLambdaDeepOptional():Boolean[1] +{ + let result = execute(|Person.all()->sortBy(p | $p.address->toOne().name + '|' + $p.firstName + '|' + $p.lastName), simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertSize($result.values, 12); + assertEquals(['Hoboken|Peter|Smith', 'Hong Kong|Oliver|Hill', 'New York|Anthony|Allen', 'New York|David|Harris', 'New York|Don|New York', 'New York|Elena|Firm B', 'New York|John|Hill', 'New York|John|Johnson', 'New York|New|York', 'New York|No firm|no Firm', 'San Fransisco|Fabrice|Roberts'], zip($result.values.address.name, zip($result.values.firstName, $result.values.lastName))->map(pair | $pair.first + '|' + $pair.second.first + '|' + $pair.second.second)); + assertEquals('select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" left outer join addressTable as "addresstable_0" on ("addresstable_0".ID = "root".ADDRESSID) order by concat("addresstable_0".NAME, \'|\', "root".FIRSTNAME, \'|\', "root".LASTNAME)', $result->sqlRemoveFormatting()); +} + +function <> {meta::pure::executionPlan::profiles::serverVersion.start='vX_X_X'} meta::relational::tests::query::sort::testSortByLambdaAndGraphFetchDeep():Boolean[1] +{ + let gft = #{ + Person{ + address + { + name + } + } + }#; + let result = execute(|Person.all()->sortBy( + p | $p.address->toOne().name + )->graphFetch( + $gft + )->serialize( + $gft + ), + simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + + assertJsonStringsEqual('[{"address":null},{"address":{"name":"Hoboken"}},{"address":{"name":"Hong Kong"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"New York"}},{"address":{"name":"San Fransisco"}}]', $result.values); +} + + +function <> meta::relational::tests::query::sort::testSortByLambda_QueryWithParameters_Plan():Boolean[1] +{ + let rawPlan = meta::pure::executionPlan::executionPlan({firstName: String[1], lastName: String[1]|Person.all()->filter(p|$p.lastName == $lastName || $p.firstName == $firstName)->sortBy(p | if($p.firstName == $firstName && $p.lastName == $lastName, | 0, |if ($p.lastName == $lastName || $p.firstName == $firstName, | 10, | 20)))}, simpleRelationalMapping, testRuntime(), meta::relational::extension::relationalExtensions()); + assertEquals('Sequence\n' + + '(\n' + + ' type = Class[impls=(meta::relational::tests::model::simple::Person | simpleRelationalMappingInc.meta_relational_tests_model_simple_Person)]\n' + + ' resultSizeRange = *\n' + + ' (\n' + + ' FunctionParametersValidationNode\n' + + ' (\n' + + ' functionParameters = [firstName:String[1], lastName:String[1]]\n' + + ' )\n' + + ' Relational\n' + + ' (\n' + + ' type = Class[impls=(meta::relational::tests::model::simple::Person | simpleRelationalMappingInc.meta_relational_tests_model_simple_Person)]\n' + + ' resultSizeRange = *\n' + + ' resultColumns = [("pk_0", INT), ("firstName", VARCHAR(200)), ("age", INT), ("lastName", VARCHAR(200))]\n' + + ' sql = select "root".ID as "pk_0", "root".FIRSTNAME as "firstName", "root".AGE as "age", "root".LASTNAME as "lastName" from personTable as "root" where ("root".LASTNAME = \'${lastName?replace("\'", "\'\'")}\' or "root".FIRSTNAME = \'${firstName?replace("\'", "\'\'")}\') order by case when ("root".FIRSTNAME = \'${firstName?replace("\'", "\'\'")}\' and "root".LASTNAME = \'${lastName?replace("\'", "\'\'")}\') then 0 else case when ("root".LASTNAME = \'${lastName?replace("\'", "\'\'")}\' or "root".FIRSTNAME = \'${firstName?replace("\'", "\'\'")}\') then 10 else 20 end end\n' + + ' connection = TestDatabaseConnection(type = "H2")\n' + + ' )\n' + + ' )\n' + + ')\n',$rawPlan->planToString(meta::relational::extension::relationalExtensions())); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure index cccadba05ef..f9016b2711d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure @@ -2487,29 +2487,74 @@ function meta::relational::functions::pureToSqlQuery::processTDSSort(f:FunctionE function meta::relational::functions::pureToSqlQuery::processSortBy(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] { - let mainQuery = processValueSpecification($f.parametersValues->at(0), $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne()->cast(@SelectWithCursor); - let mainSelect = $mainQuery.select->pushSavedFilteringOperation($extensions); - - let sortByFunc = $f.parametersValues->evaluateAndDeactivate()->at(1)->byPassRouterInfo()->cast(@InstanceValue).values->at(0); - assert($sortByFunc->instanceOf(Path), 'sortBy can currently be converted to SQL only if the parameter is a Path.'); - let path = $sortByFunc->cast(@Path); - let pathName = 'o_'+$path->buildColumnNameOutOfPath(); - let joinResult = processPath($path, $pathName, '1', $mainQuery, $vars, ^$state(inFilter=false), $nodeId, $aggFromMap, $context->shift(), $extensions).element->cast(@SelectWithCursor).select; - - let merge = $mainSelect->concatenate($joinResult)->mergeSQLQueryData($nodeId, $state, $context, $extensions); + let leftSidePure = $f.parametersValues->at(0); + let leftSideOp = processValueSpecificationReturnPropertyMapping($leftSidePure, $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne(); + let mainQuery = $leftSideOp.element->cast(@SelectWithCursor); + let mainSelect = $mainQuery.select->pushSavedFilteringOperation($extensions); + let sortByFunc = $f.parametersValues->evaluateAndDeactivate()->at(1)->byPassRouterInfo()->cast(@InstanceValue).values->at(0); + + $sortByFunc->match([ + path: Path[1] | + let pathName = 'o_'+$path->buildColumnNameOutOfPath(); + let joinResult = processPath($path, $pathName, '1', $mainQuery, $vars, ^$state(inFilter=false), $nodeId, $aggFromMap, $context->shift(), $extensions).element->cast(@SelectWithCursor).select; + + let merge = $mainSelect->concatenate($joinResult)->mergeSQLQueryData($nodeId, $state, $context, $extensions); - ^$operation( - currentTreeNode = $mainQuery.currentTreeNode->toOne()->findOneNode($mainQuery.select.data->toOne(), $merge.data->toOne()), - positionBeforeLastApplyJoinTreeNode = if ($mainQuery.positionBeforeLastApplyJoinTreeNode->isEmpty(),|[],|$mainQuery.positionBeforeLastApplyJoinTreeNode->toOne()->findOneNode($mainQuery.select.data->toOne(), $merge.data->toOne())), - select = ^SelectSQLQuery( - columns = $merge.columns, - data = $merge.data, - leftSideOfFilter = $merge.leftSideOfFilter, - filteringOperation = $merge.filteringOperation, - extraFilteringOperation = $merge.extraFilteringOperation, - orderBy = ^OrderBy(column=$joinResult.columns->at(0)->cast(@Alias).relationalElement, direction=meta::relational::metamodel::SortDirection.ASC) - ) - ); + ^$operation( + currentTreeNode = $mainQuery.currentTreeNode->toOne()->findOneNode($mainQuery.select.data->toOne(), $merge.data->toOne()), + positionBeforeLastApplyJoinTreeNode = if ($mainQuery.positionBeforeLastApplyJoinTreeNode->isEmpty(),|[],|$mainQuery.positionBeforeLastApplyJoinTreeNode->toOne()->findOneNode($mainQuery.select.data->toOne(), $merge.data->toOne())), + select = ^SelectSQLQuery( + columns = $merge.columns, + data = $merge.data, + leftSideOfFilter = $merge.leftSideOfFilter, + filteringOperation = $merge.filteringOperation, + extraFilteringOperation = $merge.extraFilteringOperation, + orderBy = ^OrderBy(column=$joinResult.columns->at(0)->cast(@Alias).relationalElement, direction=meta::relational::metamodel::SortDirection.ASC) + ) + );, + l: LambdaFunction[1] | + /* + - already have a mainQuery - select ... + - process the lambda function inside sortBy with the state to generate a sql query + select as "generated_order_key" from ... + where is a sql representation for lambda function and "generated_order_key" is a temporary alias introduced + - merge these two queries + - modify the merged query to move the sql_expr from select to order by + select ... order by + */ + + let generated_order_key = '"generated_order_key"'; + let paramValue = $f.parametersValues->at(1)->match([e:StoreMappingRoutedValueSpecification[1]|$e.value,v:ValueSpecification[1]|$v]); + let updatedState = $state->updateFunctionParamScope($paramValue.genericType.typeArguments.rawType->toOne()->cast(@FunctionType),$mainQuery); + let lambdaFunction = $f->instanceValuesAtParameter(1, $vars, $updatedState.inScopeVars)->at(0)->cast(@FunctionDefinition); + let lambdaFunctionExpression = $lambdaFunction.expressionSequence->at(0); + let inScopeVarsWithPlaceholdersState = $lambdaFunction->addPlaceHoldersForLambdaOpenVariables($vars, $updatedState); + let embeddedMapping = if($leftSideOp.currentPropertyMapping->isEmpty(), | $currentPropertyMapping, | $leftSideOp.currentPropertyMapping); + + let rightSide = processValueSpecification($lambdaFunctionExpression, $embeddedMapping, $mainQuery, $vars, ^$inScopeVarsWithPlaceholdersState(inFilter=false), JoinType.LEFT_OUTER, $nodeId, $aggFromMap, $context->shift(), $extensions)->toOne()->cast(@SelectWithCursor); + let rightSideSelect = $rightSide.select; + let rightSideModified = ^$rightSide( + select = ^$rightSideSelect( + columns = if($rightSideSelect.columns->isEmpty(), | [], |^Alias(name = $generated_order_key, relationalElement = $rightSideSelect.columns->toOne())) + ) + ); + + let revisedLeftRightSelects = pair($mainSelect,$rightSideModified.select); + let merge = [$revisedLeftRightSelects.first, $revisedLeftRightSelects.second]->cast(@SelectSQLQuery)->mergeSQLQueryData($nodeId, $updatedState, $context, $extensions); + ^$operation( + currentTreeNode = $mainQuery.currentTreeNode->toOne()->findOneNode($mainQuery.select.data->toOne(), $merge.data->toOne()), + positionBeforeLastApplyJoinTreeNode = if ($mainQuery.positionBeforeLastApplyJoinTreeNode->isEmpty(),|[],|$mainQuery.positionBeforeLastApplyJoinTreeNode->toOne()->findOneNode($mainQuery.select.data->toOne(), $merge.data->toOne())), + select = ^SelectSQLQuery( + columns = $merge.columns->filter(c|!$c->instanceOf(Alias) || $c->cast(@Alias).name != $generated_order_key), + data = $merge.data, + leftSideOfFilter = $merge.leftSideOfFilter, + filteringOperation = $merge.filteringOperation, + extraFilteringOperation = $merge.extraFilteringOperation, + orderBy = ^OrderBy(column=$rightSideModified.select.columns->toOne()->cast(@Alias).name->findAliasOrFail($merge).relationalElement, direction=meta::relational::metamodel::SortDirection.ASC) + ) + );, + a: Any[1] | fail('Invalid parameter type to sortBy ' + $sortByFunc->type().name->orElse('unknown')); $operation; + ]); } function meta::relational::functions::pureToSqlQuery::processSlice(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] From 8e935d4687246400531ab32e0e615d3ca68e80aa Mon Sep 17 00:00:00 2001 From: Abhishoya Lunavat <87463332+abhishoya-gs@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:52:09 +0530 Subject: [PATCH 3/3] fix: use parent temp table strategy in cross store node (#2481) --- .../relational/plugin/RelationalExecutionNodeExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java index 876c712bc1b..f34b6a15729 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java @@ -1909,7 +1909,7 @@ private Result executeRelationalCrossRootQueryTempTableGraphFetchExecutionNode(R { } node.parentTempTableStrategy.createTempTableNode.accept(new ExecutionNodeExecutor(this.profiles, this.executionState)); - loadValuesIntoTempTablesFromRelationalResult(node.parentTempTableStrategy.loadTempTableNode, parentRealizedRelationalResult, ((LoadFromResultSetAsValueTuplesTempTableStrategy) node.parentTempTableStrategy).tupleBatchSize, ((LoadFromResultSetAsValueTuplesTempTableStrategy) node.tempTableStrategy).quoteCharacterReplacement, databaseTimeZone, this.executionState, this.profiles); + loadValuesIntoTempTablesFromRelationalResult(node.parentTempTableStrategy.loadTempTableNode, parentRealizedRelationalResult, ((LoadFromResultSetAsValueTuplesTempTableStrategy) node.parentTempTableStrategy).tupleBatchSize, ((LoadFromResultSetAsValueTuplesTempTableStrategy) node.parentTempTableStrategy).quoteCharacterReplacement, databaseTimeZone, this.executionState, this.profiles); } else if (node.parentTempTableStrategy instanceof LoadFromTempFileTempTableStrategy) {