Skip to content

Commit

Permalink
Legend SQL - assortment of changes (#2340)
Browse files Browse the repository at this point in the history 
* Legend SQL - assortment of changes
1. allow realias of column to table column name
2. splitPart support
3. handle aggregations within case statement

* checkstyle

* checkstyle

* split h2 extension

* h2 split extension

* Commons lang dependency

* Enum test exclude alloy

* Fix concat handling

* Handling String Concat

* Handle String concat
  • Loading branch information
@gs-jp1
gs-jp1 authored Oct 4, 2023
1 parent 5ee34f8 commit bf1fa5a
Show file tree
Hide file tree
Showing 20 changed files with 356 additions and 151 deletions.
2 changes: 2 additions & 0 deletions ...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,7 @@ private void registerStrings()
register("meta::pure::functions::string::replace_String_1__String_1__String_1__String_1_", true, ps -> res("String", "one"));
register("meta::pure::functions::string::reverseString_String_1__String_1_", true, ps -> res("String", "one"));
register("meta::pure::functions::string::split_String_1__String_1__String_MANY_", true, ps -> res("String", "zeroMany"));
register("meta::pure::functions::string::splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_", false, ps -> res("String", "zeroOne"));
register(m(m(h("meta::pure::functions::string::substring_String_1__Integer_1__Integer_1__String_1_", true, ps -> res("String", "one"), ps -> ps.size() == 3)),
m(h("meta::pure::functions::string::substring_String_1__Integer_1__String_1_", true, ps -> res("String", "one"), ps -> true))));
register("meta::pure::functions::string::toLower_String_1__String_1_", true, ps -> res("String", "one"));
Expand Down Expand Up @@ -2040,6 +2041,7 @@ private Map<String, Dispatch> buildDispatch()
map.put("meta::pure::functions::string::reverseString_String_1__String_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::string::splitOnCamelCase_String_1__String_MANY_", (List<ValueSpecification> ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::string::split_String_1__String_1__String_MANY_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::string::splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_", (List<ValueSpecification> ps) -> ps.size() == 3 && matchZeroOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::string::startsWith_String_$0_1$__String_1__Boolean_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && matchZeroOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::string::startsWith_String_1__String_1__Boolean_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::string::substringAfter_String_1__String_1__String_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())));
Expand Down
1 change: 1 addition & 0 deletions ...gend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,7 @@ Class meta::legend::test::handlers::model::TestString
replace(){$this.string->replace($this.string, $this.string)}:String[1];
reverseString(){$this.string->reverseString()}:String[1];
split(){$this.string->split($this.string)}:String[*];
splitPart(){$this.string->splitPart($this.string, 0)}:String[*];
rtrim(){$this.string->rtrim()}:String[1];
substringTwo(){$this.string->substring(1,2)}:String[1];
substring(){$this.string->substring(1)}:String[1];
Expand Down
8 changes: 8 additions & 0 deletions ...e-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,14 @@ function {doc.doc = 'Un-camel case / humanize the provided string using provided
->joinStrings(' ');
}

function {doc.doc = 'Split the string and select the part'}
meta::pure::functions::string::splitPart(str:String[0..1], token:String[1], part:Integer[1]):String[0..1]
{
if ($str->isEmpty(),
| [],
| $str->toOne()->split($token)->at($part));
}

function {doc.doc = 'Split the proivded string on camel case'}
meta::pure::functions::string::splitOnCamelCase(str:String[1]):String[*]
{
Expand Down
8 changes: 8 additions & 0 deletions ...-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/stringExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,14 @@ function <<test.Test>> {test.excludePlatform = 'Java compiled'} meta::pure::func
assertEquals('abab', repeatString('ab', 2));
}

function <<test.Test>> meta::pure::functions::string::tests::testSplitPart():Boolean[1]
{
assertEquals([], splitPart([], 'a', 1));
assertEquals('abc', splitPart('abc', 'd', 0));
assertEquals('a', splitPart('abc', 'b', 0));
assertEquals('c', splitPart('abc', 'b', 1));
}

function <<test.Test>> meta::pure::functions::string::tests::testSplitOnCamelCase():Boolean[1]
{
let pairs = [
Expand Down
1 change: 1 addition & 0 deletions ...e-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,7 @@ function meta::pure::router::routing::shouldStopFunctions(extensions:meta::pure:
or_Boolean_$1_MANY$__Boolean_1_,
tdsContains_T_1__Function_MANY__TabularDataSet_1__Boolean_1_,
tdsContains_T_1__Function_MANY__String_MANY__TabularDataSet_1__Function_1__Boolean_1_,
splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_,
meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_,
meta::pure::functions::date::calendar::annualized_Date_1__String_1__Date_1__Number_$0_1$__Number_$0_1$_,
meta::pure::functions::date::calendar::cme_Date_1__String_1__Date_1__Number_$0_1$__Number_$0_1$_,
Expand Down
19 changes: 19 additions & 0 deletions ...ain/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.finos.legend.engine.shared.core.ObjectMapperFactory;
import org.h2.value.Value;
import org.h2.value.ValueBigint;
Expand Down Expand Up @@ -128,4 +129,22 @@ public static String legend_h2_extension_reverse_string(String string)
{
return string == null ? null : new StringBuilder(string).reverse().toString();
}

public static String legend_h2_extension_split_part(String string, String token, Integer part)
{
if (part < 1)
{
throw new IllegalArgumentException("Split part must be greater than zero");
}

if (string == null)
{
return null;
}

String[] parts = StringUtils.split(string, token);
int readjustedPart = part - 1;

return parts.length > readjustedPart ? parts[readjustedPart] : null;
}
}
6 changes: 4 additions & 2 deletions ...stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ private static List<String> getLegendH2ExtensionSQLs()
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_json_parse FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_json_parse\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_decode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_base64_decode\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_encode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_base64_encode\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_reverse_string\";"
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_reverse_string\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_split_part FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_split_part\";"
);
}

Expand All @@ -119,7 +120,8 @@ private static List<String> getLegendH2_1_4_200_ExtensionSQLs()
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_encode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_base64_encode\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_reverse_string\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_md5 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_md5\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_sha1 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_sha1\";"
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_sha1 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_sha1\";",
"CREATE ALIAS IF NOT EXISTS legend_h2_extension_split_part FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_split_part\";"
);
}
}
8 changes: 8 additions & 0 deletions ...t-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,13 @@
<artifactId>legend-engine-shared-core</artifactId>
</dependency>
<!-- DRIVERS -->

<!-- COMMONS-LANG-->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<!-- COMMONS-LANG -->

</dependencies>
</project>
19 changes: 19 additions & 0 deletions .../org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.codec.digest.DigestUtils;
import org.finos.legend.engine.shared.core.ObjectMapperFactory;
import org.h2.value.Value;
Expand Down Expand Up @@ -139,4 +140,22 @@ public static String legend_h2_extension_hash_sha1(String string)
{
return string == null ? null : DigestUtils.sha1Hex(string);
}

public static String legend_h2_extension_split_part(String string, String token, Integer part)
{
if (part < 1)
{
throw new IllegalArgumentException("Split part must be greater than zero");
}

if (string == null)
{
return null;
}

String[] parts = StringUtils.split(string, token);
int readjustedPart = part - 1;

return parts.length > readjustedPart ? parts[readjustedPart] : null;
}
}
30 changes: 28 additions & 2 deletions ...ore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2226,6 +2226,22 @@ function meta::relational::functions::pureToSqlQuery::processNoOp(f:FunctionExpr
processValueSpecificationReturnPropertyMapping($f.parametersValues->at(0), $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne()
}

function meta::relational::functions::pureToSqlQuery::processSplitPart(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map<VariableExpression, ValueSpecification>[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List<ColumnGroup>[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1]
{
//sql is 1 index based, pure is 0 index based
let position = $f.parametersValues->at(2)->zeroToOneBasedIndex();

let functionExpression = ^$f(parametersValues = [$f.parametersValues->at(0), $f.parametersValues->at(1), $position]);
$functionExpression->processDynaFunction($currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions);
}

function meta::relational::functions::pureToSqlQuery::zeroToOneBasedIndex(v:ValueSpecification[1]):ValueSpecification[1]
{
$v->match([
i:InstanceValue[1] | ^$i(values = $i.values->toOne()->cast(@Integer) + 1)
]);
}

function meta::relational::functions::pureToSqlQuery::processHash(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map<VariableExpression, ValueSpecification>[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List<ColumnGroup>[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1]
{
let type = $f.parametersValues->at(1)->reactivate()->cast(@meta::pure::functions::hash::HashType)->toOne();
Expand Down Expand Up @@ -4355,7 +4371,9 @@ function meta::relational::functions::pureToSqlQuery::processAggFn(v:ValueSpecif
b:Boolean[1] | ^Literal(value=$b),
f:ValueSpecification[1]|$f->processAggFn($params),
f:LambdaFunction<Any>[1]|$f.expressionSequence->toOne()->cast(@ValueSpecification)->processAggFn($params),
a:Any[1] | fail('Unsupported aggregation operation'); [];
e:Enum[1] | ^Literal(value=$e),
e:Enumeration<Enum>[1] | ^Literal(value = $v),
a:Any[1] | println($a);fail('Unsupported aggregation operation'); [];
]));
);
]
Expand Down Expand Up @@ -5031,6 +5049,13 @@ function meta::relational::functions::pureToSqlQuery::processTdsLambda(mapFn:Val
^PureFunctionTDSToRelationalFunctionPair(first = meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_, second = {|
newDynaFunction('coalesce', $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context))
}),
^PureFunctionTDSToRelationalFunctionPair(first = splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_, second = {|
newDynaFunction('splitPart', [
$f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)->toOne(),
$f.parametersValues->at(1)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)->toOne(),
$f.parametersValues->at(2)->zeroToOneBasedIndex()->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context) ->toOne()
])
}),
^PureFunctionTDSToRelationalFunctionPair(first = parseDate_String_1__Date_1_, second = {|
let format = ^InstanceValue(multiplicity = PureOne, genericType = ^GenericType(rawType=String), values = 'YYYY-MM-DD HH24:MI:SS');
newDynaFunction('toTimestamp', [
Expand Down Expand Up @@ -7584,6 +7609,7 @@ function meta::relational::functions::pureToSqlQuery::getSupportedFunctions():Ma
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::replace_String_1__String_1__String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::repeatString_String_$0_1$__Integer_1__String_$0_1$_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::reverseString_String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_,second=meta::relational::functions::pureToSqlQuery::processSplitPart_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::math::abs_Integer_1__Integer_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::math::abs_Number_1__Number_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::math::abs_Float_1__Float_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
Expand Down Expand Up @@ -7699,4 +7725,4 @@ function meta::relational::functions::pureToSqlQuery::getSupportedFunctions():Ma
^PureFunctionToRelationalFunctionPair(first=meta::pure::mutation::save_T_MANY__RootGraphFetchTree_1__Mapping_1__Runtime_1__T_MANY_, second=meta::relational::functions::pureToSqlQuery::processNoOp_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_),
^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::hash::hash_String_1__HashType_1__String_1_, second=meta::relational::functions::pureToSqlQuery::processHash_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_)
])
}
}
1 change: 1 addition & 0 deletions ...tore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1001,6 +1001,7 @@ Enum meta::relational::functions::sqlQueryToString::DynaFunctionRegistry
sign,
sin,
size,
splitPart,
sqlFalse,
sqlNull,
sqlTrue,
Expand Down
Loading

0 comments on commit bf1fa5a

Please sign in to comment.