Skip to content

Commit

Permalink
Legend SQL assortment of updates (finos#2478)
Browse files Browse the repository at this point in the history 
- handle standard aggregation functions in window context
- bool_and/bool_or support
- extra tracing
- do not fail on duplicate sources
- explicit fail handling of function orderby
  • Loading branch information
@gs-jp1 @AFine-gs
gs-jp1 authored and AFine-gs committed Jan 9, 2024
1 parent 9feef9f commit c09e90d
Show file tree
Hide file tree
Showing 33 changed files with 503 additions and 136 deletions.
4 changes: 3 additions & 1 deletion ...in/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,8 @@ public Handlers(PureModel pureModel)
m(h("meta::pure::functions::collection::and_Boolean_MANY__Boolean_1_", false, ps -> res("Boolean", "one"), ps -> true))));

register(m(m(h("meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_", true, ps -> res("Boolean", "one"), ps -> ps.size() == 2)),
m(h("meta::pure::functions::collection::or_Boolean_$1_MANY$__Boolean_1_", false, ps -> res("Boolean", "one"), ps -> true))));
m(h("meta::pure::functions::collection::or_Boolean_$1_MANY$__Boolean_1_", false, ps -> res("Boolean", "one"), ps -> ps.size() == 1 && matchOneMany(ps.get(0)._multiplicity())),
h("meta::pure::functions::collection::or_Boolean_MANY__Boolean_1_", false, ps -> res("Boolean", "one"), ps -> true))));

register("meta::pure::tds::tdsRows_TabularDataSet_1__TDSRow_MANY_", false, ps -> res("meta::pure::tds::TDSRow", "zeroMany"));
register("meta::pure::functions::boolean::not_Boolean_1__Boolean_1_", true, ps -> res("Boolean", "one"));
Expand Down Expand Up @@ -1785,6 +1786,7 @@ private Map<String, Dispatch> buildDispatch()
map.put("meta::pure::functions::collection::objectReferenceIn_Any_1__String_MANY__Boolean_1_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::oneOf_Boolean_MANY__Boolean_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Boolean".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::or_Boolean_$1_MANY$__Boolean_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && matchOneMany(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Boolean".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::or_Boolean_MANY__Boolean_1_", (List<ValueSpecification> ps) -> ps.size() == 1 && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Boolean".equals(ps.get(0)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::range_Integer_1__Integer_1__Integer_1__Integer_MANY_", (List<ValueSpecification> ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Integer".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::range_Integer_1__Integer_1__Integer_MANY_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Integer".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name())));
map.put("meta::pure::functions::collection::removeDuplicatesBy_T_MANY__Function_1__T_MANY_", (List<ValueSpecification> ps) -> ps.size() == 2 && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && check(ft._parameters().toList(), (List<? extends VariableExpression> nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity())))));
Expand Down
5 changes: 5 additions & 0 deletions ...re-code-compiled-core/src/main/resources/core/pure/corefunctions/collectionExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,11 @@ function meta::pure::functions::collection::and(vals:Boolean[*]):Boolean[1]
$vals->fold({i,a|$i && $a}, true)
}

function meta::pure::functions::collection::or(vals:Boolean[*]):Boolean[1]
{
$vals->fold({i,a|$i || $a}, false)
}

function meta::pure::functions::collection::or(vals:Boolean[1..*]):Boolean[1]
{
$vals->fold({i,a|$i || $a}, false)
Expand Down
1 change: 1 addition & 0 deletions ...e-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,7 @@ function meta::pure::router::routing::shouldStopFunctions(extensions:meta::pure:
func_FunctionDefinition_1__TdsOlapRank_1_,
and_Boolean_MANY__Boolean_1_,
or_Boolean_$1_MANY$__Boolean_1_,
or_Boolean_MANY__Boolean_1_,
tdsContains_T_1__Function_MANY__TabularDataSet_1__Boolean_1_,
tdsContains_T_1__Function_MANY__String_MANY__TabularDataSet_1__Function_1__Boolean_1_,
splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_,
Expand Down
4 changes: 3 additions & 1 deletion ...ine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,9 @@ function meta::pure::metamodel::serialization::grammar::printFunctionExpression(
);
if ($name == 'minus' && $vals->size() == 1,
| $m->get($name)->toOne() + $functionExpression.parametersValues->at(0)->printValueSpecification($space),
| '(' + $vals->at(0)->printValueSpecification($space) + ' ' + $m->get($name)->toOne() + ' ' + $vals->at(1)->printValueSpecification($space) + ')'
| if ($name->in(['and', 'or']) && $vals->size() == 1,
| $name + '(' + $vals->at(0)->printValueSpecification($space) + ')',
| '(' + $vals->at(0)->printValueSpecification($space) + ' ' + $m->get($name)->toOne() + ' ' + $vals->at(1)->printValueSpecification($space) + ')')
);
);
);
Expand Down
1 change: 1 addition & 0 deletions ...e_java_platform_binding/legendJavaPlatformBinding/planConventions/collectionsLibrary.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ function meta::pure::executionPlan::platformBinding::legendJava::library::collec

fc1(and_Boolean_MANY__Boolean_1_, {ctx,collection | $collection->j_streamOf()->js_reduce(j_true(), j_lambda([$x, $y], $x->j_and($y)))}),
fc1(or_Boolean_$1_MANY$__Boolean_1_, {ctx,collection | $collection->j_streamOf()->js_reduce(j_false(), j_lambda([$x, $y], $x->j_or($y)))}),
fc1(or_Boolean_MANY__Boolean_1_, {ctx,collection | $collection->j_streamOf()->js_reduce(j_false(), j_lambda([$x, $y], $x->j_or($y)))}),

fc1(size_Any_MANY__Integer_1_, {ctx,collection | $collection->j_streamOf()->js_count()}),
fc (count_Any_MANY__Integer_1_, fcAlias( size_Any_MANY__Integer_1_)),
Expand Down
1 change: 1 addition & 0 deletions ...ources/core_external_format_json/transformation/toBeRefactored/jsonSchemaConstraints.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ function <<access.private>> meta::external::format::json::schema::constraint::to
pair('meta::json::schema::discriminateOneOf_Any_1__Any_1__Type_MANY__DiscriminatorMapping_MANY__Boolean_1_',meta::external::format::json::schema::constraint::buildSubTypeDiscriminator_SimpleFunctionExpression_1__JSONSchemaConfigInternal_1__SchemaConstraint_1_),
pair('meta::pure::functions::string::isUUID_String_$0_1$__Boolean_1_',meta::external::format::json::schema::constraint::buildUUID_SimpleFunctionExpression_1__JSONSchemaConfigInternal_1__SchemaConstraint_1_),
pair('meta::pure::functions::collection::or_Boolean_$1_MANY$__Boolean_1_', meta::external::format::json::schema::constraint::buildOr_SimpleFunctionExpression_1__JSONSchemaConfigInternal_1__SchemaConstraint_1_),
pair('meta::pure::functions::collection::or_Boolean_MANY__Boolean_1_', meta::external::format::json::schema::constraint::buildOr_SimpleFunctionExpression_1__JSONSchemaConfigInternal_1__SchemaConstraint_1_),
pair('meta::pure::functions::math::rem_Number_1__Number_1__Number_1_', meta::external::format::json::schema::constraint::builtMultipleOf_SimpleFunctionExpression_1__JSONSchemaConfigInternal_1__SchemaConstraint_1_)


Expand Down
2 changes: 2 additions & 0 deletions ...rc/main/resources/core_relational_athena/relational/sqlQueryToString/athenaExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::athen

[
dynaFnToSql('and', $allStates, ^ToSql(format='%s', transform={p:String[*]|$p->makeString(' and ')})),
dynaFnToSql('booland', $allStates, ^ToSql(format='bool_and(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='bool_or(%s)')),
dynaFnToSql('count', $allStates, ^ToSql(format='count(%s)', transform={p:String[*]|if($p->isEmpty(),|'*',|$p)})),
dynaFnToSql('datePart', $allStates, ^ToSql(format='date(%s)')),
dynaFnToSql('equal', $allStates, ^ToSql(format='%s = %s')),
Expand Down
2 changes: 2 additions & 0 deletions ...ain/resources/core_relational_bigquery/relational/sqlQueryToString/bigQueryExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::bigQu
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='date_add(%s)', transform={p:String[3] | $p->at(0) + ', INTERVAL ' + $p->at(1) + ' ' + $p->at(2)->mapToDBUnitType()})),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='logical_and(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='logical_or(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
dynaFnToSql('datePart', $allStates, ^ToSql(format='date(%s)')),
dynaFnToSql('hour', $allStates, ^ToSql(format='extract(hour from %s)')),
Expand Down
2 changes: 2 additions & 0 deletions ...rc/main/resources/core_relational_memsql/relational/sqlQueryToString/memSQLExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::memsq
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='adddate(%s)', transform={p:String[3] | $p->at(0) + ', INTERVAL ' + $p->at(1) + ' ' + $p->at(2)->mapToDBUnitType()})),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='every(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='any(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDateMemSQL()})),
dynaFnToSql('convertVarchar128', $allStates, ^ToSql(format='convert(%s, CHAR)')),
Expand Down
2 changes: 2 additions & 0 deletions ...ain/resources/core_relational_postgres/relational/sqlQueryToString/postgresExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::postg
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='(%s + (INTERVAL \'1 %s\' * %s))', transform={p:String[3] | [$p->at(0), $p->at(2)->replace('Text', '')->removeQuotesIfExist(), $p->at(1)]})), //https://www.postgresql.org/message-id/2B704E31-B533-4159-B6F0-EC284CFE824C%40skype.net,
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='bool_and(%s)', transform={p:String[*] | joinStrings($p, ', ')})),
dynaFnToSql('boolor', $allStates, ^ToSql(format='bool_or(%s)', transform={p:String[*] | joinStrings($p, ', ')})),
dynaFnToSql('dateDiff', $allStates, ^ToSql(format='%s', transform={p:String[*]|generateDateDiffExpressionForPostgres($p)})),
dynaFnToSql('datePart', $allStates, ^ToSql(format='Date(%s)')),
dynaFnToSql('dayOfMonth', $allStates, ^ToSql(format='date_part(\'day\', %s)')),
Expand Down
2 changes: 2 additions & 0 deletions ...rc/main/resources/core_relational_presto/relational/sqlQueryToString/prestoExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::prest
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='date_add(%s)', transform={p:String[3] | '\'' + $p->at(2)->mapToDBUnitType() + '\', ' + $p->at(1) + ', ' + $p->at(0)})),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='bool_and(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='bool_or(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDatePresto()})),
dynaFnToSql('convertDateTime', $allStates, ^ToSql(format='%s' , transform={p:String[*] | $p->convertToDateTimePresto()})),
Expand Down
2 changes: 2 additions & 0 deletions ...ain/resources/core_relational_redshift/relational/sqlQueryToString/redshiftExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::redsh

[
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='bool_and(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='bool_or(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='%s', transform={p:String[*]|$p->joinStrings(' + ')})),
dynaFnToSql('datePart', $allStates, ^ToSql(format='trunc(%s)')),
dynaFnToSql('dayOfWeek', $allStates, ^ToSql(format='to_char(%s, \'FMDay\')')),
Expand Down
2 changes: 2 additions & 0 deletions ...n/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::snowf
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='dateadd(%s)', transform={p:String[3] | $p->at(2)->mapToDBUnitType() + ', ' + $p->at(1) + ', ' + $p->at(0)})),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='booland(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='boolor(%s)')),
dynaFnToSql('ceiling', $allStates, ^ToSql(format='ceil(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDateSnowflake()})),
Expand Down
2 changes: 2 additions & 0 deletions .../main/resources/core_relational_spanner/relational/sqlQueryToString/spannerExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::spann
dynaFnToSql('asin', $allStates, ^ToSql(format='asin(%s)')),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('atan', $allStates, ^ToSql(format='atan(%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='logical_and(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='logical_or(%s)')),
dynaFnToSql('average', $allStates, ^ToSql(format='avg(1.0 * %s)')),
dynaFnToSql('ceiling', $allStates, ^ToSql(format='ceiling(%s)')),
dynaFnToSql('coalesce', $allStates, ^ToSql(format='coalesce%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
Expand Down
2 changes: 2 additions & 0 deletions ...main/resources/core_relational_sybase/relational/sqlQueryToString/sybaseASEExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ function meta::relational::functions::sqlQueryToString::sybaseASE::getDynaFuncti
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='dateadd(%s)', transform={p:String[3] | $p->at(2)->mapToDBUnitType() + ', ' + $p->at(1) + ', ' + $p->at(0)})),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='%s', transform={p:String[1] | convertBoolAndOrToCaseStatement($p, true)})),
dynaFnToSql('boolor', $allStates, ^ToSql(format='%s', transform={p:String[1] | convertBoolAndOrToCaseStatement($p, false)})),
dynaFnToSql('concat', $allStates, ^ToSql(format='%s', transform={p:String[*]|$p->joinStrings(' + ')})),
dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDateIQ()})),
dynaFnToSql('convertDateTime', $allStates, ^ToSql(format='%s' , transform={p:String[*] | $p->convertToDateTimeIQ()})),
Expand Down
2 changes: 2 additions & 0 deletions ...ain/resources/core_relational_sybaseiq/relational/sqlQueryToString/sybaseIQExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ function meta::relational::functions::sqlQueryToString::sybaseIQ::getDynaFunctio
[
dynaFnToSql('adjust', $allStates, ^ToSql(format='dateadd(%s)', transform={p:String[3] | $p->at(2)->mapToDBUnitType() + ', ' + $p->at(1) + ', ' + $p->at(0)})),
dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='%s', transform={p:String[1] | convertBoolAndOrToCaseStatement($p, true)})),
dynaFnToSql('boolor', $allStates, ^ToSql(format='%s', transform={p:String[1] | convertBoolAndOrToCaseStatement($p, false)})),
dynaFnToSql('char', $allStates, ^ToSql(format='char(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='%s', transform={p:String[*]|$p->joinStrings(' + ')})),
dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDateIQ()})),
Expand Down
Loading

0 comments on commit c09e90d

Please sign in to comment.