From fbcbc426cd72b1235e893c8a3ce21d430c312a2a Mon Sep 17 00:00:00 2001 From: Pierre De Belen Date: Tue, 8 Oct 2024 08:32:59 -0400 Subject: [PATCH] Add support for a new relation asOfJoin operation (#3162) --- .../engine/repl/autocomplete/Completer.java | 3 +- .../handlers/AsOfJoinHandler.java | 83 ++ .../legend/engine/repl/TestCompleter.java | 22 +- .../toPureGraph/handlers/Handlers.java | 40 +- .../functions/transformation/asofjoin.pure | 121 ++ .../compiled/RelationExtensionCompiled.java | 1 + .../RelationNativeImplementation.java | 46 + .../relation/compiled/natives/AsOfJoin.java | 55 + .../natives/shared/TestTDSCompiled.java | 7 + .../RelationExtensionInterpreted.java | 6 +- .../interpreted/natives/AsOfJoin.java | 108 ++ .../interpreted/natives/shared/Shared.java | 18 +- .../natives/shared/TestTDSInterpreted.java | 8 + .../interpreted/pure/TestFunctionTester.java | 28 + .../external/relation/shared/TestTDS.java | 223 +++- .../java/extension/relation/TestTestTDS.java | 126 +- .../Test_JAVA_RelationFunction_PCT.java | 4 + .../pct_relational.pure | 16 +- .../sqlQueryToString/duckdbExtension.pure | 18 + ...t_Relational_H2_RelationFunctions_PCT.java | 7 +- ...tional_Postgres_RelationFunctions_PCT.java | 9 +- .../sqlQueryToString/snowflakeExtension.pure | 33 + .../pureToSQLQuery/pureToSQLQuery.pure | 107 +- .../sqlQueryToString/dbExtension.pure | 6 + .../debugPrint/debugPrintExtension.pure | 1180 ++++++++++++++++- .../dbSpecific/h2/h2Extension2_1_214.pure | 4 +- .../sqlQueryToString/extensionDefaults.pure | 18 +- pom.xml | 2 +- 28 files changed, 2168 insertions(+), 131 deletions(-) create mode 100644 legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/handlers/AsOfJoinHandler.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/transformation/asofjoin.pure create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/AsOfJoin.java create mode 100644 legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/AsOfJoin.java diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java index d8cf17c7cfb..9c33e343a12 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/Completer.java @@ -103,6 +103,7 @@ private Completer(String buildCodeContext, MutableList exten new PivotHandler(), new SortHandler(), new JoinHandler(), + new AsOfJoinHandler(), new SelectHandler(), new DistinctHandler(), new OverHandler() @@ -268,7 +269,7 @@ private MutableList getFunctionCandidates(org.finos.legend.pure.m3.corei if (org.finos.legend.pure.m3.navigation.type.Type.subTypeOf(leftType._rawType(), pureModel.getType(M3Paths.Relation), pureModel.getExecutionSupport().getProcessorSupport())) { // May want to assert the mul to 1 - return Lists.mutable.with("cast", "distinct", "drop", "select", "extend", "filter", "from", "groupBy", "pivot", "join", "limit", "rename", "size", "slice", "sort"); + return Lists.mutable.with("cast", "distinct", "drop", "select", "extend", "filter", "from", "groupBy", "pivot", "join", "asOfJoin", "limit", "rename", "size", "slice", "sort"); } else if (leftType._rawType().getName().equals("String")) { diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/handlers/AsOfJoinHandler.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/handlers/AsOfJoinHandler.java new file mode 100644 index 00000000000..f94ef8e4a27 --- /dev/null +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/autocomplete/handlers/AsOfJoinHandler.java @@ -0,0 +1,83 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.repl.autocomplete.handlers; + +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.CompileContext; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.ProcessingContext; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.ValueSpecificationBuilder; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.Variable; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.application.AppliedFunction; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.repl.autocomplete.Completer; +import org.finos.legend.engine.repl.autocomplete.CompletionItem; +import org.finos.legend.engine.repl.autocomplete.FunctionHandler; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; + +public class AsOfJoinHandler extends FunctionHandler +{ + @Override + public String functionName() + { + return "asOfJoin"; + } + + @Override + public MutableList proposedParameters(AppliedFunction currentFunc, GenericType leftType, PureModel pureModel, Completer completer, ProcessingContext processingContext, ValueSpecification currentVS) + { + if (currentFunc.parameters.size() == 2) + { + return completer.processValueSpecification(currentFunc.parameters.get(1), currentVS, pureModel, processingContext).getCompletion(); + } + return Lists.mutable.empty(); + } + + public void handleFunctionAppliedParameters(AppliedFunction currentFunc, GenericType leftType, ProcessingContext processingContext, PureModel pureModel) + { + if (currentFunc.parameters.size() == 3) + { + processLambda(currentFunc, currentFunc.parameters.get(2), leftType, processingContext, pureModel); + } + if (currentFunc.parameters.size() == 4) + { + processLambda(currentFunc, currentFunc.parameters.get(3), leftType, processingContext, pureModel); + } + } + + private static void processLambda(AppliedFunction currentFunc, ValueSpecification value, GenericType leftType, ProcessingContext processingContext, PureModel pureModel) + { + GenericType genericType = currentFunc.parameters.get(1).accept(new ValueSpecificationBuilder(new CompileContext.Builder(pureModel).build(), Lists.mutable.empty(), processingContext))._genericType()._typeArguments().getFirst(); + if (value instanceof Lambda) + { + processLambda((Lambda)value, leftType._typeArguments().getFirst(), genericType, processingContext, pureModel); + } + } + + + private static void processLambda(Lambda lambda, GenericType first, GenericType second, ProcessingContext processingContext, PureModel pureModel) + { + if (lambda != null) + { + Variable f_variable = lambda.parameters.get(0); + processingContext.addInferredVariables(f_variable.name, buildTypedVariable(f_variable, first, pureModel.getMultiplicity("one"), pureModel)); + Variable s_variable = lambda.parameters.get(1); + processingContext.addInferredVariables(s_variable.name, buildTypedVariable(s_variable, second, pureModel.getMultiplicity("one"), pureModel)); + } + } + +} diff --git a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java index 0fc6fc3a2f1..e485b3fed6b 100644 --- a/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java +++ b/legend-engine-config/legend-engine-repl/legend-engine-repl-relational/src/test/java/org/finos/legend/engine/repl/TestCompleter.java @@ -47,14 +47,14 @@ public void testAutocompleteFunctionParameter() @Test public void testArrowOnFunction() { - Assert.assertEquals("[cast , cast(], [distinct , distinct(], [drop , drop(], [select , select(], [extend , extend(], [filter , filter(], [from , from(], [groupBy , groupBy(], [pivot , pivot(], [join , join(], [limit , limit(], [rename , rename(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(x|$x.col == 'oo')->"))); + Assert.assertEquals("[cast , cast(], [distinct , distinct(], [drop , drop(], [select , select(], [extend , extend(], [filter , filter(], [from , from(], [groupBy , groupBy(], [pivot , pivot(], [join , join(], [asOfJoin , asOfJoin(], [limit , limit(], [rename , rename(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->filter(x|$x.col == 'oo')->"))); Assert.assertEquals("PARSER error at [6:1-23]: parsing error", new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->limit(10)-").getEngineException().toPretty()); } @Test public void testArrowRelation() { - Assert.assertEquals("[cast , cast(], [distinct , distinct(], [drop , drop(], [select , select(], [extend , extend(], [filter , filter(], [from , from(], [groupBy , groupBy(], [pivot , pivot(], [join , join(], [limit , limit(], [rename , rename(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->"))); + Assert.assertEquals("[cast , cast(], [distinct , distinct(], [drop , drop(], [select , select(], [extend , extend(], [filter , filter(], [from , from(], [groupBy , groupBy(], [pivot , pivot(], [join , join(], [asOfJoin , asOfJoin(], [limit , limit(], [rename , rename(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->"))); Assert.assertEquals("[select , select(], [size , size(], [slice , slice(], [sort , sort(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200)))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->s"))); } @@ -199,6 +199,24 @@ public void testJoin() Assert.assertEquals("COMPILATION error at [6:14-17]: \"The relation contains duplicates: [val, col]\"", new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->join(#>{a::A.t}#, JoinKind.INNER, {a,b|$a.val == $b.val})->").getEngineException().toPretty()); } + //------ + // AsOfJoin + //------ + @Test + public void testAsOfJoin() + { + Assert.assertEquals("[a::A , >{a::A]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>"))); + Assert.assertEquals("[t , t}]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A."))); + Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#, {a,b|$a."))); + Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#,"))); + Assert.assertEquals("[k , k], [o , o]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b."))); + Assert.assertEquals("", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b.k}, "))); + Assert.assertEquals("[k , k], [o , o]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b.k}, {b,c|$c."))); + Assert.assertEquals("[col , col], [val , val]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b.k}, {b,c|$b."))); + Assert.assertEquals("[filter , filter(]", checkResultNoException(new Completer("###Relational\nDatabase a::A(Table t2(k VARCHAR(200), o INT) Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t2}#, {a,b|$a.col == $b.k}, {b,c|$c.k == $b.col})->fil"))); + Assert.assertEquals("COMPILATION error at [6:14-21]: \"The relation contains duplicates: [val, col]\"", new Completer("###Relational\nDatabase a::A(Table t(col VARCHAR(200), val INT))", Lists.mutable.with(new RelationalCompleterExtension())).complete("#>{a::A.t}#->asOfJoin(#>{a::A.t}#, {a,b|$a.val == $b.val})->").getEngineException().toPretty()); + } + //-------- // Select //-------- diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index 01702844c37..e71d0b2684a 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -630,6 +630,21 @@ else if (Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS" return result; }; + public static final ParametersInference AsOfJoinInference = (parameters, ov, cc, pc) -> + { + ValueSpecification firstProcessedParameter = parameters.get(0).accept(new ValueSpecificationBuilder(cc, ov, pc)); + ValueSpecification secondProcessedParameter = parameters.get(1).accept(new ValueSpecificationBuilder(cc, ov, pc)); + org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity one = new org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity(1, 1); + updateTwoParamsLambdaDiffTypes(parameters.get(2), firstProcessedParameter._genericType()._typeArguments().getFirst(), secondProcessedParameter._genericType()._typeArguments().getFirst(), one, one); + if (parameters.size() == 4) + { + updateTwoParamsLambdaDiffTypes(parameters.get(3), firstProcessedParameter._genericType()._typeArguments().getFirst(), secondProcessedParameter._genericType()._typeArguments().getFirst(), one, one); + } + MutableList base = Lists.mutable.with(firstProcessedParameter).with(secondProcessedParameter).with(parameters.get(2).accept(new ValueSpecificationBuilder(cc, ov, pc))); + return parameters.size() == 4 ? base.with(parameters.get(3).accept(new ValueSpecificationBuilder(cc, ov, pc))) : base; + }; + + public static final ParametersInference JoinInference = (parameters, ov, cc, pc) -> { ValueSpecification firstProcessedParameter = parameters.get(0).accept(new ValueSpecificationBuilder(cc, ov, pc)); @@ -1380,6 +1395,16 @@ private void registerTDS() register(grp(JoinInference, h("meta::pure::functions::relation::join_Relation_1__Relation_1__JoinKind_1__Function_1__Relation_1_", true, ps -> JoinReturnInference(ps, this.pureModel), ps -> true))); + register(m( + grp(AsOfJoinInference, + h("meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_", true, ps -> JoinReturnInference(ps, this.pureModel), ps -> true) + ), + grp(AsOfJoinInference, + h("meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_", true, ps -> JoinReturnInference(ps, this.pureModel), ps -> true) + ) + ) + ); + register( m( m(h("meta::pure::functions::collection::sort_T_m__T_m_", false, ps -> res(ps.get(0)._genericType(), ps.get(0)._multiplicity()), ps -> ps.size() == 1)), @@ -1829,10 +1854,10 @@ private void registerAggregations() h("meta::pure::functions::math::sum_Number_MANY__Number_1_", false, ps -> res("Number", "one"), ps -> typeMany(ps.get(0), "Number"))); register(m(m(h("meta::pure::functions::math::wavg_Number_MANY__Number_MANY__Float_1_", false, ps -> res("Float", "one"), ps -> typeMany(ps.get(0), "Number"))), - m(h("meta::pure::functions::math::wavg_WavgRowMapper_MANY__Float_1_", false, ps -> res("Float", "one"), ps -> typeMany(ps.get(0), "meta::pure::functions::math::wavgUtility::WavgRowMapper"))))); + m(h("meta::pure::functions::math::wavg_WavgRowMapper_MANY__Float_1_", false, ps -> res("Float", "one"), ps -> typeMany(ps.get(0), "meta::pure::functions::math::wavgUtility::WavgRowMapper"))))); register(h("meta::pure::functions::math::wavgUtility::wavgRowMapper_Number_$0_1$__Number_$0_1$__WavgRowMapper_1_", false, ps -> res("meta::pure::functions::math::wavgUtility::WavgRowMapper", "one"), ps -> typeZeroOne(ps.get(0), "Number"))); - + register(h("meta::pure::functions::math::variance_Number_MANY__Boolean_1__Number_1_", false, ps -> res("Number", "one"))); register(m(m(h("meta::pure::functions::math::percentile_Number_MANY__Float_1__Boolean_1__Boolean_1__Number_$0_1$_", false, ps -> res("Number", "zeroOne"), ps -> ps.size() == 4)), @@ -2611,7 +2636,7 @@ private Map buildDispatch() map.put("meta::pure::functions::lang::match_Any_MANY__Function_$1_MANY$__T_m_", (List ps) -> ps.size() == 2 && matchOneMany(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> check(ft._parameters().toList(), (List nps) -> nps.size() == 1)))); map.put("meta::pure::functions::lang::new_Class_1__String_1__KeyExpression_MANY__T_1_", (List ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Class", "MappingClass", "ClassProjection").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "KeyExpression".equals(ps.get(2)._genericType()._rawType()._name()))); map.put("meta::pure::functions::lang::subType_Any_m__T_1__T_m_", (List ps) -> ps.size() == 2 && isOne(ps.get(1)._multiplicity())); - map.put("meta::pure::functions::math::sign_Number_1__Integer_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil","Number","Float","Integer","Decimal").contains(ps.get(0)._genericType()._rawType()._name())); + map.put("meta::pure::functions::math::sign_Number_1__Integer_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Number", "Float", "Integer", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); map.put("meta::pure::functions::math::abs_Float_1__Float_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Float".equals(ps.get(0)._genericType()._rawType()._name()))); map.put("meta::pure::functions::math::abs_Integer_1__Integer_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Integer".equals(ps.get(0)._genericType()._rawType()._name()))); map.put("meta::pure::functions::math::abs_Number_1__Number_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Number", "Integer", "Float", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); @@ -2683,7 +2708,7 @@ private Map buildDispatch() map.put("meta::pure::functions::math::stdDevPopulation_Number_MANY__Number_1_", (List ps) -> ps.size() == 1 && Sets.immutable.with("Nil", "Number", "Integer", "Float", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); map.put("meta::pure::functions::math::stdDevSample_Number_$1_MANY$__Number_1_", (List ps) -> ps.size() == 1 && matchOneMany(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Number", "Integer", "Float", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); map.put("meta::pure::functions::math::stdDevSample_Number_MANY__Number_1_", (List ps) -> ps.size() == 1 && Sets.immutable.with("Nil", "Number", "Integer", "Float", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); - map.put("meta::pure::functions::math::variance_Number_MANY__Boolean_1__Number_1_", (List ps) -> ps.size() == 2 && Sets.immutable.with("Nil","Number","Float","Integer","Decimal").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Boolean".equals(ps.get(1)._genericType()._rawType()._name()))); + map.put("meta::pure::functions::math::variance_Number_MANY__Boolean_1__Number_1_", (List ps) -> ps.size() == 2 && Sets.immutable.with("Nil", "Number", "Float", "Integer", "Decimal").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Boolean".equals(ps.get(1)._genericType()._rawType()._name()))); map.put("meta::pure::functions::math::variancePopulation_Number_MANY__Number_1_", (List ps) -> ps.size() == 1 && Sets.immutable.with("Nil", "Number", "Integer", "Float", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); map.put("meta::pure::functions::math::varianceSample_Number_MANY__Number_1_", (List ps) -> ps.size() == 1 && Sets.immutable.with("Nil", "Number", "Integer", "Float", "Decimal").contains(ps.get(0)._genericType()._rawType()._name())); map.put("meta::pure::functions::math::sum_Float_MANY__Float_1_", (List ps) -> ps.size() == 1 && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "Float".equals(ps.get(0)._genericType()._rawType()._name()))); @@ -2937,10 +2962,11 @@ private Map buildDispatch() map.put("meta::pure::functions::relation::percentRank_Relation_1___Window_1__T_1__Float_1_", (List ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "_Window".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity())); map.put("meta::pure::functions::relation::rank_Relation_1___Window_1__T_1__Integer_1_", (List ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "_Window".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity())); map.put("meta::pure::functions::relation::rowNumber_Relation_1__T_1__Integer_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity())); - map.put("meta::pure::functions::math::wavg_Number_MANY__Number_MANY__Float_1_", (List ps) -> ps.size() == 2 && Sets.immutable.with("Nil","Number","Integer","Decimal","Float").contains(ps.get(0)._genericType()._rawType()._name()) && Sets.immutable.with("Nil","Number","Integer","Decimal","Float").contains(ps.get(1)._genericType()._rawType()._name())); + map.put("meta::pure::functions::math::wavg_Number_MANY__Number_MANY__Float_1_", (List ps) -> ps.size() == 2 && Sets.immutable.with("Nil", "Number", "Integer", "Decimal", "Float").contains(ps.get(0)._genericType()._rawType()._name()) && Sets.immutable.with("Nil", "Number", "Integer", "Decimal", "Float").contains(ps.get(1)._genericType()._rawType()._name())); map.put("meta::pure::functions::math::wavg_WavgRowMapper_MANY__Float_1_", (List ps) -> ps.size() == 1 && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "WavgRowMapper".equals(ps.get(0)._genericType()._rawType()._name()))); - map.put("meta::pure::functions::math::wavgUtility::wavgRowMapper_Number_$0_1$__Number_$0_1$__WavgRowMapper_1_", (List ps) -> ps.size() == 2 && matchZeroOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil","Number","Integer","Decimal","Float").contains(ps.get(0)._genericType()._rawType()._name()) && matchZeroOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil","Number","Integer","Decimal","Float").contains(ps.get(1)._genericType()._rawType()._name())); - + map.put("meta::pure::functions::math::wavgUtility::wavgRowMapper_Number_$0_1$__Number_$0_1$__WavgRowMapper_1_", (List ps) -> ps.size() == 2 && matchZeroOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Number", "Integer", "Decimal", "Float").contains(ps.get(0)._genericType()._rawType()._name()) && matchZeroOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "Number", "Integer", "Decimal", "Float").contains(ps.get(1)._genericType()._rawType()._name())); + map.put("meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_", (List ps) -> ps.size() == 4 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(1)._genericType()._rawType()._name()) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || check(funcType(ps.get(2)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && ("Nil".equals(ft._returnType()._rawType()._name()) || "Boolean".equals(ft._returnType()._rawType()._name())) && check(ft._parameters().toList(), (List nps) -> nps.size() == 2 && isOne(nps.get(0)._multiplicity()) && isOne(nps.get(1)._multiplicity())))) && isOne(ps.get(3)._multiplicity()) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || check(funcType(ps.get(3)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && ("Nil".equals(ft._returnType()._rawType()._name()) || "Boolean".equals(ft._returnType()._rawType()._name())) && check(ft._parameters().toList(), (List nps) -> nps.size() == 2 && isOne(nps.get(0)._multiplicity()) && isOne(nps.get(1)._multiplicity()))))); + map.put("meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_", (List ps) -> ps.size() == 3 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "Relation", "RelationElementAccessor", "TDS", "RelationStoreAccessor").contains(ps.get(1)._genericType()._rawType()._name()) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || check(funcType(ps.get(2)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && ("Nil".equals(ft._returnType()._rawType()._name()) || "Boolean".equals(ft._returnType()._rawType()._name())) && check(ft._parameters().toList(), (List nps) -> nps.size() == 2 && isOne(nps.get(0)._multiplicity()) && isOne(nps.get(1)._multiplicity()))))); // ------------------------------------------------------------------------------------------------ // Please do not update the following code manually! Please check with the team when introducing // new matchers as this might be complicated by modularization diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/transformation/asofjoin.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/transformation/asofjoin.pure new file mode 100644 index 00000000000..052d5a6b77e --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-functions-relation-pure/src/main/resources/core_functions_relation/relation/functions/transformation/asofjoin.pure @@ -0,0 +1,121 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::test::pct::*; +import meta::pure::metamodel::relation::*; + +native function <> meta::pure::functions::relation::asOfJoin(rel1:Relation[1], rel2:Relation[1], match:Function<{T[1],V[1]->Boolean[1]}>[1]):Relation[1]; + +native function <> meta::pure::functions::relation::asOfJoin(rel1:Relation[1], rel2:Relation[1], match:Function<{T[1],V[1]->Boolean[1]}>[1], join:Function<{T[1],V[1]->Boolean[1]}>[1]):Relation[1]; + +function <> meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let tds1 = #TDS + key, time, value + 1, 2000-10-25T06:30:00Z, 5000 + 1, 2000-10-25T06:31:00Z, 4000 + 3, 2000-10-25T06:32:00Z, 3000 + 4, 2000-10-25T06:33:00Z, 1200 + 5, 2000-10-25T06:34:00Z, 3200 + 6, 2000-10-25T06:35:00Z, 4300 + #; + let tds2 = #TDS + key2, time2, value2 + 2, 2000-10-25T06:31:20Z, 3000 + 1, 2000-10-25T06:30:10Z, 2000 + 4, 2000-10-25T06:33:40Z, 1400 + 3, 2000-10-25T06:32:30Z, 3200 + 6, 2000-10-25T06:35:10Z, 2900 + 5, 2000-10-25T06:34:50Z, 3200 + #; + + let expr = {|$tds1->asOfJoin($tds2, {x,y|$x.time > $y.time2});}; + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' key,time,value,key2,time2,value2\n'+ + ' 1,2000-10-25T06:30:00.000+0000,5000,null,null,null\n'+ + ' 1,2000-10-25T06:31:00.000+0000,4000,1,2000-10-25T06:30:10.000+0000,2000\n'+ + ' 3,2000-10-25T06:32:00.000+0000,3000,2,2000-10-25T06:31:20.000+0000,3000\n'+ + ' 4,2000-10-25T06:33:00.000+0000,1200,3,2000-10-25T06:32:30.000+0000,3200\n'+ + ' 5,2000-10-25T06:34:00.000+0000,3200,4,2000-10-25T06:33:40.000+0000,1400\n'+ + ' 6,2000-10-25T06:35:00.000+0000,4300,5,2000-10-25T06:34:50.000+0000,3200\n'+ + '#', $res->sort([~key->ascending(), ~time->ascending()])->toString()); + + let expr2 = {|$tds1->asOfJoin($tds2, {x,y|$x.time < $y.time2});}; + let res2 = $f->eval($expr2); + + assertEquals( '#TDS\n'+ + ' key,time,value,key2,time2,value2\n'+ + ' 1,2000-10-25T06:30:00.000+0000,5000,1,2000-10-25T06:30:10.000+0000,2000\n'+ + ' 1,2000-10-25T06:31:00.000+0000,4000,2,2000-10-25T06:31:20.000+0000,3000\n'+ + ' 3,2000-10-25T06:32:00.000+0000,3000,3,2000-10-25T06:32:30.000+0000,3200\n'+ + ' 4,2000-10-25T06:33:00.000+0000,1200,4,2000-10-25T06:33:40.000+0000,1400\n'+ + ' 5,2000-10-25T06:34:00.000+0000,3200,5,2000-10-25T06:34:50.000+0000,3200\n'+ + ' 6,2000-10-25T06:35:00.000+0000,4300,6,2000-10-25T06:35:10.000+0000,2900\n'+ + '#', $res2->sort([~key->ascending(), ~time->ascending()])->toString()); + +} + + +function <> meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] +{ + let tds1 = #TDS + key, time, value + 1, 2000-10-25T06:30:00Z, 5000 + 1, 2000-10-25T06:31:00Z, 4000 + 3, 2000-10-25T06:32:00Z, 3000 + 4, 2000-10-25T06:33:00Z, 1200 + 5, 2000-10-25T06:34:00Z, 3200 + 6, 2000-10-25T06:35:00Z, 4300 + #; + let tds2 = #TDS + key2, time2, value2 + 2, 2000-10-25T06:31:20Z, 3000 + 1, 2000-10-25T06:30:10Z, 2000 + 4, 2000-10-25T06:33:40Z, 1400 + 3, 2000-10-25T06:32:30Z, 3200 + 3, 2000-10-25T06:33:30Z, 3200 + 3, 2000-10-25T06:34:30Z, 3200 + 6, 2000-10-25T06:35:10Z, 2900 + 6, 2000-10-25T06:35:50Z, 2900 + 5, 2000-10-25T06:34:50Z, 3200 + #; + + let expr = {|$tds1->asOfJoin($tds2, {x,y|$x.time > $y.time2}, {x,y|$x.key == $y.key2});}; + let res = $f->eval($expr); + + assertEquals( '#TDS\n'+ + ' key,time,value,key2,time2,value2\n'+ + ' 1,2000-10-25T06:30:00.000+0000,5000,null,null,null\n'+ + ' 1,2000-10-25T06:31:00.000+0000,4000,1,2000-10-25T06:30:10.000+0000,2000\n'+ + ' 3,2000-10-25T06:32:00.000+0000,3000,null,null,null\n'+ + ' 4,2000-10-25T06:33:00.000+0000,1200,null,null,null\n'+ + ' 5,2000-10-25T06:34:00.000+0000,3200,null,null,null\n'+ + ' 6,2000-10-25T06:35:00.000+0000,4300,null,null,null\n'+ + '#', $res->sort([~key->ascending(), ~time->ascending()])->toString()); + + let expr2 = {|$tds1->asOfJoin($tds2, {x,y|$x.time < $y.time2}, {x,y|$x.key == $y.key2});}; + let res2 = $f->eval($expr2); + + assertEquals( '#TDS\n'+ + ' key,time,value,key2,time2,value2\n'+ + ' 1,2000-10-25T06:30:00.000+0000,5000,1,2000-10-25T06:30:10.000+0000,2000\n'+ + ' 1,2000-10-25T06:31:00.000+0000,4000,null,null,null\n'+ + ' 3,2000-10-25T06:32:00.000+0000,3000,3,2000-10-25T06:32:30.000+0000,3200\n'+ + ' 4,2000-10-25T06:33:00.000+0000,1200,4,2000-10-25T06:33:40.000+0000,1400\n'+ + ' 5,2000-10-25T06:34:00.000+0000,3200,5,2000-10-25T06:34:50.000+0000,3200\n'+ + ' 6,2000-10-25T06:35:00.000+0000,4300,6,2000-10-25T06:35:10.000+0000,2900\n'+ + '#', $res2->sort([~key->ascending(), ~time->ascending()])->toString()); +} \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java index 533514352f5..0942c09627a 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java @@ -48,6 +48,7 @@ public List getExtraNatives() new Filter(), new Columns(), new Concatenate(), + new AsOfJoin(), new Join(), new Extend(), new ExtendArray(), diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java index 37b801fedbd..060a5d10b64 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java @@ -169,6 +169,52 @@ public static T offset(Relation w, T r, long offset, ExecutionS return (T) new RowContainer(RelationNativeImplementation.getTDS(w), actualOffset); } + public static Relation asOfJoin(Relation rel1, Relation rel2, Function3 pureFunction, LambdaFunction _func, ExecutionSupport es) + { + return asOfJoin(rel1, rel2, pureFunction, _func, null, es); + } + + public static Relation asOfJoin(Relation rel1, Relation rel2, Function3 matchFunction, LambdaFunction _func, Function3 onFunction, ExecutionSupport es) + { + ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); + + TestTDS tds1 = RelationNativeImplementation.getTDS(rel1).sortForOuterJoin(true, _func, ps); + TestTDS tds2 = RelationNativeImplementation.getTDS(rel2).sortForOuterJoin(false, _func, ps); + + TestTDS result = tds1.join(tds2).newEmptyTDS(); + for (int i = 0; i < tds1.getRowCount(); i++) + { + TestTDS oneRow = tds1.slice(i, i + 1); + TestTDS exploded = oneRow.join(tds2); + TestTDS res = filterTwoParam(exploded, matchFunction, es); + res = onFunction == null ? res : filterTwoParam(res, onFunction, es); + if (res.getRowCount() == 0) + { + result = result.concatenate(oneRow.join(tds2.newNullTDS())); + } + else + { + result = result.concatenate(res.slice(0, 1)); + } + } + return new TDSContainer((TestTDSCompiled) result, ps); + } + + private static TestTDS filterTwoParam(TestTDS tds, Function3 matchFunction, ExecutionSupport es) + { + MutableIntSet list = new IntHashSet(); + for (int i = 0; i < tds.getRowCount(); i++) + { + RowContainer rc = new RowContainer((TestTDSCompiled) tds, i); + if (!(boolean) matchFunction.value(rc, rc, es)) + { + list.add(i); + } + } + return tds.drop(list); + } + + public abstract static class ColFuncSpecTrans { public String newColName; diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/AsOfJoin.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/AsOfJoin.java new file mode 100644 index 00000000000..ba594e61a5b --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/AsOfJoin.java @@ -0,0 +1,55 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; + +import org.eclipse.collections.api.list.ListIterable; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; +import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; + +public class AsOfJoin extends AbstractNative implements Native +{ + public AsOfJoin() + { + super("asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_", "asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_"); + } + + @Override + public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) + { + StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.asOfJoin"); + result.append('('); + result.append(transformedParams.get(0)); + result.append(", "); + result.append(transformedParams.get(1)); + result.append(", "); + result.append("(org.eclipse.collections.api.block.function.Function3)PureCompiledLambda.getPureFunction("); + result.append(transformedParams.get(2)); + result.append(",es)\n"); + result.append(", "); + result.append(transformedParams.get(2)); + if (transformedParams.size() > 3) + { + result.append(", "); + result.append("(org.eclipse.collections.api.block.function.Function3)PureCompiledLambda.getPureFunction("); + result.append(transformedParams.get(3)); + result.append(",es)\n"); + } + result.append(", "); + result.append("es)"); + return result.toString(); + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java index 63f7089e734..4a9d104678b 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/shared/TestTDSCompiled.java @@ -28,6 +28,7 @@ import org.finos.legend.pure.m3.navigation.relation._Column; import org.finos.legend.pure.m3.navigation.relation._RelationType; import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; @@ -106,6 +107,10 @@ public Object getValueAsCoreInstance(String columnName, int rowNum) result = !isNull[rowNum] ? Double.valueOf(data[rowNum]) : null; break; } + case DATETIME_AS_LONG: + PureDate[] data = (PureDate[]) dataAsObject; + result = data[rowNum]; + break; default: throw new RuntimeException("ERROR " + columnType.get(columnName) + " not supported in getValue"); } @@ -162,6 +167,8 @@ public String convert(DataType dataType) case FLOAT: case DOUBLE: return "Float"; + case DATETIME_AS_LONG: + return "Date"; } throw new RuntimeException("To Handle " + dataType); } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java index 5c798bd394d..1a3bfa16a8b 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java @@ -22,6 +22,7 @@ import org.finos.legend.pure.m3.navigation.M3Paths; import org.finos.legend.pure.m3.navigation.ProcessorSupport; import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.AsOfJoin; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Columns; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Concatenate; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.CumulativeDistribution; @@ -106,7 +107,10 @@ public RelationExtensionInterpreted() Tuples.pair("percentRank_Relation_1___Window_1__T_1__Float_1_", PercentRank::new), Tuples.pair("denseRank_Relation_1___Window_1__T_1__Integer_1_", DenseRank::new), Tuples.pair("ntile_Relation_1__T_1__Integer_1__Integer_1_", NTile::new), - Tuples.pair("cumulativeDistribution_Relation_1___Window_1__T_1__Float_1_", CumulativeDistribution::new) + Tuples.pair("cumulativeDistribution_Relation_1___Window_1__T_1__Float_1_", CumulativeDistribution::new), + Tuples.pair("asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_", AsOfJoin::new), + Tuples.pair("asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_", AsOfJoin::new) + ); } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/AsOfJoin.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/AsOfJoin.java new file mode 100644 index 00000000000..419b2b95a63 --- /dev/null +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/AsOfJoin.java @@ -0,0 +1,108 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; + +import org.eclipse.collections.api.list.FixedSizeList; +import org.eclipse.collections.api.list.ListIterable; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.api.set.primitive.MutableIntSet; +import org.eclipse.collections.impl.factory.Lists; +import org.eclipse.collections.impl.factory.primitive.IntSets; +import org.finos.legend.pure.m3.compiler.Context; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunctionCoreInstanceWrapper; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; +import org.finos.legend.pure.m3.exception.PureExecutionException; +import org.finos.legend.pure.m3.navigation.Instance; +import org.finos.legend.pure.m3.navigation.M3Properties; +import org.finos.legend.pure.m3.navigation.PrimitiveUtilities; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.ValueSpecificationBootstrap; +import org.finos.legend.pure.m4.ModelRepository; +import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSCoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; +import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; +import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; +import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; +import org.finos.legend.pure.runtime.java.interpreted.VariableContext; +import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; +import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; + +import java.util.Stack; + +public class AsOfJoin extends Shared +{ + public AsOfJoin(FunctionExecutionInterpreted functionExecution, ModelRepository repository) + { + super(functionExecution, repository); + } + + @Override + public CoreInstance execute(ListIterable params, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException + { + CoreInstance returnGenericType = getReturnGenericType(resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, processorSupport); + RelationType relationtype = (RelationType) returnGenericType.getValueForMetaPropertyToMany("typeArguments").get(0).getValueForMetaPropertyToOne("rawType"); + + CoreInstance matchFunction = Instance.getValueForMetaPropertyToOneResolved(params.get(2), M3Properties.values, processorSupport); + + CoreInstance filterFunction = params.size() > 3 ? Instance.getValueForMetaPropertyToOneResolved(params.get(3), M3Properties.values, processorSupport) : null; + + LambdaFunction lambdaMatchFunction = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(matchFunction); + + TestTDS tds1 = getTDS(params, 0, processorSupport).sortForOuterJoin(true, lambdaMatchFunction, processorSupport); + TestTDS tds2 = getTDS(params, 1, processorSupport).sortForOuterJoin(false, lambdaMatchFunction, processorSupport); + + TestTDS result = tds1.join(tds2).newEmptyTDS(); + for (int i = 0; i < tds1.getRowCount(); i++) + { + TestTDS oneRow = tds1.slice(i, i + 1); + TestTDS exploded = oneRow.join(tds2); + TestTDS res = filter(exploded, matchFunction, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationtype); + res = filterFunction == null ? res : filter(res, filterFunction, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationtype); + if (res.getRowCount() == 0) + { + result = result.concatenate(oneRow.join(tds2.newNullTDS())); + } + else + { + result = result.concatenate(res.slice(0, 1)); + } + } + + return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(result, returnGenericType, repository, processorSupport), false, processorSupport); + } + + private TestTDS filter(TestTDS tds, CoreInstance filterFunction, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationtype) + { + LambdaFunction lambdaFunction = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(filterFunction); + VariableContext evalVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, filterFunction); + + MutableIntSet discardedRows = IntSets.mutable.empty(); + FixedSizeList parameters = Lists.fixedSize.with((CoreInstance) null, (CoreInstance) null); + for (int i = 0; i < tds.getRowCount(); i++) + { + parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, i, "", null, relationtype, -1, repository, false), true, processorSupport)); + parameters.set(1, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(tds, i, "", null, relationtype, -1, repository, false), true, processorSupport)); + CoreInstance subResult = this.functionExecution.executeFunction(false, lambdaFunction, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, evalVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); + if (!PrimitiveUtilities.getBooleanValue(Instance.getValueForMetaPropertyToOneResolved(subResult, M3Properties.values, processorSupport))) + { + discardedRows.add(i); + } + } + return tds.drop(discardedRows); + } +} diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/Shared.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/Shared.java index 0d94f45e6d7..012d7ae60ba 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/Shared.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/Shared.java @@ -69,11 +69,27 @@ public static CsvReader.Result readCsv(String csv) { try { - return CsvReader.read(CsvSpecs.csv(), new ByteArrayInputStream(csv.getBytes()), SinkFactory.arrays()); + return CsvReader.read(CsvSpecs.csv(), new ByteArrayInputStream(csv.getBytes()), makeMySinkFactory()); } catch (Exception e) { throw new RuntimeException(e); } } + + private static SinkFactory makeMySinkFactory() + { + return SinkFactory.arrays( + null, + null, + null, + null, + null, + null, + null, + null, + null, + Long.MIN_VALUE, + Long.MIN_VALUE); + } } \ No newline at end of file diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TestTDSInterpreted.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TestTDSInterpreted.java index 1be353e2381..51992a6d1be 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TestTDSInterpreted.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/shared/TestTDSInterpreted.java @@ -26,6 +26,7 @@ import org.finos.legend.pure.m3.navigation.type.Type; import org.finos.legend.pure.m4.ModelRepository; import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; import java.math.BigDecimal; @@ -120,6 +121,13 @@ public CoreInstance getValueAsCoreInstance(String columnName, int rowNum) result = !isNull[rowNum] ? newFloatLiteral(modelRepository, BigDecimal.valueOf(data[rowNum]), processorSupport) : ValueSpecificationBootstrap.wrapValueSpecification_ResultGenericTypeIsKnown(Lists.mutable.empty(), Type.wrapGenericType(_Package.getByUserPath(M3Paths.Float, processorSupport), processorSupport), true, processorSupport); break; } + case DATETIME_AS_LONG: + { + PureDate[] data = (PureDate[]) dataAsObject; + PureDate value = data[rowNum]; + result = value != null ? newDateLiteral(modelRepository, value, processorSupport) : ValueSpecificationBootstrap.wrapValueSpecification_ResultGenericTypeIsKnown(Lists.mutable.empty(), Type.wrapGenericType(_Package.getByUserPath(M3Paths.String, processorSupport), processorSupport), true, processorSupport); + break; + } default: throw new RuntimeException("ERROR " + columnType.get(columnName) + " not supported in getValue"); } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java index eef4134ce0b..11bf8513d03 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/interpreted/pure/TestFunctionTester.java @@ -60,6 +60,34 @@ protected static FunctionExecution getFunctionExecution() @org.junit.Test public void testFunction() { +// compileTestSource("fromString.pure", +// "function test():Any[*]\n" + +// "{\n" + +// " let s1 = #TDS\n" + +// " key, time, value\n" + +// " 6, 2000-10-25T06:35:00Z, 4300\n" + +// " 1, 2000-10-25T06:30:00Z, 5000\n" + +// " 4, 2000-10-25T06:33:00Z, 1200\n" + +// " 3, 2000-10-25T06:32:00Z, 3000\n" + +// " 2, 2000-10-25T06:31:00Z, 4000\n" + +// " 5, 2000-10-25T06:34:00Z, 3200\n" + +// " #;\n" + +// " let s2 = #TDS\n" + +// " key2, time2, value2\n" + +// " 2, 2000-10-25T06:31:20Z, 3000\n" + +// " 1, 2000-10-25T06:30:10Z, 2000\n" + +// " 4, 2000-10-25T06:33:40Z, 1400\n" + +// " 3, 2000-10-25T06:32:30Z, 3200\n" + +// " 6, 2000-10-25T06:35:10Z, 2900\n" + +// " 5, 2000-10-25T06:34:50Z, 3200\n" + +// " #;\n" + +// " let res = $s1->asOfJoin($s2, {x,y|$x.time < $y.time2}, {x,y|$x.key == $y.key2});\n" + +// " print($res->sort(~key->ascending())->toString(), 1);\n" + +// "}"); +// functionExecution.getConsole().setPrintStream(System.out); +// this.execute("test():Any[*]"); +// runtime.delete("fromString.pure"); + // compileTestSource("fromString.pure", // "function test():Any[*]\n" + diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java index b31dc57315b..da7a543e255 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java @@ -32,6 +32,16 @@ import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import org.eclipse.collections.impl.tuple.Tuples; import org.eclipse.collections.impl.utility.ArrayIterate; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.SimpleFunctionExpression; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.VariableExpression; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.VariableExpressionAccessor; +import org.finos.legend.pure.m3.navigation.ProcessorSupport; +import org.finos.legend.pure.m3.navigation.function.Function; +import org.finos.legend.pure.m4.coreinstance.primitive.date.DateFunctions; +import org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.window.SortDirection; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.window.SortInfo; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.window.Window; @@ -40,7 +50,10 @@ import java.io.ByteArrayInputStream; import java.lang.reflect.Array; import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.Arrays; +import java.util.Date; import java.util.Objects; public abstract class TestTDS @@ -95,6 +108,11 @@ public TestTDS newNullTDS() testTDS.dataByColumnName.put(col, new double[(int) this.rowCount]); break; } + case DATETIME_AS_LONG: + { + testTDS.dataByColumnName.put(col, new PureDate[(int) this.rowCount]); + break; + } default: throw new RuntimeException("ERROR " + columnType.get(col) + " not supported yet!"); } @@ -148,6 +166,18 @@ public TestTDS(CsvReader.Result result) ((String[]) c.data())[i] = null; } } + break; + case DATETIME_AS_LONG: + PureDate[] dates = new PureDate[(int) this.rowCount]; + dataByColumnName.put(c.name(), dates); + for (int i = 0; i < this.rowCount; i++) + { + long value = ((long[]) c.data())[i]; + dates[i] = value == Long.MIN_VALUE ? null : DateFunctions.fromDate(new Date(value / 1000000)); + } + break; + default: + throw new RuntimeException(c.dataType() + " not supported yet!"); } }); } @@ -180,9 +210,6 @@ protected TestTDS(MutableList columnOrdered, MutableMap columnOrdered, MutableMap sortInfos, int start, int } break; } + case DATETIME_AS_LONG: + { + PureDate[] src = (PureDate[]) dataAsObject; + PureDate val = src[start]; + int subStart = start; + for (int i = start; i < end; i++) + { + if (!Objects.equals(src[i], val) || (Objects.equals(src[i], val) && i == end - 1)) + { + int realEnd = (Objects.equals(src[i], val) && i == end - 1) ? end : i; + if (sortInfos.size() > 1) + { + sort(copy, sortInfos.subList(1, sortInfos.size()), subStart, realEnd, ranges); + } + else + { + ranges.add(Tuples.pair(subStart, realEnd)); + } + val = src[i]; + subStart = i; + } + } + break; + } } } if (ranges.getLast() != null) @@ -883,6 +977,22 @@ private void sortOneLevel(TestTDS copy, SortInfo sortInfo, int start, int end) this.reorder(copy, list.collect(Pair::getOne), start, end); break; } + case DATETIME_AS_LONG: + { + PureDate[] src = (PureDate[]) dataAsObject; + MutableList> list = Lists.mutable.empty(); + for (int i = start; i < end; i++) + { + list.add(Tuples.pair(i, src[i])); + } + list.sortThisBy(Pair::getTwo); + if (sortInfo.direction == SortDirection.DESC) + { + list.reverseThis(); + } + this.reorder(copy, list.collect(Pair::getOne), start, end); + break; + } default: throw new RuntimeException("ERROR " + columnType.get(columnName) + " not supported yet!"); } @@ -950,6 +1060,17 @@ private void reorder(TestTDS copy, MutableList indices, int start, int System.arraycopy(isNullResult, 0, isNull, start, end - start); break; } + case DATETIME_AS_LONG: + { + PureDate[] src = (PureDate[]) dataAsObject; + PureDate[] result = new PureDate[(int) copy.rowCount]; + for (int i = 0; i < indices.size(); i++) + { + result[i] = src[indices.get(i)]; + } + System.arraycopy(result, 0, src, start, end - start); + break; + } default: throw new RuntimeException("ERROR " + copy.columnType.get(columnName) + " not supported yet!"); } @@ -988,8 +1109,13 @@ public String toString() { return isNull[finalI] ? "NULL" : ((double[]) dataAsObject)[finalI]; } + case DATETIME_AS_LONG: + { + PureDate res = ((PureDate[]) dataAsObject)[finalI]; + return res == null ? "NULL" : DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.of("UTC")).format(res.getCalendar().toInstant()); + } default: - return ""; + throw new RuntimeException(columnType.get(columnName) + " is not supported yet!"); } }).makeString(", ")); } @@ -1070,6 +1196,11 @@ public boolean fullMatch(MutableList cols, TestTDS second, int rowFirst, valid = valid && ((double[]) firstDataAsObject)[rowFirst] == ((double[]) secondDataAsObject)[rowSecond]; break; } + case DATETIME_AS_LONG: + { + valid = valid && Objects.equals(((PureDate[]) firstDataAsObject)[rowFirst], (((PureDate[]) secondDataAsObject)[rowSecond])); + break; + } default: throw new RuntimeException("ERROR"); } @@ -1233,6 +1364,10 @@ public boolean match(TestTDS tds, int row) { return ((Double) ((double[]) tds.dataByColumnName.get(col.getOne()))[row]).toString().equals(col.getTwo()); } + case DATETIME_AS_LONG: + { + return (((PureDate[]) tds.dataByColumnName.get(col.getOne()))[row]).toString().equals(col.getTwo()); + } default: throw new RuntimeException("ERROR " + columnType + " not supported yet!"); } @@ -1273,6 +1408,11 @@ public TestTDS applyPivot(ListIterable nonTransposeColumns, ListIterable { return Tuples.pair(c, ((Double) ((double[]) valuesAsObject)[r.getOne()]).toString()); } + case DATETIME_AS_LONG: + { + return Tuples.pair(c, (((PureDate[]) valuesAsObject)[r.getOne()]).toString()); + } + default: throw new RuntimeException("ERROR " + columnType + " not supported yet!"); } @@ -1318,7 +1458,6 @@ public TestTDS applyPivot(ListIterable nonTransposeColumns, ListIterable if (newColInfo.match(sortedByNonTransposeColumns.getOne(), j)) { values[i] = ((String[]) sortedByNonTransposeColumns.getOne().dataByColumnName.get(newColInfo.getAggColumnName()))[j]; - isNull[i] = Boolean.FALSE; } } } @@ -1359,13 +1498,29 @@ public TestTDS applyPivot(ListIterable nonTransposeColumns, ListIterable dataAsObject = values; break; } + case DATETIME_AS_LONG: + { + PureDate[] values = new PureDate[size]; + for (int i = 0; i < size; i++) + { + for (int j = sortedByNonTransposeColumns.getTwo().get(i).getOne(); j < sortedByNonTransposeColumns.getTwo().get(i).getTwo(); j++) + { + if (newColInfo.match(sortedByNonTransposeColumns.getOne(), j)) + { + values[i] = ((PureDate[]) sortedByNonTransposeColumns.getOne().dataByColumnName.get(newColInfo.getAggColumnName()))[j]; + } + } + } + dataAsObject = values; + break; + } default: { throw new RuntimeException("ERROR " + newColInfo.getColumnType() + " not supported yet!"); } } tds.dataByColumnName.put(name, dataAsObject); - if (!newColInfo.getColumnType().equals(DataType.STRING)) + if (!newColInfo.getColumnType().equals(DataType.STRING) && !newColInfo.getColumnType().equals(DataType.DATETIME_AS_LONG)) { tds.isNullByColumn.put(name, isNull); } @@ -1381,4 +1536,40 @@ public MutableList> getColumnWithTypes() { return this.columnsOrdered.collect(col -> Tuples.pair(col, this.columnType.get(col))); } + + + public TestTDS sortForOuterJoin(boolean isLeft, LambdaFunction lambdaFunction, ProcessorSupport processorSupport) + { + FunctionType fType = (FunctionType) Function.computeFunctionType(lambdaFunction, processorSupport); + ValueSpecification vs = lambdaFunction._expressionSequence().getFirst(); + if (vs instanceof SimpleFunctionExpression) + { + SimpleFunctionExpression fe = (SimpleFunctionExpression) vs; + String funcName = fe._func().getName(); + String truncatedFuncName = funcName.substring(0, funcName.indexOf("_")); + if (Sets.mutable.with("lessThan", "greaterThan", "lessThanEquals", "greaterThanEquals").contains(truncatedFuncName)) + { + SortDirection sortDirection = Sets.mutable.with("lessThan", "lessThanEquals").contains(truncatedFuncName) ? SortDirection.ASC : SortDirection.DESC; + ValueSpecification left = fe._parametersValues().toList().get(0); + ValueSpecification right = fe._parametersValues().toList().get(1); + if (left instanceof SimpleFunctionExpression && right instanceof SimpleFunctionExpression) + { + SimpleFunctionExpression leftF = (SimpleFunctionExpression) left; + SimpleFunctionExpression rightF = (SimpleFunctionExpression) right; + MutableList signatureParameters = fType._parameters().collect(VariableExpressionAccessor::_name).toList(); + String leftName = (((VariableExpression) leftF._parametersValues().getFirst())._name()); + if (leftName.equals(signatureParameters.get(0))) + { + return this.sort(new SortInfo(isLeft ? leftF._func()._name() : rightF._func()._name(), sortDirection)).getOne(); + } + else + { + return this.sort(new SortInfo(isLeft ? rightF._func()._name() : leftF._func()._name(), sortDirection)).getOne(); + } + } + } + } + return this; + } + } diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java index 3a683a37973..0030c5c01e7 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-functions-relation/legend-engine-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java @@ -25,49 +25,60 @@ public class TestTestTDS @org.junit.Test public void testSort() { - String initialTDS = "id, name, otherOne\n" + - "4, Simple, D\n" + - "4, Simple, A\n" + - "3, Ephrim, C\n" + - "2, Bla, B\n" + - "3, Ok, D\n" + - "3, Nop, E\n" + - "2, Neema, F\n" + - "1, Pierre, F"; + String initialTDS = "id, name, otherOne, date\n" + + "4, Simple, D, 2002-10-25T06:30:00Z\n" + + "4, Simple, A, 2010-10-25T06:30:00Z\n" + + "3, Ephrim, C, 2000-10-10T06:30:00Z\n" + + "2, Bla, B, 2000-10-25T06:30:00Z\n" + + "3, Ok, D, 2000-01-25T06:30:00Z\n" + + "3, Nop, E, 2020-02-15T10:30:00Z\n" + + "2, Neema, F, 1975-09-29T08:30:00Z\n" + + "1, Pierre, F, 2100-02-05T02:30:00Z"; TestTDS tds = new TestTDSImpl(initialTDS); TestTDS t = tds.sort(Lists.mutable.with(new SortInfo("id", SortDirection.ASC), new SortInfo("name", SortDirection.ASC))).getOne(); - Assert.assertEquals("id, name, otherOne\n" + - "1, Pierre, F\n" + - "2, Bla, B\n" + - "2, Neema, F\n" + - "3, Ephrim, C\n" + - "3, Nop, E\n" + - "3, Ok, D\n" + - "4, Simple, D\n" + - "4, Simple, A", t.toString()); + Assert.assertEquals("id, name, otherOne, date\n" + + "1, Pierre, F, 2100-02-05T02:30:00Z\n" + + "2, Bla, B, 2000-10-25T06:30:00Z\n" + + "2, Neema, F, 1975-09-29T08:30:00Z\n" + + "3, Ephrim, C, 2000-10-10T06:30:00Z\n" + + "3, Nop, E, 2020-02-15T10:30:00Z\n" + + "3, Ok, D, 2000-01-25T06:30:00Z\n" + + "4, Simple, D, 2002-10-25T06:30:00Z\n" + + "4, Simple, A, 2010-10-25T06:30:00Z", t.toString()); TestTDS t2 = tds.sort(Lists.mutable.with(new SortInfo("id", SortDirection.ASC), new SortInfo("name", SortDirection.ASC), new SortInfo("otherOne", SortDirection.ASC))).getOne(); - Assert.assertEquals("id, name, otherOne\n" + - "1, Pierre, F\n" + - "2, Bla, B\n" + - "2, Neema, F\n" + - "3, Ephrim, C\n" + - "3, Nop, E\n" + - "3, Ok, D\n" + - "4, Simple, A\n" + - "4, Simple, D", t2.toString()); + Assert.assertEquals("id, name, otherOne, date\n" + + "1, Pierre, F, 2100-02-05T02:30:00Z\n" + + "2, Bla, B, 2000-10-25T06:30:00Z\n" + + "2, Neema, F, 1975-09-29T08:30:00Z\n" + + "3, Ephrim, C, 2000-10-10T06:30:00Z\n" + + "3, Nop, E, 2020-02-15T10:30:00Z\n" + + "3, Ok, D, 2000-01-25T06:30:00Z\n" + + "4, Simple, A, 2010-10-25T06:30:00Z\n" + + "4, Simple, D, 2002-10-25T06:30:00Z", t2.toString()); TestTDS t3 = tds.sort(Lists.mutable.with(new SortInfo("id", SortDirection.DESC), new SortInfo("name", SortDirection.ASC), new SortInfo("otherOne", SortDirection.DESC))).getOne(); - Assert.assertEquals("id, name, otherOne\n" + - "4, Simple, D\n" + - "4, Simple, A\n" + - "3, Ephrim, C\n" + - "3, Nop, E\n" + - "3, Ok, D\n" + - "2, Bla, B\n" + - "2, Neema, F\n" + - "1, Pierre, F", t3.toString()); + Assert.assertEquals("id, name, otherOne, date\n" + + "4, Simple, D, 2002-10-25T06:30:00Z\n" + + "4, Simple, A, 2010-10-25T06:30:00Z\n" + + "3, Ephrim, C, 2000-10-10T06:30:00Z\n" + + "3, Nop, E, 2020-02-15T10:30:00Z\n" + + "3, Ok, D, 2000-01-25T06:30:00Z\n" + + "2, Bla, B, 2000-10-25T06:30:00Z\n" + + "2, Neema, F, 1975-09-29T08:30:00Z\n" + + "1, Pierre, F, 2100-02-05T02:30:00Z", t3.toString()); + + TestTDS t4 = tds.sort(Lists.mutable.with(new SortInfo("date", SortDirection.DESC), new SortInfo("name", SortDirection.ASC), new SortInfo("otherOne", SortDirection.DESC))).getOne(); + Assert.assertEquals("id, name, otherOne, date\n" + + "1, Pierre, F, 2100-02-05T02:30:00Z\n" + + "3, Nop, E, 2020-02-15T10:30:00Z\n" + + "4, Simple, A, 2010-10-25T06:30:00Z\n" + + "4, Simple, D, 2002-10-25T06:30:00Z\n" + + "2, Bla, B, 2000-10-25T06:30:00Z\n" + + "3, Ephrim, C, 2000-10-10T06:30:00Z\n" + + "3, Ok, D, 2000-01-25T06:30:00Z\n" + + "2, Neema, F, 1975-09-29T08:30:00Z", t4.toString()); Assert.assertEquals(initialTDS, tds.toString()); } @@ -380,20 +391,39 @@ public void testCompensateLeft() @org.junit.Test public void testSentinel() { - String resTDS = "id,name,id2,col,other\n" + - "1,George,1,More George 1,1\n" + - "1,George,1,More George 2,2\n" + - "2,Pierre,-2147483648,null,-2147483648\n" + - "3,Sachin,-2147483648,null,-2147483648\n" + - "4,David,4,More David,1"; + String resTDS = "id,name,id2,col,other, date\n" + + "1,George,1,More George 1,1, 2000-10-31T01:30:00.000-05:00\n" + + "1,George,1,More George 2,2, 2000-10-25T01:30:00.000-05:00\n" + + "2,Pierre,-2147483648,null,-2147483648,\n" + + "3,Sachin,-2147483648,null,-2147483648,\n" + + "4,David,4,More David,1, 2020-10-31T01:30:00.000-05:00"; + + TestTDS res = new TestTDSImpl(resTDS); + + Assert.assertEquals("id, name, id2, col, other, date\n" + + "1, George, 1, More George 1, 1, 2000-10-31T06:30:00Z\n" + + "1, George, 1, More George 2, 2, 2000-10-25T06:30:00Z\n" + + "2, Pierre, NULL, NULL, NULL, NULL\n" + + "3, Sachin, NULL, NULL, NULL, NULL\n" + + "4, David, 4, More David, 1, 2020-10-31T06:30:00Z", res.toString()); + } + + + @org.junit.Test + public void testDate() + { + String resTDS = "id,date,val\n" + + "1, 2000-10-25T06:30:00Z, 1\n" + + "1, 2000-10-31T01:31:00.000-05:00, 1\n" + + "1,, 1\n" + + "1, 2000-10-31T01:33:00.000-05:00, 1\n"; TestTDS res = new TestTDSImpl(resTDS); - Assert.assertEquals("id, name, id2, col, other\n" + - "1, George, 1, More George 1, 1\n" + - "1, George, 1, More George 2, 2\n" + - "2, Pierre, NULL, NULL, NULL\n" + - "3, Sachin, NULL, NULL, NULL\n" + - "4, David, 4, More David, 1", res.toString()); + Assert.assertEquals("id, date, val\n" + + "1, 2000-10-25T06:30:00Z, 1\n" + + "1, 2000-10-31T06:31:00Z, 1\n" + + "1, NULL, 1\n" + + "1, 2000-10-31T06:33:00Z, 1", res.toString()); } } \ No newline at end of file diff --git a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java index a48b2d7ec2e..0427b1b20ed 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java +++ b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-PCT/src/test/java/org/finos/legend/engine/pure/code/core/java/binding/Test_JAVA_RelationFunction_PCT.java @@ -116,6 +116,10 @@ public class Test_JAVA_RelationFunction_PCT extends PCTReportConfiguration // Sort one("meta::pure::functions::relation::tests::sort::testSimpleSortShared_Function_1__Boolean_1_", "\"meta::pure::functions::relation::sort_Relation_1__SortInfo_MANY__Relation_1_ is not supported yet!\""), + // AsOfJoin + one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_ is not supported yet!\""), + one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_ is not supported yet!\""), + // Composition one("meta::pure::functions::relation::tests::composition::testExtendFilter_Function_1__Boolean_1_", "\"meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_ is not supported yet!\""), one("meta::pure::functions::relation::tests::composition::testFilterPostProject_Function_1__Boolean_1_", "\"meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_ is not supported yet!\""), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure index fc8dcaa7654..595fe717a18 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-PCT/legend-engine-pure-functions-relationalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure @@ -120,10 +120,11 @@ function meta::relational::tests::pct::testAdapterForRelationalExecution(f: | let tdsString = $res.result.columns->joinStrings(',') + '\n' + $res.result.rows->map(x| range($x.values->size())->map(z | if($x.values->at($z) == TDSNull, - |let type = $res.builder->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::TDSBuilder).columns->at($z).type; + | let type = $res.builder->cast(@meta::protocols::pure::vX_X_X::metamodel::invocation::execution::execute::TDSBuilder).columns->at($z).type; if ([ - pair(|$type == 'Integer', |-2147483648), - pair(|$type == 'String', |'null') + pair(|$type == 'Integer', |-2147483648), + pair(|$type == 'String', |'null'), + pair(|$type == 'DateTime', |'') ], |fail();0; );, @@ -591,7 +592,14 @@ function meta::relational::tests::pct::process::processTDS(tds:TDS[1], stat let csv = $state.schema.name+'\n' + $relAccessor.sourceElement->cast(@Table).name + '\n' + $cols->map(x|$x.name)->joinStrings(',' ) + '\n' + - $tds->map(x|$cols->map(c|$c->eval($x)->toOne()->toString())->joinStrings(',')) + $tds->map( + x | $cols->map(c|$c->eval($x)->toOne()->match( + [ + x : Date[1] | format('%t{yyyy-MM-dd"T"HH:mm:ss.SSSSSSSSS}', $x), + z : Any[1] | $z->toString() + ] + ))->joinStrings(',') + ) ->joinStrings('\n'); ^$state diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure index eede8afef60..753a16fcf94 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure @@ -1,3 +1,4 @@ +import meta::relational::metamodel::join::*; import meta::relational::functions::sqlQueryToString::duckDB::*; import meta::relational::functions::sqlQueryToString::default::*; import meta::relational::functions::sqlQueryToString::*; @@ -28,6 +29,7 @@ function <> meta::relational::functions::sqlQueryToString::duckD windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_, joinStringsProcessor = processJoinStringsOperationForDuckDB_JoinStrings_1__SqlGenerationContext_1__String_1_, literalProcessor = $literalProcessor, + joinProcessor = meta::relational::functions::sqlQueryToString::duckDB::processJoinForDuckDB_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, selectSQLQueryProcessor = processSelectSQLQueryForDuckDB_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, selectSQLQueryWithCTEsProcessor = processSelectSQLQueryWithCTEsDefault_SelectSQLQueryWithCommonTableExpressions_1__SqlGenerationContext_1__Boolean_1__String_1_, columnNameToIdentifier = columnNameToIdentifierForDuckDB_String_1__DbConfig_1__String_1_, @@ -37,6 +39,22 @@ function <> meta::relational::functions::sqlQueryToString::duckD ); } +function <> meta::relational::functions::sqlQueryToString::duckDB::processJoinForDuckDB(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + $j.join->match( + [ + join : AsOfJoin[1] | 'asof left join ' + $j.alias->map(a|^$a(name = '"' + $a.name + '"'))->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + + $format.separator() + + 'on (' + processOperation($join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')', + join : Join[1] | + $j.joinType->map(jt|$jt->processJoinType($dbConfig, $format, $extensions))->orElse('') + + $j.alias->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')' + ] + ); +} + function <> meta::relational::functions::sqlQueryToString::duckDB::getDDLCommandsTranslator(): RelationalDDLCommandsTranslator[1] { ^RelationalDDLCommandsTranslator( diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java index 8e676d08b82..3ae29a21a34 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-h2/legend-engine-xt-relationalStore-h2-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/h2/pct/Test_Relational_H2_RelationFunctions_PCT.java @@ -47,8 +47,11 @@ public class Test_Relational_H2_RelationFunctions_PCT extends PCTReportConfigura one("meta::pure::functions::relation::tests::composition::test_GroupBy_Distinct_Filter_Function_1__Boolean_1_", "org.h2.jdbc.JdbcSQLSyntaxErrorException: Column \"restrict__d#2.newCol\" not found; SQL statement"), // BUG: Column name with special characters is not properly escaped - one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema.tb") - ); + one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema.tb"), + + one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\""), + one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported by H2!\"") + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java index 619ac724c77..aac2fd525b8 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/postgres/pct/Test_Relational_Postgres_RelationFunctions_PCT.java @@ -50,8 +50,13 @@ public class Test_Relational_Postgres_RelationFunctions_PCT extends PCTReportCon one("meta::pure::functions::relation::tests::composition::test_GroupBy_Filter_Function_1__Boolean_1_", "org.postgresql.util.PSQLException: ERROR: column \"subselect.newCol\" must appear in the GROUP BY clause or be used in an aggregate function"), // BUG: Column name with special characters is not properly escaped - one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema.") - ); + one("meta::pure::functions::relation::tests::select::testSingleSelectWithQuotedColumn_Function_1__Boolean_1_", "Error while executing: Create Table leSchema."), + + // Postgres doesn't support asOf Join (May want to compensate with an OLAP equivalent if required + one("meta::pure::functions::relation::tests::asOfJoin::testAsOfJoinWithKeyMatch_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\""), + one("meta::pure::functions::relation::tests::asOfJoin::testSimpleAsOfJoin_Function_1__Boolean_1_", "\"AsOfJoins are not supported in the generic generator!\"") + + ); public static Test suite() { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure index 3b42e40750a..508eb081f3d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/sqlQueryToString/snowflakeExtension.pure @@ -38,6 +38,7 @@ function <> meta::relational::functions::sqlQueryToString::snowf semiStructuredElementProcessor = processSemiStructuredElementForSnowflake_RelationalOperationElement_1__SqlGenerationContext_1__String_1_, tableFunctionParamProcessor = processTableFunctionParamPlaceHolder_RelationalOperationElement_1__SqlGenerationContext_1__String_1_, lateralJoinProcessor = processJoinTreeNodeWithLateralJoinForSnowflake_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, + joinProcessor = meta::relational::functions::sqlQueryToString::snowflake::processJoinForSnowflake_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, joinStringsProcessor = processJoinStringsOperationForSnowflake_JoinStrings_1__SqlGenerationContext_1__String_1_, selectSQLQueryProcessor = processSelectSQLQueryForSnowflake_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, selectSQLQueryWithCTEsProcessor = processSelectSQLQueryWithCTEsDefault_SelectSQLQueryWithCommonTableExpressions_1__SqlGenerationContext_1__Boolean_1__String_1_, @@ -51,6 +52,38 @@ function <> meta::relational::functions::sqlQueryToString::snowf ); } +function <> meta::relational::functions::sqlQueryToString::snowflake::processJoinForSnowflake(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + $j.join->match( + [ + join : AsOfJoin[1] | assert($join.operation->instanceOf(DynaFunction), |'Error the "asof join" operation is not a dynaFunction'); + let dFunc = $join.operation->cast(@DynaFunction); + 'asof join ' + $j.alias->map(a|^$a(name = '"' + $a.name + '"'))->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + + $format.separator() + + if ( + [ + pair( + |['lessThan', 'lessThanEqual', 'greaterThan', 'greaterThanEqual']->contains($dFunc.name), + |'MATCH_CONDITION (' + processOperation($dFunc, $dbConfig, $format->indent(), ^Config(), $extensions) + ')' + ), + pair( + |$dFunc.name == 'and', + |'MATCH_CONDITION (' + processOperation($dFunc.parameters->at(0), $dbConfig, $format->indent(), ^Config(), $extensions) + ')' + + 'ON ('+ processOperation($dFunc.parameters->at(1), $dbConfig, $format->indent(), ^Config(), $extensions) +')' + ) + ], + | fail('The operation '+$dFunc.name+' is not supported in asOf joins'); ''; + ); + , + join : Join[1] | + $j.joinType->map(jt|$jt->processJoinType($dbConfig, $format, $extensions))->orElse('') + + $j.alias->map(a|^$a(name = '"' + $a.name + '"')) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')' + ] + ); +} + function meta::relational::functions::sqlQueryToString::snowflake::schemaIdentifierToString(identifier:String[1], dbConfig: DbConfig[1]):String[1] { $identifier->split('.')->map(s|$s->processIdentifierWithQuoteChar('"', $dbConfig))->joinStrings('.'); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure index d6cced92bd4..4c41a64589c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure @@ -5281,23 +5281,30 @@ function meta::relational::functions::pureToSqlQuery::joinKindToType(jk:JoinKind } function meta::relational::functions::pureToSqlQuery::processTdsJoin(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] +{ + meta::relational::functions::pureToSqlQuery::processTdsJoin(false, $f, $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions) +} + +function meta::relational::functions::pureToSqlQuery::processTdsAsOfJoin(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] +{ + meta::relational::functions::pureToSqlQuery::processTdsJoin(true, $f, $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions) +} + +function meta::relational::functions::pureToSqlQuery::processTdsJoin(asOfJoin:Boolean[1], f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] { let swc1 = processValueSpecification($f.parametersValues->at(0), $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne()->cast(@SelectWithCursor); let swc2 = processValueSpecification($f.parametersValues->at(1), $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne()->cast(@SelectWithCursor); let query2 = $swc2.select->pushExtraFilteringOperation($extensions); - let type = $f->instanceValuesAtParameter(2, $vars, $state.inScopeVars) - ->match([ j: JoinType[1] | $j, + let type = if($asOfJoin, + |JoinKind.LEFT, + |$f->instanceValuesAtParameter(2, $vars, $state.inScopeVars) + ->match([ j: JoinType[1] | $j, k: JoinKind[1] | $k, a: Any[1] | let jt = $a->cast(@FunctionExpression).parametersValues; extractEnumValue($jt->at(0)->cast(@InstanceValue).values->toOne()->cast(@Enumeration), $jt->at(1)->cast(@InstanceValue).values->cast(@String)->toOne()); - ]); - - let joinCondition = $f->instanceValuesAtParameter(3, $vars, $state.inScopeVars); - let joinOp = $joinCondition->cast(@FunctionDefinition); - let newOpenVars = $state.inScopeVars->putAll($joinCondition->cast(@meta::pure::metamodel::function::Function)->at(0)->evaluateAndDeactivate()->openVariableValues()); - - assert($joinOp->cast(@FunctionDefinition).expressionSequence->size() <= 1, 'Lambda with more than one expression are not supported yet'); + ]) + ); let query1WithFilter = $swc1.select->pushExtraFilteringOperation($extensions); let Q1isVarSetPlaceHolder = $query1WithFilter.data.alias.relationalElement->toOne()->instanceOf(VarSetPlaceHolder); @@ -5309,34 +5316,62 @@ function meta::relational::functions::pureToSqlQuery::processTdsJoin(f:FunctionE let aliases2 = $rightAlias->switchAliasForJoinInput($query2.columns); assertEmpty($leftAliasWithColumns->filter(n|$n.name->in($aliases2->map(a|$a.name))), 'Duplicate column names between input TDS are not supported'); - let paths = $query1->cast(@RelationalTds).paths->concatenate($query2->cast(@RelationalTds).paths); - let element_old = processTdsLambda($joinOp.expressionSequence->at(0)->cast(@ValueSpecification), $leftAliasWithColumns->concatenate($aliases2), false, $vars, ^$state(inScopeVars=$newOpenVars), $currentPropertyMapping, $paths, $context); - let element_new = $element_old->match([ - i: Literal[1] | let val = $i.value->cast(@Boolean); - if($val ,|^DynaFunction(name = 'equal', parameters = [^Literal(value=1), ^Literal(value=1)]), |^DynaFunction(name = 'equal', parameters=[^Literal(value=1), ^Literal(value=0)]));, - a: Any[1] | $a; - ])->cast(@Operation); - - let leftAliasForJoinNode = if($Q1isVarSetPlaceHolder, - | ^TableAlias(name='"joinleft_"'+$nodeId, relationalElement=^$query1(columns=[])); - , | $leftTableAlias); - - let join = ^Join(name='tdsJoin', target=$rightAlias, aliases=[pair($leftAliasForJoinNode,$rightAlias),pair($rightAlias,$leftAliasForJoinNode)],operation=$element_new->cast(@Operation)->toOne()); - let child = ^JoinTreeNode(alias=$rightAlias, join=$join, joinType=if($type->instanceOf(JoinType),|$type->cast(@JoinType),|meta::relational::functions::pureToSqlQuery::joinKindToType($type->cast(@JoinKind))), database=^Database(), joinName='tdsJoin'); - let root = ^RootJoinTreeNode(alias=$leftAliasForJoinNode, childrenData=$child); - let newAlias = ^TableAlias(name = 'tdsJoined_' + $nodeId, relationalElement=^TdsSelectSqlQuery(data=$root, columns=if($Q1isVarSetPlaceHolder,|[],|$leftAliasWithColumns->concatenate($aliases2)))); - let newData = ^RootJoinTreeNode(alias = $newAlias); + let paths = $query1->cast(@RelationalTds).paths->concatenate($query2->cast(@RelationalTds).paths); - ^$swc1(select = ^TdsSelectSqlQuery( - data = $newData, - columns = $leftAliasWithColumns->concatenate($aliases2)->map(cl| let als = $cl->cast(@Alias); - let col = $als.relationalElement->cast(@TableAliasColumn).column; - ^Alias(name=if($Q1isVarSetPlaceHolder,|$als.name->addQuotesIfNoQuotes(),|$als.name), relationalElement=^TableAliasColumn(alias=$newAlias, column=^$col(name=$als.name)));), - paths = $paths - ), - currentTreeNode = $swc1->concatenate($swc2)->filter(q|$q.currentTreeNode->isNotEmpty())->map(q|$q.currentTreeNode->toOne()->findNode($q.select.data->toOne(), $newData))->first() + let leftAliasForJoinNode = if($Q1isVarSetPlaceHolder, + | ^TableAlias(name='"joinleft_"'+$nodeId, relationalElement=^$query1(columns=[])), + | $leftTableAlias + ); - ); + let join = if ($asOfJoin, + |let op = processJoinCondition($f->instanceValuesAtParameter(2, $vars, $state.inScopeVars)->cast(@FunctionDefinition)->toOne(), $leftAliasWithColumns, $aliases2, $vars, $paths, $currentPropertyMapping, $state, $context)->toOne(); + let finalOp = if ($f.parametersValues->size() > 3, + | ^DynaFunction(name = 'and', parameters=[$op, processJoinCondition($f->instanceValuesAtParameter(3, $vars, $state.inScopeVars)->cast(@FunctionDefinition)->toOne(), $leftAliasWithColumns, $aliases2, $vars, $paths, $currentPropertyMapping, $state, $context)->toOne()]), + | $op + ); + ^AsOfJoin( + name='tdsJoin', + target=$rightAlias, + aliases=[pair($leftAliasForJoinNode,$rightAlias),pair($rightAlias,$leftAliasForJoinNode)], + operation=$finalOp + );, + |^Join( + name='tdsJoin', + target=$rightAlias, + aliases=[pair($leftAliasForJoinNode,$rightAlias),pair($rightAlias,$leftAliasForJoinNode)], + operation=processJoinCondition($f->instanceValuesAtParameter(3, $vars, $state.inScopeVars)->cast(@FunctionDefinition)->toOne(), $leftAliasWithColumns, $aliases2, $vars, $paths, $currentPropertyMapping, $state, $context)->toOne() + ) + ); + + let child = ^JoinTreeNode(alias=$rightAlias, join=$join, joinType=if($type->instanceOf(JoinType),|$type->cast(@JoinType),|meta::relational::functions::pureToSqlQuery::joinKindToType($type->cast(@JoinKind))), database=^Database(), joinName='tdsJoin'); + let root = ^RootJoinTreeNode(alias=$leftAliasForJoinNode, childrenData=$child); + let newAlias = ^TableAlias(name = 'tdsJoined_' + $nodeId, relationalElement=^TdsSelectSqlQuery(data=$root, columns=if($Q1isVarSetPlaceHolder,|[],|$leftAliasWithColumns->concatenate($aliases2)))); + let newData = ^RootJoinTreeNode(alias = $newAlias); + + ^$swc1(select = ^TdsSelectSqlQuery( + data = $newData, + columns = $leftAliasWithColumns->concatenate($aliases2)->map(cl| let als = $cl->cast(@Alias); + let col = $als.relationalElement->cast(@TableAliasColumn).column; + ^Alias(name=if($Q1isVarSetPlaceHolder,|$als.name->addQuotesIfNoQuotes(),|$als.name), relationalElement=^TableAliasColumn(alias=$newAlias, column=^$col(name=$als.name)));), + paths = $paths + ), + currentTreeNode = $swc1->concatenate($swc2)->filter(q|$q.currentTreeNode->isNotEmpty())->map(q|$q.currentTreeNode->toOne()->findNode($q.select.data->toOne(), $newData))->first() + + ); +} + +function meta::relational::functions::pureToSqlQuery::processJoinCondition(joinCondition:FunctionDefinition[1], leftAliasWithColumns:Alias[*], aliases2: Alias[*], vars:Map[1], paths:Pair[*], currentPropertyMapping:PropertyMapping[*], state:State[1], context:DebugContext[1]):Operation[1] +{ + let newOpenVars = $state.inScopeVars->putAll($joinCondition->at(0)->evaluateAndDeactivate()->openVariableValues()); + + assert($joinCondition.expressionSequence->size() <= 1, 'Lambda with more than one expression are not supported yet'); + + let element_old = processTdsLambda($joinCondition.expressionSequence->at(0)->cast(@ValueSpecification), $leftAliasWithColumns->concatenate($aliases2), false, $vars, ^$state(inScopeVars=$newOpenVars), $currentPropertyMapping, $paths, $context); + let element_new = $element_old->match([ + i: Literal[1] | let val = $i.value->cast(@Boolean); + if($val ,|^DynaFunction(name = 'equal', parameters = [^Literal(value=1), ^Literal(value=1)]), |^DynaFunction(name = 'equal', parameters=[^Literal(value=1), ^Literal(value=0)]));, + a: Any[1] | $a; + ])->cast(@Operation); } function meta::relational::functions::pureToSqlQuery::createAliasWithColumns(query:SelectSQLQuery[1],prefix:String[1]):Alias[*] @@ -8607,6 +8642,8 @@ function meta::relational::functions::pureToSqlQuery::getSupportedFunctions():Ma ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::jaroWinklerSimilarity_String_1__String_1__Float_1_, second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::levenshteinDistance_String_1__String_1__Integer_1_, second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), + ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Relation_1_, second=meta::relational::functions::pureToSqlQuery::processTdsAsOfJoin_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), + ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::relation::asOfJoin_Relation_1__Relation_1__Function_1__Function_1__Relation_1_, second=meta::relational::functions::pureToSqlQuery::processTdsAsOfJoin_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::relation::filter_Relation_1__Function_1__Relation_1_, second=meta::relational::functions::pureToSqlQuery::processTdsFilter_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::relation::distinct_Relation_1__Relation_1_, second=meta::relational::functions::pureToSqlQuery::processDistinct_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::relation::distinct_Relation_1__ColSpecArray_1__Relation_1_, second=meta::relational::functions::pureToSqlQuery::processDistinctByColumns_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure index f7e71f3e75f..e1233bb6ddc 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure @@ -87,6 +87,11 @@ Class meta::relational::functions::sqlQueryToString::DbConfig if($this.dbExtension.lateralJoinProcessor->isEmpty(), |meta::relational::functions::sqlQueryToString::default::processJoinTreeNodeWithLateralJoinDefault($j, $dbConfig, $format, $extensions), |$this.dbExtension.lateralJoinProcessor->toOne()->eval($j, $dbConfig, $format, $extensions)); }: String[1]; + joinProcessor(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]) + { + if($this.dbExtension.joinProcessor->isEmpty(),|meta::relational::functions::sqlQueryToString::default::processJoin($j, $dbConfig, $format, $extensions),|$this.dbExtension.joinProcessor->toOne()->eval($j, $dbConfig, $format, $extensions)); + }: String[1]; + joinStringsProcessor(j:JoinStrings[1], sgc:SqlGenerationContext[1]) { assert($this.dbExtension.joinStringsProcessor->isNotEmpty(), '[unsupported-api] Join strings operation not supported for Database Type: ' + $this.dbType->toString()); @@ -227,6 +232,7 @@ Class meta::relational::functions::sqlQueryToString::DbExtension semiStructuredElementProcessor: meta::pure::metamodel::function::Function<{RelationalOperationElement[1], SqlGenerationContext[1] -> String[1]}>[0..1]; tableFunctionParamProcessor: meta::pure::metamodel::function::Function<{RelationalOperationElement[1], SqlGenerationContext[1] -> String[1]}>[0..1]; lateralJoinProcessor : meta::pure::metamodel::function::Function<{JoinTreeNode[1], DbConfig[1], Format[1], Extension[*] -> String[1]}>[0..1]; + joinProcessor : meta::pure::metamodel::function::Function<{JoinTreeNode[1], DbConfig[1], Format[1], Extension[*] -> String[1]}>[0..1]; joinStringsProcessor: meta::pure::metamodel::function::Function<{JoinStrings[1], SqlGenerationContext[1] -> String[1]}>[0..1]; selectSQLQueryProcessor: meta::pure::metamodel::function::Function<{SelectSQLQuery[1], SqlGenerationContext[1], Boolean[1] -> String[1]}>[1]; selectSQLQueryWithCTEsProcessor: meta::pure::metamodel::function::Function<{SelectSQLQueryWithCommonTableExpressions[1], SqlGenerationContext[1], Boolean[1] -> String[1]}>[0..1]; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/debugPrint/debugPrintExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/debugPrint/debugPrintExtension.pure index 7dc772bbf0a..671e0f85069 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/debugPrint/debugPrintExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/debugPrint/debugPrintExtension.pure @@ -1,4 +1,6 @@ -import meta::relational::functions::sqlQueryToString::duckDB::*; +import meta::relational::functions::pureToSqlQuery::metamodel::*; +import meta::relational::metamodel::join::*; +import meta::relational::functions::sqlQueryToString::debugPrint::*; import meta::relational::functions::sqlQueryToString::default::*; import meta::relational::functions::sqlQueryToString::*; import meta::relational::metamodel::operation::*; @@ -9,7 +11,1179 @@ import meta::relational::runtime::*; import meta::pure::extension::*; import meta::relational::extension::*; -function <> meta::relational::functions::sqlQueryToString::duckDB::dbExtensionLoaderForDebugPrint():DbExtensionLoader[1] +function <> meta::relational::functions::sqlQueryToString::debugPrint::dbExtensionLoaderForDebugPrint():DbExtensionLoader[1] { - ^DbExtensionLoader(dbType = DatabaseType.DebugPrint, loader = meta::relational::functions::sqlQueryToString::h2::v1_4_200::createDbExtensionForH2__DbExtension_1_); + ^DbExtensionLoader(dbType = DatabaseType.DebugPrint, loader = meta::relational::functions::sqlQueryToString::debugPrint::createDbExtensionForDebugPrint__DbExtension_1_); +} + + +function meta::relational::functions::sqlQueryToString::debugPrint::createDbExtensionForDebugPrint():DbExtension[1] +{ + let reservedWords = h2ReservedWords(); + let literalProcessors = getDefaultLiteralProcessors(); + let literalProcessor = {type:Type[1]| $literalProcessors->get(if($type->instanceOf(Enumeration), | Enum, | $type))->toOne()}; + let dynaFuncDispatch = getDynaFunctionToSqlDefault($literalProcessor)->groupBy(d| $d.funcName)->putAll( + getDynaFunctionToSqlForH2()->groupBy(d| $d.funcName))->getDynaFunctionDispatcher(); + + ^DbExtension( + isBooleanLiteralSupported = true, + aliasLimit = 1000, + isDbReservedIdentifier = {str:String[1]| $str->toLower()->in($reservedWords)}, + literalProcessor = $literalProcessor, + windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_, + lateralJoinProcessor = processJoinTreeNodeWithLateralJoinForH2_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, + semiStructuredElementProcessor = processSemiStructuredElementForH2_RelationalOperationElement_1__SqlGenerationContext_1__String_1_, + joinStringsProcessor = processJoinStringsOperationForH2_JoinStrings_1__SqlGenerationContext_1__String_1_, + joinProcessor = processJoinForDebugPrint_JoinTreeNode_1__DbConfig_1__Format_1__Extension_MANY__String_1_, + selectSQLQueryProcessor = processSelectSQLQueryDefault_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, + selectSQLQueryWithCTEsProcessor = processSelectSQLQueryWithCTEsDefault_SelectSQLQueryWithCommonTableExpressions_1__SqlGenerationContext_1__Boolean_1__String_1_, + upsertSQLQueryProcessor = processUpsertSQLQueryForH2_UpsertSQLQuery_1__SqlGenerationContext_1__String_1_, + identifierProcessor = processIdentifierWithDoubleQuotes_String_1__DbConfig_1__String_1_, + dynaFuncDispatch = $dynaFuncDispatch, + ddlCommandsTranslator = getDDLCommandsTranslator(), + processTempTableName = processTempTableNameDefault_String_1__DatabaseConnection_1__String_1_ + ); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processJoinForDebugPrint(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + $j.join->match( + [ + join : AsOfJoin[1] | 'asof left join ' + $j.alias->map(a|^$a(name = '"' + $a.name + '"'))->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + + $format.separator() + + 'on (' + processOperation($join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')', + join : Join[1] | + $j.joinType->map(jt|$jt->processJoinType($dbConfig, $format, $extensions))->orElse('') + + $j.alias->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')' + ] + ); +} + +function meta::relational::functions::sqlQueryToString::debugPrint::getDDLCommandsTranslator(): RelationalDDLCommandsTranslator[1] +{ + ^RelationalDDLCommandsTranslator( + createSchema = translateCreateSchemaStatementDefault_CreateSchemaSQL_1__DbConfig_1__String_1_, + dropSchema = translateDropSchemaStatementDefault_DropSchemaSQL_1__DbConfig_1__String_1_, + createTable = translateCreateTableStatementForH2_CreateTableSQL_1__DbConfig_1__String_1_, + dropTable = translateDropTableStatementDefault_DropTableSQL_1__DbConfig_1__String_1_, + loadTable = loadValuesToDbTableForH2_LoadTableSQL_1__DbConfig_1__String_MANY_ + ); +} + +function meta::relational::functions::sqlQueryToString::debugPrint::translateCreateTableStatementForH2(c:CreateTableSQL[1], dbConfig: DbConfig[1]): String[1] +{ + if($c.isTempTable->isTrue(), + | 'CREATE LOCAL TEMPORARY TABLE ' + $c.table->tableToString($dbConfig) + + '(' + + $c.table.columns->map(r| $r->match([ + c: Column[1] | $c.name->processColumnName($dbConfig) + ' ' + getColumnTypeSqlTextH2($c.type), + r: RelationalOperationElement[1] | fail('Only \'Column\' types are supported when creating temporary tables, found: '+$r->type()->toOne()->elementToPath());''; + ]))->joinStrings(',') + + ');', + | $c->translateCreateTableStatementH2($dbConfig) + ) +} + +function meta::relational::functions::sqlQueryToString::debugPrint::getColumnTypeSqlTextH2(columnType: meta::relational::metamodel::datatype::DataType[1]): String[1] +{ + $columnType->match([ + s : meta::relational::metamodel::datatype::SemiStructured[1] | 'VARCHAR(4000)', + a : Any[*] | dataTypeToSqlTextH2($columnType) + ]) +} + +function meta::relational::functions::sqlQueryToString::debugPrint::translateCreateTableStatementH2(createTableSQL:CreateTableSQL[1], dbConfig:DbConfig[1]): String[1] +{ + let t= $createTableSQL.table; + let applyConstraints = $createTableSQL.applyConstraints; + 'Create Table '+if($t.schema.name == 'default',|'',|$t.schema.name+'.')+$t.name+ + + '(' + + $t.columns->cast(@meta::relational::metamodel::Column) + ->map(c | $c.name->processColumnName($dbConfig) + ' ' + getColumnTypeSqlTextH2($c.type) + if($c.nullable->isEmpty() || $applyConstraints == false, | '', | if($c.nullable == true , | ' NULL', | ' NOT NULL'))) + ->joinStrings(',') + + if ($t.primaryKey->isEmpty() || $applyConstraints == false, | '', | ', PRIMARY KEY(' + $t.primaryKey->map(c | $c.name)->joinStrings(',') + ')') + +');'; +} + +function meta::relational::functions::sqlQueryToString::debugPrint::loadValuesToDbTableForH2(l:LoadTableSQL[1], dbConfig: DbConfig[1]): String[*] +{ + if($l.absolutePathToFile->isNotEmpty(), + | 'INSERT INTO ' + $l.table->tableToString($dbConfig) + ' SELECT * FROM CSVREAD(\''+$l.absolutePathToFile->toOne()->processOperation($dbConfig.dbType, []) + '\');', + | $l->meta::relational::functions::sqlQueryToString::debugPrint::loadValuesToDbTableH2($dbConfig) + ) +} + +function meta::relational::functions::sqlQueryToString::debugPrint::loadValuesToDbTableH2(loadTableSQL: LoadTableSQL[1], dbConfig: DbConfig[1]): String[*] +{ + $loadTableSQL.parsedData.values->map(row| let sql = + 'insert into ' + if($loadTableSQL.table.schema.name == 'default', | '' , | $loadTableSQL.table.schema.name + '.') + $loadTableSQL.table.name + + ' (' + + $loadTableSQL.columnsToLoad.name->map(colName | $colName->processColumnName($dbConfig))->joinStrings(',') + +') ' + + 'values (' + + $row.values->meta::relational::functions::sqlQueryToString::debugPrint::convertValuesToCsvH2($loadTableSQL.columnsToLoad.type) + + ');'; + ); +} + +function meta::relational::functions::sqlQueryToString::debugPrint::convertValuesToCsvH2(str: String[*], types: Any[*]): String[1] +{ + let stringToken = map(range($types->size()), {x| if($str->at($x) == '' || $str->at($x) == '---null---', |'null', |$types->at($x)->match([ + s: meta::relational::metamodel::datatype::Varchar[*] | '\'' + $str->at($x)->replace('\'', '\'\'') + '\'', + s: meta::relational::metamodel::datatype::SemiStructured[*] | '\'' + $str->at($x)->replace('\'', '\'\'') + '\'', + s: meta::relational::metamodel::datatype::Char[*] | '\'' + $str->at($x)->replace('\'', '\'\'') + '\'', + d: meta::relational::metamodel::datatype::Date[*] | '\'' + $str->at($x) + '\'', + t: meta::relational::metamodel::datatype::Timestamp[*] | '\'' + if($str->at($x)->length() > 10, |$str->at($x), |$str->at($x)) + '\'', + b: meta::relational::metamodel::datatype::Bit[1] | bitValueFromString($str->at($x)), + a: Any[*] | $str->at($x) + ]))})->joinStrings(','); +} + +function meta::relational::functions::sqlQueryToString::debugPrint::bitValueFromString(s: String[1]): String[1] +{ + let truesList = ['y', '1', 'true']; + let falsesList = ['n', '0', 'false']; + let bitStr = $s->trim()->toLower(); + + let bitVal = if($bitStr->in($truesList), + | 'true', + | if($bitStr->in($falsesList), + | 'false', + | $s + ) + ); + + $bitVal; +} + +function meta::relational::functions::sqlQueryToString::debugPrint::getLiteralProcessorsForH2():Map[1] +{ + newMap([ + pair(Boolean, ^LiteralProcessor(format = '%s', transform = toString_Any_1__String_1_->literalTransform())), + pair(Float, ^LiteralProcessor(format = 'CAST(%s AS FLOAT)', transform = toString_Any_1__String_1_->literalTransform())), + pair(StrictDate, ^LiteralProcessor(format = 'DATE\'%s\'', transform = {d:StrictDate[1], dbTimeZone:String[0..1] | $d->convertDateToSqlString($dbTimeZone)})), + pair(DateTime, ^LiteralProcessor(format = 'TIMESTAMP\'%s\'', transform = {d:DateTime[1], dbTimeZone:String[0..1] | $d->convertDateToSqlString($dbTimeZone)})), + pair(Date, ^LiteralProcessor(format = 'TIMESTAMP\'%s\'', transform = {d:Date[1], dbTimeZone:String[0..1] | $d->convertDateToSqlString($dbTimeZone)})) + ]); +} + +function meta::relational::functions::sqlQueryToString::debugPrint::dataTypeToSqlTextH2(type: meta::relational::metamodel::datatype::DataType[1]):String[1] +{ + let MAX_CHAR_LENGTH = 1000000000; + let MIN_CHAR_LENGTH = 1; + let MIN_PRECISION = 1; + + $type->match([ + v : meta::relational::metamodel::datatype::Varchar[1] | format('VARCHAR(%d)', max([$MIN_CHAR_LENGTH, min([$v.size, $MAX_CHAR_LENGTH])])), + c : meta::relational::metamodel::datatype::Char[1] | format('CHAR(%d)', max([$MIN_CHAR_LENGTH, min([$c.size, $MAX_CHAR_LENGTH])])), // H2 now pads characters to hit stated size + //b : meta::relational::metamodel::datatype::Bit[1] | 'TINYINT', // allows comparisons to Booleans with new H2 + n : meta::relational::metamodel::datatype::Numeric[1] | format('NUMERIC(%d, %d)', [max([$n.precision, $MIN_PRECISION]), $n.scale]), + d : meta::relational::metamodel::datatype::DataType[1] | getColumnTypeSqlTextDefault($d) + ]); +} + +// words found in ParserUtil.KEYWORDS of h2database EXCEPT for anything listed explicitly as NON_KEYWORD for compatibility +function meta::relational::functions::sqlQueryToString::debugPrint::h2ReservedWords():String[*] +{ + [ + 'all','and','array','as','between','case','check','constraint','cross','current_catalog', + 'current_date','current_schema','current_time','current_timestamp','current_user','distinct', + 'except','exists','false','fetch','for','foreign','from','full','group','having','if','in', + 'inner','intersect','interval','is','join','left','like','limit','localtime','localtimestamp', + 'minus','natural','not','null','offset','on','or','order','primary','qualify','row','rownum', + 'select','table','true','union','unique','unknown','using','values','where','window','with', + '_rowid_','both','groups','ilike','leading','over','partition','range','regexp','rows','top', + 'trailing' + ]; +} + +function meta::relational::functions::sqlQueryToString::debugPrint::getDynaFunctionToSqlForH2(): DynaFunctionToSql[*] +{ + let allStates = allGenerationStates(); + + [ + dynaFnToSql('adjust', $allStates, ^ToSql(format='dateadd(%s)', transform={p:String[3] | $p->at(2)->mapToDBUnitType() + ', ' + $p->at(1) + ', ' + $p->at(0)})), + dynaFnToSql('booland', $allStates, ^ToSql(format='every(%s)')), + dynaFnToSql('boolor', $allStates, ^ToSql(format='any(%s)')), + dynaFnToSql('char', $allStates, ^ToSql(format='char(%s)')), + dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), + dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDateH2()})), + dynaFnToSql('castBoolean', $allStates, ^ToSql(format='cast(%s as boolean)')), + dynaFnToSql('convertDateTime', $allStates, ^ToSql(format='%s' , transform={p:String[*] | $p->convertToDateTimeH2()})), + dynaFnToSql('convertVarchar128', $allStates, ^ToSql(format='convert(%s, VARCHAR(128))')), + dynaFnToSql('dateDiff', $allStates, ^ToSql(format='datediff(%s,%s,%s)', transform={p:String[*]|[$p->at(2)->replace('\'', '')->processDateDiffDurationUnitForH2(),$p->at(0),$p->at(1)]})), + dynaFnToSql('datePart', $allStates, ^ToSql(format='cast(truncate(%s) as date)')), + dynaFnToSql('dayOfMonth', $allStates, ^ToSql(format='DAY_OF_MONTH(%s)')), + dynaFnToSql('dayOfWeek', $allStates, ^ToSql(format='dayname(%s)')), + dynaFnToSql('dayOfWeekNumber', $allStates, ^ToSql(format='%s',transform={p:String[1..2]| if($p->size()==1,| 'DAY_OF_WEEK('+$p->at(0)+')',|$p->dayOfWeekNumberH2());})), + dynaFnToSql('dayOfYear', $allStates, ^ToSql(format='DAY_OF_YEAR(%s)')), + dynaFnToSql('decodeBase64', $allStates, ^ToSql(format='legend_h2_extension_base64_decode(%s)')), + dynaFnToSql('encodeBase64', $allStates, ^ToSql(format='legend_h2_extension_base64_encode(%s)')), + dynaFnToSql('extractFromSemiStructured', $allStates, ^ToSql(format='%s', transform={p:String[3]|$p->processExtractFromSemiStructuredParamsForH2()})), + dynaFnToSql('firstDayOfMonth', $allStates, ^ToSql(format='dateadd(DAY, -(dayofmonth(%s) - 1), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstDayOfQuarter', $allStates, ^ToSql(format='dateadd(MONTH, 3 * quarter(%s) - 3, dateadd(DAY, -(dayofyear(%s) - 1), %s))', transform={p:String[1] | $p->repeat(3)})), + dynaFnToSql('firstDayOfThisMonth', $allStates, ^ToSql(format='dateadd(DAY, -(dayofmonth(current_date()) - 1), current_date())')), + dynaFnToSql('firstDayOfThisQuarter', $allStates, ^ToSql(format='dateadd(MONTH, 3 * quarter(current_date) - 3, dateadd(DAY, -(dayofyear(current_date()) - 1), current_date()))')), + dynaFnToSql('firstDayOfThisYear', $allStates, ^ToSql(format='dateadd(DAY, -(dayofyear(current_date()) - 1), current_date())')), + dynaFnToSql('firstDayOfWeek', $allStates, ^ToSql(format='dateadd(DAY, -(mod(dayofweek(%s)+5, 7)), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstDayOfYear', $allStates, ^ToSql(format='dateadd(DAY, -(dayofyear(%s) - 1), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstHourOfDay', $allStates, ^ToSql(format='date_trunc(\'day\', %s)')), + dynaFnToSql('firstMillisecondOfSecond', $allStates, ^ToSql(format='date_trunc(\'second\', %s)')), + dynaFnToSql('firstMinuteOfHour', $allStates, ^ToSql(format='date_trunc(\'hour\', %s)')), + dynaFnToSql('firstSecondOfMinute', $allStates, ^ToSql(format='date_trunc(\'minute\', %s)')), + dynaFnToSql('hour', $allStates, ^ToSql(format='hour(%s)')), + dynaFnToSql('indexOf', $allStates, ^ToSql(format='LOCATE(%s)', transform={p:String[2] | $p->at(1) + ', ' + $p->at(0)})), + dynaFnToSql('isNumeric', $allStates, ^ToSql(format='(lower(%s) = upper(%s))')), + dynaFnToSql('isAlphaNumeric', $allStates, ^ToSql(format=regexpPattern('%s'), transform={p:String[1]|$p->transformAlphaNumericParamsDefault()})), + dynaFnToSql('jaroWinklerSimilarity', $allStates, ^ToSql(format='legend_h2_extension_jaro_winkler_similarity(%s, %s)')), + dynaFnToSql('joinStrings', $allStates, ^ToSql(format='group_concat(%s separator %s)')), + dynaFnToSql('length', $allStates, ^ToSql(format='char_length(%s)')), + dynaFnToSql('levenshteinDistance', $allStates, ^ToSql(format='legend_h2_extension_edit_distance(%s, %s)')), + dynaFnToSql('matches', $allStates, ^ToSql(format=regexpPattern('%s'), transform={p:String[2]|$p->transformRegexpParams()})), + dynaFnToSql('md5', $allStates, ^ToSql(format='rawtohex(hash(\'MD5\', %s))')), + dynaFnToSql('minute', $allStates, ^ToSql(format='minute(%s)')), + dynaFnToSql('month', $allStates, ^ToSql(format='month(%s)')), + dynaFnToSql('monthNumber', $allStates, ^ToSql(format='month(%s)')), + dynaFnToSql('monthName', $allStates, ^ToSql(format='monthname(%s)')), + dynaFnToSql('mostRecentDayOfWeek', $allStates, ^ToSql(format='dateadd(DAY, case when %s - DAY_OF_WEEK(%s) > 0 then %s - DAY_OF_WEEK(%s) - 7 else %s - DAY_OF_WEEK(%s) end, %s)', transform={p:String[1..2] | $p->formatMostRecentH2('current_date()')}, parametersWithinWhenClause = [false, false])), + dynaFnToSql('now', $allStates, ^ToSql(format='current_timestamp()')), + dynaFnToSql('parseDate', $allStates, ^ToSql(format='cast(parsedatetime(%s,%s) as timestamp)')), + dynaFnToSql('parseDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')), + dynaFnToSql('parseFloat', $allStates, ^ToSql(format='cast(%s as float)')), + dynaFnToSql('parseInteger', $allStates, ^ToSql(format='cast(%s as integer)')), + dynaFnToSql('parseJson', $allStates, ^ToSql(format='legend_h2_extension_json_parse(%s)')), + dynaFnToSql('position', $allStates, ^ToSql(format='position(%s, %s)')), + dynaFnToSql('previousDayOfWeek', $allStates, ^ToSql(format='dateadd(DAY, case when %s - DAY_OF_WEEK(%s) >= 0 then %s - DAY_OF_WEEK(%s) - 7 else %s - DAY_OF_WEEK(%s) end, %s)', transform={p:String[1..2] | $p->formatMostRecentH2('current_date()')}, parametersWithinWhenClause = [false, false])), + dynaFnToSql('quarter', $allStates, ^ToSql(format='quarter(%s)')), + dynaFnToSql('quarterNumber', $allStates, ^ToSql(format='quarter(%s)')), + dynaFnToSql('reverseString', $allStates, ^ToSql(format='legend_h2_extension_reverse_string(%s)')), + dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)), + dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')), + dynaFnToSql('sha1', $allStates, ^ToSql(format='rawtohex(hash(\'SHA-1\', %s))')), + dynaFnToSql('sha256', $allStates, ^ToSql(format='rawtohex(hash(\'SHA-256\', %s))')), + dynaFnToSql('splitPart', $allStates, ^ToSql(format='legend_h2_extension_split_part(%s, %s, %s)')), + dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), + dynaFnToSql('stdDevPopulation', $allStates, ^ToSql(format='stddev_pop(%s)')), + dynaFnToSql('stdDevSample', $allStates, ^ToSql(format='stddev_samp(%s)')), + dynaFnToSql('today', $allStates, ^ToSql(format='current_date()')), + dynaFnToSql('toString', $allStates, ^ToSql(format='cast(%s as varchar)')), + dynaFnToSql('toDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')), + dynaFnToSql('toFloat', $allStates, ^ToSql(format='cast(%s as double precision)')), + dynaFnToSql('toTimestamp', $allStates, ^ToSql(format='%s', transform={p:String[2] | $p->transformToTimestampH2()})), + dynaFnToSql('weekOfYear', $allStates, ^ToSql(format='week(%s)')), + dynaFnToSql('year', $allStates, ^ToSql(format='year(%s)')), + dynaFnToSql('convertTimeZone', $allStates, ^ToSql(format='%s', transform={p:String[3] | $p->transformConvertTimeZone()})) + + ]; +} + + +function <> meta::relational::functions::sqlQueryToString::debugPrint::transformConvertTimeZone(params:String[3]):String[1] +{ + let unWrappedfmt = $params->at(2)->substring(1, $params->at(2)->length()-1); + assert($unWrappedfmt->validateDateTimeFormat(),'Found an invalid date format'); + let formatpairs = meta::relational::functions::sqlQueryToString::default::defaultJavaToSQLTimeParts(); + let datefmt = $formatpairs->fold( {sub, date| $date->toOne()->replace($sub.first,$sub.second)},$params->at(2)); + format('TO_CHAR(legend_h2_extension_convertTimeZone(\'%s\',%s),%s)',[$params->at(0),$params->at(1),$datefmt]); +} +function meta::relational::functions::sqlQueryToString::debugPrint::processUpsertSQLQueryForH2(upsertQuery: UpsertSQLQuery[1], sgc: SqlGenerationContext[1]): String[1] +{ + // Map of Column to Literals of VarPlaceHolder + let keyValues = $upsertQuery.equalityStatements->keyValues()->sortBy(kv | $kv.first); + let columnNames = $keyValues->map(kv | $kv.first)->joinStrings(', '); + let literalValues = $keyValues->map(kv | meta::relational::functions::sqlQueryToString::processLiteralValue($kv.second.value, $sgc.dbConfig))->joinStrings(', '); + + 'merge into ' + $upsertQuery.data.name + ' (' + $columnNames + ') values (' + $literalValues + ')'; +} + +function meta::relational::functions::sqlQueryToString::debugPrint::processJoinTreeNodeWithLateralJoinForH2(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + // keeping consistent with snowflake + assert(processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) == '1 = 1', | 'Filter in column projections is not supported. Use a Post Filter if filtering is necessary'); + + assert($j.alias.relationalElement->instanceOf(SemiStructuredArrayFlatten), | 'Lateral join in H2 should be followed by flatten operation'); + + let lhs = ^TableAliasColumn(column = ^Column(name = '__INPUT__', type = ^meta::relational::metamodel::datatype::SemiStructured()),alias = $j.alias); + let rhs = $j.alias.relationalElement->cast(@SemiStructuredArrayFlatten).navigation->cast(@SemiStructuredObjectNavigation); + let joinOperation = ^DynaFunction(name= 'equal', parameters = [$lhs, ^$rhs(returnType=String, avoidCastIfPrimitive=false)]); + + ' ' + $format.separator() + 'left outer join ' + + $j.alias + ->map(a|^$a(name = '"' + $a.name + '"')) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($joinOperation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')'; +} + +function meta::relational::functions::sqlQueryToString::debugPrint::processExtractFromSemiStructuredParamsForH2(params:String[3]):String[1] +{ + let baseRelationalOp = $params->at(0); + let pathNavigation = $params->at(1); + let returnType = $params->at(2); + + let parsedPathNavigation = $pathNavigation->parseSemiStructuredPathNavigation(); + let relationalPropertyAccess = $parsedPathNavigation->fold({property,relational | $relational->semiStructuredPathAccessForH2($property)}, $baseRelationalOp); + + let castTo = if ($returnType->in(['CHAR', 'VARCHAR', 'STRING']), | 'varchar', | + if ($returnType->in(['DATETIME', 'TIMESTAMP']), | 'timestamp', | + if ($returnType == 'DATE', | 'date', | + if ($returnType == 'BOOLEAN', | 'boolean', | + if ($returnType == 'FLOAT', | 'float', | + if ($returnType == 'INTEGER', | 'integer', | + $returnType)))))); + + format('cast(%s as %s)', [$relationalPropertyAccess, $castTo]); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::semiStructuredPathAccessForH2(elementAccess: String[1], property: String[1]): String[1] +{ + if($property->isDigit(), | $elementAccess->semiStructuredArrayElementAccessForH2($property), | $elementAccess->semiStructuredPropertyAccessForH2($property->substring(1, $property->length()-1))); // remove double quotes +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::semiStructuredPropertyAccessForH2(elementAccess: String[1], property: String[1]): String[1] +{ + format('legend_h2_extension_json_navigate(%s, \'%s\', null)', [$elementAccess, $property]); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::semiStructuredArrayElementAccessForH2(elementAccess: String[1], index: String[1]): String[1] +{ + format('legend_h2_extension_json_navigate(%s, null, %s)', [$elementAccess, $index]); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSemiStructuredElementForH2(s:RelationalOperationElement[1], sgc:SqlGenerationContext[1]): String[1] +{ + $s->match([ + o:SemiStructuredObjectNavigation[1] | $o->processSemiStructuredObjectNavigationForH2($sgc), + a:SemiStructuredArrayFlatten[1] | $a->processSemiStructuredArrayFlattenForH2($sgc), + a:SemiStructuredArrayFlattenOutput[1] | $a->processSemiStructuredArrayFlattenOutputForH2($sgc) + ]) +} + +/* +* returns property accesses to extract the semi structured property starting from root +*/ +function <> meta::relational::functions::sqlQueryToString::debugPrint::propertyAccessForSemiStructuredObjectNavigationH2(z:SemiStructuredObjectNavigation[1], sgc:SqlGenerationContext[1]): String[*] +{ + let elementAccess = $z->match([ + p: SemiStructuredPropertyAccess[1] | + let propertyAccess = '"' + $p.property->cast(@Literal).value->cast(@String) + '"'; + if ($p.index->isNotEmpty(), + | $propertyAccess->concatenate($p.index->toOne()->cast(@Literal).value->toString()), + | $propertyAccess + );, + a: SemiStructuredArrayElementAccess[1] | $a.index->toOne()->cast(@Literal).value->toString() + ]); + $z.operand->match([ + s: SemiStructuredObjectNavigation[1] | $s->propertyAccessForSemiStructuredObjectNavigationH2($sgc), + a: SemiStructuredArrayFlatten[1] | $a.navigation->cast(@SemiStructuredObjectNavigation)->propertyAccessForSemiStructuredObjectNavigationH2($sgc)->concatenate('"*"'), + s: SemiStructuredArrayFlattenOutput[1] | let flattening = $s.tableAliasColumn.alias.relationalElement->cast(@SemiStructuredArrayFlatten); + $flattening.navigation->cast(@SemiStructuredObjectNavigation)->propertyAccessForSemiStructuredObjectNavigationH2($sgc)->concatenate('"*"');, + a: Any[1] | [] + ])->concatenate($elementAccess); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSemiStructuredArrayFlattenForH2(s:SemiStructuredArrayFlatten[1], sgc:SqlGenerationContext[1]): String[1] +{ + let rootTableAndColumnName = $s->meta::relational::functions::pureToSqlQuery::findTableForColumnInAlias([]); + + let jsonPaths = $s.navigation->match([ // assumes input to ssaf is always sson + s: SemiStructuredObjectNavigation[1] | $s->propertyAccessForSemiStructuredObjectNavigationH2($sgc) + ]); + + let schema = $rootTableAndColumnName.first.schema.name; + + let processedNavigation = $s.navigation->processOperation($sgc); + 'legend_h2_extension_flatten_array(' + '\'' + $rootTableAndColumnName.first->processOperation($sgc) + '\',\'' + $rootTableAndColumnName.second->processColumnName($sgc.dbConfig) + '\',ARRAY[\'' + $jsonPaths->joinStrings('\',\'') + '\'])'; +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSemiStructuredArrayFlattenOutputForH2(s:SemiStructuredArrayFlattenOutput[1], sgc:SqlGenerationContext[1]): String[1] +{ + let doubleQuote = if($sgc.config.useQuotesForTableAliasColumn == false, |'', |'"'); + let processedIdentifier = $sgc.dbConfig.identifierProcessor($doubleQuote + $s.tableAliasColumn.alias.name->toOne() + $doubleQuote); + let elementAccess = $processedIdentifier + '.' + processColumnName('VALUE', $sgc.dbConfig); + $elementAccess->castToReturnTypeForSemiStructuredData($s.returnType); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::castToReturnTypeForSemiStructuredData(elementAccess:String[1], returnType:Type[0..1]): String[1] +{ + if ($returnType == String, | 'cast(' + $elementAccess + ' as varchar)', | + if ($returnType == Boolean, | 'cast(' + $elementAccess + ' as boolean)', | + if ($returnType == Float, | 'cast(' + $elementAccess + ' as float)', | + if ($returnType == Integer, | 'cast(' + $elementAccess + ' as integer)', | + if ($returnType == StrictDate, | 'cast(' + $elementAccess + ' as date)', | + if ($returnType->isNotEmpty() && $returnType->toOne()->_subTypeOf(Date), | 'cast(' + $elementAccess + ' as timestamp)', | + if ($returnType->isNotEmpty() && $returnType->toOne()->instanceOf(Enumeration), | 'cast(' + $elementAccess + ' as varchar)', | + $elementAccess))))))); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSemiStructuredObjectNavigationForH2(s:SemiStructuredObjectNavigation[1], sgc:SqlGenerationContext[1]): String[1] +{ + // Use a user defined function for H2 (testing purpose) + + let processedOperand = $s.operand->processOperation($sgc); + + let udfName = 'legend_h2_extension_json_navigate'; + + let elementAccess = $s->match([ + p: SemiStructuredPropertyAccess[1] | + let propertyAccess = semiStructuredPropertyAccessForH2($processedOperand, $p.property->cast(@Literal).value->cast(@String)); + if ($p.index->isNotEmpty(), + | semiStructuredArrayElementAccessForH2($propertyAccess, $p.index->toOne()->cast(@Literal).value->toString()), + | $propertyAccess + );, + + a: SemiStructuredArrayElementAccess[1] | semiStructuredArrayElementAccessForH2($processedOperand, $a.index->cast(@Literal).value->toString()) + ]); + + if($s.avoidCastIfPrimitive == true, | $elementAccess, | $elementAccess->castToReturnTypeForSemiStructuredData($s.returnType)); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processJoinStringsOperationForH2(js:JoinStrings[1], sgc:SqlGenerationContext[1]): String[1] +{ + processJoinStringsOperation($js, $sgc, {col, sep| 'group_concat(' + $col + if($sep == '\'\'', |'', |' separator ' + $sep) + ' )'}, + {strs, sep| $strs->joinStrings('concat(', if('\'\'' == $sep, |', ', |',' + $sep + ',') , ')')}); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::convertToDateH2(params:String[*]):String[1] +{ + + assert( 2 - $params->size() >= 0,'Incorrect number of parameters for convertDate: convertDate(column,[dateformat])'); + let dateFormat = if( $params->size() == 1,|'\'yyyy-MM-dd\'' ,| let normalizedFormat = $params->at(1)->normalizeH2DateFormat(); + assert(dateFormatsH2()->contains($normalizedFormat->replace('\'', '')), $normalizedFormat +' not supported '); + $normalizedFormat; + ); + // FIXME: we currently allow MMMyyyy as a dateformat which requires the following hack similar to IQ + if($dateFormat == '\'MMMyyyy\'', + | 'cast( parseDateTime(concat(\'01\', %s), \'%s\') as date)'->format([ + $params->at(0), + joinStrings(['dd', $dateFormat->replace('\'', '')]) + ]), + | 'cast( parseDateTime('+$params->at(0)+','+$dateFormat +') as date)' + ); + +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::normalizeH2DateFormat(params:String[1]):String[1] +{ + [ + pair('YYYY', 'yyyy'), + pair('DD', 'dd') + ]->fold({e, a| $a->replace($e.first,$e.second)}, $params); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::dateFormatsH2():String[*] +{ + ['yyyy-MM-dd', 'MMMyyyy', 'yyyyMMdd']; +} + +// Overrides convertToDateTime acceptable formats +function <> meta::relational::functions::sqlQueryToString::debugPrint::convertToDateTimeH2(params:String[*]):String[1] +{ + $params->convertDateTimeFunctionHasCorrectParamsH2(); + + let dateTimeFormat = if( $params->size() == 1, + |'\'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS][.SSSSSSSS][.SSSSSSS][.SSSSSS][.SSSSS][.SSSS][.SSS][.SS][.S]\'', + | $params->at(1) + ); + 'parseDateTime('+$params->at(0)+','+$dateTimeFormat +')'; +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::convertDateTimeFunctionHasCorrectParamsH2(params:String[*]):Boolean[1] +{ + assert( 2 - $params->size() >= 0,'Incorrect number of parameters for convertDateTime: convertDateTime(column,[dateTimeformat])'); + + let supportedDateTimeFormat = if($params->size() == 2, + | let userFormat = $params->at(1); + dateTimeFormatsH2()->get($userFormat->normalizeH2DateTimeFormat()->replace('\'', ''));, + | [] + ); + assert($params->size() ==1 || $supportedDateTimeFormat->size() == 1 , | $params->at(1) +' not supported '); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::normalizeH2DateTimeFormat(userFormat: String[1]): String[1] +{ + $userFormat->normalizeH2DateFormat()->normalizeH2TimeFormat() +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::normalizeH2TimeFormat(userFormat: String[1]): String[1] +{ + [pair('hh', 'HH')]->fold({e, a| $a->replace($e.first,$e.second)}, $userFormat) +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::dateTimeFormatsH2():Map[1] +{ + newMap([ + pair('yyyy-MM-dd HH:mm:ss',120), // no decimal use-case compatibility + pair('yyyy-MM-dd HH:mm:ss[.SSSSSSSSS][.SSSSSSSS][.SSSSSSS][.SSSSSS][.SSSSS][.SSSS][.SSS][.SS][.S]',121) + ]); +} + +// FIXME: Are datetime and timestamp not treated the same in the backend (because parsedatetime always returns TIMESTAMP_WITH_TIMEZONE)? +// If so, this and the above logic (convertToDateTimeH2) should be identical +function <> meta::relational::functions::sqlQueryToString::debugPrint::transformToTimestampH2(params:String[2]):String[1] +{ + // Standardizing the format as per Postgres specification, will include mappings for the formats in future. + assert($params->at(1)->replace('\'', '') == 'YYYY-MM-DD HH24:MI:SS', | $params->at(1) +' not supported '); + let timestampFormat = '\'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS][.SSSSSSSS][.SSSSSSS][.SSSSSS][.SSSSS][.SSSS][.SSS][.SS][.S]\''; + 'cast(parsedatetime('+$params->at(0)+','+ $timestampFormat+') as timestamp)'; +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processDateDiffDurationUnitForH2(durationUnit:String[1]):String[1] +{ + let durationEnumNames = [DurationUnit.YEARS,DurationUnit.MONTHS,DurationUnit.WEEKS,DurationUnit.DAYS,DurationUnit.HOURS,DurationUnit.MINUTES,DurationUnit.SECONDS,DurationUnit.MILLISECONDS]->map(e|$e->toString()); + let durationDbNames = ['year', 'month', 'week', 'day', 'hour', 'minute', 'second', 'millisecond']; + $durationEnumNames->zip($durationDbNames)->filter(h | $h.first == $durationUnit).second->toOne(); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::formatMostRecentH2(p:String[1..2], defaultDay:String[1]):String[*] +{ + let day = $p->last()->toOne()->mapToDBDayOfWeekNumber()->toString(); + let current = if ($p->size() == 2, | $p->first()->toOne(), | $defaultDay); + [$day, $current, $day, $current, $day, $current, $current]; +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::dayOfWeekNumberH2(dayOfWeek: String[*]):String[1] +{ + assert(or($dayOfWeek->at(1)=='Sunday',$dayOfWeek->at(1)=='Monday'),'DayOfWeekNumber Function requires either Sunday or Monday as First Day of Week'); + if($dayOfWeek->at(1)=='Sunday',|'DAY_OF_WEEK('+$dayOfWeek->at(0)+')',|'ISO_DAY_OF_WEEK('+$dayOfWeek->at(0)+')'); +} + +// Need to override how limit is processed +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSelectSQLQueryForH2(s:SelectSQLQuery[1], sgc:SqlGenerationContext[1], isSubSelect:Boolean[1]):String[1] +{ + $s->processSelectSQLQueryForH2($sgc.dbConfig, $sgc.format, $sgc.config, $isSubSelect, $sgc.extensions); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSelectSQLQueryForH2( + s:SelectSQLQuery[1], + dbConfig : DbConfig[1], + format:Format[1], + config:Config[1], + isSubSelect : Boolean[1], + extensions:Extension[*] +): String[1] +{ + assertEmpty($s.pivot, 'pivot is not supported'); + if($s.data.childrenData->match([jtn:JoinTreeNode[1]|$jtn.joinType == JoinType.FULL_OUTER, a:Any[*]|false]) , + | + // H2 doesn't support FULL_OUTER join, so we need to convert/emulate it so people can write unit tests... + $s->meta::relational::functions::sqlQueryToString::default::convertFullOuterJoinToLeftRightOuter()->processOperation($dbConfig, $format->indent(), $extensions); + , + | + let opStr = + if($s.filteringOperation->isEmpty(), + | '', + | $s.filteringOperation->map(s | $s->wrapH2Boolean($extensions)->processOperation($dbConfig, $format->indent(), ^$config(callingFromFilter = true), $extensions))->filter(s|$s != '')->joinStrings(' <||> ') + ); + let havingStr = + if($s.havingOperation->isEmpty(), + | '', + | $s.havingOperation->map(s|$s->wrapH2Boolean($extensions)->processOperation($dbConfig, $format->indent(), $config, $extensions))->filter(s|$s != '')->joinStrings(' <||> ') + ); + + $format.separator + 'select ' + processTop($s, $format, $dbConfig, $extensions) + if($s.distinct == true,|'distinct ',|'') + + processSelectColumnsH2($s.columns, $dbConfig, $format->indent(), true, $extensions) + + if($s.data == [],|'',| ' ' + $format.separator + 'from ' + $s.data->toOne()->processJoinTreeNodeH2([], $dbConfig, $format->indent(), [], $extensions)) + + if (eq($opStr, ''), |'', | ' ' + $format.separator + 'where ' + $opStr) + + if ($s.groupBy->isEmpty(),|'',| ' ' + $format.separator + 'group by '+$s.groupBy->processGroupByColumns($dbConfig, $format->indent(), true, $extensions)->makeString(','))+ + if (eq($havingStr, ''), |'', | ' ' + $format.separator + 'having ' + $havingStr) + + if ($s.orderBy->isEmpty(),|'',| ' ' + $format.separator + 'order by '+ $s.orderBy->processOrderBy($dbConfig, $format->indent(), $config, $extensions)->makeString(','))+ + + processLimit($s, $dbConfig, $format, $extensions, [], processSliceOrDropForH2_SelectSQLQuery_1__Format_1__DbConfig_1__Extension_MANY__Any_1__String_1_); + ); +} + +/* +* ifs in column creation gets translated to cases and so should also be candidates for wrapping +*/ +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSelectColumnsH2( + s: RelationalOperationElement[*], + dbConfig: DbConfig[1], + format: Format[1], + conditionalExprAllowed: Boolean[1], + extensions: Extension[*] +): String[1] +{ + if($s->size() == 0, + |'*', + | $format.separator + $s->map(r| $r->match([ + a: Alias[1] | + let shouldWrapWithCase = $a.relationalElement->isBooleanOperation($extensions) && !$conditionalExprAllowed; + if($shouldWrapWithCase, + | 'case when (' + + $a.relationalElement->wrapH2Boolean($extensions)->processOperation($dbConfig, $format, ^GenerationState(generationSide = GenerationSide.Select, withinWhenClause = true), $extensions) + + ') then \'true\' else \'false\' end as ' + $a.name, + | $r->wrapH2Boolean($extensions)->processOperation($dbConfig, $format, ^GenerationState(generationSide = GenerationSide.Select, withinWhenClause = false), $extensions) + );, + + z: RelationalOperationElement[1] | $z->wrapH2Boolean($extensions)->processOperation($dbConfig, $format, ^GenerationState(generationSide = GenerationSide.Select, withinWhenClause = false), $extensions) + ]))->joinStrings(', ' + $format.separator); + ); +} + +/* +TODO: +- what to do with freemarker placeholder operations? They are also RelationalOpElements +*/ +// To be used to wrap filter conditions and their arguments to compare boolean to boolean +function <> meta::relational::functions::sqlQueryToString::debugPrint::wrapH2Boolean( + op: RelationalOperationElement[1], extensions: Extension[*] +): RelationalOperationElement[1] +{ + // Base case: at an equals sign whose children are case and a booleanExpr OR we are a solitary case node + // Tail recurse: if we are at oneOf ['and', 'or', 'not', 'group'] or a optional placeholder, then recurse left and right + $op->match([ + d: DynaFunction[1] | if(isCastableDyna($d), + | ^DynaFunction(name='castBoolean', parameters=[$d]), + | if(isEqualComparingCastableDynaAndBoolean($d), + | ^$d(parameters=$d.parameters->map(p| ^DynaFunction(name='castBoolean', parameters=[$p]))), + | if(atRecursibleOperation($d), + | ^$d(parameters=$d.parameters->map(p| $p->wrapH2Boolean($extensions))), + | $d + ) + ) + ), + // Casting alias is required for wrapping logic within projections + a: Alias[1] | ^$a(relationalElement = wrapH2Boolean($a.relationalElement, $extensions)), + f: FreeMarkerOperationHolder[1] | if($f.name->in(['optionalVarPlaceHolderOpSelector']), + | ^$f(parameters=$f.parameters->map(p| $p->wrapH2Boolean($extensions))), + | $f + ), + e: RelationalOperationElement[1] | $e + ]); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::isEqualComparingCastableDynaAndBoolean( + d: DynaFunction[1] +): Boolean[1] +{ + $d.name == 'equal' && $d.parameters->at(0)->isCastableDyna() && $d.parameters->at(1)->isBooleanExpr(); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::isCastableDyna( + op: RelationalOperationElement[1] +): Boolean[1] +{ + // if and case both get converted to case in SQL + $op->instanceOf(DynaFunction) && + (cast($op, @DynaFunction).name->in(['case', 'if'])) + && cast($op, @DynaFunction).parameters->slice(1,3)->filter(x| $x->isTrueFalseString())->isNotEmpty() +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::isTrueFalseString( + op: RelationalOperationElement[1] +): Boolean[1] +{ + $op->instanceOf(Literal) && cast($op, @Literal).value->match([ + s: String[1] | $s->toLower()->in(['false','true']), + o: Any[1] | false + ]) +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::atRecursibleOperation( + op: DynaFunction[1] +): Boolean[1] +{ + $op.name->in(['and', 'or', 'not', 'group', 'if']) +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::isBooleanExpr(op: RelationalOperationElement[1]): Boolean[1] +{ + $op->instanceOf(Literal) && cast($op, @Literal).value->match([ + b: Boolean[1] | true, + v: VarPlaceHolder[1] | $v.type == Boolean, + a: Any[1] | false + ]) +} + +// Required so that any join conditions also have correct wrapping +function <> meta::relational::functions::sqlQueryToString::debugPrint::processJoinTreeNodeH2( + joinTreeNode:RelationalTreeNode[1], + parent:TableAlias[0..1], + dbConfig : DbConfig[1], + format:Format[1], + joinOrder:JoinType[*], + extensions:Extension[*] +):String[1] +{ + let tableAlias = $joinTreeNode->match( + [ + r:RootJoinTreeNode[1] | $r.alias, + j:JoinTreeNode[1] | $j.join->otherTableFromAlias($parent->toOne()); + ] + ); + + let val = $joinTreeNode->match( + [ + r:RootJoinTreeNode[1] | + $tableAlias->toOne() + ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) + ->processOperation($dbConfig, $format->indent(), $extensions), + j:JoinTreeNode[1] | + if($j.joinType == JoinType.FULL_OUTER, + | + // This should have been converted earlier to avoid a FULL_OUTER reaching this point + fail($j.joinType->toOne()->toString() + ' join not supported in H2'); ''; + , + | + if($j.lateral == true, + | $dbConfig.lateralJoinProcessor($j, $dbConfig, $format, $extensions), + | $j.joinType->map(jt|$jt->meta::relational::functions::sqlQueryToString::default::processJoinType($dbConfig, $format, $extensions))->orElse('') + + $j.alias + ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + $j.join.operation->wrapH2Boolean($extensions)->processOperation($dbConfig, $format->indent(), ^Config(), $extensions) + ')'; + )), + a:Any[1] | '' + ] + ); + let children = if($joinOrder->isNotEmpty(), + |$joinTreeNode->children()->sortBy({node | if($node.joinType->isEmpty(), |2, |$joinOrder->indexOf($node.joinType->toOne()))}), + |$joinTreeNode->children()); + $children->map(child | processJoinTreeNodeH2($child, $tableAlias->cast(@TableAlias), $dbConfig, $format, $joinOrder, $extensions))->joinStrings($val, '', ''); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::processSliceOrDropForH2(s:SelectSQLQuery[1], format:Format[1], dbConfig: DbConfig[1], extensions:Extension[*], size:Any[1]):String[1] +{ + '%s offset %s rows'->format([$format.separator, $s.fromRow->toOne()->getValueForTake($format, $dbConfig, $extensions)]) + if ($size == -1, | '', | ' fetch next %s rows only'->format($size)); +} + + +/* Test: + "root".RG_FUNCTION_ID = 2 + and ((case when "root".MEMBERSHIP_SOURCE = 'CUSTOM' then 'true' else 'false' end = true and + case when "root".IS_CATCH_ALL_GROUP = 1 then 'true' else 'false' end = false) and "root".STATUS = 'A') + and "root".IN_Z <= DATE '8888-01-01' + and "root".OUT_Z > DATE '8888-01-01' +*/ + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testSomeAST_thenIsWrapped(): Boolean[1] +{ + let inner = ^DynaFunction(name='group', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='group', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='equal', parameters=[ + ^DynaFunction(name='if', parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='CUSTOM'), ^Literal(value='CUSTOM')]), + ^Literal(value='true'), + ^Literal(value='false') + ]), + ^Literal(value=true) + ]), + ^DynaFunction(name='equal', parameters=[ + ^DynaFunction(name='if', parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value=1), ^Literal(value=2)]), + ^Literal(value='true'), + ^Literal(value='false') + ]), + ^Literal(value=false) + ]) + ]) + ]), + ^DynaFunction(name='equal', parameters=[^Literal(value='A'), ^Literal(value='A')]) + ]) + ]); + + let op = ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value=2), ^Literal(value=2)]), + $inner + ]), + ^DynaFunction(name='lessThanEqual', parameters=[^Literal(value='8888-01-01'), ^Literal(value='8888-01-01')]) + ]), + ^DynaFunction(name='greaterThan', parameters=[^Literal(value='8888-01-02'), ^Literal(value='8888-01-01')]) + ]); + + let castedInner = ^DynaFunction(name='group', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='group', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='equal', parameters=[ + ^DynaFunction(name='castBoolean', parameters=[ + ^DynaFunction(name='if', parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='CUSTOM'), ^Literal(value='CUSTOM')]), + ^Literal(value='true'), + ^Literal(value='false') + ]) + ]), + ^DynaFunction(name='castBoolean', parameters=[^Literal(value=true)]) + ]), + ^DynaFunction(name='equal', parameters=[ + ^DynaFunction(name='castBoolean', parameters=[ + ^DynaFunction(name='if', parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value=1), ^Literal(value=2)]), + ^Literal(value='true'), + ^Literal(value='false') + ]) + ]), + ^DynaFunction(name='castBoolean', parameters=[^Literal(value=false)]) + ]) + ]) + ]), + ^DynaFunction(name='equal', parameters=[^Literal(value='A'), ^Literal(value='A')]) + ]) + ]); + + let expected = ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='and', parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value=2), ^Literal(value=2)]), + $castedInner + ]), + ^DynaFunction(name='lessThanEqual', parameters=[^Literal(value='8888-01-01'), ^Literal(value='8888-01-01')]) + ]), + ^DynaFunction(name='greaterThan', parameters=[^Literal(value='8888-01-02'), ^Literal(value='8888-01-01')]) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + print($wrappedOp); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenJustCase_thenIsWrapped(): Boolean[1] +{ + let op = ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ); + + let expected = ^DynaFunction( + name= 'castBoolean', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ) + ] + ); + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseNestedByAnd_thenIsWrapped(): Boolean[1] +{ + let op = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let expected = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name= 'castBoolean', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ) + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseEqualBooleanLit_thenBothWrapped(): Boolean[1] +{ + let op = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value=true) + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let expected = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name= 'castBoolean', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ) + ] + ), + ^DynaFunction(name= 'castBoolean', parameters=[^Literal(value=true)]) + ] + ), + ^DynaFunction(name='equal', parameters=[^Literal(value=1), ^Literal(value=1)]) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseEqualTrue_thenNoOp(): Boolean[1] +{ + let op = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value='true') + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let expected = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value='true') + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseEqualYesNoLit_thenNoOp(): Boolean[1] +{ + let op = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value='Y') + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let expected = ^DynaFunction(name='and', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value='Y') + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseNestedByNot_thenIsWrapped(): Boolean[1] +{ + let op = ^DynaFunction(name='not', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value=true) + ] + ) + ]); + + let expected = ^DynaFunction(name='not', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='castBoolean', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ) + ] + ), + ^DynaFunction(name= 'castBoolean', parameters=[^Literal(value=true)]) + ] + ) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseNestedByOr_thenIsWrapped(): Boolean[1] +{ + let op = ^DynaFunction(name='or', parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let expected = ^DynaFunction(name='or', parameters=[ + ^DynaFunction( + name= 'castBoolean', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ) + ] + ), + ^DynaFunction( + name='equal', + parameters=[^Literal(value=1), ^Literal(value=1)] + ) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); +} + +function <> meta::relational::functions::sqlQueryToString::debugPrint::testWhenCaseNestedByGroup_thenIsWrapped(): Boolean[1] +{ + let op = ^DynaFunction(name='group', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ), + ^Literal(value=true) + ] + ) + ]); + + let expected = ^DynaFunction(name='group', parameters=[ + ^DynaFunction( + name='equal', + parameters=[ + ^DynaFunction( + name='castBoolean', + parameters=[ + ^DynaFunction( + name='case', + parameters=[ + ^DynaFunction(name='equal', parameters=[^Literal(value='Y'), ^Literal(value='Y')]), + ^Literal(value='true'), + ^Literal(value='false') + ] + ) + ] + ), + ^DynaFunction(name='castBoolean', parameters=[^Literal(value=true)]) + ] + ) + ]); + + let wrappedOp = wrapH2Boolean($op, []); + assertEquals($expected, $wrappedOp); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure index acb738133cf..ec6535b5a33 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure @@ -707,8 +707,8 @@ function <> meta::relational::functions::sqlQueryToString::h2::v | if($j.lateral == true, | $dbConfig.lateralJoinProcessor($j, $dbConfig, $format, $extensions), - | - $j.joinType->map(jt|$jt->meta::relational::functions::sqlQueryToString::default::processJoinType($dbConfig, $format, $extensions))->orElse('') + | assert(!$j.join->instanceOf(AsOfJoin), |'AsOfJoins are not supported by H2!'); + $j.joinType->map(jt|$jt->meta::relational::functions::sqlQueryToString::default::processJoinType($dbConfig, $format, $extensions))->orElse('') + $j.alias ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure index ed8ef787ef0..46c65311699 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure @@ -423,6 +423,16 @@ function meta::relational::functions::sqlQueryToString::default::processJoinTree + ' ' + 'on (' + processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')'; } +function meta::relational::functions::sqlQueryToString::default::processJoin(j:JoinTreeNode[1], dbConfig : DbConfig[1], format:Format[1], extensions:Extension[*]):String[1] +{ + assert(!$j.join->instanceOf(AsOfJoin), |'AsOfJoins are not supported in the generic generator!'); + $j.joinType->map(jt|$jt->processJoinType($dbConfig, $format, $extensions))->orElse('') + + $j.alias + ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) + ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() + + ' ' + 'on (' + processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')'; +} + function meta::relational::functions::sqlQueryToString::default::processJoinTreeNode(joinTreeNode:RelationalTreeNode[1], parent:TableAlias[0..1], dbConfig : DbConfig[1], format:Format[1], joinOrder:JoinType[*], extensions:Extension[*]):String[1] { let tableAlias = $joinTreeNode->match( @@ -436,15 +446,11 @@ function meta::relational::functions::sqlQueryToString::default::processJoinTree r:RootJoinTreeNode[1] | $tableAlias->toOne() ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) - ->processOperation($dbConfig, $format->indent(), $extensions), + ->processOperation($dbConfig, $format->indent(), $extensions);, j:JoinTreeNode[1] | if($j.lateral == true, | $dbConfig.lateralJoinProcessor($j, $dbConfig, $format, $extensions), - | $j.joinType->map(jt|$jt->processJoinType($dbConfig, $format, $extensions))->orElse('') - + $j.alias - ->map(a|^$a(name = '"' + $a.name + '"')) //Not sure why this is necessary, but it's retained to keep the generated SQL the same as previously (and does no real harm) - ->toOne()->processOperation($dbConfig, $format->indent(), $extensions) + $format.separator() - + ' ' + 'on (' + processOperation($j.join.operation, $dbConfig, $format->indent(), ^Config(), $extensions) + ')'; + | $dbConfig.joinProcessor($j, $dbConfig, $format, $extensions) );, a:Any[1] | '' ] diff --git a/pom.xml b/pom.xml index a5a6158aa7d..7aa6d093319 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ - 5.19.0 + 5.20.0 0.25.6 12.46.0