diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index 7f7bdc87b7f..fc9ea7725e1 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -2078,8 +2078,6 @@ private Map buildDispatch() map.put("meta::pure::tds::distinct_TabularDataSet_1__TabularDataSet_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name())); map.put("meta::pure::tds::drop_TabularDataSet_1__Integer_1__TabularDataSet_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "Integer".equals(ps.get(1)._genericType()._rawType()._name()))); map.put("meta::pure::tds::extend_TabularDataSet_1__BasicColumnSpecification_MANY__TabularDataSet_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "BasicColumnSpecification".equals(ps.get(1)._genericType()._rawType()._name()))); - map.put("meta::pure::tds::extensions::columnValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", (List ps) -> ps.size() == 5 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(1)._genericType()._rawType()._name()) && matchOneMany(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "String".equals(ps.get(2)._genericType()._rawType()._name())) && matchOneMany(ps.get(3)._multiplicity()) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name())) && matchOneMany(ps.get(4)._multiplicity()) && ("Nil".equals(ps.get(4)._genericType()._rawType()._name()) || "String".equals(ps.get(4)._genericType()._rawType()._name()))); - map.put("meta::pure::tds::extensions::columnValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", (List ps) -> ps.size() == 4 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(1)._genericType()._rawType()._name()) && matchOneMany(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "String".equals(ps.get(2)._genericType()._rawType()._name())) && matchOneMany(ps.get(3)._multiplicity()) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name()))); map.put("meta::pure::tds::filter_TabularDataSet_1__Function_1__TabularDataSet_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && Sets.immutable.with("Nil", "TabularDataSet", "TabularDataSetImplementation", "TableTDS").contains(ps.get(0)._genericType()._rawType()._name()) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> isOne(ft._returnMultiplicity()) && ("Nil".equals(ft._returnType()._rawType()._name()) || "Boolean".equals(ft._returnType()._rawType()._name())) && check(ft._parameters().toList(), (List nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity()) && Sets.immutable.with("TDSRow", "Any").contains(nps.get(0)._genericType()._rawType()._name()))))); map.put("meta::pure::tds::groupByWithWindowSubset_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__String_MANY__String_MANY__TabularDataSet_1_", (List ps) -> ps.size() == 6 && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> check(ft._parameters().toList(), (List nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity())))) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggregateValue".equals(ps.get(2)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(4)._genericType()._rawType()._name()) || "String".equals(ps.get(4)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(5)._genericType()._rawType()._name()) || "String".equals(ps.get(5)._genericType()._rawType()._name()))); map.put("meta::pure::tds::groupBy_K_MANY__Function_MANY__AggregateValue_MANY__String_MANY__TabularDataSet_1_", (List ps) -> ps.size() == 4 && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || check(funcType(ps.get(1)._genericType()), (FunctionType ft) -> check(ft._parameters().toList(), (List nps) -> nps.size() == 1 && isOne(nps.get(0)._multiplicity())))) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "AggregateValue".equals(ps.get(2)._genericType()._rawType()._name())) && ("Nil".equals(ps.get(3)._genericType()._rawType()._name()) || "String".equals(ps.get(3)._genericType()._rawType()._name()))); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java index 51eadfd2a4d..1dec37f6acd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java @@ -525,6 +525,12 @@ public List>> handlers.m(handlers.m(handlers.h("meta::pure::tds::extensions::rowValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 5)), handlers.m(handlers.h("meta::pure::tds::extensions::rowValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 4))) ), + new FunctionExpressionBuilderRegistrationInfo(null, + handlers.m(handlers.h("meta::pure::tds::extensions::zScore_TabularDataSet_1__String_MANY__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res(ps.get(0)._genericType(), "one"), ps -> ps.size() == 4)) + ), + new FunctionExpressionBuilderRegistrationInfo(null, + handlers.m(handlers.h("meta::pure::tds::extensions::iqrClassify_TabularDataSet_1__String_MANY__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res(ps.get(0)._genericType(), "one"), ps -> ps.size() == 4)) + ), new FunctionExpressionBuilderRegistrationInfo(Lists.mutable.with(3), handlers.m(handlers.h("meta::pure::functions::asserts::assertEq_Any_1__Any_1__Function_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 3 && !handlers.typeOne(ps.get(2), "String"))) ) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestRelationalFunctionHandler.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestRelationalFunctionHandler.java index bddc3c64d81..43d26492fac 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestRelationalFunctionHandler.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestRelationalFunctionHandler.java @@ -94,4 +94,79 @@ public void testRowValueDifferenceCompile() " }:meta::pure::tds::TabularDataSet[1];" + "}"); } + + @Test + public void testCompile_zScore() + { + // Faulty stereotype + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" + + "function xx::myFunc() : Boolean[1] {\n" + + " ^Pair(first = 'student1', second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])->meta::pure::tds::extensions::zScore([], ['score'], ['score zScore']);\n" + + " true;\n" + + "}\n"); + } + + @Test + public void testCompile_iqrClassify() + { + // Faulty stereotype + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" + + "function xx::myFunc() : Boolean[1] {\n" + + " ^Pair(first = 'student1', second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])->meta::pure::tds::extensions::iqrClassify([], 'score', 'irq_classification');\n" + + " true;\n" + + "}\n"); + } + + @Test + public void testCompile_columnValueDifference1() + { + // Faulty stereotype + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" + + "function xx::myFunc() : Boolean[1] {\n" + + " ^Pair(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" + + " ->meta::pure::tds::extensions::columnValueDifference(\n" + + " ^Pair(first = 1, second = 6)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')]),\n" + + " ['student'], ['score']);\n" + + " true;\n" + + "}\n"); + } + + @Test + public void testCompile_columnValueDifference2() + { + // Faulty stereotype + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" + + "function xx::myFunc() : Boolean[1] {\n" + + " ^Pair(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" + + " ->meta::pure::tds::extensions::columnValueDifference(\n" + + " ^Pair(first = 1, second = 6)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')]),\n" + + " ['student'], ['score'], ['score diff']);\n" + + " true;\n" + + "}\n"); + } + + @Test + public void testCompile_extendWithDigestOnColumns1() + { + // Faulty stereotype + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" + + "function xx::myFunc() : Boolean[1] {\n" + + " ^Pair(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" + + " ->meta::pure::tds::extensions::extendWithDigestOnColumns('digest');\n" + + " true;\n" + + "}\n"); + } + + @Test + public void testCompile_extendWithDigestOnColumns2() + { + // Faulty stereotype + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test("###Pure\n" + + "function xx::myFunc() : Boolean[1] {\n" + + " ^Pair(first = 1, second = 5)->project([col(x|$x.first, 'student'), col(x|$x.second, 'score')])\n" + + " ->meta::pure::tds::extensions::extendWithDigestOnColumns(['student'], meta::pure::functions::hash::HashType.MD5, 'digest');\n" + + " true;\n" + + "}\n"); + } + }